sync with latest version

update_checker.py

 #!/usr/bin/python3
 import json
 import netaddr
+import gzip
+import re
+import os
+import sys
+import pprint
+import requests
+MAC_URL = 'http://macvendors.co/api/%s'
 
-branch_to_be_checked = ["nightly"]
-#release_to_be_watched = "1.6+2019-09-14-g.9f827678-s.5904ca9"
-release_to_be_watched = "1.7+2019-09-15-g.43e01839-s.300ff17"
+def getHardwareModelFromEntry(d):
+    hardware_model = "UNKNOWN"
+    if d != None:
+        if "hardware" in d:
+            if "model" in d["hardware"]:
+                hardware_model = d["hardware"]["model"]
+    return hardware_model
+
+def ipv62mac(ipv6):
+    # remove subnet info if given
+    subnetIndex = ipv6.find("/")
+    if subnetIndex != -1:
+        ipv6 = ipv6[:subnetIndex]
+
+    ipv6Parts = ipv6.split(":")
+    macParts = []
+    for ipv6Part in ipv6Parts[-4:]:
+        while len(ipv6Part) < 4:
+            ipv6Part = "0" + ipv6Part
+        macParts.append(ipv6Part[:2])
+        macParts.append(ipv6Part[-2:])
 
-def getNodesDownloadedUpdate(access,segments):
-    downloaded = []
-    for line in access.split("\n"):
-        ipv6 = line.split(" ")[0]
-        for segment in segments:
-            for branch in branch_to_be_checked:
-                if ("Wget" in line or "Gluon Autoupdater (using libuclient)" in line ) and branch in line and ipv6.startswith("fd21:b4dc:4b%02i"%(segment)) and "gluon-ffs" in line and release_to_be_watched in line:
-                    #print(line)
-                    if netaddr.IPAddress(ipv6) not in downloaded:
-                        downloaded.append(netaddr.IPAddress(ipv6))
-    return downloaded
+    # modify parts to match MAC value
+    macParts[0] = "%02x" % (int(macParts[0], 16) ^ 2)
+    del macParts[4]
+    del macParts[3]
 
+    return ":".join(macParts)
 
-access = open("/var/log/nginx/access.log.1").read()
+class LogRecord:
+    def __init__(self,record):
+        record = record.replace("%2B","+")
+        regex = r'(fd21.*) - - \[(.*) "GET /gluon/(.*)/sysupgrade/gluon-ffs-([0-9]\.[0-9]\+[0-9]+-[0-9]+-[0-9]+-g\.[0-9a-z]+-s\.[0-9a-z]+)-(.*)\ HTTP/1.1" ([0-9]+) [0-9]+ "-" "(.*)"'
+        result = re.search(regex,record)
+        try:
+            groups = result.groups()
+        except:
+            print(record)
+            print("error parsing record")
+            sys.exit(1)
+        self.ipv6 = groups[0]
+        self.date = groups[1]
+        self.branch = groups[2]
+        self.release = groups[3]
+        self.model = groups[4]
+        self.status = groups[5]
+        self.agent = groups[6]
+        self.segment = int(self.ipv6.split(":")[2][2:])
+
+try:
+    allFirmwareDownloads = json.load(open("firmwareDownloads.json","r"))
+except:
+    allFirmwareDownloads = []
+
+access = ""
+
+try:
+    #for i in range(2,15):
+    #   access += gzip.open("/var/log/nginx/access.log.%i.gz"%(i),"rt").read()
+    access += open("/var/log/nginx/access.log.1").read()
     access += open("/var/log/nginx/access.log").read()
-data = json.load(open("/home/www/html/netinfo/json/nodesdb.json"))
 
+except:
+    print("not using /var/log/nginx/access.log")
+try:
+    access += open("access.log").read()
+except:
+    pass
+
+allDownloads = access.strip().split("\n")
+allNewFirmwareDownloads = [l for l in allDownloads if l.startswith("fd21:") and "sysupgrade" in l and "manifest" not in l]
 
+for f in allNewFirmwareDownloads:
+    if f not in allFirmwareDownloads:
+        allFirmwareDownloads.append(f)
 
-countWaiting = 0
-countUpdated = 0
-countOfflineAfterUpdate = 0
-segmentsWatched = range(1,33)
-downloads = getNodesDownloadedUpdate(access,segmentsWatched)
-updated = []
-for mac in data:
+json.dump(allFirmwareDownloads,open("firmwareDownloads.json","w"),sort_keys=True, indent=4, separators=(',', ': '))
+
+
+nodesdbFilename = "/home/www/html/netinfo/json/nodesdb.json"
+if not os.path.isfile(nodesdbFilename):
+    nodesdbFilename = "nodesdb.json"
+data = json.load(open(nodesdbFilename))
+
+for download in allFirmwareDownloads:
+    r = LogRecord(download)
+    mac = ipv62mac(r.ipv6)
+
+    if mac in data:
         d = data[mac]
-    software = d["software"]
-    branch = d["software"]["autoupdater"]["branch"]
-    autoupdater_enabled= d["software"]["autoupdater"]["enabled"]
-    release = d["software"]["firmware"]["release"]
-    hardware_model = "UNKNOWN"
-    if "model" in d["hardware"]:
-        hardware_model = d["hardware"]["model"]
-    if "addresses" in d["network"]:
-        addresses = d["network"]["addresses"]
-        for a in addresses:
-            if a.startswith("fd21:b4dc:4b"):
-                ipv6 = netaddr.IPAddress(a)
-    else:
-        ipv6 = None
-    if "segment" in d:
-        segment = d["segment"]
-    else:
-        segment = 0
-    hasDownloaded = ""
-    if ipv6 in downloads:
-        hasDownloaded = "UPDATING"
-    online = d["status"]
-    for segmentWatched in segmentsWatched:
-        if segment == segmentWatched and release >= release_to_be_watched :
-            updated.append(ipv6)
-        if segment == segmentWatched and branch in branch_to_be_checked and autoupdater_enabled == True and release == release_to_be_watched :
-            countUpdated+=1
-        if segment == segmentWatched and online=="online" and branch in branch_to_be_checked and autoupdater_enabled == True:
-            if release >= release_to_be_watched:
-                if ipv6 not in downloads:
-                    print ("Not expected %s %s %s %s %2i %s"%(mac,online,branch,release,segment,ipv6)) 
+        currentRelease = d["software"]["firmware"]["release"]
+        status = d["status"]
+        hostname = d["hostname"]
+        hardware_model = getHardwareModelFromEntry(d)
+        if currentRelease < r.release:
+            print("%s (%s) %s -> %s Segment %i %s status %s @ %s" % (mac, hostname, currentRelease, r.release, r.segment, hardware_model, status, r.date))
     else:
-                print ("%s %s %s %s %2i %s %s %s"%(mac,online,branch,release,segment,ipv6,hardware_model,hasDownloaded)) 
-                countWaiting+=1
-        if segment == segmentWatched and online!="online" and release != release_to_be_watched and branch in branch_to_be_checked and autoupdater_enabled == True and ipv6 in downloads:
-            print ("!!!! %s %s %s %s %2i %s %s"%(mac,online,branch,release,segment,ipv6,hasDownloaded)) 
-            countOfflineAfterUpdate+=1
-
-print("Waiting: %i"%countWaiting)
-print("Updated: %i"%countUpdated)
-print("Offline after Update: %i"%countOfflineAfterUpdate)
-print("Downloads: %i"%(len(downloads)))
-print("Sum: %i"%(countWaiting+countUpdated))
-print("Nodes with upgrade in progress:")
-for d in downloads:
-    if d not in updated:
-        print(d)
+        #request = requests.get(MAC_URL % mac)
+        #pprint.pprint(request.json())
+        print("%s %s with agent %s seems not to be a node @ %s"%(r.ipv6,mac,r.agent,r.date))