From d7c25b25a0422b1d993d188b98de71ed815a8c11 Mon Sep 17 00:00:00 2001
From: Leonard Penzer <leonard@penzer.de>
Date: Mon, 6 Jan 2020 09:42:17 +0000
Subject: [PATCH] sync with latest version
---
update_checker.py | 181 +++++++++++++++++++++++++++-------------------
1 file changed, 105 insertions(+), 76 deletions(-)
diff --git a/update_checker.py b/update_checker.py
index c95396c..ce973c9 100755
--- a/update_checker.py
+++ b/update_checker.py
@@ -1,82 +1,111 @@
#!/usr/bin/python3
import json
import netaddr
+import gzip
+import re
+import os
+import sys
+import pprint
+import requests
+MAC_URL = 'http://macvendors.co/api/%s'
-branch_to_be_checked = ["nightly"]
-#release_to_be_watched = "1.6+2019-09-14-g.9f827678-s.5904ca9"
-release_to_be_watched = "1.7+2019-09-15-g.43e01839-s.300ff17"
-
-def getNodesDownloadedUpdate(access,segments):
- downloaded = []
- for line in access.split("\n"):
- ipv6 = line.split(" ")[0]
- for segment in segments:
- for branch in branch_to_be_checked:
- if ("Wget" in line or "Gluon Autoupdater (using libuclient)" in line ) and branch in line and ipv6.startswith("fd21:b4dc:4b%02i"%(segment)) and "gluon-ffs" in line and release_to_be_watched in line:
- #print(line)
- if netaddr.IPAddress(ipv6) not in downloaded:
- downloaded.append(netaddr.IPAddress(ipv6))
- return downloaded
-
-
-access = open("/var/log/nginx/access.log.1").read()
-access += open("/var/log/nginx/access.log").read()
-data = json.load(open("/home/www/html/netinfo/json/nodesdb.json"))
-
-
-
-countWaiting = 0
-countUpdated = 0
-countOfflineAfterUpdate = 0
-segmentsWatched = range(1,33)
-downloads = getNodesDownloadedUpdate(access,segmentsWatched)
-updated = []
-for mac in data:
- d = data[mac]
- software = d["software"]
- branch = d["software"]["autoupdater"]["branch"]
- autoupdater_enabled= d["software"]["autoupdater"]["enabled"]
- release = d["software"]["firmware"]["release"]
+def getHardwareModelFromEntry(d):
hardware_model = "UNKNOWN"
- if "model" in d["hardware"]:
- hardware_model = d["hardware"]["model"]
- if "addresses" in d["network"]:
- addresses = d["network"]["addresses"]
- for a in addresses:
- if a.startswith("fd21:b4dc:4b"):
- ipv6 = netaddr.IPAddress(a)
- else:
- ipv6 = None
- if "segment" in d:
- segment = d["segment"]
+ if d != None:
+ if "hardware" in d:
+ if "model" in d["hardware"]:
+ hardware_model = d["hardware"]["model"]
+ return hardware_model
+
+def ipv62mac(ipv6):
+ # remove subnet info if given
+ subnetIndex = ipv6.find("/")
+ if subnetIndex != -1:
+ ipv6 = ipv6[:subnetIndex]
+
+ ipv6Parts = ipv6.split(":")
+ macParts = []
+ for ipv6Part in ipv6Parts[-4:]:
+ while len(ipv6Part) < 4:
+ ipv6Part = "0" + ipv6Part
+ macParts.append(ipv6Part[:2])
+ macParts.append(ipv6Part[-2:])
+
+ # modify parts to match MAC value
+ macParts[0] = "%02x" % (int(macParts[0], 16) ^ 2)
+ del macParts[4]
+ del macParts[3]
+
+ return ":".join(macParts)
+
+class LogRecord:
+ def __init__(self,record):
+ record = record.replace("%2B","+")
+ regex = r'(fd21.*) - - \[(.*) "GET /gluon/(.*)/sysupgrade/gluon-ffs-([0-9]\.[0-9]\+[0-9]+-[0-9]+-[0-9]+-g\.[0-9a-z]+-s\.[0-9a-z]+)-(.*)\ HTTP/1.1" ([0-9]+) [0-9]+ "-" "(.*)"'
+ result = re.search(regex,record)
+ try:
+ groups = result.groups()
+ except:
+ print(record)
+ print("error parsing record")
+ sys.exit(1)
+ self.ipv6 = groups[0]
+ self.date = groups[1]
+ self.branch = groups[2]
+ self.release = groups[3]
+ self.model = groups[4]
+ self.status = groups[5]
+ self.agent = groups[6]
+ self.segment = int(self.ipv6.split(":")[2][2:])
+
+try:
+ allFirmwareDownloads = json.load(open("firmwareDownloads.json","r"))
+except:
+ allFirmwareDownloads = []
+
+access = ""
+
+try:
+ #for i in range(2,15):
+ # access += gzip.open("/var/log/nginx/access.log.%i.gz"%(i),"rt").read()
+ access += open("/var/log/nginx/access.log.1").read()
+ access += open("/var/log/nginx/access.log").read()
+
+except:
+ print("not using /var/log/nginx/access.log")
+try:
+ access += open("access.log").read()
+except:
+ pass
+
+allDownloads = access.strip().split("\n")
+allNewFirmwareDownloads = [l for l in allDownloads if l.startswith("fd21:") and "sysupgrade" in l and "manifest" not in l]
+
+for f in allNewFirmwareDownloads:
+ if f not in allFirmwareDownloads:
+ allFirmwareDownloads.append(f)
+
+json.dump(allFirmwareDownloads,open("firmwareDownloads.json","w"),sort_keys=True, indent=4, separators=(',', ': '))
+
+
+nodesdbFilename = "/home/www/html/netinfo/json/nodesdb.json"
+if not os.path.isfile(nodesdbFilename):
+ nodesdbFilename = "nodesdb.json"
+data = json.load(open(nodesdbFilename))
+
+for download in allFirmwareDownloads:
+ r = LogRecord(download)
+ mac = ipv62mac(r.ipv6)
+
+ if mac in data:
+ d = data[mac]
+ currentRelease = d["software"]["firmware"]["release"]
+ status = d["status"]
+ hostname = d["hostname"]
+ hardware_model = getHardwareModelFromEntry(d)
+ if currentRelease < r.release:
+ print("%s (%s) %s -> %s Segment %i %s status %s @ %s" % (mac, hostname, currentRelease, r.release, r.segment, hardware_model, status, r.date))
else:
- segment = 0
- hasDownloaded = ""
- if ipv6 in downloads:
- hasDownloaded = "UPDATING"
- online = d["status"]
- for segmentWatched in segmentsWatched:
- if segment == segmentWatched and release >= release_to_be_watched :
- updated.append(ipv6)
- if segment == segmentWatched and branch in branch_to_be_checked and autoupdater_enabled == True and release == release_to_be_watched :
- countUpdated+=1
- if segment == segmentWatched and online=="online" and branch in branch_to_be_checked and autoupdater_enabled == True:
- if release >= release_to_be_watched:
- if ipv6 not in downloads:
- print ("Not expected %s %s %s %s %2i %s"%(mac,online,branch,release,segment,ipv6))
- else:
- print ("%s %s %s %s %2i %s %s %s"%(mac,online,branch,release,segment,ipv6,hardware_model,hasDownloaded))
- countWaiting+=1
- if segment == segmentWatched and online!="online" and release != release_to_be_watched and branch in branch_to_be_checked and autoupdater_enabled == True and ipv6 in downloads:
- print ("!!!! %s %s %s %s %2i %s %s"%(mac,online,branch,release,segment,ipv6,hasDownloaded))
- countOfflineAfterUpdate+=1
-
-print("Waiting: %i"%countWaiting)
-print("Updated: %i"%countUpdated)
-print("Offline after Update: %i"%countOfflineAfterUpdate)
-print("Downloads: %i"%(len(downloads)))
-print("Sum: %i"%(countWaiting+countUpdated))
-print("Nodes with upgrade in progress:")
-for d in downloads:
- if d not in updated:
- print(d)
+ #request = requests.get(MAC_URL % mac)
+ #pprint.pprint(request.json())
+ print("%s %s with agent %s seems not to be a node @ %s"%(r.ipv6,mac,r.agent,r.date))
--
GitLab