update_checker.py

#!/usr/bin/python3
import json
import netaddr
import gzip
import re
import os
import sys
import pprint
import requests
import argparse
import datetime
MAC_URL = 'http://macvendors.co/api/%s'

ap = argparse.ArgumentParser()
ap.add_argument("--raw", type=argparse.FileType("r", encoding="utf-8"), required=True)
args = ap.parse_args()

def getHardwareModelFromEntry(d):
    try: 
        hardware_model = d["nodeinfo"]["hardware"]["model"]
    except KeyError:
        hardware_model = "UNKNOWN"

    return hardware_model

def ipv62mac(ipv6):
    # remove subnet info if given
    subnetIndex = ipv6.find("/")
    if subnetIndex != -1:
        ipv6 = ipv6[:subnetIndex]

    ipv6Parts = ipv6.split(":")
    macParts = []
    for ipv6Part in ipv6Parts[-4:]:
        while len(ipv6Part) < 4:
            ipv6Part = "0" + ipv6Part
        macParts.append(ipv6Part[:2])
        macParts.append(ipv6Part[-2:])

    # modify parts to match MAC value
    macParts[0] = "%02x" % (int(macParts[0], 16) ^ 2)
    del macParts[4]
    del macParts[3]

    return ":".join(macParts)

class LogRecordParseError(Exception):
    def __init__(self, record_line):
        self.record_line = record_line

class LogRecord:
    def __init__(self,record):
        record = record.replace("%2B","+")
        regex = r'(fd21.*) - - \[(.*) "GET /gluon/(.*)/sysupgrade/gluon-ffs-([0-9]\.[0-9]\+[0-9]+-[0-9]+-[0-9]+-g\.[0-9a-z]+-s\.[0-9a-z]+)-(.*)\ HTTP/1.1" ([0-9]+) [0-9]+ "-" "(.*)"'
        result = re.search(regex,record)
        try:
            groups = result.groups()
        except:
            raise LogRecordParseError(record)
        self.ipv6 = groups[0]
        self.date = groups[1]
        self.branch = groups[2]
        self.release = groups[3]
        self.model = groups[4]
        self.status = groups[5]
        self.agent = groups[6]
        self.segment = int(self.ipv6.split(":")[2][2:])

try:
    allFirmwareDownloads = json.load(open("firmwareDownloads.json","r"))
except:
    allFirmwareDownloads = []

access = ""

try:
    #for i in range(2,15):
    #   access += gzip.open("/var/log/nginx/access.log.%i.gz"%(i),"rt").read()
    access += open("/var/log/nginx/access.log.1").read()
    access += open("/var/log/nginx/access.log").read()

except:
    print("not using /var/log/nginx/access.log")
try:
    access += open("access.log").read()
except:
    pass

allDownloads = access.strip().split("\n")
allNewFirmwareDownloads = [l for l in allDownloads if l.startswith("fd21:") and "sysupgrade" in l and "manifest" not in l]

for f in allNewFirmwareDownloads:
    if f not in allFirmwareDownloads:
        allFirmwareDownloads.append(f)

json.dump(allFirmwareDownloads,open("firmwareDownloads.json","w"),sort_keys=True, indent=4, separators=(',', ': '))


data = json.load(args.raw)

for download in allFirmwareDownloads:
    try:
        r = LogRecord(download)
    except LogRecordParseError as e:
        print("error parsing line, skipping: {}".format(e.record_line))
        continue

    mac = ipv62mac(r.ipv6)

    nodes_with_mac = [node for node in data["nodes"] if node["nodeinfo"]["network"]["mac"] == mac]

    if len(nodes_with_mac) == 1:
        d = nodes_with_mac[0]
        currentRelease = d["nodeinfo"]["software"]["firmware"]["release"]
        status = d["online"]
        hostname = d["nodeinfo"]["hostname"]
        hardware_model = getHardwareModelFromEntry(d)
        if currentRelease < r.release:
            print("%s (%s) %s -> %s Segment %i %s status %s @ %s" % (mac, hostname, currentRelease, r.release, r.segment, hardware_model, status, r.date))
    else:
        #request = requests.get(MAC_URL % mac)
        #pprint.pprint(request.json())
        # raw.json only contains data from 14 days - do not print message if we encounter older log entries
        download_ts = datetime.datetime.strptime(r.date, "%d/%b/%Y:%H:%M:%S %z]")
        if download_ts - datetime.datetime.now(tz=datetime.timezone.utc) > datetime.timedelta(days=14):
            print("%s %s with agent %s seems not to be a node @ %s"%(r.ipv6,mac,r.agent,r.date))