diff --git a/support/scripts/pkg-stats b/support/scripts/pkg-stats index ae1a9aa5e4..f67be0063f 100755 --- a/support/scripts/pkg-stats +++ b/support/scripts/pkg-stats @@ -27,12 +27,16 @@ import re import subprocess import json import sys +import time +import gzip +import xml.etree.ElementTree +import requests brpath = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "..")) sys.path.append(os.path.join(brpath, "utils")) from getdeveloperlib import parse_developers # noqa: E402 -from cpedb import CPEDB # noqa: E402 +from cpedb import CPEDB_URL # noqa: E402 INFRA_RE = re.compile(r"\$\(eval \$\(([a-z-]*)-package\)\)") URL_RE = re.compile(r"\s*https?://\S*\s*$") @@ -624,12 +628,41 @@ def check_package_cves(nvd_path, packages): def check_package_cpes(nvd_path, packages): - cpedb = CPEDB(nvd_path) - cpedb.get_xml_dict() + class CpeXmlParser: + cpes = [] + + def start(self, tag, attrib): + if tag == "{http://scap.nist.gov/schema/cpe-extension/2.3}cpe23-item": + self.cpes.append(attrib['name']) + + def close(self): + return self.cpes + + print("CPE: Setting up NIST dictionary") + if not os.path.exists(os.path.join(nvd_path, "cpe")): + os.makedirs(os.path.join(nvd_path, "cpe")) + + cpe_dict_local = os.path.join(nvd_path, "cpe", os.path.basename(CPEDB_URL)) + if not os.path.exists(cpe_dict_local) or os.stat(cpe_dict_local).st_mtime < time.time() - 86400: + print("CPE: Fetching xml manifest from [" + CPEDB_URL + "]") + cpe_dict = requests.get(CPEDB_URL) + open(cpe_dict_local, "wb").write(cpe_dict.content) + + print("CPE: Unzipping xml manifest...") + nist_cpe_file = gzip.GzipFile(fileobj=open(cpe_dict_local, 'rb')) + + parser = xml.etree.ElementTree.XMLParser(target=CpeXmlParser()) + while True: + c = nist_cpe_file.read(1024*1024) + if not c: + break + parser.feed(c) + cpes = parser.close() + for p in packages: if not p.cpeid: continue - if cpedb.find(p.cpeid): + if p.cpeid in cpes: p.status['cpe'] = ("ok", "verified CPE identifier") else: p.status['cpe'] = ("error", "CPE version unknown in CPE database")