support/scripts/pkg-stats: replace with new Python version
Rename pkg-stats-new to pkg-stats. Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
This commit is contained in:
Thomas Petazzoni
parent
338c7b5e62
commit
f955a4eac0
@ -1,4 +1,4 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env python
|
||||||
|
|
||||||
# Copyright (C) 2009 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
|
# Copyright (C) 2009 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
|
||||||
#
|
#
|
||||||
@ -16,16 +16,275 @@
|
|||||||
# along with this program; if not, write to the Free Software
|
# along with this program; if not, write to the Free Software
|
||||||
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||||
|
|
||||||
# This script generates an HTML file that contains a report about all
|
import argparse
|
||||||
# Buildroot packages, their usage of the different package
|
import datetime
|
||||||
# infrastructure and possible cleanup actions
|
import fnmatch
|
||||||
#
|
import os
|
||||||
# Run the script from the Buildroot toplevel directory:
|
from collections import defaultdict
|
||||||
#
|
import re
|
||||||
# ./support/scripts/pkg-stats > /tmp/pkg.html
|
import subprocess
|
||||||
#
|
import sys
|
||||||
|
|
||||||
echo "<head>
|
INFRA_RE = re.compile("\$\(eval \$\(([a-z-]*)-package\)\)")
|
||||||
|
|
||||||
|
|
||||||
|
class Package:
|
||||||
|
all_licenses = list()
|
||||||
|
all_license_files = list()
|
||||||
|
all_versions = dict()
|
||||||
|
|
||||||
|
def __init__(self, name, path):
|
||||||
|
self.name = name
|
||||||
|
self.path = path
|
||||||
|
self.infras = None
|
||||||
|
self.has_license = False
|
||||||
|
self.has_license_files = False
|
||||||
|
self.has_hash = False
|
||||||
|
self.patch_count = 0
|
||||||
|
self.warnings = 0
|
||||||
|
self.current_version = None
|
||||||
|
|
||||||
|
def pkgvar(self):
|
||||||
|
return self.name.upper().replace("-", "_")
|
||||||
|
|
||||||
|
def set_infra(self):
|
||||||
|
"""
|
||||||
|
Fills in the .infras field
|
||||||
|
"""
|
||||||
|
self.infras = list()
|
||||||
|
with open(self.path, 'r') as f:
|
||||||
|
lines = f.readlines()
|
||||||
|
for l in lines:
|
||||||
|
match = INFRA_RE.match(l)
|
||||||
|
if not match:
|
||||||
|
continue
|
||||||
|
infra = match.group(1)
|
||||||
|
if infra.startswith("host-"):
|
||||||
|
self.infras.append(("host", infra[5:]))
|
||||||
|
else:
|
||||||
|
self.infras.append(("target", infra))
|
||||||
|
|
||||||
|
def set_license(self):
|
||||||
|
"""
|
||||||
|
Fills in the .has_license and .has_license_files fields
|
||||||
|
"""
|
||||||
|
var = self.pkgvar()
|
||||||
|
if var in self.all_licenses:
|
||||||
|
self.has_license = True
|
||||||
|
if var in self.all_license_files:
|
||||||
|
self.has_license_files = True
|
||||||
|
|
||||||
|
def set_hash_info(self):
|
||||||
|
"""
|
||||||
|
Fills in the .has_hash field
|
||||||
|
"""
|
||||||
|
hashpath = self.path.replace(".mk", ".hash")
|
||||||
|
self.has_hash = os.path.exists(hashpath)
|
||||||
|
|
||||||
|
def set_patch_count(self):
|
||||||
|
"""
|
||||||
|
Fills in the .patch_count field
|
||||||
|
"""
|
||||||
|
self.patch_count = 0
|
||||||
|
pkgdir = os.path.dirname(self.path)
|
||||||
|
for subdir, _, _ in os.walk(pkgdir):
|
||||||
|
self.patch_count += len(fnmatch.filter(os.listdir(subdir), '*.patch'))
|
||||||
|
|
||||||
|
def set_current_version(self):
|
||||||
|
"""
|
||||||
|
Fills in the .current_version field
|
||||||
|
"""
|
||||||
|
var = self.pkgvar()
|
||||||
|
if var in self.all_versions:
|
||||||
|
self.current_version = self.all_versions[var]
|
||||||
|
|
||||||
|
def set_check_package_warnings(self):
|
||||||
|
"""
|
||||||
|
Fills in the .warnings field
|
||||||
|
"""
|
||||||
|
cmd = ["./utils/check-package"]
|
||||||
|
pkgdir = os.path.dirname(self.path)
|
||||||
|
for root, dirs, files in os.walk(pkgdir):
|
||||||
|
for f in files:
|
||||||
|
if f.endswith(".mk") or f.endswith(".hash") or f == "Config.in" or f == "Config.in.host":
|
||||||
|
cmd.append(os.path.join(root, f))
|
||||||
|
o = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[1]
|
||||||
|
lines = o.splitlines()
|
||||||
|
for line in lines:
|
||||||
|
m = re.match("^([0-9]*) warnings generated", line)
|
||||||
|
if m:
|
||||||
|
self.warnings = int(m.group(1))
|
||||||
|
return
|
||||||
|
|
||||||
|
def __eq__(self, other):
|
||||||
|
return self.path == other.path
|
||||||
|
|
||||||
|
def __lt__(self, other):
|
||||||
|
return self.path < other.path
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return "%s (path='%s', license='%s', license_files='%s', hash='%s', patches=%d)" % \
|
||||||
|
(self.name, self.path, self.has_license, self.has_license_files, self.has_hash, self.patch_count)
|
||||||
|
|
||||||
|
|
||||||
|
def get_pkglist(npackages, package_list):
|
||||||
|
"""
|
||||||
|
Builds the list of Buildroot packages, returning a list of Package
|
||||||
|
objects. Only the .name and .path fields of the Package object are
|
||||||
|
initialized.
|
||||||
|
|
||||||
|
npackages: limit to N packages
|
||||||
|
package_list: limit to those packages in this list
|
||||||
|
"""
|
||||||
|
WALK_USEFUL_SUBDIRS = ["boot", "linux", "package", "toolchain"]
|
||||||
|
WALK_EXCLUDES = ["boot/common.mk",
|
||||||
|
"linux/linux-ext-.*.mk",
|
||||||
|
"package/freescale-imx/freescale-imx.mk",
|
||||||
|
"package/gcc/gcc.mk",
|
||||||
|
"package/gstreamer/gstreamer.mk",
|
||||||
|
"package/gstreamer1/gstreamer1.mk",
|
||||||
|
"package/gtk2-themes/gtk2-themes.mk",
|
||||||
|
"package/matchbox/matchbox.mk",
|
||||||
|
"package/opengl/opengl.mk",
|
||||||
|
"package/qt5/qt5.mk",
|
||||||
|
"package/x11r7/x11r7.mk",
|
||||||
|
"package/doc-asciidoc.mk",
|
||||||
|
"package/pkg-.*.mk",
|
||||||
|
"package/nvidia-tegra23/nvidia-tegra23.mk",
|
||||||
|
"toolchain/toolchain-external/pkg-toolchain-external.mk",
|
||||||
|
"toolchain/toolchain-external/toolchain-external.mk",
|
||||||
|
"toolchain/toolchain.mk",
|
||||||
|
"toolchain/helpers.mk",
|
||||||
|
"toolchain/toolchain-wrapper.mk"]
|
||||||
|
packages = list()
|
||||||
|
count = 0
|
||||||
|
for root, dirs, files in os.walk("."):
|
||||||
|
rootdir = root.split("/")
|
||||||
|
if len(rootdir) < 2:
|
||||||
|
continue
|
||||||
|
if rootdir[1] not in WALK_USEFUL_SUBDIRS:
|
||||||
|
continue
|
||||||
|
for f in files:
|
||||||
|
if not f.endswith(".mk"):
|
||||||
|
continue
|
||||||
|
# Strip ending ".mk"
|
||||||
|
pkgname = f[:-3]
|
||||||
|
if package_list and pkgname not in package_list:
|
||||||
|
continue
|
||||||
|
pkgpath = os.path.join(root, f)
|
||||||
|
skip = False
|
||||||
|
for exclude in WALK_EXCLUDES:
|
||||||
|
# pkgpath[2:] strips the initial './'
|
||||||
|
if re.match(exclude, pkgpath[2:]):
|
||||||
|
skip = True
|
||||||
|
continue
|
||||||
|
if skip:
|
||||||
|
continue
|
||||||
|
p = Package(pkgname, pkgpath)
|
||||||
|
packages.append(p)
|
||||||
|
count += 1
|
||||||
|
if npackages and count == npackages:
|
||||||
|
return packages
|
||||||
|
return packages
|
||||||
|
|
||||||
|
|
||||||
|
def package_init_make_info():
|
||||||
|
# Licenses
|
||||||
|
o = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y",
|
||||||
|
"-s", "printvars", "VARS=%_LICENSE"])
|
||||||
|
for l in o.splitlines():
|
||||||
|
# Get variable name and value
|
||||||
|
pkgvar, value = l.split("=")
|
||||||
|
|
||||||
|
# If present, strip HOST_ from variable name
|
||||||
|
if pkgvar.startswith("HOST_"):
|
||||||
|
pkgvar = pkgvar[5:]
|
||||||
|
|
||||||
|
# Strip _LICENSE
|
||||||
|
pkgvar = pkgvar[:-8]
|
||||||
|
|
||||||
|
# If value is "unknown", no license details available
|
||||||
|
if value == "unknown":
|
||||||
|
continue
|
||||||
|
Package.all_licenses.append(pkgvar)
|
||||||
|
|
||||||
|
# License files
|
||||||
|
o = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y",
|
||||||
|
"-s", "printvars", "VARS=%_LICENSE_FILES"])
|
||||||
|
for l in o.splitlines():
|
||||||
|
# Get variable name and value
|
||||||
|
pkgvar, value = l.split("=")
|
||||||
|
|
||||||
|
# If present, strip HOST_ from variable name
|
||||||
|
if pkgvar.startswith("HOST_"):
|
||||||
|
pkgvar = pkgvar[5:]
|
||||||
|
|
||||||
|
if pkgvar.endswith("_MANIFEST_LICENSE_FILES"):
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Strip _LICENSE_FILES
|
||||||
|
pkgvar = pkgvar[:-14]
|
||||||
|
|
||||||
|
Package.all_license_files.append(pkgvar)
|
||||||
|
|
||||||
|
# Version
|
||||||
|
o = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y",
|
||||||
|
"-s", "printvars", "VARS=%_VERSION"])
|
||||||
|
|
||||||
|
# We process first the host package VERSION, and then the target
|
||||||
|
# package VERSION. This means that if a package exists in both
|
||||||
|
# target and host variants, with different version numbers
|
||||||
|
# (unlikely), we'll report the target version number.
|
||||||
|
version_list = o.splitlines()
|
||||||
|
version_list = [x for x in version_list if x.startswith("HOST_")] + \
|
||||||
|
[x for x in version_list if not x.startswith("HOST_")]
|
||||||
|
for l in version_list:
|
||||||
|
# Get variable name and value
|
||||||
|
pkgvar, value = l.split("=")
|
||||||
|
|
||||||
|
# If present, strip HOST_ from variable name
|
||||||
|
if pkgvar.startswith("HOST_"):
|
||||||
|
pkgvar = pkgvar[5:]
|
||||||
|
|
||||||
|
if pkgvar.endswith("_DL_VERSION"):
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Strip _VERSION
|
||||||
|
pkgvar = pkgvar[:-8]
|
||||||
|
|
||||||
|
Package.all_versions[pkgvar] = value
|
||||||
|
|
||||||
|
|
||||||
|
def calculate_stats(packages):
|
||||||
|
stats = defaultdict(int)
|
||||||
|
for pkg in packages:
|
||||||
|
# If packages have multiple infra, take the first one. For the
|
||||||
|
# vast majority of packages, the target and host infra are the
|
||||||
|
# same. There are very few packages that use a different infra
|
||||||
|
# for the host and target variants.
|
||||||
|
if len(pkg.infras) > 0:
|
||||||
|
infra = pkg.infras[0][1]
|
||||||
|
stats["infra-%s" % infra] += 1
|
||||||
|
else:
|
||||||
|
stats["infra-unknown"] += 1
|
||||||
|
if pkg.has_license:
|
||||||
|
stats["license"] += 1
|
||||||
|
else:
|
||||||
|
stats["no-license"] += 1
|
||||||
|
if pkg.has_license_files:
|
||||||
|
stats["license-files"] += 1
|
||||||
|
else:
|
||||||
|
stats["no-license-files"] += 1
|
||||||
|
if pkg.has_hash:
|
||||||
|
stats["hash"] += 1
|
||||||
|
else:
|
||||||
|
stats["no-hash"] += 1
|
||||||
|
stats["patches"] += pkg.patch_count
|
||||||
|
return stats
|
||||||
|
|
||||||
|
|
||||||
|
html_header = """
|
||||||
|
<head>
|
||||||
<script src=\"https://www.kryogenix.org/code/browser/sorttable/sorttable.js\"></script>
|
<script src=\"https://www.kryogenix.org/code/browser/sorttable/sorttable.js\"></script>
|
||||||
<style type=\"text/css\">
|
<style type=\"text/css\">
|
||||||
table {
|
table {
|
||||||
@ -46,11 +305,9 @@ td.correct {
|
|||||||
td.nopatches {
|
td.nopatches {
|
||||||
background: #d2ffc4;
|
background: #d2ffc4;
|
||||||
}
|
}
|
||||||
|
|
||||||
td.somepatches {
|
td.somepatches {
|
||||||
background: #ffd870;
|
background: #ffd870;
|
||||||
}
|
}
|
||||||
|
|
||||||
td.lotsofpatches {
|
td.lotsofpatches {
|
||||||
background: #ff9a69;
|
background: #ff9a69;
|
||||||
}
|
}
|
||||||
@ -61,395 +318,209 @@ td.lotsofpatches {
|
|||||||
<a href=\"#results\">Results</a><br/>
|
<a href=\"#results\">Results</a><br/>
|
||||||
|
|
||||||
<p id=\"sortable_hint\"></p>
|
<p id=\"sortable_hint\"></p>
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
html_footer = """
|
||||||
|
</body>
|
||||||
|
<script>
|
||||||
|
if (typeof sorttable === \"object\") {
|
||||||
|
document.getElementById(\"sortable_hint\").innerHTML =
|
||||||
|
\"hint: the table can be sorted by clicking the column headers\"
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
</html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def infra_str(infra_list):
|
||||||
|
if not infra_list:
|
||||||
|
return "Unknown"
|
||||||
|
elif len(infra_list) == 1:
|
||||||
|
return "<b>%s</b><br/>%s" % (infra_list[0][1], infra_list[0][0])
|
||||||
|
elif infra_list[0][1] == infra_list[1][1]:
|
||||||
|
return "<b>%s</b><br/>%s + %s" % \
|
||||||
|
(infra_list[0][1], infra_list[0][0], infra_list[1][0])
|
||||||
|
else:
|
||||||
|
return "<b>%s</b> (%s)<br/><b>%s</b> (%s)" % \
|
||||||
|
(infra_list[0][1], infra_list[0][0],
|
||||||
|
infra_list[1][1], infra_list[1][0])
|
||||||
|
|
||||||
|
|
||||||
|
def boolean_str(b):
|
||||||
|
if b:
|
||||||
|
return "Yes"
|
||||||
|
else:
|
||||||
|
return "No"
|
||||||
|
|
||||||
|
|
||||||
|
def dump_html_pkg(f, pkg):
|
||||||
|
f.write(" <tr>\n")
|
||||||
|
f.write(" <td>%s</td>\n" % pkg.path[2:])
|
||||||
|
|
||||||
|
# Patch count
|
||||||
|
td_class = ["centered"]
|
||||||
|
if pkg.patch_count == 0:
|
||||||
|
td_class.append("nopatches")
|
||||||
|
elif pkg.patch_count < 5:
|
||||||
|
td_class.append("somepatches")
|
||||||
|
else:
|
||||||
|
td_class.append("lotsofpatches")
|
||||||
|
f.write(" <td class=\"%s\">%s</td>\n" %
|
||||||
|
(" ".join(td_class), str(pkg.patch_count)))
|
||||||
|
|
||||||
|
# Infrastructure
|
||||||
|
infra = infra_str(pkg.infras)
|
||||||
|
td_class = ["centered"]
|
||||||
|
if infra == "Unknown":
|
||||||
|
td_class.append("wrong")
|
||||||
|
else:
|
||||||
|
td_class.append("correct")
|
||||||
|
f.write(" <td class=\"%s\">%s</td>\n" %
|
||||||
|
(" ".join(td_class), infra_str(pkg.infras)))
|
||||||
|
|
||||||
|
# License
|
||||||
|
td_class = ["centered"]
|
||||||
|
if pkg.has_license:
|
||||||
|
td_class.append("correct")
|
||||||
|
else:
|
||||||
|
td_class.append("wrong")
|
||||||
|
f.write(" <td class=\"%s\">%s</td>\n" %
|
||||||
|
(" ".join(td_class), boolean_str(pkg.has_license)))
|
||||||
|
|
||||||
|
# License files
|
||||||
|
td_class = ["centered"]
|
||||||
|
if pkg.has_license_files:
|
||||||
|
td_class.append("correct")
|
||||||
|
else:
|
||||||
|
td_class.append("wrong")
|
||||||
|
f.write(" <td class=\"%s\">%s</td>\n" %
|
||||||
|
(" ".join(td_class), boolean_str(pkg.has_license_files)))
|
||||||
|
|
||||||
|
# Hash
|
||||||
|
td_class = ["centered"]
|
||||||
|
if pkg.has_hash:
|
||||||
|
td_class.append("correct")
|
||||||
|
else:
|
||||||
|
td_class.append("wrong")
|
||||||
|
f.write(" <td class=\"%s\">%s</td>\n" %
|
||||||
|
(" ".join(td_class), boolean_str(pkg.has_hash)))
|
||||||
|
|
||||||
|
# Current version
|
||||||
|
if len(pkg.current_version) > 20:
|
||||||
|
current_version = pkg.current_version[:20] + "..."
|
||||||
|
else:
|
||||||
|
current_version = pkg.current_version
|
||||||
|
f.write(" <td class=\"centered\">%s</td>\n" % current_version)
|
||||||
|
|
||||||
|
# Warnings
|
||||||
|
td_class = ["centered"]
|
||||||
|
if pkg.warnings == 0:
|
||||||
|
td_class.append("correct")
|
||||||
|
else:
|
||||||
|
td_class.append("wrong")
|
||||||
|
f.write(" <td class=\"%s\">%d</td>\n" %
|
||||||
|
(" ".join(td_class), pkg.warnings))
|
||||||
|
|
||||||
|
f.write(" </tr>\n")
|
||||||
|
|
||||||
|
|
||||||
|
def dump_html_all_pkgs(f, packages):
|
||||||
|
f.write("""
|
||||||
<table class=\"sortable\">
|
<table class=\"sortable\">
|
||||||
<tr>
|
<tr>
|
||||||
<td>Id</td>
|
|
||||||
<td>Package</td>
|
<td>Package</td>
|
||||||
<td class=\"centered\">Patch count</td>
|
<td class=\"centered\">Patch count</td>
|
||||||
<td class=\"centered\">Infrastructure</td>
|
<td class=\"centered\">Infrastructure</td>
|
||||||
<td class=\"centered\">License</td>
|
<td class=\"centered\">License</td>
|
||||||
<td class=\"centered\">License files</td>
|
<td class=\"centered\">License files</td>
|
||||||
<td class=\"centered\">Hash file</td>
|
<td class=\"centered\">Hash file</td>
|
||||||
|
<td class=\"centered\">Current version</td>
|
||||||
<td class=\"centered\">Warnings</td>
|
<td class=\"centered\">Warnings</td>
|
||||||
</tr>
|
</tr>
|
||||||
"
|
""")
|
||||||
|
for pkg in sorted(packages):
|
||||||
autotools_packages=0
|
dump_html_pkg(f, pkg)
|
||||||
cmake_packages=0
|
f.write("</table>")
|
||||||
kconfig_packages=0
|
|
||||||
luarocks_package=0
|
|
||||||
perl_packages=0
|
|
||||||
python_packages=0
|
|
||||||
rebar_packages=0
|
|
||||||
virtual_packages=0
|
|
||||||
generic_packages=0
|
|
||||||
waf_packages=0
|
|
||||||
manual_packages=0
|
|
||||||
packages_with_licence=0
|
|
||||||
packages_without_licence=0
|
|
||||||
packages_with_license_files=0
|
|
||||||
packages_without_license_files=0
|
|
||||||
packages_with_hash_file=0
|
|
||||||
packages_without_hash_file=0
|
|
||||||
total_patch_count=0
|
|
||||||
cnt=0
|
|
||||||
|
|
||||||
for i in $(find boot/ linux/ package/ toolchain/ -name '*.mk' | sort) ; do
|
|
||||||
|
|
||||||
if test \
|
|
||||||
$i = "boot/common.mk" -o \
|
|
||||||
$i = "linux/linux-ext-ev3dev-linux-drivers.mk" -o \
|
|
||||||
$i = "linux/linux-ext-fbtft.mk" -o \
|
|
||||||
$i = "linux/linux-ext-xenomai.mk" -o \
|
|
||||||
$i = "linux/linux-ext-rtai.mk" -o \
|
|
||||||
$i = "package/freescale-imx/freescale-imx.mk" -o \
|
|
||||||
$i = "package/gcc/gcc.mk" -o \
|
|
||||||
$i = "package/gstreamer/gstreamer.mk" -o \
|
|
||||||
$i = "package/gstreamer1/gstreamer1.mk" -o \
|
|
||||||
$i = "package/gtk2-themes/gtk2-themes.mk" -o \
|
|
||||||
$i = "package/matchbox/matchbox.mk" -o \
|
|
||||||
$i = "package/opengl/opengl.mk" -o \
|
|
||||||
$i = "package/qt5/qt5.mk" -o \
|
|
||||||
$i = "package/x11r7/x11r7.mk" -o \
|
|
||||||
$i = "package/doc-asciidoc.mk" -o \
|
|
||||||
$i = "package/pkg-autotools.mk" -o \
|
|
||||||
$i = "package/pkg-cmake.mk" -o \
|
|
||||||
$i = "package/pkg-kconfig.mk" -o \
|
|
||||||
$i = "package/pkg-luarocks.mk" -o \
|
|
||||||
$i = "package/pkg-perl.mk" -o \
|
|
||||||
$i = "package/pkg-python.mk" -o \
|
|
||||||
$i = "package/pkg-rebar.mk" -o \
|
|
||||||
$i = "package/pkg-virtual.mk" -o \
|
|
||||||
$i = "package/pkg-download.mk" -o \
|
|
||||||
$i = "package/pkg-generic.mk" -o \
|
|
||||||
$i = "package/pkg-waf.mk" -o \
|
|
||||||
$i = "package/pkg-kernel-module.mk" -o \
|
|
||||||
$i = "package/pkg-utils.mk" -o \
|
|
||||||
$i = "package/nvidia-tegra23/nvidia-tegra23.mk" -o \
|
|
||||||
$i = "toolchain/toolchain-external/pkg-toolchain-external.mk" -o \
|
|
||||||
$i = "toolchain/toolchain-external/toolchain-external.mk" -o \
|
|
||||||
$i = "toolchain/toolchain.mk" -o \
|
|
||||||
$i = "toolchain/helpers.mk" -o \
|
|
||||||
$i = "toolchain/toolchain-wrapper.mk" ; then
|
|
||||||
echo "skipping $i" 1>&2
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
cnt=$((cnt+1))
|
|
||||||
|
|
||||||
hashost=0
|
|
||||||
hastarget=0
|
|
||||||
infratype=""
|
|
||||||
|
|
||||||
# Determine package infrastructure
|
|
||||||
if grep -E "\(host-autotools-package\)" $i > /dev/null ; then
|
|
||||||
infratype="autotools"
|
|
||||||
hashost=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "\(autotools-package\)" $i > /dev/null ; then
|
|
||||||
infratype="autotools"
|
|
||||||
hastarget=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "\(kconfig-package\)" $i > /dev/null ; then
|
|
||||||
infratype="kconfig"
|
|
||||||
hastarget=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "\(host-luarocks-package\)" $i > /dev/null ; then
|
|
||||||
infratype="luarocks"
|
|
||||||
hashost=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "\(luarocks-package\)" $i > /dev/null ; then
|
|
||||||
infratype="luarocks"
|
|
||||||
hastarget=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "\(host-perl-package\)" $i > /dev/null ; then
|
|
||||||
infratype="perl"
|
|
||||||
hashost=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "\(perl-package\)" $i > /dev/null ; then
|
|
||||||
infratype="perl"
|
|
||||||
hastarget=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "\(host-python-package\)" $i > /dev/null ; then
|
|
||||||
infratype="python"
|
|
||||||
hashost=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "\(python-package\)" $i > /dev/null ; then
|
|
||||||
infratype="python"
|
|
||||||
hastarget=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "\(host-rebar-package\)" $i > /dev/null ; then
|
|
||||||
infratype="rebar"
|
|
||||||
hashost=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "\(rebar-package\)" $i > /dev/null ; then
|
|
||||||
infratype="rebar"
|
|
||||||
hastarget=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "\(host-virtual-package\)" $i > /dev/null ; then
|
|
||||||
infratype="virtual"
|
|
||||||
hashost=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "\(virtual-package\)" $i > /dev/null ; then
|
|
||||||
infratype="virtual"
|
|
||||||
hastarget=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "\(host-generic-package\)" $i > /dev/null ; then
|
|
||||||
infratype="generic"
|
|
||||||
hashost=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "\(generic-package\)" $i > /dev/null ; then
|
|
||||||
infratype="generic"
|
|
||||||
hastarget=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "\(host-cmake-package\)" $i > /dev/null ; then
|
|
||||||
infratype="cmake"
|
|
||||||
hashost=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "\(cmake-package\)" $i > /dev/null ; then
|
|
||||||
infratype="cmake"
|
|
||||||
hastarget=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "\(toolchain-external-package\)" $i > /dev/null ; then
|
|
||||||
infratype="toolchain-external"
|
|
||||||
hastarget=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -E "\(waf-package\)" $i > /dev/null ; then
|
|
||||||
infratype="waf"
|
|
||||||
hastarget=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
pkg=$(basename $i)
|
|
||||||
dir=$(dirname $i)
|
|
||||||
pkg=${pkg%.mk}
|
|
||||||
pkgvariable=$(echo ${pkg} | tr "a-z-" "A-Z_")
|
|
||||||
|
|
||||||
|
|
||||||
# Count packages per infrastructure
|
def dump_html_stats(f, stats):
|
||||||
if [ -z ${infratype} ] ; then
|
f.write("<a id=\"results\"></a>\n")
|
||||||
infratype="manual"
|
f.write("<table>\n")
|
||||||
manual_packages=$(($manual_packages+1))
|
infras = [infra[6:] for infra in stats.keys() if infra.startswith("infra-")]
|
||||||
elif [ ${infratype} = "autotools" ]; then
|
for infra in infras:
|
||||||
autotools_packages=$(($autotools_packages+1))
|
f.write(" <tr><td>Packages using the <i>%s</i> infrastructure</td><td>%s</td></tr>\n" %
|
||||||
elif [ ${infratype} = "cmake" ]; then
|
(infra, stats["infra-%s" % infra]))
|
||||||
cmake_packages=$(($cmake_packages+1))
|
f.write(" <tr><td>Packages having license information</td><td>%s</td></tr>\n" %
|
||||||
elif [ ${infratype} = "kconfig" ]; then
|
stats["license"])
|
||||||
kconfig_packages=$(($kconfig_packages+1))
|
f.write(" <tr><td>Packages not having license information</td><td>%s</td></tr>\n" %
|
||||||
elif [ ${infratype} = "luarocks" ]; then
|
stats["no-license"])
|
||||||
luarocks_packages=$(($luarocks_packages+1))
|
f.write(" <tr><td>Packages having license files information</td><td>%s</td></tr>\n" %
|
||||||
elif [ ${infratype} = "perl" ]; then
|
stats["license-files"])
|
||||||
perl_packages=$(($perl_packages+1))
|
f.write(" <tr><td>Packages not having license files information</td><td>%s</td></tr>\n" %
|
||||||
elif [ ${infratype} = "python" ]; then
|
stats["no-license-files"])
|
||||||
python_packages=$(($python_packages+1))
|
f.write(" <tr><td>Packages having a hash file</td><td>%s</td></tr>\n" %
|
||||||
elif [ ${infratype} = "rebar" ]; then
|
stats["hash"])
|
||||||
rebar_packages=$(($rebar_packages+1))
|
f.write(" <tr><td>Packages not having a hash file</td><td>%s</td></tr>\n" %
|
||||||
elif [ ${infratype} = "virtual" ]; then
|
stats["no-hash"])
|
||||||
virtual_packages=$(($virtual_packages+1))
|
f.write(" <tr><td>Total number of patches</td><td>%s</td></tr>\n" %
|
||||||
elif [ ${infratype} = "generic" ]; then
|
stats["patches"])
|
||||||
generic_packages=$(($generic_packages+1))
|
f.write("</table>\n")
|
||||||
elif [ ${infratype} = "waf" ]; then
|
|
||||||
waf_packages=$(($waf_packages+1))
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -qE "^${pkgvariable}_LICENSE[ ]*=" $i ; then
|
|
||||||
packages_with_license=$(($packages_with_license+1))
|
|
||||||
license=1
|
|
||||||
else
|
|
||||||
packages_without_license=$(($packages_without_license+1))
|
|
||||||
license=0
|
|
||||||
fi
|
|
||||||
|
|
||||||
if grep -qE "^${pkgvariable}_LICENSE_FILES[ ]*=" $i ; then
|
def dump_gen_info(f):
|
||||||
packages_with_license_files=$(($packages_with_license_files+1))
|
# Updated on Mon Feb 19 08:12:08 CET 2018, Git commit aa77030b8f5e41f1c53eb1c1ad664b8c814ba032
|
||||||
license_files=1
|
o = subprocess.check_output(["git", "log", "master", "-n", "1", "--pretty=format:%H"])
|
||||||
else
|
git_commit = o.splitlines()[0]
|
||||||
packages_without_license_files=$(($packages_without_license_files+1))
|
f.write("<p><i>Updated on %s, git commit %s</i></p>\n" %
|
||||||
license_files=0
|
(str(datetime.datetime.utcnow()), git_commit))
|
||||||
fi
|
|
||||||
|
|
||||||
if test -f ${dir}/${pkg}.hash; then
|
|
||||||
packages_with_hash_file=$(($packages_with_hash_file+1))
|
|
||||||
hash_file=1
|
|
||||||
else
|
|
||||||
packages_without_hash_file=$(($packages_without_hash_file+1))
|
|
||||||
hash_file=0
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "<tr>"
|
def dump_html(packages, stats, output):
|
||||||
|
with open(output, 'w') as f:
|
||||||
|
f.write(html_header)
|
||||||
|
dump_html_all_pkgs(f, packages)
|
||||||
|
dump_html_stats(f, stats)
|
||||||
|
dump_gen_info(f)
|
||||||
|
f.write(html_footer)
|
||||||
|
|
||||||
echo "<td>$cnt</td>"
|
|
||||||
echo "<td>$i</td>"
|
|
||||||
|
|
||||||
package_dir=$(dirname $i)
|
def parse_args():
|
||||||
patch_count=$(find ${package_dir} -name '*.patch' | wc -l)
|
parser = argparse.ArgumentParser()
|
||||||
total_patch_count=$(($total_patch_count+$patch_count))
|
parser.add_argument('-o', dest='output', action='store', required=True,
|
||||||
|
help='HTML output file')
|
||||||
|
parser.add_argument('-n', dest='npackages', type=int, action='store',
|
||||||
|
help='Number of packages')
|
||||||
|
parser.add_argument('-p', dest='packages', action='store',
|
||||||
|
help='List of packages (comma separated)')
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
if test $patch_count -lt 1 ; then
|
|
||||||
patch_count_class="nopatches"
|
|
||||||
elif test $patch_count -lt 5 ; then
|
|
||||||
patch_count_class="somepatches"
|
|
||||||
else
|
|
||||||
patch_count_class="lotsofpatches"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "<td class=\"centered ${patch_count_class}\">"
|
def __main__():
|
||||||
echo "<b>$patch_count</b>"
|
args = parse_args()
|
||||||
echo "</td>"
|
if args.npackages and args.packages:
|
||||||
|
print "ERROR: -n and -p are mutually exclusive"
|
||||||
|
sys.exit(1)
|
||||||
|
if args.packages:
|
||||||
|
package_list = args.packages.split(",")
|
||||||
|
else:
|
||||||
|
package_list = None
|
||||||
|
print "Build package list ..."
|
||||||
|
packages = get_pkglist(args.npackages, package_list)
|
||||||
|
print "Getting package make info ..."
|
||||||
|
package_init_make_info()
|
||||||
|
print "Getting package details ..."
|
||||||
|
for pkg in packages:
|
||||||
|
pkg.set_infra()
|
||||||
|
pkg.set_license()
|
||||||
|
pkg.set_hash_info()
|
||||||
|
pkg.set_patch_count()
|
||||||
|
pkg.set_check_package_warnings()
|
||||||
|
pkg.set_current_version()
|
||||||
|
print "Calculate stats"
|
||||||
|
stats = calculate_stats(packages)
|
||||||
|
print "Write HTML"
|
||||||
|
dump_html(packages, stats, args.output)
|
||||||
|
|
||||||
if [ ${infratype} = "manual" ] ; then
|
|
||||||
echo "<td class=\"centered wrong\"><b>manual</b></td>"
|
|
||||||
else
|
|
||||||
echo "<td class=\"centered correct\">"
|
|
||||||
echo "<b>${infratype}</b><br/>"
|
|
||||||
if [ ${hashost} -eq 1 -a ${hastarget} -eq 1 ]; then
|
|
||||||
echo "target + host"
|
|
||||||
elif [ ${hashost} -eq 1 ]; then
|
|
||||||
echo "host"
|
|
||||||
else
|
|
||||||
echo "target"
|
|
||||||
fi
|
|
||||||
echo "</td>"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ ${license} -eq 0 ] ; then
|
__main__()
|
||||||
echo "<td class=\"centered wrong\">No</td>"
|
|
||||||
else
|
|
||||||
echo "<td class=\"centered correct\">Yes</td>"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ ${license_files} -eq 0 ] ; then
|
|
||||||
echo "<td class=\"centered wrong\">No</td>"
|
|
||||||
else
|
|
||||||
echo "<td class=\"centered correct\">Yes</td>"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ ${hash_file} -eq 0 ] ; then
|
|
||||||
echo "<td class=\"centered wrong\">No</td>"
|
|
||||||
else
|
|
||||||
echo "<td class=\"centered correct\">Yes</td>"
|
|
||||||
fi
|
|
||||||
|
|
||||||
file_list=$(find ${package_dir} -name '*.mk' -o -name '*.in*' -o -name '*.hash')
|
|
||||||
nwarnings=$(./utils/check-package ${file_list} 2>&1 | sed '/\([0-9]*\) warnings generated/!d; s//\1/')
|
|
||||||
if [ ${nwarnings} -eq 0 ] ; then
|
|
||||||
echo "<td class=\"centered correct\">${nwarnings}</td>"
|
|
||||||
else
|
|
||||||
echo "<td class=\"centered wrong\">${nwarnings}</td>"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "</tr>"
|
|
||||||
|
|
||||||
done
|
|
||||||
echo "</table>"
|
|
||||||
|
|
||||||
echo "<a id="results"></a>"
|
|
||||||
echo "<table>"
|
|
||||||
echo "<tr>"
|
|
||||||
echo "<td>Packages using the <i>generic</i> infrastructure</td>"
|
|
||||||
echo "<td>$generic_packages</td>"
|
|
||||||
echo "</tr>"
|
|
||||||
echo "<tr>"
|
|
||||||
echo "<td>Packages using the <i>cmake</i> infrastructure</td>"
|
|
||||||
echo "<td>$cmake_packages</td>"
|
|
||||||
echo "</tr>"
|
|
||||||
echo "<tr>"
|
|
||||||
echo "<td>Packages using the <i>autotools</i> infrastructure</td>"
|
|
||||||
echo "<td>$autotools_packages</td>"
|
|
||||||
echo "</tr>"
|
|
||||||
echo "<tr>"
|
|
||||||
echo "<td>Packages using the <i>luarocks</i> infrastructure</td>"
|
|
||||||
echo "<td>$luarocks_packages</td>"
|
|
||||||
echo "</tr>"
|
|
||||||
echo "<tr>"
|
|
||||||
echo "<td>Packages using the <i>kconfig</i> infrastructure</td>"
|
|
||||||
echo "<td>$kconfig_packages</td>"
|
|
||||||
echo "</tr>"
|
|
||||||
echo "<tr>"
|
|
||||||
echo "<td>Packages using the <i>perl</i> infrastructure</td>"
|
|
||||||
echo "<td>$perl_packages</td>"
|
|
||||||
echo "</tr>"
|
|
||||||
echo "<tr>"
|
|
||||||
echo "<td>Packages using the <i>python</i> infrastructure</td>"
|
|
||||||
echo "<td>$python_packages</td>"
|
|
||||||
echo "</tr>"
|
|
||||||
echo "<tr>"
|
|
||||||
echo "<td>Packages using the <i>rebar</i> infrastructure</td>"
|
|
||||||
echo "<td>$rebar_packages</td>"
|
|
||||||
echo "</tr>"
|
|
||||||
echo "<tr>"
|
|
||||||
echo "<td>Packages using the <i>virtual</i> infrastructure</td>"
|
|
||||||
echo "<td>$virtual_packages</td>"
|
|
||||||
echo "</tr>"
|
|
||||||
echo "<tr>"
|
|
||||||
echo "<td>Packages using the <i>waf</i> infrastructure</td>"
|
|
||||||
echo "<td>$waf_packages</td>"
|
|
||||||
echo "</tr>"
|
|
||||||
echo "<tr>"
|
|
||||||
echo "<td>Packages not using any infrastructure</td>"
|
|
||||||
echo "<td>$manual_packages</td>"
|
|
||||||
echo "</tr>"
|
|
||||||
echo "<tr>"
|
|
||||||
echo "<td>Packages having license information</td>"
|
|
||||||
echo "<td>$packages_with_license</td>"
|
|
||||||
echo "</tr>"
|
|
||||||
echo "<tr>"
|
|
||||||
echo "<td>Packages not having licence information</td>"
|
|
||||||
echo "<td>$packages_without_license</td>"
|
|
||||||
echo "</tr>"
|
|
||||||
echo "<tr>"
|
|
||||||
echo "<td>Packages having license files information</td>"
|
|
||||||
echo "<td>$packages_with_license_files</td>"
|
|
||||||
echo "</tr>"
|
|
||||||
echo "<tr>"
|
|
||||||
echo "<td>Packages not having licence files information</td>"
|
|
||||||
echo "<td>$packages_without_license_files</td>"
|
|
||||||
echo "</tr>"
|
|
||||||
echo "<tr>"
|
|
||||||
echo "<td>Packages having hash file</td>"
|
|
||||||
echo "<td>$packages_with_hash_file</td>"
|
|
||||||
echo "</tr>"
|
|
||||||
echo "<tr>"
|
|
||||||
echo "<td>Packages not having hash file</td>"
|
|
||||||
echo "<td>$packages_without_hash_file</td>"
|
|
||||||
echo "</tr>"
|
|
||||||
echo "<tr>"
|
|
||||||
echo "<td>Number of patches in all packages</td>"
|
|
||||||
echo "<td>$total_patch_count</td>"
|
|
||||||
echo "</tr>"
|
|
||||||
echo "<tr>"
|
|
||||||
echo "<td>TOTAL</td>"
|
|
||||||
echo "<td>$cnt</td>"
|
|
||||||
echo "</tr>"
|
|
||||||
echo "</table>"
|
|
||||||
|
|
||||||
echo "<hr/>"
|
|
||||||
echo "<i>Updated on $(LANG=C date), Git commit $(git log master -n 1 --pretty=format:%H)</i>"
|
|
||||||
echo "</body>"
|
|
||||||
|
|
||||||
echo "<script>
|
|
||||||
if (typeof sorttable === \"object\") {
|
|
||||||
document.getElementById(\"sortable_hint\").innerHTML =
|
|
||||||
\"hint: the table can be sorted by clicking the column headers\"
|
|
||||||
}
|
|
||||||
</script>
|
|
||||||
"
|
|
||||||
echo "</html>"
|
|
||||||
|
|||||||
@ -1,526 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
# Copyright (C) 2009 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
|
|
||||||
#
|
|
||||||
# This program is free software; you can redistribute it and/or modify
|
|
||||||
# it under the terms of the GNU General Public License as published by
|
|
||||||
# the Free Software Foundation; either version 2 of the License, or
|
|
||||||
# (at your option) any later version.
|
|
||||||
#
|
|
||||||
# This program is distributed in the hope that it will be useful,
|
|
||||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
||||||
# General Public License for more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU General Public License
|
|
||||||
# along with this program; if not, write to the Free Software
|
|
||||||
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import datetime
|
|
||||||
import fnmatch
|
|
||||||
import os
|
|
||||||
from collections import defaultdict
|
|
||||||
import re
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
|
|
||||||
INFRA_RE = re.compile("\$\(eval \$\(([a-z-]*)-package\)\)")
|
|
||||||
|
|
||||||
|
|
||||||
class Package:
|
|
||||||
all_licenses = list()
|
|
||||||
all_license_files = list()
|
|
||||||
all_versions = dict()
|
|
||||||
|
|
||||||
def __init__(self, name, path):
|
|
||||||
self.name = name
|
|
||||||
self.path = path
|
|
||||||
self.infras = None
|
|
||||||
self.has_license = False
|
|
||||||
self.has_license_files = False
|
|
||||||
self.has_hash = False
|
|
||||||
self.patch_count = 0
|
|
||||||
self.warnings = 0
|
|
||||||
self.current_version = None
|
|
||||||
|
|
||||||
def pkgvar(self):
|
|
||||||
return self.name.upper().replace("-", "_")
|
|
||||||
|
|
||||||
def set_infra(self):
|
|
||||||
"""
|
|
||||||
Fills in the .infras field
|
|
||||||
"""
|
|
||||||
self.infras = list()
|
|
||||||
with open(self.path, 'r') as f:
|
|
||||||
lines = f.readlines()
|
|
||||||
for l in lines:
|
|
||||||
match = INFRA_RE.match(l)
|
|
||||||
if not match:
|
|
||||||
continue
|
|
||||||
infra = match.group(1)
|
|
||||||
if infra.startswith("host-"):
|
|
||||||
self.infras.append(("host", infra[5:]))
|
|
||||||
else:
|
|
||||||
self.infras.append(("target", infra))
|
|
||||||
|
|
||||||
def set_license(self):
|
|
||||||
"""
|
|
||||||
Fills in the .has_license and .has_license_files fields
|
|
||||||
"""
|
|
||||||
var = self.pkgvar()
|
|
||||||
if var in self.all_licenses:
|
|
||||||
self.has_license = True
|
|
||||||
if var in self.all_license_files:
|
|
||||||
self.has_license_files = True
|
|
||||||
|
|
||||||
def set_hash_info(self):
|
|
||||||
"""
|
|
||||||
Fills in the .has_hash field
|
|
||||||
"""
|
|
||||||
hashpath = self.path.replace(".mk", ".hash")
|
|
||||||
self.has_hash = os.path.exists(hashpath)
|
|
||||||
|
|
||||||
def set_patch_count(self):
|
|
||||||
"""
|
|
||||||
Fills in the .patch_count field
|
|
||||||
"""
|
|
||||||
self.patch_count = 0
|
|
||||||
pkgdir = os.path.dirname(self.path)
|
|
||||||
for subdir, _, _ in os.walk(pkgdir):
|
|
||||||
self.patch_count += len(fnmatch.filter(os.listdir(subdir), '*.patch'))
|
|
||||||
|
|
||||||
def set_current_version(self):
|
|
||||||
"""
|
|
||||||
Fills in the .current_version field
|
|
||||||
"""
|
|
||||||
var = self.pkgvar()
|
|
||||||
if var in self.all_versions:
|
|
||||||
self.current_version = self.all_versions[var]
|
|
||||||
|
|
||||||
def set_check_package_warnings(self):
|
|
||||||
"""
|
|
||||||
Fills in the .warnings field
|
|
||||||
"""
|
|
||||||
cmd = ["./utils/check-package"]
|
|
||||||
pkgdir = os.path.dirname(self.path)
|
|
||||||
for root, dirs, files in os.walk(pkgdir):
|
|
||||||
for f in files:
|
|
||||||
if f.endswith(".mk") or f.endswith(".hash") or f == "Config.in" or f == "Config.in.host":
|
|
||||||
cmd.append(os.path.join(root, f))
|
|
||||||
o = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[1]
|
|
||||||
lines = o.splitlines()
|
|
||||||
for line in lines:
|
|
||||||
m = re.match("^([0-9]*) warnings generated", line)
|
|
||||||
if m:
|
|
||||||
self.warnings = int(m.group(1))
|
|
||||||
return
|
|
||||||
|
|
||||||
def __eq__(self, other):
|
|
||||||
return self.path == other.path
|
|
||||||
|
|
||||||
def __lt__(self, other):
|
|
||||||
return self.path < other.path
|
|
||||||
|
|
||||||
def __str__(self):
|
|
||||||
return "%s (path='%s', license='%s', license_files='%s', hash='%s', patches=%d)" % \
|
|
||||||
(self.name, self.path, self.has_license, self.has_license_files, self.has_hash, self.patch_count)
|
|
||||||
|
|
||||||
|
|
||||||
def get_pkglist(npackages, package_list):
|
|
||||||
"""
|
|
||||||
Builds the list of Buildroot packages, returning a list of Package
|
|
||||||
objects. Only the .name and .path fields of the Package object are
|
|
||||||
initialized.
|
|
||||||
|
|
||||||
npackages: limit to N packages
|
|
||||||
package_list: limit to those packages in this list
|
|
||||||
"""
|
|
||||||
WALK_USEFUL_SUBDIRS = ["boot", "linux", "package", "toolchain"]
|
|
||||||
WALK_EXCLUDES = ["boot/common.mk",
|
|
||||||
"linux/linux-ext-.*.mk",
|
|
||||||
"package/freescale-imx/freescale-imx.mk",
|
|
||||||
"package/gcc/gcc.mk",
|
|
||||||
"package/gstreamer/gstreamer.mk",
|
|
||||||
"package/gstreamer1/gstreamer1.mk",
|
|
||||||
"package/gtk2-themes/gtk2-themes.mk",
|
|
||||||
"package/matchbox/matchbox.mk",
|
|
||||||
"package/opengl/opengl.mk",
|
|
||||||
"package/qt5/qt5.mk",
|
|
||||||
"package/x11r7/x11r7.mk",
|
|
||||||
"package/doc-asciidoc.mk",
|
|
||||||
"package/pkg-.*.mk",
|
|
||||||
"package/nvidia-tegra23/nvidia-tegra23.mk",
|
|
||||||
"toolchain/toolchain-external/pkg-toolchain-external.mk",
|
|
||||||
"toolchain/toolchain-external/toolchain-external.mk",
|
|
||||||
"toolchain/toolchain.mk",
|
|
||||||
"toolchain/helpers.mk",
|
|
||||||
"toolchain/toolchain-wrapper.mk"]
|
|
||||||
packages = list()
|
|
||||||
count = 0
|
|
||||||
for root, dirs, files in os.walk("."):
|
|
||||||
rootdir = root.split("/")
|
|
||||||
if len(rootdir) < 2:
|
|
||||||
continue
|
|
||||||
if rootdir[1] not in WALK_USEFUL_SUBDIRS:
|
|
||||||
continue
|
|
||||||
for f in files:
|
|
||||||
if not f.endswith(".mk"):
|
|
||||||
continue
|
|
||||||
# Strip ending ".mk"
|
|
||||||
pkgname = f[:-3]
|
|
||||||
if package_list and pkgname not in package_list:
|
|
||||||
continue
|
|
||||||
pkgpath = os.path.join(root, f)
|
|
||||||
skip = False
|
|
||||||
for exclude in WALK_EXCLUDES:
|
|
||||||
# pkgpath[2:] strips the initial './'
|
|
||||||
if re.match(exclude, pkgpath[2:]):
|
|
||||||
skip = True
|
|
||||||
continue
|
|
||||||
if skip:
|
|
||||||
continue
|
|
||||||
p = Package(pkgname, pkgpath)
|
|
||||||
packages.append(p)
|
|
||||||
count += 1
|
|
||||||
if npackages and count == npackages:
|
|
||||||
return packages
|
|
||||||
return packages
|
|
||||||
|
|
||||||
|
|
||||||
def package_init_make_info():
|
|
||||||
# Licenses
|
|
||||||
o = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y",
|
|
||||||
"-s", "printvars", "VARS=%_LICENSE"])
|
|
||||||
for l in o.splitlines():
|
|
||||||
# Get variable name and value
|
|
||||||
pkgvar, value = l.split("=")
|
|
||||||
|
|
||||||
# If present, strip HOST_ from variable name
|
|
||||||
if pkgvar.startswith("HOST_"):
|
|
||||||
pkgvar = pkgvar[5:]
|
|
||||||
|
|
||||||
# Strip _LICENSE
|
|
||||||
pkgvar = pkgvar[:-8]
|
|
||||||
|
|
||||||
# If value is "unknown", no license details available
|
|
||||||
if value == "unknown":
|
|
||||||
continue
|
|
||||||
Package.all_licenses.append(pkgvar)
|
|
||||||
|
|
||||||
# License files
|
|
||||||
o = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y",
|
|
||||||
"-s", "printvars", "VARS=%_LICENSE_FILES"])
|
|
||||||
for l in o.splitlines():
|
|
||||||
# Get variable name and value
|
|
||||||
pkgvar, value = l.split("=")
|
|
||||||
|
|
||||||
# If present, strip HOST_ from variable name
|
|
||||||
if pkgvar.startswith("HOST_"):
|
|
||||||
pkgvar = pkgvar[5:]
|
|
||||||
|
|
||||||
if pkgvar.endswith("_MANIFEST_LICENSE_FILES"):
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Strip _LICENSE_FILES
|
|
||||||
pkgvar = pkgvar[:-14]
|
|
||||||
|
|
||||||
Package.all_license_files.append(pkgvar)
|
|
||||||
|
|
||||||
# Version
|
|
||||||
o = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y",
|
|
||||||
"-s", "printvars", "VARS=%_VERSION"])
|
|
||||||
|
|
||||||
# We process first the host package VERSION, and then the target
|
|
||||||
# package VERSION. This means that if a package exists in both
|
|
||||||
# target and host variants, with different version numbers
|
|
||||||
# (unlikely), we'll report the target version number.
|
|
||||||
version_list = o.splitlines()
|
|
||||||
version_list = [x for x in version_list if x.startswith("HOST_")] + \
|
|
||||||
[x for x in version_list if not x.startswith("HOST_")]
|
|
||||||
for l in version_list:
|
|
||||||
# Get variable name and value
|
|
||||||
pkgvar, value = l.split("=")
|
|
||||||
|
|
||||||
# If present, strip HOST_ from variable name
|
|
||||||
if pkgvar.startswith("HOST_"):
|
|
||||||
pkgvar = pkgvar[5:]
|
|
||||||
|
|
||||||
if pkgvar.endswith("_DL_VERSION"):
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Strip _VERSION
|
|
||||||
pkgvar = pkgvar[:-8]
|
|
||||||
|
|
||||||
Package.all_versions[pkgvar] = value
|
|
||||||
|
|
||||||
|
|
||||||
def calculate_stats(packages):
|
|
||||||
stats = defaultdict(int)
|
|
||||||
for pkg in packages:
|
|
||||||
# If packages have multiple infra, take the first one. For the
|
|
||||||
# vast majority of packages, the target and host infra are the
|
|
||||||
# same. There are very few packages that use a different infra
|
|
||||||
# for the host and target variants.
|
|
||||||
if len(pkg.infras) > 0:
|
|
||||||
infra = pkg.infras[0][1]
|
|
||||||
stats["infra-%s" % infra] += 1
|
|
||||||
else:
|
|
||||||
stats["infra-unknown"] += 1
|
|
||||||
if pkg.has_license:
|
|
||||||
stats["license"] += 1
|
|
||||||
else:
|
|
||||||
stats["no-license"] += 1
|
|
||||||
if pkg.has_license_files:
|
|
||||||
stats["license-files"] += 1
|
|
||||||
else:
|
|
||||||
stats["no-license-files"] += 1
|
|
||||||
if pkg.has_hash:
|
|
||||||
stats["hash"] += 1
|
|
||||||
else:
|
|
||||||
stats["no-hash"] += 1
|
|
||||||
stats["patches"] += pkg.patch_count
|
|
||||||
return stats
|
|
||||||
|
|
||||||
|
|
||||||
html_header = """
|
|
||||||
<head>
|
|
||||||
<script src=\"https://www.kryogenix.org/code/browser/sorttable/sorttable.js\"></script>
|
|
||||||
<style type=\"text/css\">
|
|
||||||
table {
|
|
||||||
width: 100%;
|
|
||||||
}
|
|
||||||
td {
|
|
||||||
border: 1px solid black;
|
|
||||||
}
|
|
||||||
td.centered {
|
|
||||||
text-align: center;
|
|
||||||
}
|
|
||||||
td.wrong {
|
|
||||||
background: #ff9a69;
|
|
||||||
}
|
|
||||||
td.correct {
|
|
||||||
background: #d2ffc4;
|
|
||||||
}
|
|
||||||
td.nopatches {
|
|
||||||
background: #d2ffc4;
|
|
||||||
}
|
|
||||||
td.somepatches {
|
|
||||||
background: #ffd870;
|
|
||||||
}
|
|
||||||
td.lotsofpatches {
|
|
||||||
background: #ff9a69;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
<title>Statistics of Buildroot packages</title>
|
|
||||||
</head>
|
|
||||||
|
|
||||||
<a href=\"#results\">Results</a><br/>
|
|
||||||
|
|
||||||
<p id=\"sortable_hint\"></p>
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
html_footer = """
|
|
||||||
</body>
|
|
||||||
<script>
|
|
||||||
if (typeof sorttable === \"object\") {
|
|
||||||
document.getElementById(\"sortable_hint\").innerHTML =
|
|
||||||
\"hint: the table can be sorted by clicking the column headers\"
|
|
||||||
}
|
|
||||||
</script>
|
|
||||||
</html>
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
def infra_str(infra_list):
|
|
||||||
if not infra_list:
|
|
||||||
return "Unknown"
|
|
||||||
elif len(infra_list) == 1:
|
|
||||||
return "<b>%s</b><br/>%s" % (infra_list[0][1], infra_list[0][0])
|
|
||||||
elif infra_list[0][1] == infra_list[1][1]:
|
|
||||||
return "<b>%s</b><br/>%s + %s" % \
|
|
||||||
(infra_list[0][1], infra_list[0][0], infra_list[1][0])
|
|
||||||
else:
|
|
||||||
return "<b>%s</b> (%s)<br/><b>%s</b> (%s)" % \
|
|
||||||
(infra_list[0][1], infra_list[0][0],
|
|
||||||
infra_list[1][1], infra_list[1][0])
|
|
||||||
|
|
||||||
|
|
||||||
def boolean_str(b):
|
|
||||||
if b:
|
|
||||||
return "Yes"
|
|
||||||
else:
|
|
||||||
return "No"
|
|
||||||
|
|
||||||
|
|
||||||
def dump_html_pkg(f, pkg):
|
|
||||||
f.write(" <tr>\n")
|
|
||||||
f.write(" <td>%s</td>\n" % pkg.path[2:])
|
|
||||||
|
|
||||||
# Patch count
|
|
||||||
td_class = ["centered"]
|
|
||||||
if pkg.patch_count == 0:
|
|
||||||
td_class.append("nopatches")
|
|
||||||
elif pkg.patch_count < 5:
|
|
||||||
td_class.append("somepatches")
|
|
||||||
else:
|
|
||||||
td_class.append("lotsofpatches")
|
|
||||||
f.write(" <td class=\"%s\">%s</td>\n" %
|
|
||||||
(" ".join(td_class), str(pkg.patch_count)))
|
|
||||||
|
|
||||||
# Infrastructure
|
|
||||||
infra = infra_str(pkg.infras)
|
|
||||||
td_class = ["centered"]
|
|
||||||
if infra == "Unknown":
|
|
||||||
td_class.append("wrong")
|
|
||||||
else:
|
|
||||||
td_class.append("correct")
|
|
||||||
f.write(" <td class=\"%s\">%s</td>\n" %
|
|
||||||
(" ".join(td_class), infra_str(pkg.infras)))
|
|
||||||
|
|
||||||
# License
|
|
||||||
td_class = ["centered"]
|
|
||||||
if pkg.has_license:
|
|
||||||
td_class.append("correct")
|
|
||||||
else:
|
|
||||||
td_class.append("wrong")
|
|
||||||
f.write(" <td class=\"%s\">%s</td>\n" %
|
|
||||||
(" ".join(td_class), boolean_str(pkg.has_license)))
|
|
||||||
|
|
||||||
# License files
|
|
||||||
td_class = ["centered"]
|
|
||||||
if pkg.has_license_files:
|
|
||||||
td_class.append("correct")
|
|
||||||
else:
|
|
||||||
td_class.append("wrong")
|
|
||||||
f.write(" <td class=\"%s\">%s</td>\n" %
|
|
||||||
(" ".join(td_class), boolean_str(pkg.has_license_files)))
|
|
||||||
|
|
||||||
# Hash
|
|
||||||
td_class = ["centered"]
|
|
||||||
if pkg.has_hash:
|
|
||||||
td_class.append("correct")
|
|
||||||
else:
|
|
||||||
td_class.append("wrong")
|
|
||||||
f.write(" <td class=\"%s\">%s</td>\n" %
|
|
||||||
(" ".join(td_class), boolean_str(pkg.has_hash)))
|
|
||||||
|
|
||||||
# Current version
|
|
||||||
if len(pkg.current_version) > 20:
|
|
||||||
current_version = pkg.current_version[:20] + "..."
|
|
||||||
else:
|
|
||||||
current_version = pkg.current_version
|
|
||||||
f.write(" <td class=\"centered\">%s</td>\n" % current_version)
|
|
||||||
|
|
||||||
# Warnings
|
|
||||||
td_class = ["centered"]
|
|
||||||
if pkg.warnings == 0:
|
|
||||||
td_class.append("correct")
|
|
||||||
else:
|
|
||||||
td_class.append("wrong")
|
|
||||||
f.write(" <td class=\"%s\">%d</td>\n" %
|
|
||||||
(" ".join(td_class), pkg.warnings))
|
|
||||||
|
|
||||||
f.write(" </tr>\n")
|
|
||||||
|
|
||||||
|
|
||||||
def dump_html_all_pkgs(f, packages):
|
|
||||||
f.write("""
|
|
||||||
<table class=\"sortable\">
|
|
||||||
<tr>
|
|
||||||
<td>Package</td>
|
|
||||||
<td class=\"centered\">Patch count</td>
|
|
||||||
<td class=\"centered\">Infrastructure</td>
|
|
||||||
<td class=\"centered\">License</td>
|
|
||||||
<td class=\"centered\">License files</td>
|
|
||||||
<td class=\"centered\">Hash file</td>
|
|
||||||
<td class=\"centered\">Current version</td>
|
|
||||||
<td class=\"centered\">Warnings</td>
|
|
||||||
</tr>
|
|
||||||
""")
|
|
||||||
for pkg in sorted(packages):
|
|
||||||
dump_html_pkg(f, pkg)
|
|
||||||
f.write("</table>")
|
|
||||||
|
|
||||||
|
|
||||||
def dump_html_stats(f, stats):
|
|
||||||
f.write("<a id=\"results\"></a>\n")
|
|
||||||
f.write("<table>\n")
|
|
||||||
infras = [infra[6:] for infra in stats.keys() if infra.startswith("infra-")]
|
|
||||||
for infra in infras:
|
|
||||||
f.write(" <tr><td>Packages using the <i>%s</i> infrastructure</td><td>%s</td></tr>\n" %
|
|
||||||
(infra, stats["infra-%s" % infra]))
|
|
||||||
f.write(" <tr><td>Packages having license information</td><td>%s</td></tr>\n" %
|
|
||||||
stats["license"])
|
|
||||||
f.write(" <tr><td>Packages not having license information</td><td>%s</td></tr>\n" %
|
|
||||||
stats["no-license"])
|
|
||||||
f.write(" <tr><td>Packages having license files information</td><td>%s</td></tr>\n" %
|
|
||||||
stats["license-files"])
|
|
||||||
f.write(" <tr><td>Packages not having license files information</td><td>%s</td></tr>\n" %
|
|
||||||
stats["no-license-files"])
|
|
||||||
f.write(" <tr><td>Packages having a hash file</td><td>%s</td></tr>\n" %
|
|
||||||
stats["hash"])
|
|
||||||
f.write(" <tr><td>Packages not having a hash file</td><td>%s</td></tr>\n" %
|
|
||||||
stats["no-hash"])
|
|
||||||
f.write(" <tr><td>Total number of patches</td><td>%s</td></tr>\n" %
|
|
||||||
stats["patches"])
|
|
||||||
f.write("</table>\n")
|
|
||||||
|
|
||||||
|
|
||||||
def dump_gen_info(f):
|
|
||||||
# Updated on Mon Feb 19 08:12:08 CET 2018, Git commit aa77030b8f5e41f1c53eb1c1ad664b8c814ba032
|
|
||||||
o = subprocess.check_output(["git", "log", "master", "-n", "1", "--pretty=format:%H"])
|
|
||||||
git_commit = o.splitlines()[0]
|
|
||||||
f.write("<p><i>Updated on %s, git commit %s</i></p>\n" %
|
|
||||||
(str(datetime.datetime.utcnow()), git_commit))
|
|
||||||
|
|
||||||
|
|
||||||
def dump_html(packages, stats, output):
|
|
||||||
with open(output, 'w') as f:
|
|
||||||
f.write(html_header)
|
|
||||||
dump_html_all_pkgs(f, packages)
|
|
||||||
dump_html_stats(f, stats)
|
|
||||||
dump_gen_info(f)
|
|
||||||
f.write(html_footer)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_args():
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument('-o', dest='output', action='store', required=True,
|
|
||||||
help='HTML output file')
|
|
||||||
parser.add_argument('-n', dest='npackages', type=int, action='store',
|
|
||||||
help='Number of packages')
|
|
||||||
parser.add_argument('-p', dest='packages', action='store',
|
|
||||||
help='List of packages (comma separated)')
|
|
||||||
return parser.parse_args()
|
|
||||||
|
|
||||||
|
|
||||||
def __main__():
|
|
||||||
args = parse_args()
|
|
||||||
if args.npackages and args.packages:
|
|
||||||
print "ERROR: -n and -p are mutually exclusive"
|
|
||||||
sys.exit(1)
|
|
||||||
if args.packages:
|
|
||||||
package_list = args.packages.split(",")
|
|
||||||
else:
|
|
||||||
package_list = None
|
|
||||||
print "Build package list ..."
|
|
||||||
packages = get_pkglist(args.npackages, package_list)
|
|
||||||
print "Getting package make info ..."
|
|
||||||
package_init_make_info()
|
|
||||||
print "Getting package details ..."
|
|
||||||
for pkg in packages:
|
|
||||||
pkg.set_infra()
|
|
||||||
pkg.set_license()
|
|
||||||
pkg.set_hash_info()
|
|
||||||
pkg.set_patch_count()
|
|
||||||
pkg.set_check_package_warnings()
|
|
||||||
pkg.set_current_version()
|
|
||||||
print "Calculate stats"
|
|
||||||
stats = calculate_stats(packages)
|
|
||||||
print "Write HTML"
|
|
||||||
dump_html(packages, stats, args.output)
|
|
||||||
|
|
||||||
|
|
||||||
__main__()
|
|
||||||
Loading…
Reference in New Issue
Block a user