From f955a4eac0909154ee0d73b47261c088287db6cb Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Wed, 4 Apr 2018 22:05:33 +0200 Subject: [PATCH] support/scripts/pkg-stats: replace with new Python version Rename pkg-stats-new to pkg-stats. Signed-off-by: Thomas Petazzoni --- support/scripts/pkg-stats | 859 ++++++++++++++++++---------------- support/scripts/pkg-stats-new | 526 --------------------- 2 files changed, 465 insertions(+), 920 deletions(-) delete mode 100755 support/scripts/pkg-stats-new diff --git a/support/scripts/pkg-stats b/support/scripts/pkg-stats index 48a2cc29a1..43f7e8d543 100755 --- a/support/scripts/pkg-stats +++ b/support/scripts/pkg-stats @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env python # Copyright (C) 2009 by Thomas Petazzoni # @@ -16,16 +16,275 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# This script generates an HTML file that contains a report about all -# Buildroot packages, their usage of the different package -# infrastructure and possible cleanup actions -# -# Run the script from the Buildroot toplevel directory: -# -# ./support/scripts/pkg-stats > /tmp/pkg.html -# - -echo " +import argparse +import datetime +import fnmatch +import os +from collections import defaultdict +import re +import subprocess +import sys + +INFRA_RE = re.compile("\$\(eval \$\(([a-z-]*)-package\)\)") + + +class Package: + all_licenses = list() + all_license_files = list() + all_versions = dict() + + def __init__(self, name, path): + self.name = name + self.path = path + self.infras = None + self.has_license = False + self.has_license_files = False + self.has_hash = False + self.patch_count = 0 + self.warnings = 0 + self.current_version = None + + def pkgvar(self): + return self.name.upper().replace("-", "_") + + def set_infra(self): + """ + Fills in the .infras field + """ + self.infras = list() + with open(self.path, 'r') as f: + lines = f.readlines() + for l in lines: + match = INFRA_RE.match(l) + if not match: + continue + infra = match.group(1) + if infra.startswith("host-"): + self.infras.append(("host", infra[5:])) + else: + self.infras.append(("target", infra)) + + def set_license(self): + """ + Fills in the .has_license and .has_license_files fields + """ + var = self.pkgvar() + if var in self.all_licenses: + self.has_license = True + if var in self.all_license_files: + self.has_license_files = True + + def set_hash_info(self): + """ + Fills in the .has_hash field + """ + hashpath = self.path.replace(".mk", ".hash") + self.has_hash = os.path.exists(hashpath) + + def set_patch_count(self): + """ + Fills in the .patch_count field + """ + self.patch_count = 0 + pkgdir = os.path.dirname(self.path) + for subdir, _, _ in os.walk(pkgdir): + self.patch_count += len(fnmatch.filter(os.listdir(subdir), '*.patch')) + + def set_current_version(self): + """ + Fills in the .current_version field + """ + var = self.pkgvar() + if var in self.all_versions: + self.current_version = self.all_versions[var] + + def set_check_package_warnings(self): + """ + Fills in the .warnings field + """ + cmd = ["./utils/check-package"] + pkgdir = os.path.dirname(self.path) + for root, dirs, files in os.walk(pkgdir): + for f in files: + if f.endswith(".mk") or f.endswith(".hash") or f == "Config.in" or f == "Config.in.host": + cmd.append(os.path.join(root, f)) + o = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[1] + lines = o.splitlines() + for line in lines: + m = re.match("^([0-9]*) warnings generated", line) + if m: + self.warnings = int(m.group(1)) + return + + def __eq__(self, other): + return self.path == other.path + + def __lt__(self, other): + return self.path < other.path + + def __str__(self): + return "%s (path='%s', license='%s', license_files='%s', hash='%s', patches=%d)" % \ + (self.name, self.path, self.has_license, self.has_license_files, self.has_hash, self.patch_count) + + +def get_pkglist(npackages, package_list): + """ + Builds the list of Buildroot packages, returning a list of Package + objects. Only the .name and .path fields of the Package object are + initialized. + + npackages: limit to N packages + package_list: limit to those packages in this list + """ + WALK_USEFUL_SUBDIRS = ["boot", "linux", "package", "toolchain"] + WALK_EXCLUDES = ["boot/common.mk", + "linux/linux-ext-.*.mk", + "package/freescale-imx/freescale-imx.mk", + "package/gcc/gcc.mk", + "package/gstreamer/gstreamer.mk", + "package/gstreamer1/gstreamer1.mk", + "package/gtk2-themes/gtk2-themes.mk", + "package/matchbox/matchbox.mk", + "package/opengl/opengl.mk", + "package/qt5/qt5.mk", + "package/x11r7/x11r7.mk", + "package/doc-asciidoc.mk", + "package/pkg-.*.mk", + "package/nvidia-tegra23/nvidia-tegra23.mk", + "toolchain/toolchain-external/pkg-toolchain-external.mk", + "toolchain/toolchain-external/toolchain-external.mk", + "toolchain/toolchain.mk", + "toolchain/helpers.mk", + "toolchain/toolchain-wrapper.mk"] + packages = list() + count = 0 + for root, dirs, files in os.walk("."): + rootdir = root.split("/") + if len(rootdir) < 2: + continue + if rootdir[1] not in WALK_USEFUL_SUBDIRS: + continue + for f in files: + if not f.endswith(".mk"): + continue + # Strip ending ".mk" + pkgname = f[:-3] + if package_list and pkgname not in package_list: + continue + pkgpath = os.path.join(root, f) + skip = False + for exclude in WALK_EXCLUDES: + # pkgpath[2:] strips the initial './' + if re.match(exclude, pkgpath[2:]): + skip = True + continue + if skip: + continue + p = Package(pkgname, pkgpath) + packages.append(p) + count += 1 + if npackages and count == npackages: + return packages + return packages + + +def package_init_make_info(): + # Licenses + o = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y", + "-s", "printvars", "VARS=%_LICENSE"]) + for l in o.splitlines(): + # Get variable name and value + pkgvar, value = l.split("=") + + # If present, strip HOST_ from variable name + if pkgvar.startswith("HOST_"): + pkgvar = pkgvar[5:] + + # Strip _LICENSE + pkgvar = pkgvar[:-8] + + # If value is "unknown", no license details available + if value == "unknown": + continue + Package.all_licenses.append(pkgvar) + + # License files + o = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y", + "-s", "printvars", "VARS=%_LICENSE_FILES"]) + for l in o.splitlines(): + # Get variable name and value + pkgvar, value = l.split("=") + + # If present, strip HOST_ from variable name + if pkgvar.startswith("HOST_"): + pkgvar = pkgvar[5:] + + if pkgvar.endswith("_MANIFEST_LICENSE_FILES"): + continue + + # Strip _LICENSE_FILES + pkgvar = pkgvar[:-14] + + Package.all_license_files.append(pkgvar) + + # Version + o = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y", + "-s", "printvars", "VARS=%_VERSION"]) + + # We process first the host package VERSION, and then the target + # package VERSION. This means that if a package exists in both + # target and host variants, with different version numbers + # (unlikely), we'll report the target version number. + version_list = o.splitlines() + version_list = [x for x in version_list if x.startswith("HOST_")] + \ + [x for x in version_list if not x.startswith("HOST_")] + for l in version_list: + # Get variable name and value + pkgvar, value = l.split("=") + + # If present, strip HOST_ from variable name + if pkgvar.startswith("HOST_"): + pkgvar = pkgvar[5:] + + if pkgvar.endswith("_DL_VERSION"): + continue + + # Strip _VERSION + pkgvar = pkgvar[:-8] + + Package.all_versions[pkgvar] = value + + +def calculate_stats(packages): + stats = defaultdict(int) + for pkg in packages: + # If packages have multiple infra, take the first one. For the + # vast majority of packages, the target and host infra are the + # same. There are very few packages that use a different infra + # for the host and target variants. + if len(pkg.infras) > 0: + infra = pkg.infras[0][1] + stats["infra-%s" % infra] += 1 + else: + stats["infra-unknown"] += 1 + if pkg.has_license: + stats["license"] += 1 + else: + stats["no-license"] += 1 + if pkg.has_license_files: + stats["license-files"] += 1 + else: + stats["no-license-files"] += 1 + if pkg.has_hash: + stats["hash"] += 1 + else: + stats["no-hash"] += 1 + stats["patches"] += pkg.patch_count + return stats + + +html_header = """ + -Statistics of Buildroot packages - - -Results
- -

-""" - - -html_footer = """ - - - -""" - - -def infra_str(infra_list): - if not infra_list: - return "Unknown" - elif len(infra_list) == 1: - return "%s
%s" % (infra_list[0][1], infra_list[0][0]) - elif infra_list[0][1] == infra_list[1][1]: - return "%s
%s + %s" % \ - (infra_list[0][1], infra_list[0][0], infra_list[1][0]) - else: - return "%s (%s)
%s (%s)" % \ - (infra_list[0][1], infra_list[0][0], - infra_list[1][1], infra_list[1][0]) - - -def boolean_str(b): - if b: - return "Yes" - else: - return "No" - - -def dump_html_pkg(f, pkg): - f.write(" \n") - f.write(" %s\n" % pkg.path[2:]) - - # Patch count - td_class = ["centered"] - if pkg.patch_count == 0: - td_class.append("nopatches") - elif pkg.patch_count < 5: - td_class.append("somepatches") - else: - td_class.append("lotsofpatches") - f.write(" %s\n" % - (" ".join(td_class), str(pkg.patch_count))) - - # Infrastructure - infra = infra_str(pkg.infras) - td_class = ["centered"] - if infra == "Unknown": - td_class.append("wrong") - else: - td_class.append("correct") - f.write(" %s\n" % - (" ".join(td_class), infra_str(pkg.infras))) - - # License - td_class = ["centered"] - if pkg.has_license: - td_class.append("correct") - else: - td_class.append("wrong") - f.write(" %s\n" % - (" ".join(td_class), boolean_str(pkg.has_license))) - - # License files - td_class = ["centered"] - if pkg.has_license_files: - td_class.append("correct") - else: - td_class.append("wrong") - f.write(" %s\n" % - (" ".join(td_class), boolean_str(pkg.has_license_files))) - - # Hash - td_class = ["centered"] - if pkg.has_hash: - td_class.append("correct") - else: - td_class.append("wrong") - f.write(" %s\n" % - (" ".join(td_class), boolean_str(pkg.has_hash))) - - # Current version - if len(pkg.current_version) > 20: - current_version = pkg.current_version[:20] + "..." - else: - current_version = pkg.current_version - f.write(" %s\n" % current_version) - - # Warnings - td_class = ["centered"] - if pkg.warnings == 0: - td_class.append("correct") - else: - td_class.append("wrong") - f.write(" %d\n" % - (" ".join(td_class), pkg.warnings)) - - f.write(" \n") - - -def dump_html_all_pkgs(f, packages): - f.write(""" - - - - - - - - - - - -""") - for pkg in sorted(packages): - dump_html_pkg(f, pkg) - f.write("
PackagePatch countInfrastructureLicenseLicense filesHash fileCurrent versionWarnings
") - - -def dump_html_stats(f, stats): - f.write("\n") - f.write("\n") - infras = [infra[6:] for infra in stats.keys() if infra.startswith("infra-")] - for infra in infras: - f.write(" \n" % - (infra, stats["infra-%s" % infra])) - f.write(" \n" % - stats["license"]) - f.write(" \n" % - stats["no-license"]) - f.write(" \n" % - stats["license-files"]) - f.write(" \n" % - stats["no-license-files"]) - f.write(" \n" % - stats["hash"]) - f.write(" \n" % - stats["no-hash"]) - f.write(" \n" % - stats["patches"]) - f.write("
Packages using the %s infrastructure%s
Packages having license information%s
Packages not having license information%s
Packages having license files information%s
Packages not having license files information%s
Packages having a hash file%s
Packages not having a hash file%s
Total number of patches%s
\n") - - -def dump_gen_info(f): - # Updated on Mon Feb 19 08:12:08 CET 2018, Git commit aa77030b8f5e41f1c53eb1c1ad664b8c814ba032 - o = subprocess.check_output(["git", "log", "master", "-n", "1", "--pretty=format:%H"]) - git_commit = o.splitlines()[0] - f.write("

Updated on %s, git commit %s

\n" % - (str(datetime.datetime.utcnow()), git_commit)) - - -def dump_html(packages, stats, output): - with open(output, 'w') as f: - f.write(html_header) - dump_html_all_pkgs(f, packages) - dump_html_stats(f, stats) - dump_gen_info(f) - f.write(html_footer) - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-o', dest='output', action='store', required=True, - help='HTML output file') - parser.add_argument('-n', dest='npackages', type=int, action='store', - help='Number of packages') - parser.add_argument('-p', dest='packages', action='store', - help='List of packages (comma separated)') - return parser.parse_args() - - -def __main__(): - args = parse_args() - if args.npackages and args.packages: - print "ERROR: -n and -p are mutually exclusive" - sys.exit(1) - if args.packages: - package_list = args.packages.split(",") - else: - package_list = None - print "Build package list ..." - packages = get_pkglist(args.npackages, package_list) - print "Getting package make info ..." - package_init_make_info() - print "Getting package details ..." - for pkg in packages: - pkg.set_infra() - pkg.set_license() - pkg.set_hash_info() - pkg.set_patch_count() - pkg.set_check_package_warnings() - pkg.set_current_version() - print "Calculate stats" - stats = calculate_stats(packages) - print "Write HTML" - dump_html(packages, stats, args.output) - - -__main__() -- 2.30.2