From: Yann E. MORIN Date: Sat, 1 Jul 2017 14:31:03 +0000 (+0200) Subject: tools: move check-package out of support/scripts/ X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=e518b38936fb836bebf7f2274c6d6cdc704e119a;p=buildroot.git tools: move check-package out of support/scripts/ Move it to the top-level tools/ directory, so that it is easier to find for users. Signed-off-by: "Yann E. MORIN" Cc: Ricardo Martincoski Cc: Thomas Petazzoni Cc: Arnout Vandecappelle Signed-off-by: Thomas Petazzoni --- diff --git a/DEVELOPERS b/DEVELOPERS index 47637fe6cd..cd726f5b61 100644 --- a/DEVELOPERS +++ b/DEVELOPERS @@ -1391,7 +1391,8 @@ N: Rhys Williams F: package/lirc-tools/ N: Ricardo Martincoski -F: support/scripts/check*package* +F: tools/check-package +F: tools/checkpackagelib/ N: Richard Braun F: package/curlftpfs/ diff --git a/support/scripts/check-package b/support/scripts/check-package deleted file mode 100755 index 74ea46f819..0000000000 --- a/support/scripts/check-package +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env python -# See support/scripts/checkpackagelib/readme.txt before editing this file. - -from __future__ import print_function -import argparse -import inspect -import re -import sys - -import checkpackagelib.lib_config -import checkpackagelib.lib_hash -import checkpackagelib.lib_mk -import checkpackagelib.lib_patch - -VERBOSE_LEVEL_TO_SHOW_IGNORED_FILES = 3 -flags = None # Command line arguments. - - -def parse_args(): - parser = argparse.ArgumentParser() - - # Do not use argparse.FileType("r") here because only files with known - # format will be open based on the filename. - parser.add_argument("files", metavar="F", type=str, nargs="*", - help="list of files") - - parser.add_argument("--manual-url", action="store", - default="http://nightly.buildroot.org/", - help="default: %(default)s") - parser.add_argument("--verbose", "-v", action="count", default=0) - - # Now the debug options in the order they are processed. - parser.add_argument("--include-only", dest="include_list", action="append", - help="run only the specified functions (debug)") - parser.add_argument("--exclude", dest="exclude_list", action="append", - help="do not run the specified functions (debug)") - parser.add_argument("--dry-run", action="store_true", help="print the " - "functions that would be called for each file (debug)") - - return parser.parse_args() - - -CONFIG_IN_FILENAME = re.compile("/Config\.\S*$") -FILE_IS_FROM_A_PACKAGE = re.compile("package/[^/]*/") - - -def get_lib_from_filename(fname): - if FILE_IS_FROM_A_PACKAGE.search(fname) is None: - return None - if CONFIG_IN_FILENAME.search(fname): - return checkpackagelib.lib_config - if fname.endswith(".hash"): - return checkpackagelib.lib_hash - if fname.endswith(".mk"): - return checkpackagelib.lib_mk - if fname.endswith(".patch"): - return checkpackagelib.lib_patch - return None - - -def is_a_check_function(m): - if not inspect.isclass(m): - return False - # do not call the base class - if m.__name__.startswith("_"): - return False - if flags.include_list and m.__name__ not in flags.include_list: - return False - if flags.exclude_list and m.__name__ in flags.exclude_list: - return False - return True - - -def print_warnings(warnings): - # Avoid the need to use 'return []' at the end of every check function. - if warnings is None: - return 0 # No warning generated. - - for level, message in enumerate(warnings): - if flags.verbose >= level: - print(message.replace("\t", "< tab >").rstrip()) - return 1 # One more warning to count. - - -def check_file_using_lib(fname): - # Count number of warnings generated and lines processed. - nwarnings = 0 - nlines = 0 - - lib = get_lib_from_filename(fname) - if not lib: - if flags.verbose >= VERBOSE_LEVEL_TO_SHOW_IGNORED_FILES: - print("{}: ignored".format(fname)) - return nwarnings, nlines - classes = inspect.getmembers(lib, is_a_check_function) - - if flags.dry_run: - functions_to_run = [c[0] for c in classes] - print("{}: would run: {}".format(fname, functions_to_run)) - return nwarnings, nlines - - objects = [c[1](fname, flags.manual_url) for c in classes] - - for cf in objects: - nwarnings += print_warnings(cf.before()) - for lineno, text in enumerate(open(fname, "r").readlines()): - nlines += 1 - for cf in objects: - nwarnings += print_warnings(cf.check_line(lineno + 1, text)) - for cf in objects: - nwarnings += print_warnings(cf.after()) - - return nwarnings, nlines - - -def __main__(): - global flags - flags = parse_args() - - if len(flags.files) == 0: - print("No files to check style") - sys.exit(1) - - # Accumulate number of warnings generated and lines processed. - total_warnings = 0 - total_lines = 0 - - for fname in flags.files: - nwarnings, nlines = check_file_using_lib(fname) - total_warnings += nwarnings - total_lines += nlines - - # The warning messages are printed to stdout and can be post-processed - # (e.g. counted by 'wc'), so for stats use stderr. Wait all warnings are - # printed, for the case there are many of them, before printing stats. - sys.stdout.flush() - print("{} lines processed".format(total_lines), file=sys.stderr) - print("{} warnings generated".format(total_warnings), file=sys.stderr) - - if total_warnings > 0: - sys.exit(1) - - -__main__() diff --git a/support/scripts/checkpackagelib/__init__.py b/support/scripts/checkpackagelib/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/support/scripts/checkpackagelib/base.py b/support/scripts/checkpackagelib/base.py deleted file mode 100644 index 7669775f06..0000000000 --- a/support/scripts/checkpackagelib/base.py +++ /dev/null @@ -1,16 +0,0 @@ -# See support/scripts/checkpackagelib/readme.txt before editing this file. - - -class _CheckFunction(object): - def __init__(self, filename, url_to_manual): - self.filename = filename - self.url_to_manual = url_to_manual - - def before(self): - pass - - def check_line(self, lineno, text): - pass - - def after(self): - pass diff --git a/support/scripts/checkpackagelib/lib.py b/support/scripts/checkpackagelib/lib.py deleted file mode 100644 index 3077f518b3..0000000000 --- a/support/scripts/checkpackagelib/lib.py +++ /dev/null @@ -1,54 +0,0 @@ -# See support/scripts/checkpackagelib/readme.txt before editing this file. - -from base import _CheckFunction - - -class ConsecutiveEmptyLines(_CheckFunction): - def before(self): - self.lastline = "non empty" - - def check_line(self, lineno, text): - if text.strip() == "" == self.lastline.strip(): - return ["{}:{}: consecutive empty lines" - .format(self.filename, lineno)] - self.lastline = text - - -class EmptyLastLine(_CheckFunction): - def before(self): - self.lastlineno = 0 - self.lastline = "non empty" - - def check_line(self, lineno, text): - self.lastlineno = lineno - self.lastline = text - - def after(self): - if self.lastline.strip() == "": - return ["{}:{}: empty line at end of file" - .format(self.filename, self.lastlineno)] - - -class NewlineAtEof(_CheckFunction): - def before(self): - self.lastlineno = 0 - self.lastline = "\n" - - def check_line(self, lineno, text): - self.lastlineno = lineno - self.lastline = text - - def after(self): - if self.lastline == self.lastline.rstrip("\r\n"): - return ["{}:{}: missing newline at end of file" - .format(self.filename, self.lastlineno), - self.lastline] - - -class TrailingSpace(_CheckFunction): - def check_line(self, lineno, text): - line = text.rstrip("\r\n") - if line != line.rstrip(): - return ["{}:{}: line contains trailing whitespace" - .format(self.filename, lineno), - text] diff --git a/support/scripts/checkpackagelib/lib_config.py b/support/scripts/checkpackagelib/lib_config.py deleted file mode 100644 index 8f49224e95..0000000000 --- a/support/scripts/checkpackagelib/lib_config.py +++ /dev/null @@ -1,138 +0,0 @@ -# See support/scripts/checkpackagelib/readme.txt before editing this file. -# Kconfig generates errors if someone introduces a typo like "boool" instead of -# "bool", so below check functions don't need to check for things already -# checked by running "make menuconfig". - -import re - -from base import _CheckFunction -# Notice: ignore 'imported but unused' from pyflakes for check functions. -from lib import ConsecutiveEmptyLines -from lib import EmptyLastLine -from lib import NewlineAtEof -from lib import TrailingSpace - - -def _empty_or_comment(text): - line = text.strip() - # ignore empty lines and comment lines indented or not - return line == "" or line.startswith("#") - - -def _part_of_help_text(text): - return text.startswith("\t ") - - -# used in more than one check -entries_that_should_not_be_indented = [ - "choice", "comment", "config", "endchoice", "endif", "endmenu", "if", - "menu", "menuconfig", "source"] - - -class AttributesOrder(_CheckFunction): - attributes_order_convention = { - "bool": 1, "prompt": 1, "string": 1, "default": 2, "depends": 3, - "select": 4, "help": 5} - - def before(self): - self.state = 0 - - def check_line(self, lineno, text): - if _empty_or_comment(text) or _part_of_help_text(text): - return - - attribute = text.split()[0] - - if attribute in entries_that_should_not_be_indented: - self.state = 0 - return - if attribute not in self.attributes_order_convention.keys(): - return - new_state = self.attributes_order_convention[attribute] - wrong_order = self.state > new_state - - # save to process next line - self.state = new_state - - if wrong_order: - return ["{}:{}: attributes order: type, default, depends on," - " select, help ({}#_config_files)" - .format(self.filename, lineno, self.url_to_manual), - text] - - -class HelpText(_CheckFunction): - HELP_TEXT_FORMAT = re.compile("^\t .{,62}$") - URL_ONLY = re.compile("^(http|https|git)://\S*$") - - def before(self): - self.help_text = False - - def check_line(self, lineno, text): - if _empty_or_comment(text): - return - - entry = text.split()[0] - - if entry in entries_that_should_not_be_indented: - self.help_text = False - return - if text.strip() == "help": - self.help_text = True - return - - if not self.help_text: - return - - if self.HELP_TEXT_FORMAT.match(text.rstrip()): - return - if self.URL_ONLY.match(text.strip()): - return - return ["{}:{}: help text: <2 spaces><62 chars>" - " ({}#writing-rules-config-in)" - .format(self.filename, lineno, self.url_to_manual), - text, - "\t " + "123456789 " * 6 + "12"] - - -class Indent(_CheckFunction): - ENDS_WITH_BACKSLASH = re.compile(r"^[^#].*\\$") - entries_that_should_be_indented = [ - "bool", "default", "depends", "help", "prompt", "select", "string"] - - def before(self): - self.backslash = False - - def check_line(self, lineno, text): - if _empty_or_comment(text) or _part_of_help_text(text): - self.backslash = False - return - - entry = text.split()[0] - - last_line_ends_in_backslash = self.backslash - - # calculate for next line - if self.ENDS_WITH_BACKSLASH.search(text): - self.backslash = True - else: - self.backslash = False - - if last_line_ends_in_backslash: - if text.startswith("\t"): - return - return ["{}:{}: continuation line should be indented using tabs" - .format(self.filename, lineno), - text] - - if entry in self.entries_that_should_be_indented: - if not text.startswith("\t{}".format(entry)): - return ["{}:{}: should be indented with one tab" - " ({}#_config_files)" - .format(self.filename, lineno, self.url_to_manual), - text] - elif entry in entries_that_should_not_be_indented: - if not text.startswith(entry): - return ["{}:{}: should not be indented" - .format(self.filename, lineno), - text] diff --git a/support/scripts/checkpackagelib/lib_hash.py b/support/scripts/checkpackagelib/lib_hash.py deleted file mode 100644 index c76abb43f1..0000000000 --- a/support/scripts/checkpackagelib/lib_hash.py +++ /dev/null @@ -1,72 +0,0 @@ -# See support/scripts/checkpackagelib/readme.txt before editing this file. -# The validity of the hashes itself is checked when building, so below check -# functions don't need to check for things already checked by running -# "make package-dirclean package-source". - -import re - -from base import _CheckFunction -# Notice: ignore 'imported but unused' from pyflakes for check functions. -from lib import ConsecutiveEmptyLines -from lib import EmptyLastLine -from lib import NewlineAtEof -from lib import TrailingSpace - - -def _empty_line_or_comment(text): - return text.strip() == "" or text.startswith("#") - - -class HashFilename(_CheckFunction): - def check_line(self, lineno, text): - if _empty_line_or_comment(text): - return - - fields = text.split() - if len(fields) < 3: - return - - if '/' in fields[2]: - return ["{}:{}: use filename without directory component" - " ({}#adding-packages-hash)" - .format(self.filename, lineno, self.url_to_manual), - text] - - -class HashNumberOfFields(_CheckFunction): - def check_line(self, lineno, text): - if _empty_line_or_comment(text): - return - - fields = text.split() - if len(fields) != 3: - return ["{}:{}: expected three fields ({}#adding-packages-hash)" - .format(self.filename, lineno, self.url_to_manual), - text] - - -class HashType(_CheckFunction): - len_of_hash = {"md5": 32, "sha1": 40, "sha224": 56, "sha256": 64, - "sha384": 96, "sha512": 128} - - def check_line(self, lineno, text): - if _empty_line_or_comment(text): - return - - fields = text.split() - if len(fields) < 2: - return - - htype, hexa = fields[:2] - if htype == "none": - return - if htype not in self.len_of_hash.keys(): - return ["{}:{}: unexpected type of hash ({}#adding-packages-hash)" - .format(self.filename, lineno, self.url_to_manual), - text] - if not re.match("^[0-9A-Fa-f]{%s}$" % self.len_of_hash[htype], hexa): - return ["{}:{}: hash size does not match type " - "({}#adding-packages-hash)" - .format(self.filename, lineno, self.url_to_manual), - text, - "expected {} hex digits".format(self.len_of_hash[htype])] diff --git a/support/scripts/checkpackagelib/lib_mk.py b/support/scripts/checkpackagelib/lib_mk.py deleted file mode 100644 index a51e0e3ce6..0000000000 --- a/support/scripts/checkpackagelib/lib_mk.py +++ /dev/null @@ -1,223 +0,0 @@ -# See support/scripts/checkpackagelib/readme.txt before editing this file. -# There are already dependency checks during the build, so below check -# functions don't need to check for things already checked by exploring the -# menu options using "make menuconfig" and by running "make" with appropriate -# packages enabled. - -import re - -from base import _CheckFunction -# Notice: ignore 'imported but unused' from pyflakes for check functions. -from lib import ConsecutiveEmptyLines -from lib import EmptyLastLine -from lib import NewlineAtEof -from lib import TrailingSpace - - -class Indent(_CheckFunction): - COMMENT = re.compile("^\s*#") - CONDITIONAL = re.compile("^\s*(ifeq|ifneq|endif)\s") - ENDS_WITH_BACKSLASH = re.compile(r"^[^#].*\\$") - END_DEFINE = re.compile("^\s*endef\s") - MAKEFILE_TARGET = re.compile("^[^# \t]+:\s") - START_DEFINE = re.compile("^\s*define\s") - - def before(self): - self.define = False - self.backslash = False - self.makefile_target = False - - def check_line(self, lineno, text): - if self.START_DEFINE.search(text): - self.define = True - return - if self.END_DEFINE.search(text): - self.define = False - return - - expect_tabs = False - if self.define or self.backslash or self.makefile_target: - expect_tabs = True - if self.CONDITIONAL.search(text): - expect_tabs = False - - # calculate for next line - if self.ENDS_WITH_BACKSLASH.search(text): - self.backslash = True - else: - self.backslash = False - - if self.MAKEFILE_TARGET.search(text): - self.makefile_target = True - return - if text.strip() == "": - self.makefile_target = False - return - - # comment can be indented or not inside define ... endef, so ignore it - if self.define and self.COMMENT.search(text): - return - - if expect_tabs: - if not text.startswith("\t"): - return ["{}:{}: expected indent with tabs" - .format(self.filename, lineno), - text] - else: - if text.startswith("\t"): - return ["{}:{}: unexpected indent with tabs" - .format(self.filename, lineno), - text] - - -class PackageHeader(_CheckFunction): - def before(self): - self.skip = False - - def check_line(self, lineno, text): - if self.skip or lineno > 6: - return - - if lineno in [1, 5]: - if lineno == 1 and text.startswith("include "): - self.skip = True - return - if text.rstrip() != "#" * 80: - return ["{}:{}: should be 80 hashes ({}#writing-rules-mk)" - .format(self.filename, lineno, self.url_to_manual), - text, - "#" * 80] - elif lineno in [2, 4]: - if text.rstrip() != "#": - return ["{}:{}: should be 1 hash ({}#writing-rules-mk)" - .format(self.filename, lineno, self.url_to_manual), - text] - elif lineno == 6: - if text.rstrip() != "": - return ["{}:{}: should be a blank line ({}#writing-rules-mk)" - .format(self.filename, lineno, self.url_to_manual), - text] - - -class SpaceBeforeBackslash(_CheckFunction): - TAB_OR_MULTIPLE_SPACES_BEFORE_BACKSLASH = re.compile(r"^.*( |\t)\\$") - - def check_line(self, lineno, text): - if self.TAB_OR_MULTIPLE_SPACES_BEFORE_BACKSLASH.match(text.rstrip()): - return ["{}:{}: use only one space before backslash" - .format(self.filename, lineno), - text] - - -class TrailingBackslash(_CheckFunction): - ENDS_WITH_BACKSLASH = re.compile(r"^[^#].*\\$") - - def before(self): - self.backslash = False - - def check_line(self, lineno, text): - last_line_ends_in_backslash = self.backslash - - # calculate for next line - if self.ENDS_WITH_BACKSLASH.search(text): - self.backslash = True - self.lastline = text - return - self.backslash = False - - if last_line_ends_in_backslash and text.strip() == "": - return ["{}:{}: remove trailing backslash" - .format(self.filename, lineno - 1), - self.lastline] - - -class TypoInPackageVariable(_CheckFunction): - ALLOWED = re.compile("|".join([ - "ACLOCAL_DIR", - "ACLOCAL_HOST_DIR", - "BR_CCACHE_INITIAL_SETUP", - "BR_NO_CHECK_HASH_FOR", - "LINUX_POST_PATCH_HOOKS", - "LINUX_TOOLS", - "LUA_RUN", - "MKFS_JFFS2", - "MKIMAGE_ARCH", - "PKG_CONFIG_HOST_BINARY", - "TARGET_FINALIZE_HOOKS", - "XTENSA_CORE_NAME"])) - PACKAGE_NAME = re.compile("/([^/]+)\.mk") - VARIABLE = re.compile("^([A-Z0-9_]+_[A-Z0-9_]+)\s*(\+|)=") - - def before(self): - package = self.PACKAGE_NAME.search(self.filename).group(1) - package = package.replace("-", "_").upper() - # linux tools do not use LINUX_TOOL_ prefix for variables - package = package.replace("LINUX_TOOL_", "") - self.package = package - self.REGEX = re.compile("^(HOST_)?({}_[A-Z0-9_]+)".format(package)) - self.FIND_VIRTUAL = re.compile( - "^{}_PROVIDES\s*(\+|)=\s*(.*)".format(package)) - self.virtual = [] - - def check_line(self, lineno, text): - m = self.VARIABLE.search(text) - if m is None: - return - - variable = m.group(1) - - # allow to set variables for virtual package this package provides - v = self.FIND_VIRTUAL.search(text) - if v: - self.virtual += v.group(2).upper().split() - return - for virtual in self.virtual: - if variable.startswith("{}_".format(virtual)): - return - - if self.ALLOWED.match(variable): - return - if self.REGEX.search(text) is None: - return ["{}:{}: possible typo: {} -> *{}*" - .format(self.filename, lineno, variable, self.package), - text] - - -class UselessFlag(_CheckFunction): - DEFAULT_AUTOTOOLS_FLAG = re.compile("^.*{}".format("|".join([ - "_AUTORECONF\s*=\s*NO", - "_LIBTOOL_PATCH\s*=\s*YES"]))) - DEFAULT_GENERIC_FLAG = re.compile("^.*{}".format("|".join([ - "_INSTALL_IMAGES\s*=\s*NO", - "_INSTALL_REDISTRIBUTE\s*=\s*YES", - "_INSTALL_STAGING\s*=\s*NO", - "_INSTALL_TARGET\s*=\s*YES"]))) - END_CONDITIONAL = re.compile("^\s*(endif)") - START_CONDITIONAL = re.compile("^\s*(ifeq|ifneq)") - - def before(self): - self.conditional = 0 - - def check_line(self, lineno, text): - if self.START_CONDITIONAL.search(text): - self.conditional += 1 - return - if self.END_CONDITIONAL.search(text): - self.conditional -= 1 - return - - # allow non-default conditionally overridden by default - if self.conditional > 0: - return - - if self.DEFAULT_GENERIC_FLAG.search(text): - return ["{}:{}: useless default value ({}#" - "_infrastructure_for_packages_with_specific_build_systems)" - .format(self.filename, lineno, self.url_to_manual), - text] - - if self.DEFAULT_AUTOTOOLS_FLAG.search(text): - return ["{}:{}: useless default value " - "({}#_infrastructure_for_autotools_based_packages)" - .format(self.filename, lineno, self.url_to_manual), - text] diff --git a/support/scripts/checkpackagelib/lib_patch.py b/support/scripts/checkpackagelib/lib_patch.py deleted file mode 100644 index c191d262e2..0000000000 --- a/support/scripts/checkpackagelib/lib_patch.py +++ /dev/null @@ -1,62 +0,0 @@ -# See support/scripts/checkpackagelib/readme.txt before editing this file. -# The format of the patch files is tested during the build, so below check -# functions don't need to check for things already checked by running -# "make package-dirclean package-patch". - -import re - -from base import _CheckFunction -# Notice: ignore 'imported but unused' from pyflakes for check functions. -from lib import NewlineAtEof - - -class ApplyOrder(_CheckFunction): - APPLY_ORDER = re.compile("/\d{1,4}-[^/]*$") - - def before(self): - if not self.APPLY_ORDER.search(self.filename): - return ["{}:0: use name -.patch " - "({}#_providing_patches)" - .format(self.filename, self.url_to_manual)] - - -class NumberedSubject(_CheckFunction): - NUMBERED_PATCH = re.compile("Subject:\s*\[PATCH\s*\d+/\d+\]") - - def before(self): - self.git_patch = False - self.lineno = 0 - self.text = None - - def check_line(self, lineno, text): - if text.startswith("diff --git"): - self.git_patch = True - return - if self.NUMBERED_PATCH.search(text): - self.lineno = lineno - self.text = text - - def after(self): - if self.git_patch and self.text: - return ["{}:{}: generate your patches with 'git format-patch -N'" - .format(self.filename, self.lineno), - self.text] - - -class Sob(_CheckFunction): - SOB_ENTRY = re.compile("^Signed-off-by: .*$") - - def before(self): - self.found = False - - def check_line(self, lineno, text): - if self.found: - return - if self.SOB_ENTRY.search(text): - self.found = True - - def after(self): - if not self.found: - return ["{}:0: missing Signed-off-by in the header " - "({}#_format_and_licensing_of_the_package_patches)" - .format(self.filename, self.url_to_manual)] diff --git a/support/scripts/checkpackagelib/readme.txt b/support/scripts/checkpackagelib/readme.txt deleted file mode 100644 index 0444d17992..0000000000 --- a/support/scripts/checkpackagelib/readme.txt +++ /dev/null @@ -1,75 +0,0 @@ -How the scripts are structured: -- check-package is the main engine, called by the user. - For each input file, this script decides which parser should be used and it - collects all classes declared in the library file and instantiates them. - The main engine opens the input files and it serves each raw line (including - newline!) to the method check_line() of every check object. - Two special methods before() and after() are used to call the initialization - of variables (for the case it needs to keep data across calls) and the - equivalent finalization (e.g. for the case a warning must be issued if some - pattern is not in the input file). -- base.py contains the base class for all check functions. -- lib.py contains the classes for common check functions. - Each check function is explicitly included in a given type-parsing library. - Do not include every single check function in this file, a class that will - only parse hash files should be implemented in the hash-parsing library. - When a warning must be issued, the check function returns an array of strings. - Each string is a warning message and is displayed if the corresponding verbose - level is active. When the script is called without --verbose only the first - warning in the returned array is printed; when called with --verbose both - first and second warnings are printed; when called with -vv until the third - warning is printed; an so on. - Helper functions can be defined and will not be called by the main script. -- lib_type.py contains check functions specific to files of this type. - -Some hints when changing this code: -- prefer O(n) algorithms, where n is the total number of lines in the files - processed. -- when there is no other reason for ordering, use alphabetical order (e.g. keep - the check functions in alphabetical order, keep the imports in alphabetical - order, and so on). -- use pyflakes to detect and fix potential problems. -- use pep8 formatting. -- keep in mind that for every class the method before() will be called before - any line is served to be checked by the method check_line(). A class that - checks the filename should only implement the method before(). A function that - needs to keep data across calls (e.g. keep the last line before the one being - processed) should initialize all variables using this method. -- keep in mind that for every class the method after() will be called after all - lines were served to be checked by the method check_line(). A class that - checks the absence of a pattern in the file will need to use this method. -- try to avoid false warnings. It's better to not issue a warning message to a - corner case than have too many false warnings. The second can make users stop - using the script. -- do not check spacing in the input line in every single function. Trailing - whitespace and wrong indentation should be checked by separate functions. -- avoid duplicate tests. Try to test only one thing in each function. -- in the warning message, include the url to a section from the manual, when - applicable. It potentially will make more people know the manual. -- use short sentences in the warning messages. A complete explanation can be - added to show when --verbose is used. -- when testing, verify the error message is displayed when the error pattern is - found, but also verify the error message is not displayed for few - well-formatted packages... there are many of these, just pick your favorite - as golden package that should not trigger any warning message. -- check the url displayed by the warning message works. - -Usage examples: -- to get a list of check functions that would be called without actually - calling them you can use the --dry-run option: -$ support/scripts/check-package --dry-run package/yourfavorite/* - -- when you just added a new check function, e.g. Something, check how it behaves - for all current packages: -$ support/scripts/check-package --include-only Something $(find package -type f) - -- the effective processing time (when the .pyc were already generated and all - files to be processed are cached in the RAM) should stay in the order of few - seconds: -$ support/scripts/check-package $(find package -type f) >/dev/null ; \ - time support/scripts/check-package $(find package -type f) >/dev/null - -- vim users can navigate the warnings (most editors probably have similar - function) since warnings are generated in the form 'path/file:line: warning': -$ find package/ -name 'Config.*' > filelist && vim -c \ - 'set makeprg=support/scripts/check-package\ $(cat\ filelist)' -c make -c copen diff --git a/support/scripts/pkg-stats b/support/scripts/pkg-stats index 4cf1f82518..f827877052 100755 --- a/support/scripts/pkg-stats +++ b/support/scripts/pkg-stats @@ -337,7 +337,7 @@ for i in $(find boot/ linux/ package/ toolchain/ -name '*.mk' | sort) ; do fi file_list=$(find ${package_dir} -name '*.mk' -o -name '*.in*' -o -name '*.hash') - nwarnings=$(./support/scripts/check-package ${file_list} 2>&1 | sed '/\([0-9]*\) warnings generated/!d; s//\1/') + nwarnings=$(./tools/check-package ${file_list} 2>&1 | sed '/\([0-9]*\) warnings generated/!d; s//\1/') if [ ${nwarnings} -eq 0 ] ; then echo "${nwarnings}" else diff --git a/tools/check-package b/tools/check-package new file mode 100755 index 0000000000..cdbac94929 --- /dev/null +++ b/tools/check-package @@ -0,0 +1,144 @@ +#!/usr/bin/env python +# See tools/checkpackagelib/readme.txt before editing this file. + +from __future__ import print_function +import argparse +import inspect +import re +import sys + +import checkpackagelib.lib_config +import checkpackagelib.lib_hash +import checkpackagelib.lib_mk +import checkpackagelib.lib_patch + +VERBOSE_LEVEL_TO_SHOW_IGNORED_FILES = 3 +flags = None # Command line arguments. + + +def parse_args(): + parser = argparse.ArgumentParser() + + # Do not use argparse.FileType("r") here because only files with known + # format will be open based on the filename. + parser.add_argument("files", metavar="F", type=str, nargs="*", + help="list of files") + + parser.add_argument("--manual-url", action="store", + default="http://nightly.buildroot.org/", + help="default: %(default)s") + parser.add_argument("--verbose", "-v", action="count", default=0) + + # Now the debug options in the order they are processed. + parser.add_argument("--include-only", dest="include_list", action="append", + help="run only the specified functions (debug)") + parser.add_argument("--exclude", dest="exclude_list", action="append", + help="do not run the specified functions (debug)") + parser.add_argument("--dry-run", action="store_true", help="print the " + "functions that would be called for each file (debug)") + + return parser.parse_args() + + +CONFIG_IN_FILENAME = re.compile("/Config\.\S*$") +FILE_IS_FROM_A_PACKAGE = re.compile("package/[^/]*/") + + +def get_lib_from_filename(fname): + if FILE_IS_FROM_A_PACKAGE.search(fname) is None: + return None + if CONFIG_IN_FILENAME.search(fname): + return checkpackagelib.lib_config + if fname.endswith(".hash"): + return checkpackagelib.lib_hash + if fname.endswith(".mk"): + return checkpackagelib.lib_mk + if fname.endswith(".patch"): + return checkpackagelib.lib_patch + return None + + +def is_a_check_function(m): + if not inspect.isclass(m): + return False + # do not call the base class + if m.__name__.startswith("_"): + return False + if flags.include_list and m.__name__ not in flags.include_list: + return False + if flags.exclude_list and m.__name__ in flags.exclude_list: + return False + return True + + +def print_warnings(warnings): + # Avoid the need to use 'return []' at the end of every check function. + if warnings is None: + return 0 # No warning generated. + + for level, message in enumerate(warnings): + if flags.verbose >= level: + print(message.replace("\t", "< tab >").rstrip()) + return 1 # One more warning to count. + + +def check_file_using_lib(fname): + # Count number of warnings generated and lines processed. + nwarnings = 0 + nlines = 0 + + lib = get_lib_from_filename(fname) + if not lib: + if flags.verbose >= VERBOSE_LEVEL_TO_SHOW_IGNORED_FILES: + print("{}: ignored".format(fname)) + return nwarnings, nlines + classes = inspect.getmembers(lib, is_a_check_function) + + if flags.dry_run: + functions_to_run = [c[0] for c in classes] + print("{}: would run: {}".format(fname, functions_to_run)) + return nwarnings, nlines + + objects = [c[1](fname, flags.manual_url) for c in classes] + + for cf in objects: + nwarnings += print_warnings(cf.before()) + for lineno, text in enumerate(open(fname, "r").readlines()): + nlines += 1 + for cf in objects: + nwarnings += print_warnings(cf.check_line(lineno + 1, text)) + for cf in objects: + nwarnings += print_warnings(cf.after()) + + return nwarnings, nlines + + +def __main__(): + global flags + flags = parse_args() + + if len(flags.files) == 0: + print("No files to check style") + sys.exit(1) + + # Accumulate number of warnings generated and lines processed. + total_warnings = 0 + total_lines = 0 + + for fname in flags.files: + nwarnings, nlines = check_file_using_lib(fname) + total_warnings += nwarnings + total_lines += nlines + + # The warning messages are printed to stdout and can be post-processed + # (e.g. counted by 'wc'), so for stats use stderr. Wait all warnings are + # printed, for the case there are many of them, before printing stats. + sys.stdout.flush() + print("{} lines processed".format(total_lines), file=sys.stderr) + print("{} warnings generated".format(total_warnings), file=sys.stderr) + + if total_warnings > 0: + sys.exit(1) + + +__main__() diff --git a/tools/checkpackagelib/__init__.py b/tools/checkpackagelib/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/checkpackagelib/base.py b/tools/checkpackagelib/base.py new file mode 100644 index 0000000000..4b376f0597 --- /dev/null +++ b/tools/checkpackagelib/base.py @@ -0,0 +1,16 @@ +# See tools/checkpackagelib/readme.txt before editing this file. + + +class _CheckFunction(object): + def __init__(self, filename, url_to_manual): + self.filename = filename + self.url_to_manual = url_to_manual + + def before(self): + pass + + def check_line(self, lineno, text): + pass + + def after(self): + pass diff --git a/tools/checkpackagelib/lib.py b/tools/checkpackagelib/lib.py new file mode 100644 index 0000000000..1a7db44b38 --- /dev/null +++ b/tools/checkpackagelib/lib.py @@ -0,0 +1,54 @@ +# See tools/checkpackagelib/readme.txt before editing this file. + +from base import _CheckFunction + + +class ConsecutiveEmptyLines(_CheckFunction): + def before(self): + self.lastline = "non empty" + + def check_line(self, lineno, text): + if text.strip() == "" == self.lastline.strip(): + return ["{}:{}: consecutive empty lines" + .format(self.filename, lineno)] + self.lastline = text + + +class EmptyLastLine(_CheckFunction): + def before(self): + self.lastlineno = 0 + self.lastline = "non empty" + + def check_line(self, lineno, text): + self.lastlineno = lineno + self.lastline = text + + def after(self): + if self.lastline.strip() == "": + return ["{}:{}: empty line at end of file" + .format(self.filename, self.lastlineno)] + + +class NewlineAtEof(_CheckFunction): + def before(self): + self.lastlineno = 0 + self.lastline = "\n" + + def check_line(self, lineno, text): + self.lastlineno = lineno + self.lastline = text + + def after(self): + if self.lastline == self.lastline.rstrip("\r\n"): + return ["{}:{}: missing newline at end of file" + .format(self.filename, self.lastlineno), + self.lastline] + + +class TrailingSpace(_CheckFunction): + def check_line(self, lineno, text): + line = text.rstrip("\r\n") + if line != line.rstrip(): + return ["{}:{}: line contains trailing whitespace" + .format(self.filename, lineno), + text] diff --git a/tools/checkpackagelib/lib_config.py b/tools/checkpackagelib/lib_config.py new file mode 100644 index 0000000000..9e93c05eb8 --- /dev/null +++ b/tools/checkpackagelib/lib_config.py @@ -0,0 +1,138 @@ +# See tools/checkpackagelib/readme.txt before editing this file. +# Kconfig generates errors if someone introduces a typo like "boool" instead of +# "bool", so below check functions don't need to check for things already +# checked by running "make menuconfig". + +import re + +from base import _CheckFunction +# Notice: ignore 'imported but unused' from pyflakes for check functions. +from lib import ConsecutiveEmptyLines +from lib import EmptyLastLine +from lib import NewlineAtEof +from lib import TrailingSpace + + +def _empty_or_comment(text): + line = text.strip() + # ignore empty lines and comment lines indented or not + return line == "" or line.startswith("#") + + +def _part_of_help_text(text): + return text.startswith("\t ") + + +# used in more than one check +entries_that_should_not_be_indented = [ + "choice", "comment", "config", "endchoice", "endif", "endmenu", "if", + "menu", "menuconfig", "source"] + + +class AttributesOrder(_CheckFunction): + attributes_order_convention = { + "bool": 1, "prompt": 1, "string": 1, "default": 2, "depends": 3, + "select": 4, "help": 5} + + def before(self): + self.state = 0 + + def check_line(self, lineno, text): + if _empty_or_comment(text) or _part_of_help_text(text): + return + + attribute = text.split()[0] + + if attribute in entries_that_should_not_be_indented: + self.state = 0 + return + if attribute not in self.attributes_order_convention.keys(): + return + new_state = self.attributes_order_convention[attribute] + wrong_order = self.state > new_state + + # save to process next line + self.state = new_state + + if wrong_order: + return ["{}:{}: attributes order: type, default, depends on," + " select, help ({}#_config_files)" + .format(self.filename, lineno, self.url_to_manual), + text] + + +class HelpText(_CheckFunction): + HELP_TEXT_FORMAT = re.compile("^\t .{,62}$") + URL_ONLY = re.compile("^(http|https|git)://\S*$") + + def before(self): + self.help_text = False + + def check_line(self, lineno, text): + if _empty_or_comment(text): + return + + entry = text.split()[0] + + if entry in entries_that_should_not_be_indented: + self.help_text = False + return + if text.strip() == "help": + self.help_text = True + return + + if not self.help_text: + return + + if self.HELP_TEXT_FORMAT.match(text.rstrip()): + return + if self.URL_ONLY.match(text.strip()): + return + return ["{}:{}: help text: <2 spaces><62 chars>" + " ({}#writing-rules-config-in)" + .format(self.filename, lineno, self.url_to_manual), + text, + "\t " + "123456789 " * 6 + "12"] + + +class Indent(_CheckFunction): + ENDS_WITH_BACKSLASH = re.compile(r"^[^#].*\\$") + entries_that_should_be_indented = [ + "bool", "default", "depends", "help", "prompt", "select", "string"] + + def before(self): + self.backslash = False + + def check_line(self, lineno, text): + if _empty_or_comment(text) or _part_of_help_text(text): + self.backslash = False + return + + entry = text.split()[0] + + last_line_ends_in_backslash = self.backslash + + # calculate for next line + if self.ENDS_WITH_BACKSLASH.search(text): + self.backslash = True + else: + self.backslash = False + + if last_line_ends_in_backslash: + if text.startswith("\t"): + return + return ["{}:{}: continuation line should be indented using tabs" + .format(self.filename, lineno), + text] + + if entry in self.entries_that_should_be_indented: + if not text.startswith("\t{}".format(entry)): + return ["{}:{}: should be indented with one tab" + " ({}#_config_files)" + .format(self.filename, lineno, self.url_to_manual), + text] + elif entry in entries_that_should_not_be_indented: + if not text.startswith(entry): + return ["{}:{}: should not be indented" + .format(self.filename, lineno), + text] diff --git a/tools/checkpackagelib/lib_hash.py b/tools/checkpackagelib/lib_hash.py new file mode 100644 index 0000000000..cc1a5e43c3 --- /dev/null +++ b/tools/checkpackagelib/lib_hash.py @@ -0,0 +1,72 @@ +# See tools/checkpackagelib/readme.txt before editing this file. +# The validity of the hashes itself is checked when building, so below check +# functions don't need to check for things already checked by running +# "make package-dirclean package-source". + +import re + +from base import _CheckFunction +# Notice: ignore 'imported but unused' from pyflakes for check functions. +from lib import ConsecutiveEmptyLines +from lib import EmptyLastLine +from lib import NewlineAtEof +from lib import TrailingSpace + + +def _empty_line_or_comment(text): + return text.strip() == "" or text.startswith("#") + + +class HashFilename(_CheckFunction): + def check_line(self, lineno, text): + if _empty_line_or_comment(text): + return + + fields = text.split() + if len(fields) < 3: + return + + if '/' in fields[2]: + return ["{}:{}: use filename without directory component" + " ({}#adding-packages-hash)" + .format(self.filename, lineno, self.url_to_manual), + text] + + +class HashNumberOfFields(_CheckFunction): + def check_line(self, lineno, text): + if _empty_line_or_comment(text): + return + + fields = text.split() + if len(fields) != 3: + return ["{}:{}: expected three fields ({}#adding-packages-hash)" + .format(self.filename, lineno, self.url_to_manual), + text] + + +class HashType(_CheckFunction): + len_of_hash = {"md5": 32, "sha1": 40, "sha224": 56, "sha256": 64, + "sha384": 96, "sha512": 128} + + def check_line(self, lineno, text): + if _empty_line_or_comment(text): + return + + fields = text.split() + if len(fields) < 2: + return + + htype, hexa = fields[:2] + if htype == "none": + return + if htype not in self.len_of_hash.keys(): + return ["{}:{}: unexpected type of hash ({}#adding-packages-hash)" + .format(self.filename, lineno, self.url_to_manual), + text] + if not re.match("^[0-9A-Fa-f]{%s}$" % self.len_of_hash[htype], hexa): + return ["{}:{}: hash size does not match type " + "({}#adding-packages-hash)" + .format(self.filename, lineno, self.url_to_manual), + text, + "expected {} hex digits".format(self.len_of_hash[htype])] diff --git a/tools/checkpackagelib/lib_mk.py b/tools/checkpackagelib/lib_mk.py new file mode 100644 index 0000000000..8cbb358b1b --- /dev/null +++ b/tools/checkpackagelib/lib_mk.py @@ -0,0 +1,223 @@ +# See tools/checkpackagelib/readme.txt before editing this file. +# There are already dependency checks during the build, so below check +# functions don't need to check for things already checked by exploring the +# menu options using "make menuconfig" and by running "make" with appropriate +# packages enabled. + +import re + +from base import _CheckFunction +# Notice: ignore 'imported but unused' from pyflakes for check functions. +from lib import ConsecutiveEmptyLines +from lib import EmptyLastLine +from lib import NewlineAtEof +from lib import TrailingSpace + + +class Indent(_CheckFunction): + COMMENT = re.compile("^\s*#") + CONDITIONAL = re.compile("^\s*(ifeq|ifneq|endif)\s") + ENDS_WITH_BACKSLASH = re.compile(r"^[^#].*\\$") + END_DEFINE = re.compile("^\s*endef\s") + MAKEFILE_TARGET = re.compile("^[^# \t]+:\s") + START_DEFINE = re.compile("^\s*define\s") + + def before(self): + self.define = False + self.backslash = False + self.makefile_target = False + + def check_line(self, lineno, text): + if self.START_DEFINE.search(text): + self.define = True + return + if self.END_DEFINE.search(text): + self.define = False + return + + expect_tabs = False + if self.define or self.backslash or self.makefile_target: + expect_tabs = True + if self.CONDITIONAL.search(text): + expect_tabs = False + + # calculate for next line + if self.ENDS_WITH_BACKSLASH.search(text): + self.backslash = True + else: + self.backslash = False + + if self.MAKEFILE_TARGET.search(text): + self.makefile_target = True + return + if text.strip() == "": + self.makefile_target = False + return + + # comment can be indented or not inside define ... endef, so ignore it + if self.define and self.COMMENT.search(text): + return + + if expect_tabs: + if not text.startswith("\t"): + return ["{}:{}: expected indent with tabs" + .format(self.filename, lineno), + text] + else: + if text.startswith("\t"): + return ["{}:{}: unexpected indent with tabs" + .format(self.filename, lineno), + text] + + +class PackageHeader(_CheckFunction): + def before(self): + self.skip = False + + def check_line(self, lineno, text): + if self.skip or lineno > 6: + return + + if lineno in [1, 5]: + if lineno == 1 and text.startswith("include "): + self.skip = True + return + if text.rstrip() != "#" * 80: + return ["{}:{}: should be 80 hashes ({}#writing-rules-mk)" + .format(self.filename, lineno, self.url_to_manual), + text, + "#" * 80] + elif lineno in [2, 4]: + if text.rstrip() != "#": + return ["{}:{}: should be 1 hash ({}#writing-rules-mk)" + .format(self.filename, lineno, self.url_to_manual), + text] + elif lineno == 6: + if text.rstrip() != "": + return ["{}:{}: should be a blank line ({}#writing-rules-mk)" + .format(self.filename, lineno, self.url_to_manual), + text] + + +class SpaceBeforeBackslash(_CheckFunction): + TAB_OR_MULTIPLE_SPACES_BEFORE_BACKSLASH = re.compile(r"^.*( |\t)\\$") + + def check_line(self, lineno, text): + if self.TAB_OR_MULTIPLE_SPACES_BEFORE_BACKSLASH.match(text.rstrip()): + return ["{}:{}: use only one space before backslash" + .format(self.filename, lineno), + text] + + +class TrailingBackslash(_CheckFunction): + ENDS_WITH_BACKSLASH = re.compile(r"^[^#].*\\$") + + def before(self): + self.backslash = False + + def check_line(self, lineno, text): + last_line_ends_in_backslash = self.backslash + + # calculate for next line + if self.ENDS_WITH_BACKSLASH.search(text): + self.backslash = True + self.lastline = text + return + self.backslash = False + + if last_line_ends_in_backslash and text.strip() == "": + return ["{}:{}: remove trailing backslash" + .format(self.filename, lineno - 1), + self.lastline] + + +class TypoInPackageVariable(_CheckFunction): + ALLOWED = re.compile("|".join([ + "ACLOCAL_DIR", + "ACLOCAL_HOST_DIR", + "BR_CCACHE_INITIAL_SETUP", + "BR_NO_CHECK_HASH_FOR", + "LINUX_POST_PATCH_HOOKS", + "LINUX_TOOLS", + "LUA_RUN", + "MKFS_JFFS2", + "MKIMAGE_ARCH", + "PKG_CONFIG_HOST_BINARY", + "TARGET_FINALIZE_HOOKS", + "XTENSA_CORE_NAME"])) + PACKAGE_NAME = re.compile("/([^/]+)\.mk") + VARIABLE = re.compile("^([A-Z0-9_]+_[A-Z0-9_]+)\s*(\+|)=") + + def before(self): + package = self.PACKAGE_NAME.search(self.filename).group(1) + package = package.replace("-", "_").upper() + # linux tools do not use LINUX_TOOL_ prefix for variables + package = package.replace("LINUX_TOOL_", "") + self.package = package + self.REGEX = re.compile("^(HOST_)?({}_[A-Z0-9_]+)".format(package)) + self.FIND_VIRTUAL = re.compile( + "^{}_PROVIDES\s*(\+|)=\s*(.*)".format(package)) + self.virtual = [] + + def check_line(self, lineno, text): + m = self.VARIABLE.search(text) + if m is None: + return + + variable = m.group(1) + + # allow to set variables for virtual package this package provides + v = self.FIND_VIRTUAL.search(text) + if v: + self.virtual += v.group(2).upper().split() + return + for virtual in self.virtual: + if variable.startswith("{}_".format(virtual)): + return + + if self.ALLOWED.match(variable): + return + if self.REGEX.search(text) is None: + return ["{}:{}: possible typo: {} -> *{}*" + .format(self.filename, lineno, variable, self.package), + text] + + +class UselessFlag(_CheckFunction): + DEFAULT_AUTOTOOLS_FLAG = re.compile("^.*{}".format("|".join([ + "_AUTORECONF\s*=\s*NO", + "_LIBTOOL_PATCH\s*=\s*YES"]))) + DEFAULT_GENERIC_FLAG = re.compile("^.*{}".format("|".join([ + "_INSTALL_IMAGES\s*=\s*NO", + "_INSTALL_REDISTRIBUTE\s*=\s*YES", + "_INSTALL_STAGING\s*=\s*NO", + "_INSTALL_TARGET\s*=\s*YES"]))) + END_CONDITIONAL = re.compile("^\s*(endif)") + START_CONDITIONAL = re.compile("^\s*(ifeq|ifneq)") + + def before(self): + self.conditional = 0 + + def check_line(self, lineno, text): + if self.START_CONDITIONAL.search(text): + self.conditional += 1 + return + if self.END_CONDITIONAL.search(text): + self.conditional -= 1 + return + + # allow non-default conditionally overridden by default + if self.conditional > 0: + return + + if self.DEFAULT_GENERIC_FLAG.search(text): + return ["{}:{}: useless default value ({}#" + "_infrastructure_for_packages_with_specific_build_systems)" + .format(self.filename, lineno, self.url_to_manual), + text] + + if self.DEFAULT_AUTOTOOLS_FLAG.search(text): + return ["{}:{}: useless default value " + "({}#_infrastructure_for_autotools_based_packages)" + .format(self.filename, lineno, self.url_to_manual), + text] diff --git a/tools/checkpackagelib/lib_patch.py b/tools/checkpackagelib/lib_patch.py new file mode 100644 index 0000000000..3e1dae5f98 --- /dev/null +++ b/tools/checkpackagelib/lib_patch.py @@ -0,0 +1,62 @@ +# See tools/checkpackagelib/readme.txt before editing this file. +# The format of the patch files is tested during the build, so below check +# functions don't need to check for things already checked by running +# "make package-dirclean package-patch". + +import re + +from base import _CheckFunction +# Notice: ignore 'imported but unused' from pyflakes for check functions. +from lib import NewlineAtEof + + +class ApplyOrder(_CheckFunction): + APPLY_ORDER = re.compile("/\d{1,4}-[^/]*$") + + def before(self): + if not self.APPLY_ORDER.search(self.filename): + return ["{}:0: use name -.patch " + "({}#_providing_patches)" + .format(self.filename, self.url_to_manual)] + + +class NumberedSubject(_CheckFunction): + NUMBERED_PATCH = re.compile("Subject:\s*\[PATCH\s*\d+/\d+\]") + + def before(self): + self.git_patch = False + self.lineno = 0 + self.text = None + + def check_line(self, lineno, text): + if text.startswith("diff --git"): + self.git_patch = True + return + if self.NUMBERED_PATCH.search(text): + self.lineno = lineno + self.text = text + + def after(self): + if self.git_patch and self.text: + return ["{}:{}: generate your patches with 'git format-patch -N'" + .format(self.filename, self.lineno), + self.text] + + +class Sob(_CheckFunction): + SOB_ENTRY = re.compile("^Signed-off-by: .*$") + + def before(self): + self.found = False + + def check_line(self, lineno, text): + if self.found: + return + if self.SOB_ENTRY.search(text): + self.found = True + + def after(self): + if not self.found: + return ["{}:0: missing Signed-off-by in the header " + "({}#_format_and_licensing_of_the_package_patches)" + .format(self.filename, self.url_to_manual)] diff --git a/tools/checkpackagelib/readme.txt b/tools/checkpackagelib/readme.txt new file mode 100644 index 0000000000..8012c72e39 --- /dev/null +++ b/tools/checkpackagelib/readme.txt @@ -0,0 +1,75 @@ +How the scripts are structured: +- check-package is the main engine, called by the user. + For each input file, this script decides which parser should be used and it + collects all classes declared in the library file and instantiates them. + The main engine opens the input files and it serves each raw line (including + newline!) to the method check_line() of every check object. + Two special methods before() and after() are used to call the initialization + of variables (for the case it needs to keep data across calls) and the + equivalent finalization (e.g. for the case a warning must be issued if some + pattern is not in the input file). +- base.py contains the base class for all check functions. +- lib.py contains the classes for common check functions. + Each check function is explicitly included in a given type-parsing library. + Do not include every single check function in this file, a class that will + only parse hash files should be implemented in the hash-parsing library. + When a warning must be issued, the check function returns an array of strings. + Each string is a warning message and is displayed if the corresponding verbose + level is active. When the script is called without --verbose only the first + warning in the returned array is printed; when called with --verbose both + first and second warnings are printed; when called with -vv until the third + warning is printed; an so on. + Helper functions can be defined and will not be called by the main script. +- lib_type.py contains check functions specific to files of this type. + +Some hints when changing this code: +- prefer O(n) algorithms, where n is the total number of lines in the files + processed. +- when there is no other reason for ordering, use alphabetical order (e.g. keep + the check functions in alphabetical order, keep the imports in alphabetical + order, and so on). +- use pyflakes to detect and fix potential problems. +- use pep8 formatting. +- keep in mind that for every class the method before() will be called before + any line is served to be checked by the method check_line(). A class that + checks the filename should only implement the method before(). A function that + needs to keep data across calls (e.g. keep the last line before the one being + processed) should initialize all variables using this method. +- keep in mind that for every class the method after() will be called after all + lines were served to be checked by the method check_line(). A class that + checks the absence of a pattern in the file will need to use this method. +- try to avoid false warnings. It's better to not issue a warning message to a + corner case than have too many false warnings. The second can make users stop + using the script. +- do not check spacing in the input line in every single function. Trailing + whitespace and wrong indentation should be checked by separate functions. +- avoid duplicate tests. Try to test only one thing in each function. +- in the warning message, include the url to a section from the manual, when + applicable. It potentially will make more people know the manual. +- use short sentences in the warning messages. A complete explanation can be + added to show when --verbose is used. +- when testing, verify the error message is displayed when the error pattern is + found, but also verify the error message is not displayed for few + well-formatted packages... there are many of these, just pick your favorite + as golden package that should not trigger any warning message. +- check the url displayed by the warning message works. + +Usage examples: +- to get a list of check functions that would be called without actually + calling them you can use the --dry-run option: +$ tools/check-package --dry-run package/yourfavorite/* + +- when you just added a new check function, e.g. Something, check how it behaves + for all current packages: +$ tools/check-package --include-only Something $(find package -type f) + +- the effective processing time (when the .pyc were already generated and all + files to be processed are cached in the RAM) should stay in the order of few + seconds: +$ tools/check-package $(find package -type f) >/dev/null ; \ + time tools/check-package $(find package -type f) >/dev/null + +- vim users can navigate the warnings (most editors probably have similar + function) since warnings are generated in the form 'path/file:line: warning': +$ find package/ -name 'Config.*' > filelist && vim -c \ + 'set makeprg=tools/check-package\ $(cat\ filelist)' -c make -c copen diff --git a/tools/readme.txt b/tools/readme.txt index 1439a0a99f..fc1108fded 100644 --- a/tools/readme.txt +++ b/tools/readme.txt @@ -2,6 +2,10 @@ This directory contains various useful scripts and tools for working with Buildroot. You need not add this directory in your PATH to use any of those tools, but you may do so if you want. +check-package + a script that checks the coding style of a package's Config.in and + .mk files, and also tests them for various types of typoes. + get-developpers a script to return the list of people interested in a specific part of Buildroot, so they can be Cc:ed on a mail. Accepts a patch as