From: Ricardo Martincoski Date: Wed, 19 Apr 2017 18:06:21 +0000 (-0300) Subject: check-package: move parts to subdirectory X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=7b394c4926dcb860a356eeb46b1f1f5d807041f7;p=buildroot.git check-package: move parts to subdirectory Currently the check-package script uses many files in the same directory. This commit keeps the main script in support/scripts/ and moves the rest into a subdirectory. The modules were previously prefixed to make it easy to identify which script they belong to. This is no longer needed when using a subdirectory, so the prefix is removed. Note: if this commit is checked out and the script is run, and later on a previous version is checked out, the file support/scripts/checkpackagelib/__init__.pyc needs to be manually removed to prevent Python interpreter to look for checkpackagelib package when only the checkpackagelib module is available. Reported-by: Thomas Petazzoni Signed-off-by: Ricardo Martincoski Signed-off-by: Thomas Petazzoni --- diff --git a/support/scripts/check-package b/support/scripts/check-package index 2add4712cd..74ea46f819 100755 --- a/support/scripts/check-package +++ b/support/scripts/check-package @@ -1,5 +1,5 @@ #!/usr/bin/env python -# See support/scripts/check-package.txt before editing this file. +# See support/scripts/checkpackagelib/readme.txt before editing this file. from __future__ import print_function import argparse @@ -7,10 +7,10 @@ import inspect import re import sys -import checkpackagelib_config -import checkpackagelib_hash -import checkpackagelib_mk -import checkpackagelib_patch +import checkpackagelib.lib_config +import checkpackagelib.lib_hash +import checkpackagelib.lib_mk +import checkpackagelib.lib_patch VERBOSE_LEVEL_TO_SHOW_IGNORED_FILES = 3 flags = None # Command line arguments. @@ -48,13 +48,13 @@ def get_lib_from_filename(fname): if FILE_IS_FROM_A_PACKAGE.search(fname) is None: return None if CONFIG_IN_FILENAME.search(fname): - return checkpackagelib_config + return checkpackagelib.lib_config if fname.endswith(".hash"): - return checkpackagelib_hash + return checkpackagelib.lib_hash if fname.endswith(".mk"): - return checkpackagelib_mk + return checkpackagelib.lib_mk if fname.endswith(".patch"): - return checkpackagelib_patch + return checkpackagelib.lib_patch return None diff --git a/support/scripts/check-package.txt b/support/scripts/check-package.txt deleted file mode 100644 index 630cd04f07..0000000000 --- a/support/scripts/check-package.txt +++ /dev/null @@ -1,76 +0,0 @@ -How the scripts are structured: -- check-package is the main engine, called by the user. - For each input file, this script decides which parser should be used and it - collects all classes declared in the library file and instantiates them. - The main engine opens the input files and it serves each raw line (including - newline!) to the method check_line() of every check object. - Two special methods before() and after() are used to call the initialization - of variables (for the case it needs to keep data across calls) and the - equivalent finalization (e.g. for the case a warning must be issued if some - pattern is not in the input file). -- checkpackagebase.py contains the base class for all check functions. -- checkpackagelib.py contains the classes for common check functions. - Each check function is explicitly included in a given type-parsing library. - Do not include every single check function in this file, a class that will - only parse hash files should be implemented in the hash-parsing library. - When a warning must be issued, the check function returns an array of strings. - Each string is a warning message and is displayed if the corresponding verbose - level is active. When the script is called without --verbose only the first - warning in the returned array is printed; when called with --verbose both - first and second warnings are printed; when called with -vv until the third - warning is printed; an so on. - Helper functions can be defined and will not be called by the main script. -- checkpackagelib_type.py contains check functions specific to files of this - type. - -Some hints when changing this code: -- prefer O(n) algorithms, where n is the total number of lines in the files - processed. -- when there is no other reason for ordering, use alphabetical order (e.g. keep - the check functions in alphabetical order, keep the imports in alphabetical - order, and so on). -- use pyflakes to detect and fix potential problems. -- use pep8 formatting. -- keep in mind that for every class the method before() will be called before - any line is served to be checked by the method check_line(). A class that - checks the filename should only implement the method before(). A function that - needs to keep data across calls (e.g. keep the last line before the one being - processed) should initialize all variables using this method. -- keep in mind that for every class the method after() will be called after all - lines were served to be checked by the method check_line(). A class that - checks the absence of a pattern in the file will need to use this method. -- try to avoid false warnings. It's better to not issue a warning message to a - corner case than have too many false warnings. The second can make users stop - using the script. -- do not check spacing in the input line in every single function. Trailing - whitespace and wrong indentation should be checked by separate functions. -- avoid duplicate tests. Try to test only one thing in each function. -- in the warning message, include the url to a section from the manual, when - applicable. It potentially will make more people know the manual. -- use short sentences in the warning messages. A complete explanation can be - added to show when --verbose is used. -- when testing, verify the error message is displayed when the error pattern is - found, but also verify the error message is not displayed for few - well-formatted packages... there are many of these, just pick your favorite - as golden package that should not trigger any warning message. -- check the url displayed by the warning message works. - -Usage examples: -- to get a list of check functions that would be called without actually - calling them you can use the --dry-run option: -$ support/scripts/check-package --dry-run package/yourfavorite/* - -- when you just added a new check function, e.g. Something, check how it behaves - for all current packages: -$ support/scripts/check-package --include-only Something $(find package -type f) - -- the effective processing time (when the .pyc were already generated and all - files to be processed are cached in the RAM) should stay in the order of few - seconds: -$ support/scripts/check-package $(find package -type f) >/dev/null ; \ - time support/scripts/check-package $(find package -type f) >/dev/null - -- vim users can navigate the warnings (most editors probably have similar - function) since warnings are generated in the form 'path/file:line: warning': -$ find package/ -name 'Config.*' > filelist && vim -c \ - 'set makeprg=support/scripts/check-package\ $(cat\ filelist)' -c make -c copen diff --git a/support/scripts/checkpackagebase.py b/support/scripts/checkpackagebase.py deleted file mode 100644 index e4c664d24c..0000000000 --- a/support/scripts/checkpackagebase.py +++ /dev/null @@ -1,16 +0,0 @@ -# See support/scripts/check-package.txt before editing this file. - - -class _CheckFunction(object): - def __init__(self, filename, url_to_manual): - self.filename = filename - self.url_to_manual = url_to_manual - - def before(self): - pass - - def check_line(self, lineno, text): - pass - - def after(self): - pass diff --git a/support/scripts/checkpackagelib.py b/support/scripts/checkpackagelib.py deleted file mode 100644 index 280084575b..0000000000 --- a/support/scripts/checkpackagelib.py +++ /dev/null @@ -1,54 +0,0 @@ -# See support/scripts/check-package.txt before editing this file. - -from checkpackagebase import _CheckFunction - - -class ConsecutiveEmptyLines(_CheckFunction): - def before(self): - self.lastline = "non empty" - - def check_line(self, lineno, text): - if text.strip() == "" == self.lastline.strip(): - return ["{}:{}: consecutive empty lines" - .format(self.filename, lineno)] - self.lastline = text - - -class EmptyLastLine(_CheckFunction): - def before(self): - self.lastlineno = 0 - self.lastline = "non empty" - - def check_line(self, lineno, text): - self.lastlineno = lineno - self.lastline = text - - def after(self): - if self.lastline.strip() == "": - return ["{}:{}: empty line at end of file" - .format(self.filename, self.lastlineno)] - - -class NewlineAtEof(_CheckFunction): - def before(self): - self.lastlineno = 0 - self.lastline = "\n" - - def check_line(self, lineno, text): - self.lastlineno = lineno - self.lastline = text - - def after(self): - if self.lastline == self.lastline.rstrip("\r\n"): - return ["{}:{}: missing newline at end of file" - .format(self.filename, self.lastlineno), - self.lastline] - - -class TrailingSpace(_CheckFunction): - def check_line(self, lineno, text): - line = text.rstrip("\r\n") - if line != line.rstrip(): - return ["{}:{}: line contains trailing whitespace" - .format(self.filename, lineno), - text] diff --git a/support/scripts/checkpackagelib/__init__.py b/support/scripts/checkpackagelib/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/support/scripts/checkpackagelib/base.py b/support/scripts/checkpackagelib/base.py new file mode 100644 index 0000000000..7669775f06 --- /dev/null +++ b/support/scripts/checkpackagelib/base.py @@ -0,0 +1,16 @@ +# See support/scripts/checkpackagelib/readme.txt before editing this file. + + +class _CheckFunction(object): + def __init__(self, filename, url_to_manual): + self.filename = filename + self.url_to_manual = url_to_manual + + def before(self): + pass + + def check_line(self, lineno, text): + pass + + def after(self): + pass diff --git a/support/scripts/checkpackagelib/lib.py b/support/scripts/checkpackagelib/lib.py new file mode 100644 index 0000000000..3077f518b3 --- /dev/null +++ b/support/scripts/checkpackagelib/lib.py @@ -0,0 +1,54 @@ +# See support/scripts/checkpackagelib/readme.txt before editing this file. + +from base import _CheckFunction + + +class ConsecutiveEmptyLines(_CheckFunction): + def before(self): + self.lastline = "non empty" + + def check_line(self, lineno, text): + if text.strip() == "" == self.lastline.strip(): + return ["{}:{}: consecutive empty lines" + .format(self.filename, lineno)] + self.lastline = text + + +class EmptyLastLine(_CheckFunction): + def before(self): + self.lastlineno = 0 + self.lastline = "non empty" + + def check_line(self, lineno, text): + self.lastlineno = lineno + self.lastline = text + + def after(self): + if self.lastline.strip() == "": + return ["{}:{}: empty line at end of file" + .format(self.filename, self.lastlineno)] + + +class NewlineAtEof(_CheckFunction): + def before(self): + self.lastlineno = 0 + self.lastline = "\n" + + def check_line(self, lineno, text): + self.lastlineno = lineno + self.lastline = text + + def after(self): + if self.lastline == self.lastline.rstrip("\r\n"): + return ["{}:{}: missing newline at end of file" + .format(self.filename, self.lastlineno), + self.lastline] + + +class TrailingSpace(_CheckFunction): + def check_line(self, lineno, text): + line = text.rstrip("\r\n") + if line != line.rstrip(): + return ["{}:{}: line contains trailing whitespace" + .format(self.filename, lineno), + text] diff --git a/support/scripts/checkpackagelib/lib_config.py b/support/scripts/checkpackagelib/lib_config.py new file mode 100644 index 0000000000..8f49224e95 --- /dev/null +++ b/support/scripts/checkpackagelib/lib_config.py @@ -0,0 +1,138 @@ +# See support/scripts/checkpackagelib/readme.txt before editing this file. +# Kconfig generates errors if someone introduces a typo like "boool" instead of +# "bool", so below check functions don't need to check for things already +# checked by running "make menuconfig". + +import re + +from base import _CheckFunction +# Notice: ignore 'imported but unused' from pyflakes for check functions. +from lib import ConsecutiveEmptyLines +from lib import EmptyLastLine +from lib import NewlineAtEof +from lib import TrailingSpace + + +def _empty_or_comment(text): + line = text.strip() + # ignore empty lines and comment lines indented or not + return line == "" or line.startswith("#") + + +def _part_of_help_text(text): + return text.startswith("\t ") + + +# used in more than one check +entries_that_should_not_be_indented = [ + "choice", "comment", "config", "endchoice", "endif", "endmenu", "if", + "menu", "menuconfig", "source"] + + +class AttributesOrder(_CheckFunction): + attributes_order_convention = { + "bool": 1, "prompt": 1, "string": 1, "default": 2, "depends": 3, + "select": 4, "help": 5} + + def before(self): + self.state = 0 + + def check_line(self, lineno, text): + if _empty_or_comment(text) or _part_of_help_text(text): + return + + attribute = text.split()[0] + + if attribute in entries_that_should_not_be_indented: + self.state = 0 + return + if attribute not in self.attributes_order_convention.keys(): + return + new_state = self.attributes_order_convention[attribute] + wrong_order = self.state > new_state + + # save to process next line + self.state = new_state + + if wrong_order: + return ["{}:{}: attributes order: type, default, depends on," + " select, help ({}#_config_files)" + .format(self.filename, lineno, self.url_to_manual), + text] + + +class HelpText(_CheckFunction): + HELP_TEXT_FORMAT = re.compile("^\t .{,62}$") + URL_ONLY = re.compile("^(http|https|git)://\S*$") + + def before(self): + self.help_text = False + + def check_line(self, lineno, text): + if _empty_or_comment(text): + return + + entry = text.split()[0] + + if entry in entries_that_should_not_be_indented: + self.help_text = False + return + if text.strip() == "help": + self.help_text = True + return + + if not self.help_text: + return + + if self.HELP_TEXT_FORMAT.match(text.rstrip()): + return + if self.URL_ONLY.match(text.strip()): + return + return ["{}:{}: help text: <2 spaces><62 chars>" + " ({}#writing-rules-config-in)" + .format(self.filename, lineno, self.url_to_manual), + text, + "\t " + "123456789 " * 6 + "12"] + + +class Indent(_CheckFunction): + ENDS_WITH_BACKSLASH = re.compile(r"^[^#].*\\$") + entries_that_should_be_indented = [ + "bool", "default", "depends", "help", "prompt", "select", "string"] + + def before(self): + self.backslash = False + + def check_line(self, lineno, text): + if _empty_or_comment(text) or _part_of_help_text(text): + self.backslash = False + return + + entry = text.split()[0] + + last_line_ends_in_backslash = self.backslash + + # calculate for next line + if self.ENDS_WITH_BACKSLASH.search(text): + self.backslash = True + else: + self.backslash = False + + if last_line_ends_in_backslash: + if text.startswith("\t"): + return + return ["{}:{}: continuation line should be indented using tabs" + .format(self.filename, lineno), + text] + + if entry in self.entries_that_should_be_indented: + if not text.startswith("\t{}".format(entry)): + return ["{}:{}: should be indented with one tab" + " ({}#_config_files)" + .format(self.filename, lineno, self.url_to_manual), + text] + elif entry in entries_that_should_not_be_indented: + if not text.startswith(entry): + return ["{}:{}: should not be indented" + .format(self.filename, lineno), + text] diff --git a/support/scripts/checkpackagelib/lib_hash.py b/support/scripts/checkpackagelib/lib_hash.py new file mode 100644 index 0000000000..c76abb43f1 --- /dev/null +++ b/support/scripts/checkpackagelib/lib_hash.py @@ -0,0 +1,72 @@ +# See support/scripts/checkpackagelib/readme.txt before editing this file. +# The validity of the hashes itself is checked when building, so below check +# functions don't need to check for things already checked by running +# "make package-dirclean package-source". + +import re + +from base import _CheckFunction +# Notice: ignore 'imported but unused' from pyflakes for check functions. +from lib import ConsecutiveEmptyLines +from lib import EmptyLastLine +from lib import NewlineAtEof +from lib import TrailingSpace + + +def _empty_line_or_comment(text): + return text.strip() == "" or text.startswith("#") + + +class HashFilename(_CheckFunction): + def check_line(self, lineno, text): + if _empty_line_or_comment(text): + return + + fields = text.split() + if len(fields) < 3: + return + + if '/' in fields[2]: + return ["{}:{}: use filename without directory component" + " ({}#adding-packages-hash)" + .format(self.filename, lineno, self.url_to_manual), + text] + + +class HashNumberOfFields(_CheckFunction): + def check_line(self, lineno, text): + if _empty_line_or_comment(text): + return + + fields = text.split() + if len(fields) != 3: + return ["{}:{}: expected three fields ({}#adding-packages-hash)" + .format(self.filename, lineno, self.url_to_manual), + text] + + +class HashType(_CheckFunction): + len_of_hash = {"md5": 32, "sha1": 40, "sha224": 56, "sha256": 64, + "sha384": 96, "sha512": 128} + + def check_line(self, lineno, text): + if _empty_line_or_comment(text): + return + + fields = text.split() + if len(fields) < 2: + return + + htype, hexa = fields[:2] + if htype == "none": + return + if htype not in self.len_of_hash.keys(): + return ["{}:{}: unexpected type of hash ({}#adding-packages-hash)" + .format(self.filename, lineno, self.url_to_manual), + text] + if not re.match("^[0-9A-Fa-f]{%s}$" % self.len_of_hash[htype], hexa): + return ["{}:{}: hash size does not match type " + "({}#adding-packages-hash)" + .format(self.filename, lineno, self.url_to_manual), + text, + "expected {} hex digits".format(self.len_of_hash[htype])] diff --git a/support/scripts/checkpackagelib/lib_mk.py b/support/scripts/checkpackagelib/lib_mk.py new file mode 100644 index 0000000000..a51e0e3ce6 --- /dev/null +++ b/support/scripts/checkpackagelib/lib_mk.py @@ -0,0 +1,223 @@ +# See support/scripts/checkpackagelib/readme.txt before editing this file. +# There are already dependency checks during the build, so below check +# functions don't need to check for things already checked by exploring the +# menu options using "make menuconfig" and by running "make" with appropriate +# packages enabled. + +import re + +from base import _CheckFunction +# Notice: ignore 'imported but unused' from pyflakes for check functions. +from lib import ConsecutiveEmptyLines +from lib import EmptyLastLine +from lib import NewlineAtEof +from lib import TrailingSpace + + +class Indent(_CheckFunction): + COMMENT = re.compile("^\s*#") + CONDITIONAL = re.compile("^\s*(ifeq|ifneq|endif)\s") + ENDS_WITH_BACKSLASH = re.compile(r"^[^#].*\\$") + END_DEFINE = re.compile("^\s*endef\s") + MAKEFILE_TARGET = re.compile("^[^# \t]+:\s") + START_DEFINE = re.compile("^\s*define\s") + + def before(self): + self.define = False + self.backslash = False + self.makefile_target = False + + def check_line(self, lineno, text): + if self.START_DEFINE.search(text): + self.define = True + return + if self.END_DEFINE.search(text): + self.define = False + return + + expect_tabs = False + if self.define or self.backslash or self.makefile_target: + expect_tabs = True + if self.CONDITIONAL.search(text): + expect_tabs = False + + # calculate for next line + if self.ENDS_WITH_BACKSLASH.search(text): + self.backslash = True + else: + self.backslash = False + + if self.MAKEFILE_TARGET.search(text): + self.makefile_target = True + return + if text.strip() == "": + self.makefile_target = False + return + + # comment can be indented or not inside define ... endef, so ignore it + if self.define and self.COMMENT.search(text): + return + + if expect_tabs: + if not text.startswith("\t"): + return ["{}:{}: expected indent with tabs" + .format(self.filename, lineno), + text] + else: + if text.startswith("\t"): + return ["{}:{}: unexpected indent with tabs" + .format(self.filename, lineno), + text] + + +class PackageHeader(_CheckFunction): + def before(self): + self.skip = False + + def check_line(self, lineno, text): + if self.skip or lineno > 6: + return + + if lineno in [1, 5]: + if lineno == 1 and text.startswith("include "): + self.skip = True + return + if text.rstrip() != "#" * 80: + return ["{}:{}: should be 80 hashes ({}#writing-rules-mk)" + .format(self.filename, lineno, self.url_to_manual), + text, + "#" * 80] + elif lineno in [2, 4]: + if text.rstrip() != "#": + return ["{}:{}: should be 1 hash ({}#writing-rules-mk)" + .format(self.filename, lineno, self.url_to_manual), + text] + elif lineno == 6: + if text.rstrip() != "": + return ["{}:{}: should be a blank line ({}#writing-rules-mk)" + .format(self.filename, lineno, self.url_to_manual), + text] + + +class SpaceBeforeBackslash(_CheckFunction): + TAB_OR_MULTIPLE_SPACES_BEFORE_BACKSLASH = re.compile(r"^.*( |\t)\\$") + + def check_line(self, lineno, text): + if self.TAB_OR_MULTIPLE_SPACES_BEFORE_BACKSLASH.match(text.rstrip()): + return ["{}:{}: use only one space before backslash" + .format(self.filename, lineno), + text] + + +class TrailingBackslash(_CheckFunction): + ENDS_WITH_BACKSLASH = re.compile(r"^[^#].*\\$") + + def before(self): + self.backslash = False + + def check_line(self, lineno, text): + last_line_ends_in_backslash = self.backslash + + # calculate for next line + if self.ENDS_WITH_BACKSLASH.search(text): + self.backslash = True + self.lastline = text + return + self.backslash = False + + if last_line_ends_in_backslash and text.strip() == "": + return ["{}:{}: remove trailing backslash" + .format(self.filename, lineno - 1), + self.lastline] + + +class TypoInPackageVariable(_CheckFunction): + ALLOWED = re.compile("|".join([ + "ACLOCAL_DIR", + "ACLOCAL_HOST_DIR", + "BR_CCACHE_INITIAL_SETUP", + "BR_NO_CHECK_HASH_FOR", + "LINUX_POST_PATCH_HOOKS", + "LINUX_TOOLS", + "LUA_RUN", + "MKFS_JFFS2", + "MKIMAGE_ARCH", + "PKG_CONFIG_HOST_BINARY", + "TARGET_FINALIZE_HOOKS", + "XTENSA_CORE_NAME"])) + PACKAGE_NAME = re.compile("/([^/]+)\.mk") + VARIABLE = re.compile("^([A-Z0-9_]+_[A-Z0-9_]+)\s*(\+|)=") + + def before(self): + package = self.PACKAGE_NAME.search(self.filename).group(1) + package = package.replace("-", "_").upper() + # linux tools do not use LINUX_TOOL_ prefix for variables + package = package.replace("LINUX_TOOL_", "") + self.package = package + self.REGEX = re.compile("^(HOST_)?({}_[A-Z0-9_]+)".format(package)) + self.FIND_VIRTUAL = re.compile( + "^{}_PROVIDES\s*(\+|)=\s*(.*)".format(package)) + self.virtual = [] + + def check_line(self, lineno, text): + m = self.VARIABLE.search(text) + if m is None: + return + + variable = m.group(1) + + # allow to set variables for virtual package this package provides + v = self.FIND_VIRTUAL.search(text) + if v: + self.virtual += v.group(2).upper().split() + return + for virtual in self.virtual: + if variable.startswith("{}_".format(virtual)): + return + + if self.ALLOWED.match(variable): + return + if self.REGEX.search(text) is None: + return ["{}:{}: possible typo: {} -> *{}*" + .format(self.filename, lineno, variable, self.package), + text] + + +class UselessFlag(_CheckFunction): + DEFAULT_AUTOTOOLS_FLAG = re.compile("^.*{}".format("|".join([ + "_AUTORECONF\s*=\s*NO", + "_LIBTOOL_PATCH\s*=\s*YES"]))) + DEFAULT_GENERIC_FLAG = re.compile("^.*{}".format("|".join([ + "_INSTALL_IMAGES\s*=\s*NO", + "_INSTALL_REDISTRIBUTE\s*=\s*YES", + "_INSTALL_STAGING\s*=\s*NO", + "_INSTALL_TARGET\s*=\s*YES"]))) + END_CONDITIONAL = re.compile("^\s*(endif)") + START_CONDITIONAL = re.compile("^\s*(ifeq|ifneq)") + + def before(self): + self.conditional = 0 + + def check_line(self, lineno, text): + if self.START_CONDITIONAL.search(text): + self.conditional += 1 + return + if self.END_CONDITIONAL.search(text): + self.conditional -= 1 + return + + # allow non-default conditionally overridden by default + if self.conditional > 0: + return + + if self.DEFAULT_GENERIC_FLAG.search(text): + return ["{}:{}: useless default value ({}#" + "_infrastructure_for_packages_with_specific_build_systems)" + .format(self.filename, lineno, self.url_to_manual), + text] + + if self.DEFAULT_AUTOTOOLS_FLAG.search(text): + return ["{}:{}: useless default value " + "({}#_infrastructure_for_autotools_based_packages)" + .format(self.filename, lineno, self.url_to_manual), + text] diff --git a/support/scripts/checkpackagelib/lib_patch.py b/support/scripts/checkpackagelib/lib_patch.py new file mode 100644 index 0000000000..c191d262e2 --- /dev/null +++ b/support/scripts/checkpackagelib/lib_patch.py @@ -0,0 +1,62 @@ +# See support/scripts/checkpackagelib/readme.txt before editing this file. +# The format of the patch files is tested during the build, so below check +# functions don't need to check for things already checked by running +# "make package-dirclean package-patch". + +import re + +from base import _CheckFunction +# Notice: ignore 'imported but unused' from pyflakes for check functions. +from lib import NewlineAtEof + + +class ApplyOrder(_CheckFunction): + APPLY_ORDER = re.compile("/\d{1,4}-[^/]*$") + + def before(self): + if not self.APPLY_ORDER.search(self.filename): + return ["{}:0: use name -.patch " + "({}#_providing_patches)" + .format(self.filename, self.url_to_manual)] + + +class NumberedSubject(_CheckFunction): + NUMBERED_PATCH = re.compile("Subject:\s*\[PATCH\s*\d+/\d+\]") + + def before(self): + self.git_patch = False + self.lineno = 0 + self.text = None + + def check_line(self, lineno, text): + if text.startswith("diff --git"): + self.git_patch = True + return + if self.NUMBERED_PATCH.search(text): + self.lineno = lineno + self.text = text + + def after(self): + if self.git_patch and self.text: + return ["{}:{}: generate your patches with 'git format-patch -N'" + .format(self.filename, self.lineno), + self.text] + + +class Sob(_CheckFunction): + SOB_ENTRY = re.compile("^Signed-off-by: .*$") + + def before(self): + self.found = False + + def check_line(self, lineno, text): + if self.found: + return + if self.SOB_ENTRY.search(text): + self.found = True + + def after(self): + if not self.found: + return ["{}:0: missing Signed-off-by in the header " + "({}#_format_and_licensing_of_the_package_patches)" + .format(self.filename, self.url_to_manual)] diff --git a/support/scripts/checkpackagelib/readme.txt b/support/scripts/checkpackagelib/readme.txt new file mode 100644 index 0000000000..0444d17992 --- /dev/null +++ b/support/scripts/checkpackagelib/readme.txt @@ -0,0 +1,75 @@ +How the scripts are structured: +- check-package is the main engine, called by the user. + For each input file, this script decides which parser should be used and it + collects all classes declared in the library file and instantiates them. + The main engine opens the input files and it serves each raw line (including + newline!) to the method check_line() of every check object. + Two special methods before() and after() are used to call the initialization + of variables (for the case it needs to keep data across calls) and the + equivalent finalization (e.g. for the case a warning must be issued if some + pattern is not in the input file). +- base.py contains the base class for all check functions. +- lib.py contains the classes for common check functions. + Each check function is explicitly included in a given type-parsing library. + Do not include every single check function in this file, a class that will + only parse hash files should be implemented in the hash-parsing library. + When a warning must be issued, the check function returns an array of strings. + Each string is a warning message and is displayed if the corresponding verbose + level is active. When the script is called without --verbose only the first + warning in the returned array is printed; when called with --verbose both + first and second warnings are printed; when called with -vv until the third + warning is printed; an so on. + Helper functions can be defined and will not be called by the main script. +- lib_type.py contains check functions specific to files of this type. + +Some hints when changing this code: +- prefer O(n) algorithms, where n is the total number of lines in the files + processed. +- when there is no other reason for ordering, use alphabetical order (e.g. keep + the check functions in alphabetical order, keep the imports in alphabetical + order, and so on). +- use pyflakes to detect and fix potential problems. +- use pep8 formatting. +- keep in mind that for every class the method before() will be called before + any line is served to be checked by the method check_line(). A class that + checks the filename should only implement the method before(). A function that + needs to keep data across calls (e.g. keep the last line before the one being + processed) should initialize all variables using this method. +- keep in mind that for every class the method after() will be called after all + lines were served to be checked by the method check_line(). A class that + checks the absence of a pattern in the file will need to use this method. +- try to avoid false warnings. It's better to not issue a warning message to a + corner case than have too many false warnings. The second can make users stop + using the script. +- do not check spacing in the input line in every single function. Trailing + whitespace and wrong indentation should be checked by separate functions. +- avoid duplicate tests. Try to test only one thing in each function. +- in the warning message, include the url to a section from the manual, when + applicable. It potentially will make more people know the manual. +- use short sentences in the warning messages. A complete explanation can be + added to show when --verbose is used. +- when testing, verify the error message is displayed when the error pattern is + found, but also verify the error message is not displayed for few + well-formatted packages... there are many of these, just pick your favorite + as golden package that should not trigger any warning message. +- check the url displayed by the warning message works. + +Usage examples: +- to get a list of check functions that would be called without actually + calling them you can use the --dry-run option: +$ support/scripts/check-package --dry-run package/yourfavorite/* + +- when you just added a new check function, e.g. Something, check how it behaves + for all current packages: +$ support/scripts/check-package --include-only Something $(find package -type f) + +- the effective processing time (when the .pyc were already generated and all + files to be processed are cached in the RAM) should stay in the order of few + seconds: +$ support/scripts/check-package $(find package -type f) >/dev/null ; \ + time support/scripts/check-package $(find package -type f) >/dev/null + +- vim users can navigate the warnings (most editors probably have similar + function) since warnings are generated in the form 'path/file:line: warning': +$ find package/ -name 'Config.*' > filelist && vim -c \ + 'set makeprg=support/scripts/check-package\ $(cat\ filelist)' -c make -c copen diff --git a/support/scripts/checkpackagelib_config.py b/support/scripts/checkpackagelib_config.py deleted file mode 100644 index be52d94327..0000000000 --- a/support/scripts/checkpackagelib_config.py +++ /dev/null @@ -1,138 +0,0 @@ -# See support/scripts/check-package.txt before editing this file. -# Kconfig generates errors if someone introduces a typo like "boool" instead of -# "bool", so below check functions don't need to check for things already -# checked by running "make menuconfig". - -import re - -from checkpackagebase import _CheckFunction -# Notice: ignore 'imported but unused' from pyflakes for check functions. -from checkpackagelib import ConsecutiveEmptyLines -from checkpackagelib import EmptyLastLine -from checkpackagelib import NewlineAtEof -from checkpackagelib import TrailingSpace - - -def _empty_or_comment(text): - line = text.strip() - # ignore empty lines and comment lines indented or not - return line == "" or line.startswith("#") - - -def _part_of_help_text(text): - return text.startswith("\t ") - - -# used in more than one check -entries_that_should_not_be_indented = [ - "choice", "comment", "config", "endchoice", "endif", "endmenu", "if", - "menu", "menuconfig", "source"] - - -class AttributesOrder(_CheckFunction): - attributes_order_convention = { - "bool": 1, "prompt": 1, "string": 1, "default": 2, "depends": 3, - "select": 4, "help": 5} - - def before(self): - self.state = 0 - - def check_line(self, lineno, text): - if _empty_or_comment(text) or _part_of_help_text(text): - return - - attribute = text.split()[0] - - if attribute in entries_that_should_not_be_indented: - self.state = 0 - return - if attribute not in self.attributes_order_convention.keys(): - return - new_state = self.attributes_order_convention[attribute] - wrong_order = self.state > new_state - - # save to process next line - self.state = new_state - - if wrong_order: - return ["{}:{}: attributes order: type, default, depends on," - " select, help ({}#_config_files)" - .format(self.filename, lineno, self.url_to_manual), - text] - - -class HelpText(_CheckFunction): - HELP_TEXT_FORMAT = re.compile("^\t .{,62}$") - URL_ONLY = re.compile("^(http|https|git)://\S*$") - - def before(self): - self.help_text = False - - def check_line(self, lineno, text): - if _empty_or_comment(text): - return - - entry = text.split()[0] - - if entry in entries_that_should_not_be_indented: - self.help_text = False - return - if text.strip() == "help": - self.help_text = True - return - - if not self.help_text: - return - - if self.HELP_TEXT_FORMAT.match(text.rstrip()): - return - if self.URL_ONLY.match(text.strip()): - return - return ["{}:{}: help text: <2 spaces><62 chars>" - " ({}#writing-rules-config-in)" - .format(self.filename, lineno, self.url_to_manual), - text, - "\t " + "123456789 " * 6 + "12"] - - -class Indent(_CheckFunction): - ENDS_WITH_BACKSLASH = re.compile(r"^[^#].*\\$") - entries_that_should_be_indented = [ - "bool", "default", "depends", "help", "prompt", "select", "string"] - - def before(self): - self.backslash = False - - def check_line(self, lineno, text): - if _empty_or_comment(text) or _part_of_help_text(text): - self.backslash = False - return - - entry = text.split()[0] - - last_line_ends_in_backslash = self.backslash - - # calculate for next line - if self.ENDS_WITH_BACKSLASH.search(text): - self.backslash = True - else: - self.backslash = False - - if last_line_ends_in_backslash: - if text.startswith("\t"): - return - return ["{}:{}: continuation line should be indented using tabs" - .format(self.filename, lineno), - text] - - if entry in self.entries_that_should_be_indented: - if not text.startswith("\t{}".format(entry)): - return ["{}:{}: should be indented with one tab" - " ({}#_config_files)" - .format(self.filename, lineno, self.url_to_manual), - text] - elif entry in entries_that_should_not_be_indented: - if not text.startswith(entry): - return ["{}:{}: should not be indented" - .format(self.filename, lineno), - text] diff --git a/support/scripts/checkpackagelib_hash.py b/support/scripts/checkpackagelib_hash.py deleted file mode 100644 index 47fe18756a..0000000000 --- a/support/scripts/checkpackagelib_hash.py +++ /dev/null @@ -1,72 +0,0 @@ -# See support/scripts/check-package.txt before editing this file. -# The validity of the hashes itself is checked when building, so below check -# functions don't need to check for things already checked by running -# "make package-dirclean package-source". - -import re - -from checkpackagebase import _CheckFunction -# Notice: ignore 'imported but unused' from pyflakes for check functions. -from checkpackagelib import ConsecutiveEmptyLines -from checkpackagelib import EmptyLastLine -from checkpackagelib import NewlineAtEof -from checkpackagelib import TrailingSpace - - -def _empty_line_or_comment(text): - return text.strip() == "" or text.startswith("#") - - -class HashFilename(_CheckFunction): - def check_line(self, lineno, text): - if _empty_line_or_comment(text): - return - - fields = text.split() - if len(fields) < 3: - return - - if '/' in fields[2]: - return ["{}:{}: use filename without directory component" - " ({}#adding-packages-hash)" - .format(self.filename, lineno, self.url_to_manual), - text] - - -class HashNumberOfFields(_CheckFunction): - def check_line(self, lineno, text): - if _empty_line_or_comment(text): - return - - fields = text.split() - if len(fields) != 3: - return ["{}:{}: expected three fields ({}#adding-packages-hash)" - .format(self.filename, lineno, self.url_to_manual), - text] - - -class HashType(_CheckFunction): - len_of_hash = {"md5": 32, "sha1": 40, "sha224": 56, "sha256": 64, - "sha384": 96, "sha512": 128} - - def check_line(self, lineno, text): - if _empty_line_or_comment(text): - return - - fields = text.split() - if len(fields) < 2: - return - - htype, hexa = fields[:2] - if htype == "none": - return - if htype not in self.len_of_hash.keys(): - return ["{}:{}: unexpected type of hash ({}#adding-packages-hash)" - .format(self.filename, lineno, self.url_to_manual), - text] - if not re.match("^[0-9A-Fa-f]{%s}$" % self.len_of_hash[htype], hexa): - return ["{}:{}: hash size does not match type " - "({}#adding-packages-hash)" - .format(self.filename, lineno, self.url_to_manual), - text, - "expected {} hex digits".format(self.len_of_hash[htype])] diff --git a/support/scripts/checkpackagelib_mk.py b/support/scripts/checkpackagelib_mk.py deleted file mode 100644 index 0740a37f79..0000000000 --- a/support/scripts/checkpackagelib_mk.py +++ /dev/null @@ -1,223 +0,0 @@ -# See support/scripts/check-package.txt before editing this file. -# There are already dependency checks during the build, so below check -# functions don't need to check for things already checked by exploring the -# menu options using "make menuconfig" and by running "make" with appropriate -# packages enabled. - -import re - -from checkpackagebase import _CheckFunction -# Notice: ignore 'imported but unused' from pyflakes for check functions. -from checkpackagelib import ConsecutiveEmptyLines -from checkpackagelib import EmptyLastLine -from checkpackagelib import NewlineAtEof -from checkpackagelib import TrailingSpace - - -class Indent(_CheckFunction): - COMMENT = re.compile("^\s*#") - CONDITIONAL = re.compile("^\s*(ifeq|ifneq|endif)\s") - ENDS_WITH_BACKSLASH = re.compile(r"^[^#].*\\$") - END_DEFINE = re.compile("^\s*endef\s") - MAKEFILE_TARGET = re.compile("^[^# \t]+:\s") - START_DEFINE = re.compile("^\s*define\s") - - def before(self): - self.define = False - self.backslash = False - self.makefile_target = False - - def check_line(self, lineno, text): - if self.START_DEFINE.search(text): - self.define = True - return - if self.END_DEFINE.search(text): - self.define = False - return - - expect_tabs = False - if self.define or self.backslash or self.makefile_target: - expect_tabs = True - if self.CONDITIONAL.search(text): - expect_tabs = False - - # calculate for next line - if self.ENDS_WITH_BACKSLASH.search(text): - self.backslash = True - else: - self.backslash = False - - if self.MAKEFILE_TARGET.search(text): - self.makefile_target = True - return - if text.strip() == "": - self.makefile_target = False - return - - # comment can be indented or not inside define ... endef, so ignore it - if self.define and self.COMMENT.search(text): - return - - if expect_tabs: - if not text.startswith("\t"): - return ["{}:{}: expected indent with tabs" - .format(self.filename, lineno), - text] - else: - if text.startswith("\t"): - return ["{}:{}: unexpected indent with tabs" - .format(self.filename, lineno), - text] - - -class PackageHeader(_CheckFunction): - def before(self): - self.skip = False - - def check_line(self, lineno, text): - if self.skip or lineno > 6: - return - - if lineno in [1, 5]: - if lineno == 1 and text.startswith("include "): - self.skip = True - return - if text.rstrip() != "#" * 80: - return ["{}:{}: should be 80 hashes ({}#writing-rules-mk)" - .format(self.filename, lineno, self.url_to_manual), - text, - "#" * 80] - elif lineno in [2, 4]: - if text.rstrip() != "#": - return ["{}:{}: should be 1 hash ({}#writing-rules-mk)" - .format(self.filename, lineno, self.url_to_manual), - text] - elif lineno == 6: - if text.rstrip() != "": - return ["{}:{}: should be a blank line ({}#writing-rules-mk)" - .format(self.filename, lineno, self.url_to_manual), - text] - - -class SpaceBeforeBackslash(_CheckFunction): - TAB_OR_MULTIPLE_SPACES_BEFORE_BACKSLASH = re.compile(r"^.*( |\t)\\$") - - def check_line(self, lineno, text): - if self.TAB_OR_MULTIPLE_SPACES_BEFORE_BACKSLASH.match(text.rstrip()): - return ["{}:{}: use only one space before backslash" - .format(self.filename, lineno), - text] - - -class TrailingBackslash(_CheckFunction): - ENDS_WITH_BACKSLASH = re.compile(r"^[^#].*\\$") - - def before(self): - self.backslash = False - - def check_line(self, lineno, text): - last_line_ends_in_backslash = self.backslash - - # calculate for next line - if self.ENDS_WITH_BACKSLASH.search(text): - self.backslash = True - self.lastline = text - return - self.backslash = False - - if last_line_ends_in_backslash and text.strip() == "": - return ["{}:{}: remove trailing backslash" - .format(self.filename, lineno - 1), - self.lastline] - - -class TypoInPackageVariable(_CheckFunction): - ALLOWED = re.compile("|".join([ - "ACLOCAL_DIR", - "ACLOCAL_HOST_DIR", - "BR_CCACHE_INITIAL_SETUP", - "BR_NO_CHECK_HASH_FOR", - "LINUX_POST_PATCH_HOOKS", - "LINUX_TOOLS", - "LUA_RUN", - "MKFS_JFFS2", - "MKIMAGE_ARCH", - "PKG_CONFIG_HOST_BINARY", - "TARGET_FINALIZE_HOOKS", - "XTENSA_CORE_NAME"])) - PACKAGE_NAME = re.compile("/([^/]+)\.mk") - VARIABLE = re.compile("^([A-Z0-9_]+_[A-Z0-9_]+)\s*(\+|)=") - - def before(self): - package = self.PACKAGE_NAME.search(self.filename).group(1) - package = package.replace("-", "_").upper() - # linux tools do not use LINUX_TOOL_ prefix for variables - package = package.replace("LINUX_TOOL_", "") - self.package = package - self.REGEX = re.compile("^(HOST_)?({}_[A-Z0-9_]+)".format(package)) - self.FIND_VIRTUAL = re.compile( - "^{}_PROVIDES\s*(\+|)=\s*(.*)".format(package)) - self.virtual = [] - - def check_line(self, lineno, text): - m = self.VARIABLE.search(text) - if m is None: - return - - variable = m.group(1) - - # allow to set variables for virtual package this package provides - v = self.FIND_VIRTUAL.search(text) - if v: - self.virtual += v.group(2).upper().split() - return - for virtual in self.virtual: - if variable.startswith("{}_".format(virtual)): - return - - if self.ALLOWED.match(variable): - return - if self.REGEX.search(text) is None: - return ["{}:{}: possible typo: {} -> *{}*" - .format(self.filename, lineno, variable, self.package), - text] - - -class UselessFlag(_CheckFunction): - DEFAULT_AUTOTOOLS_FLAG = re.compile("^.*{}".format("|".join([ - "_AUTORECONF\s*=\s*NO", - "_LIBTOOL_PATCH\s*=\s*YES"]))) - DEFAULT_GENERIC_FLAG = re.compile("^.*{}".format("|".join([ - "_INSTALL_IMAGES\s*=\s*NO", - "_INSTALL_REDISTRIBUTE\s*=\s*YES", - "_INSTALL_STAGING\s*=\s*NO", - "_INSTALL_TARGET\s*=\s*YES"]))) - END_CONDITIONAL = re.compile("^\s*(endif)") - START_CONDITIONAL = re.compile("^\s*(ifeq|ifneq)") - - def before(self): - self.conditional = 0 - - def check_line(self, lineno, text): - if self.START_CONDITIONAL.search(text): - self.conditional += 1 - return - if self.END_CONDITIONAL.search(text): - self.conditional -= 1 - return - - # allow non-default conditionally overridden by default - if self.conditional > 0: - return - - if self.DEFAULT_GENERIC_FLAG.search(text): - return ["{}:{}: useless default value ({}#" - "_infrastructure_for_packages_with_specific_build_systems)" - .format(self.filename, lineno, self.url_to_manual), - text] - - if self.DEFAULT_AUTOTOOLS_FLAG.search(text): - return ["{}:{}: useless default value " - "({}#_infrastructure_for_autotools_based_packages)" - .format(self.filename, lineno, self.url_to_manual), - text] diff --git a/support/scripts/checkpackagelib_patch.py b/support/scripts/checkpackagelib_patch.py deleted file mode 100644 index ee46efb70f..0000000000 --- a/support/scripts/checkpackagelib_patch.py +++ /dev/null @@ -1,62 +0,0 @@ -# See support/scripts/check-package.txt before editing this file. -# The format of the patch files is tested during the build, so below check -# functions don't need to check for things already checked by running -# "make package-dirclean package-patch". - -import re - -from checkpackagebase import _CheckFunction -# Notice: ignore 'imported but unused' from pyflakes for check functions. -from checkpackagelib import NewlineAtEof - - -class ApplyOrder(_CheckFunction): - APPLY_ORDER = re.compile("/\d{1,4}-[^/]*$") - - def before(self): - if not self.APPLY_ORDER.search(self.filename): - return ["{}:0: use name -.patch " - "({}#_providing_patches)" - .format(self.filename, self.url_to_manual)] - - -class NumberedSubject(_CheckFunction): - NUMBERED_PATCH = re.compile("Subject:\s*\[PATCH\s*\d+/\d+\]") - - def before(self): - self.git_patch = False - self.lineno = 0 - self.text = None - - def check_line(self, lineno, text): - if text.startswith("diff --git"): - self.git_patch = True - return - if self.NUMBERED_PATCH.search(text): - self.lineno = lineno - self.text = text - - def after(self): - if self.git_patch and self.text: - return ["{}:{}: generate your patches with 'git format-patch -N'" - .format(self.filename, self.lineno), - self.text] - - -class Sob(_CheckFunction): - SOB_ENTRY = re.compile("^Signed-off-by: .*$") - - def before(self): - self.found = False - - def check_line(self, lineno, text): - if self.found: - return - if self.SOB_ENTRY.search(text): - self.found = True - - def after(self): - if not self.found: - return ["{}:0: missing Signed-off-by in the header " - "({}#_format_and_licensing_of_the_package_patches)" - .format(self.filename, self.url_to_manual)]