util/style/verifiers.py

   1 #!/usr/bin/env python3
   2 #
   3 # Copyright (c) 2014, 2016 ARM Limited
   4 # All rights reserved
   5 #
   6 # The license below extends only to copyright in the software and shall
   7 # not be construed as granting a license to any other intellectual
   8 # property including but not limited to intellectual property relating
   9 # to a hardware implementation of the functionality of the software
  10 # licensed hereunder.  You may use the software subject to the license
  11 # terms below provided that you ensure that this notice is replicated
  12 # unmodified and in its entirety in all distributions of the software,
  13 # modified or unmodified, in source code or in binary form.
  14 #
  15 # Copyright (c) 2006 The Regents of The University of Michigan
  16 # Copyright (c) 2007,2011 The Hewlett-Packard Development Company
  17 # Copyright (c) 2016 Advanced Micro Devices, Inc.
  18 # All rights reserved.
  19 #
  20 # Redistribution and use in source and binary forms, with or without
  21 # modification, are permitted provided that the following conditions are
  22 # met: redistributions of source code must retain the above copyright
  23 # notice, this list of conditions and the following disclaimer;
  24 # redistributions in binary form must reproduce the above copyright
  25 # notice, this list of conditions and the following disclaimer in the
  26 # documentation and/or other materials provided with the distribution;
  27 # neither the name of the copyright holders nor the names of its
  28 # contributors may be used to endorse or promote products derived from
  29 # this software without specific prior written permission.
  30 #
  31 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  32 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  33 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  34 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  35 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  36 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  37 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  38 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  39 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  40 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  41 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  42
  43 from abc import ABCMeta, abstractmethod
  44 from difflib import SequenceMatcher
  45 import inspect
  46 import os
  47 import re
  48 import sys
  49
  50 from . import style
  51 from . import sort_includes
  52 from .region import *
  53 from .file_types import lang_type
  54
  55
  56 def safefix(fix_func):
  57     """ Decorator for the fix functions of the Verifier class.
  58         This function wraps the fix function and creates a backup file
  59         just in case there is an error.
  60     """
  61     def safefix_wrapper(*args, **kwargs):
  62         # Check to be sure that this is decorating a function we expect:
  63         # a class method with filename as the first argument (after self)
  64         assert(os.path.exists(args[1]))
  65         self = args[0]
  66         assert(is_verifier(self.__class__))
  67         filename = args[1]
  68
  69         # Now, Let's make a backup file.
  70         from shutil import copyfile
  71         backup_name = filename+'.bak'
  72         copyfile(filename, backup_name)
  73
  74         # Try to apply the fix. If it fails, then we revert the file
  75         # Either way, we need to clean up our backup file
  76         try:
  77             fix_func(*args, **kwargs)
  78         except Exception as e:
  79             # Restore the original file to the backup file
  80             self.ui.write("Error! Restoring the original file.\n")
  81             copyfile(backup_name, filename)
  82             raise
  83         finally:
  84             # Clean up the backup file
  85             os.remove(backup_name)
  86
  87     return safefix_wrapper
  88
  89 def _modified_regions(old, new):
  90     try:
  91         m = SequenceMatcher(a=old, b=new, autojunk=False)
  92     except TypeError:
  93         # autojunk was introduced in Python 2.7. We need a fallback
  94         # mechanism to support old Python versions.
  95         m = SequenceMatcher(a=old, b=new)
  96     regions = Regions()
  97     for tag, i1, i2, j1, j2 in m.get_opcodes():
  98         if tag != "equal":
  99             regions.extend(Region(i1, i2))
 100     return regions
 101
 102
 103 class Verifier(object, metaclass=ABCMeta):
 104     """Base class for style verifiers
 105
 106     Verifiers check for style violations and optionally fix such
 107     violations. Implementations should either inherit from this class
 108     (Verifier) if they need to work on entire files or LineVerifier if
 109     they operate on a line-by-line basis.
 110
 111     Subclasses must define these class attributes:
 112       languages = set of strings identifying applicable languages
 113       test_name = long descriptive name of test, will be used in
 114                   messages such as "error in <foo>" or "invalid <foo>"
 115       opt_name = short name used to generate command-line options to
 116                  control the test (--fix-<foo>, --ignore-<foo>, etc.)
 117
 118     """
 119
 120
 121     def __init__(self, ui, opts, base=None):
 122         self.ui = ui
 123         self.base = base
 124
 125         # opt_name must be defined as a class attribute of derived classes.
 126         # Check test-specific opts first as these have precedence.
 127         self.opt_fix = opts.get('fix_' + self.opt_name, False)
 128         self.opt_ignore = opts.get('ignore_' + self.opt_name, False)
 129         self.opt_skip = opts.get('skip_' + self.opt_name, False)
 130         # If no test-specific opts were set, then set based on "-all" opts.
 131         if not (self.opt_fix or self.opt_ignore or self.opt_skip):
 132             self.opt_fix = opts.get('fix_all', False)
 133             self.opt_ignore = opts.get('ignore_all', False)
 134             self.opt_skip = opts.get('skip_all', False)
 135
 136     def normalize_filename(self, name):
 137         abs_name = os.path.abspath(name)
 138         if self.base is None:
 139             return abs_name
 140
 141         abs_base = os.path.abspath(self.base)
 142         return os.path.relpath(abs_name, start=abs_base)
 143
 144     def open(self, filename, mode):
 145         try:
 146             f = open(filename, mode)
 147         except OSError as msg:
 148             print('could not open file {}: {}'.format(filename, msg))
 149             return None
 150
 151         return f
 152
 153     def skip(self, filename):
 154         # We never want to handle symlinks, so always skip them: If the
 155         # location pointed to is a directory, skip it. If the location is a
 156         # file inside the gem5 directory, it will be checked as a file, so
 157         # symlink can be skipped. If the location is a file outside gem5, we
 158         # don't want to check it anyway.
 159         if os.path.islink(filename):
 160             return True
 161         return lang_type(filename) not in self.languages
 162
 163     def apply(self, filename, regions=all_regions):
 164         """Possibly apply to specified regions of file 'filename'.
 165
 166         Verifier is skipped if --skip-<test> option was provided or if
 167         file is not of an applicable type.  Otherwise file is checked
 168         and error messages printed.  Errors are fixed or ignored if
 169         the corresponding --fix-<test> or --ignore-<test> options were
 170         provided.  If neither, the user is prompted for an action.
 171
 172         Returns True to abort, False otherwise.
 173         """
 174         if not (self.opt_skip or self.skip(filename)):
 175             errors = self.check(filename, regions)
 176             if errors and not self.opt_ignore:
 177                 if self.opt_fix:
 178                     self.fix(filename, regions)
 179                 else:
 180                     result = self.ui.prompt("(a)bort, (i)gnore, or (f)ix?",
 181                                             'aif', 'a')
 182                     if result == 'f':
 183                         self.fix(filename, regions)
 184                     elif result == 'a':
 185                         return True # abort
 186
 187         return False
 188
 189     @abstractmethod
 190     def check(self, filename, regions=all_regions, fobj=None, silent=False):
 191         """Check specified regions of file 'filename'.
 192
 193         Given that it is possible that the current contents of the file
 194         differ from the file as 'staged to commit', for those cases, and
 195         maybe others, the argument fobj should be a file object open and reset
 196         with the contents matching what the file would look like after the
 197         commit. This is needed keep the messages using 'filename' meaningful.
 198
 199         The argument silent is useful to prevent output when we run check in
 200         the staged file vs the actual file to detect if the user forgot
 201         staging fixes to the commit. This way, we prevent reporting errors
 202         twice in stderr.
 203
 204         Line-by-line checks can simply provide a check_line() method
 205         that returns True if the line is OK and False if it has an
 206         error.  Verifiers that need a multi-line view (like
 207         SortedIncludes) must override this entire function.
 208
 209         Returns a count of errors (0 if none), though actual non-zero
 210         count value is not currently used anywhere.
 211         """
 212         pass
 213
 214     @abstractmethod
 215     def fix(self, filename, regions=all_regions):
 216         """Fix specified regions of file 'filename'.
 217
 218         Line-by-line fixes can simply provide a fix_line() method that
 219         returns the fixed line. Verifiers that need a multi-line view
 220         (like SortedIncludes) must override this entire function.
 221         """
 222         pass
 223
 224 class LineVerifier(Verifier):
 225     def check(self, filename, regions=all_regions, fobj=None, silent=False):
 226         close = False
 227         if fobj is None:
 228             fobj = self.open(filename, 'rb')
 229             close = True
 230
 231         lang = lang_type(filename)
 232         assert lang in self.languages
 233
 234         errors = 0
 235         for num,line in enumerate(fobj):
 236             if num not in regions:
 237                 continue
 238             s_line = line.decode('utf-8').rstrip('\n')
 239             if not self.check_line(s_line, language=lang):
 240                 if not silent:
 241                     self.ui.write("invalid %s in %s:%d\n" % \
 242                                   (self.test_name, filename, num + 1))
 243                     if self.ui.verbose:
 244                         self.ui.write(">>%s<<\n" % s_line[:-1])
 245                 errors += 1
 246         if close:
 247             fobj.close()
 248         return errors
 249
 250     @safefix
 251     def fix(self, filename, regions=all_regions):
 252         f = self.open(filename, 'r+')
 253
 254         lang = lang_type(filename)
 255         assert lang in self.languages
 256
 257         lines = list(f)
 258
 259         f.seek(0)
 260         f.truncate()
 261
 262         for i,line in enumerate(lines):
 263             line = line.rstrip('\n')
 264             if i in regions:
 265                 line = self.fix_line(line, language=lang)
 266
 267             f.write(line)
 268             f.write("\n")
 269         f.close()
 270         self.current_language = None
 271
 272     @abstractmethod
 273     def check_line(self, line, **kwargs):
 274         pass
 275
 276     @abstractmethod
 277     def fix_line(self, line, **kwargs):
 278         pass
 279
 280 class Whitespace(LineVerifier):
 281     """Check whitespace.
 282
 283     Specifically:
 284     - No tabs used for indent
 285     - No trailing whitespace
 286     """
 287
 288     languages = set(('C', 'C++', 'swig', 'python', 'asm', 'isa', 'scons',
 289                      'make', 'dts'))
 290     trail_only = set(('make', 'dts'))
 291
 292     test_name = 'whitespace'
 293     opt_name = 'white'
 294
 295     _lead = re.compile(r'^([ \t]+)')
 296     _trail = re.compile(r'([ \t]+)$')
 297
 298
 299     def skip_lead(self, language):
 300         return language in Whitespace.trail_only
 301
 302     def check_line(self, line, language):
 303         if not self.skip_lead(language):
 304             match = Whitespace._lead.search(line)
 305             if match and match.group(1).find('\t') != -1:
 306                 return False
 307
 308         match = Whitespace._trail.search(line)
 309         if match:
 310             return False
 311
 312         return True
 313
 314     def fix_line(self, line, language):
 315         if not self.skip_lead(language) and Whitespace._lead.search(line):
 316             newline = ''
 317             for i,c in enumerate(line):
 318                 if c == ' ':
 319                     newline += ' '
 320                 elif c == '\t':
 321                     newline += ' ' * (style.tabsize - \
 322                                       len(newline) % style.tabsize)
 323                 else:
 324                     newline += line[i:]
 325                     break
 326
 327             line = newline
 328
 329         return line.rstrip()
 330
 331
 332 class SortedIncludes(Verifier):
 333     """Check for proper sorting of include statements"""
 334
 335     languages = sort_includes.default_languages
 336     test_name = 'include file order'
 337     opt_name = 'include'
 338
 339     def __init__(self, *args, **kwargs):
 340         super(SortedIncludes, self).__init__(*args, **kwargs)
 341         self.sort_includes = sort_includes.SortIncludes()
 342
 343     def check(self, filename, regions=all_regions, fobj=None, silent=False):
 344         close = False
 345         if fobj is None:
 346             fobj = self.open(filename, 'rb')
 347             close = True
 348         norm_fname = self.normalize_filename(filename)
 349
 350         old = [ l.decode('utf-8').rstrip('\n') for l in fobj ]
 351         if close:
 352             fobj.close()
 353
 354         if len(old) == 0:
 355             return 0
 356
 357         language = lang_type(filename, old[0])
 358         new = list(self.sort_includes(old, norm_fname, language))
 359
 360         modified = _modified_regions(old, new) & regions
 361
 362         if modified:
 363             if not silent:
 364                 self.ui.write("invalid sorting of includes in %s\n"
 365                                 % (filename))
 366                 if self.ui.verbose:
 367                     for start, end in modified.regions:
 368                         self.ui.write("bad region [%d, %d)\n" % (start, end))
 369             return 1
 370
 371         return 0
 372
 373     @safefix
 374     def fix(self, filename, regions=all_regions):
 375         f = self.open(filename, 'r+')
 376         norm_fname = self.normalize_filename(filename)
 377
 378         old = f.readlines()
 379         lines = [ l.rstrip('\n') for l in old ]
 380         language = lang_type(filename, lines[0])
 381         sort_lines = list(self.sort_includes(lines, norm_fname, language))
 382         new = ''.join(line + '\n' for line in sort_lines)
 383
 384         f.seek(0)
 385         f.truncate()
 386
 387         for i,line in enumerate(sort_lines):
 388             f.write(line)
 389             f.write('\n')
 390         f.close()
 391
 392
 393 class ControlSpace(LineVerifier):
 394     """Check for exactly one space after if/while/for"""
 395
 396     languages = set(('C', 'C++'))
 397     test_name = 'spacing after if/while/for'
 398     opt_name = 'control'
 399
 400     _any_control = re.compile(r'\b(if|while|for)([ \t]*)\(')
 401
 402     def check_line(self, line, **kwargs):
 403         match = ControlSpace._any_control.search(line)
 404         return not (match and match.group(2) != " ")
 405
 406     def fix_line(self, line, **kwargs):
 407         new_line = ControlSpace._any_control.sub(r'\1 (', line)
 408         return new_line
 409
 410
 411 class LineLength(LineVerifier):
 412     languages = set(('C', 'C++', 'swig', 'python', 'asm', 'isa', 'scons'))
 413     test_name = 'line length'
 414     opt_name = 'length'
 415
 416     def check_line(self, line, **kwargs):
 417         return style.normalized_len(line) <= 79
 418
 419     def fix(self, filename, regions=all_regions, **kwargs):
 420         self.ui.write("Warning: cannot automatically fix overly long lines.\n")
 421
 422     def fix_line(self, line):
 423         pass
 424
 425 class ControlCharacters(LineVerifier):
 426     languages = set(('C', 'C++', 'swig', 'python', 'asm', 'isa', 'scons'))
 427     test_name = 'control character'
 428     opt_name = 'ascii'
 429
 430     invalid = "".join([chr(i) for i in range(0, 0x20) \
 431         if chr(i) not in ('\n', '\t')])
 432
 433     def check_line(self, line, **kwargs):
 434         return self.fix_line(line) == line
 435
 436     def fix_line(self, line, **kwargs):
 437         return ''.join(c for c in line if c not in ControlCharacters.invalid)
 438
 439 class BoolCompare(LineVerifier):
 440     languages = set(('C', 'C++', 'python'))
 441     test_name = 'boolean comparison'
 442     opt_name = 'boolcomp'
 443
 444     regex = re.compile(r'\s*==\s*([Tt]rue|[Ff]alse)\b')
 445
 446     def check_line(self, line, **kwargs):
 447         return self.regex.search(line) == None
 448
 449     def fix_line(self, line, **kwargs):
 450         match = self.regex.search(line)
 451         if match:
 452             if match.group(1) in ('true', 'True'):
 453                 line = self.regex.sub('', line)
 454             else:
 455                 self.ui.write("Warning: cannot automatically fix "
 456                               "comparisons with false/False.\n")
 457         return line
 458
 459 def is_verifier(cls):
 460     """Determine if a class is a Verifier that can be instantiated"""
 461
 462     return inspect.isclass(cls) and issubclass(cls, Verifier) and \
 463         not inspect.isabstract(cls)
 464
 465 # list of all verifier classes
 466 all_verifiers = [ v for n, v in \
 467                   inspect.getmembers(sys.modules[__name__], is_verifier) ]