util/style/verifiers.py

   1 #!/usr/bin/env python2.7
   2 #
   3 # Copyright (c) 2014, 2016 ARM Limited
   4 # All rights reserved
   5 #
   6 # The license below extends only to copyright in the software and shall
   7 # not be construed as granting a license to any other intellectual
   8 # property including but not limited to intellectual property relating
   9 # to a hardware implementation of the functionality of the software
  10 # licensed hereunder.  You may use the software subject to the license
  11 # terms below provided that you ensure that this notice is replicated
  12 # unmodified and in its entirety in all distributions of the software,
  13 # modified or unmodified, in source code or in binary form.
  14 #
  15 # Copyright (c) 2006 The Regents of The University of Michigan
  16 # Copyright (c) 2007,2011 The Hewlett-Packard Development Company
  17 # Copyright (c) 2016 Advanced Micro Devices, Inc.
  18 # All rights reserved.
  19 #
  20 # Redistribution and use in source and binary forms, with or without
  21 # modification, are permitted provided that the following conditions are
  22 # met: redistributions of source code must retain the above copyright
  23 # notice, this list of conditions and the following disclaimer;
  24 # redistributions in binary form must reproduce the above copyright
  25 # notice, this list of conditions and the following disclaimer in the
  26 # documentation and/or other materials provided with the distribution;
  27 # neither the name of the copyright holders nor the names of its
  28 # contributors may be used to endorse or promote products derived from
  29 # this software without specific prior written permission.
  30 #
  31 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  32 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  33 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  34 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  35 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  36 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  37 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  38 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  39 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  40 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  41 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  42 #
  43 # Authors: Nathan Binkert
  44 #          Steve Reinhardt
  45 #          Andreas Sandberg
  46
  47 from abc import ABCMeta, abstractmethod
  48 from difflib import SequenceMatcher
  49 import inspect
  50 import os
  51 import re
  52 import sys
  53
  54 import style
  55 import sort_includes
  56 from region import *
  57 from file_types import lang_type
  58
  59
  60 def safefix(fix_func):
  61     """ Decorator for the fix functions of the Verifier class.
  62         This function wraps the fix function and creates a backup file
  63         just in case there is an error.
  64     """
  65     def safefix_wrapper(*args, **kwargs):
  66         # Check to be sure that this is decorating a function we expect:
  67         # a class method with filename as the first argument (after self)
  68         assert(os.path.exists(args[1]))
  69         self = args[0]
  70         assert(is_verifier(self.__class__))
  71         filename = args[1]
  72
  73         # Now, Let's make a backup file.
  74         from shutil import copyfile
  75         backup_name = filename+'.bak'
  76         copyfile(filename, backup_name)
  77
  78         # Try to apply the fix. If it fails, then we revert the file
  79         # Either way, we need to clean up our backup file
  80         try:
  81             fix_func(*args, **kwargs)
  82         except Exception as e:
  83             # Restore the original file to the backup file
  84             self.ui.write("Error! Restoring the original file.\n")
  85             copyfile(backup_name, filename)
  86             raise
  87         finally:
  88             # Clean up the backup file
  89             os.remove(backup_name)
  90
  91     return safefix_wrapper
  92
  93 def _modified_regions(old, new):
  94     try:
  95         m = SequenceMatcher(a=old, b=new, autojunk=False)
  96     except TypeError:
  97         # autojunk was introduced in Python 2.7. We need a fallback
  98         # mechanism to support old Python versions.
  99         m = SequenceMatcher(a=old, b=new)
 100     regions = Regions()
 101     for tag, i1, i2, j1, j2 in m.get_opcodes():
 102         if tag != "equal":
 103             regions.extend(Region(i1, i2))
 104     return regions
 105
 106
 107 class Verifier(object):
 108     """Base class for style verifiers
 109
 110     Verifiers check for style violations and optionally fix such
 111     violations. Implementations should either inherit from this class
 112     (Verifier) if they need to work on entire files or LineVerifier if
 113     they operate on a line-by-line basis.
 114
 115     Subclasses must define these class attributes:
 116       languages = set of strings identifying applicable languages
 117       test_name = long descriptive name of test, will be used in
 118                   messages such as "error in <foo>" or "invalid <foo>"
 119       opt_name = short name used to generate command-line options to
 120                  control the test (--fix-<foo>, --ignore-<foo>, etc.)
 121
 122     """
 123
 124     __metaclass__ = ABCMeta
 125
 126     def __init__(self, ui, opts, base=None):
 127         self.ui = ui
 128         self.base = base
 129
 130         # opt_name must be defined as a class attribute of derived classes.
 131         # Check test-specific opts first as these have precedence.
 132         self.opt_fix = opts.get('fix_' + self.opt_name, False)
 133         self.opt_ignore = opts.get('ignore_' + self.opt_name, False)
 134         self.opt_skip = opts.get('skip_' + self.opt_name, False)
 135         # If no test-specific opts were set, then set based on "-all" opts.
 136         if not (self.opt_fix or self.opt_ignore or self.opt_skip):
 137             self.opt_fix = opts.get('fix_all', False)
 138             self.opt_ignore = opts.get('ignore_all', False)
 139             self.opt_skip = opts.get('skip_all', False)
 140
 141     def normalize_filename(self, name):
 142         abs_name = os.path.abspath(name)
 143         if self.base is None:
 144             return abs_name
 145
 146         abs_base = os.path.abspath(self.base)
 147         return os.path.relpath(abs_name, start=abs_base)
 148
 149     def open(self, filename, mode):
 150         try:
 151             f = file(filename, mode)
 152         except OSError, msg:
 153             print 'could not open file %s: %s' % (filename, msg)
 154             return None
 155
 156         return f
 157
 158     def skip(self, filename):
 159         # We never want to handle symlinks, so always skip them: If the
 160         # location pointed to is a directory, skip it. If the location is a
 161         # file inside the gem5 directory, it will be checked as a file, so
 162         # symlink can be skipped. If the location is a file outside gem5, we
 163         # don't want to check it anyway.
 164         if os.path.islink(filename):
 165             return True
 166         return lang_type(filename) not in self.languages
 167
 168     def apply(self, filename, regions=all_regions):
 169         """Possibly apply to specified regions of file 'filename'.
 170
 171         Verifier is skipped if --skip-<test> option was provided or if
 172         file is not of an applicable type.  Otherwise file is checked
 173         and error messages printed.  Errors are fixed or ignored if
 174         the corresponding --fix-<test> or --ignore-<test> options were
 175         provided.  If neither, the user is prompted for an action.
 176
 177         Returns True to abort, False otherwise.
 178         """
 179         if not (self.opt_skip or self.skip(filename)):
 180             errors = self.check(filename, regions)
 181             if errors and not self.opt_ignore:
 182                 if self.opt_fix:
 183                     self.fix(filename, regions)
 184                 else:
 185                     result = self.ui.prompt("(a)bort, (i)gnore, or (f)ix?",
 186                                             'aif', 'a')
 187                     if result == 'f':
 188                         self.fix(filename, regions)
 189                     elif result == 'a':
 190                         return True # abort
 191
 192         return False
 193
 194     @abstractmethod
 195     def check(self, filename, regions=all_regions, fobj=None, silent=False):
 196         """Check specified regions of file 'filename'.
 197
 198         Given that it is possible that the current contents of the file
 199         differ from the file as 'staged to commit', for those cases, and
 200         maybe others, the argument fobj should be a file object open and reset
 201         with the contents matching what the file would look like after the
 202         commit. This is needed keep the messages using 'filename' meaningful.
 203
 204         The argument silent is useful to prevent output when we run check in
 205         the staged file vs the actual file to detect if the user forgot
 206         staging fixes to the commit. This way, we prevent reporting errors
 207         twice in stderr.
 208
 209         Line-by-line checks can simply provide a check_line() method
 210         that returns True if the line is OK and False if it has an
 211         error.  Verifiers that need a multi-line view (like
 212         SortedIncludes) must override this entire function.
 213
 214         Returns a count of errors (0 if none), though actual non-zero
 215         count value is not currently used anywhere.
 216         """
 217         pass
 218
 219     @abstractmethod
 220     def fix(self, filename, regions=all_regions):
 221         """Fix specified regions of file 'filename'.
 222
 223         Line-by-line fixes can simply provide a fix_line() method that
 224         returns the fixed line. Verifiers that need a multi-line view
 225         (like SortedIncludes) must override this entire function.
 226         """
 227         pass
 228
 229 class LineVerifier(Verifier):
 230     def check(self, filename, regions=all_regions, fobj=None, silent=False):
 231         close = False
 232         if fobj is None:
 233             fobj = self.open(filename, 'r')
 234             close = True
 235
 236         lang = lang_type(filename)
 237         assert lang in self.languages
 238
 239         errors = 0
 240         for num,line in enumerate(fobj):
 241             if num not in regions:
 242                 continue
 243             line = line.rstrip('\n')
 244             if not self.check_line(line, language=lang):
 245                 if not silent:
 246                     self.ui.write("invalid %s in %s:%d\n" % \
 247                                   (self.test_name, filename, num + 1))
 248                     if self.ui.verbose:
 249                         self.ui.write(">>%s<<\n" % line[:-1])
 250                 errors += 1
 251         if close:
 252             fobj.close()
 253         return errors
 254
 255     @safefix
 256     def fix(self, filename, regions=all_regions):
 257         f = self.open(filename, 'r+')
 258
 259         lang = lang_type(filename)
 260         assert lang in self.languages
 261
 262         lines = list(f)
 263
 264         f.seek(0)
 265         f.truncate()
 266
 267         for i,line in enumerate(lines):
 268             line = line.rstrip('\n')
 269             if i in regions:
 270                 line = self.fix_line(line, language=lang)
 271
 272             f.write(line)
 273             f.write("\n")
 274         f.close()
 275         self.current_language = None
 276
 277     @abstractmethod
 278     def check_line(self, line, **kwargs):
 279         pass
 280
 281     @abstractmethod
 282     def fix_line(self, line, **kwargs):
 283         pass
 284
 285 class Whitespace(LineVerifier):
 286     """Check whitespace.
 287
 288     Specifically:
 289     - No tabs used for indent
 290     - No trailing whitespace
 291     """
 292
 293     languages = set(('C', 'C++', 'swig', 'python', 'asm', 'isa', 'scons',
 294                      'make', 'dts'))
 295     trail_only = set(('make', 'dts'))
 296
 297     test_name = 'whitespace'
 298     opt_name = 'white'
 299
 300     _lead = re.compile(r'^([ \t]+)')
 301     _trail = re.compile(r'([ \t]+)$')
 302
 303
 304     def skip_lead(self, language):
 305         return language in Whitespace.trail_only
 306
 307     def check_line(self, line, language):
 308         if not self.skip_lead(language):
 309             match = Whitespace._lead.search(line)
 310             if match and match.group(1).find('\t') != -1:
 311                 return False
 312
 313         match = Whitespace._trail.search(line)
 314         if match:
 315             return False
 316
 317         return True
 318
 319     def fix_line(self, line, language):
 320         if not self.skip_lead(language) and Whitespace._lead.search(line):
 321             newline = ''
 322             for i,c in enumerate(line):
 323                 if c == ' ':
 324                     newline += ' '
 325                 elif c == '\t':
 326                     newline += ' ' * (style.tabsize - \
 327                                       len(newline) % style.tabsize)
 328                 else:
 329                     newline += line[i:]
 330                     break
 331
 332             line = newline
 333
 334         return line.rstrip()
 335
 336
 337 class SortedIncludes(Verifier):
 338     """Check for proper sorting of include statements"""
 339
 340     languages = sort_includes.default_languages
 341     test_name = 'include file order'
 342     opt_name = 'include'
 343
 344     def __init__(self, *args, **kwargs):
 345         super(SortedIncludes, self).__init__(*args, **kwargs)
 346         self.sort_includes = sort_includes.SortIncludes()
 347
 348     def check(self, filename, regions=all_regions, fobj=None, silent=False):
 349         close = False
 350         if fobj is None:
 351             fobj = self.open(filename, 'r')
 352             close = True
 353         norm_fname = self.normalize_filename(filename)
 354
 355         old = [ l.rstrip('\n') for l in fobj.xreadlines() ]
 356         if close:
 357             fobj.close()
 358
 359         if len(old) == 0:
 360             return 0
 361
 362         language = lang_type(filename, old[0])
 363         new = list(self.sort_includes(old, norm_fname, language))
 364
 365         modified = _modified_regions(old, new) & regions
 366
 367         if modified:
 368             if not silent:
 369                 self.ui.write("invalid sorting of includes in %s\n"
 370                                 % (filename))
 371                 if self.ui.verbose:
 372                     for start, end in modified.regions:
 373                         self.ui.write("bad region [%d, %d)\n" % (start, end))
 374             return 1
 375
 376         return 0
 377
 378     @safefix
 379     def fix(self, filename, regions=all_regions):
 380         f = self.open(filename, 'r+')
 381         norm_fname = self.normalize_filename(filename)
 382
 383         old = f.readlines()
 384         lines = [ l.rstrip('\n') for l in old ]
 385         language = lang_type(filename, lines[0])
 386         sort_lines = list(self.sort_includes(lines, norm_fname, language))
 387         new = ''.join(line + '\n' for line in sort_lines)
 388
 389         f.seek(0)
 390         f.truncate()
 391
 392         for i,line in enumerate(sort_lines):
 393             f.write(line)
 394             f.write('\n')
 395         f.close()
 396
 397
 398 class ControlSpace(LineVerifier):
 399     """Check for exactly one space after if/while/for"""
 400
 401     languages = set(('C', 'C++'))
 402     test_name = 'spacing after if/while/for'
 403     opt_name = 'control'
 404
 405     _any_control = re.compile(r'\b(if|while|for)([ \t]*)\(')
 406
 407     def check_line(self, line, **kwargs):
 408         match = ControlSpace._any_control.search(line)
 409         return not (match and match.group(2) != " ")
 410
 411     def fix_line(self, line, **kwargs):
 412         new_line = ControlSpace._any_control.sub(r'\1 (', line)
 413         return new_line
 414
 415
 416 class LineLength(LineVerifier):
 417     languages = set(('C', 'C++', 'swig', 'python', 'asm', 'isa', 'scons'))
 418     test_name = 'line length'
 419     opt_name = 'length'
 420
 421     def check_line(self, line, **kwargs):
 422         return style.normalized_len(line) <= 79
 423
 424     def fix(self, filename, regions=all_regions, **kwargs):
 425         self.ui.write("Warning: cannot automatically fix overly long lines.\n")
 426
 427     def fix_line(self, line):
 428         pass
 429
 430 class ControlCharacters(LineVerifier):
 431     languages = set(('C', 'C++', 'swig', 'python', 'asm', 'isa', 'scons'))
 432     test_name = 'control character'
 433     opt_name = 'ascii'
 434
 435     valid = ('\n', '\t')
 436     invalid = "".join([chr(i) for i in range(0, 0x20) if chr(i) not in valid])
 437
 438     def check_line(self, line, **kwargs):
 439         return self.fix_line(line) == line
 440
 441     def fix_line(self, line, **kwargs):
 442         return line.translate(None, ControlCharacters.invalid)
 443
 444 class BoolCompare(LineVerifier):
 445     languages = set(('C', 'C++', 'python'))
 446     test_name = 'boolean comparison'
 447     opt_name = 'boolcomp'
 448
 449     regex = re.compile(r'\s*==\s*([Tt]rue|[Ff]alse)\b')
 450
 451     def check_line(self, line, **kwargs):
 452         return self.regex.search(line) == None
 453
 454     def fix_line(self, line, **kwargs):
 455         match = self.regex.search(line)
 456         if match:
 457             if match.group(1) in ('true', 'True'):
 458                 line = self.regex.sub('', line)
 459             else:
 460                 self.ui.write("Warning: cannot automatically fix "
 461                               "comparisons with false/False.\n")
 462         return line
 463
 464 def is_verifier(cls):
 465     """Determine if a class is a Verifier that can be instantiated"""
 466
 467     return inspect.isclass(cls) and issubclass(cls, Verifier) and \
 468         not inspect.isabstract(cls)
 469
 470 # list of all verifier classes
 471 all_verifiers = [ v for n, v in \
 472                   inspect.getmembers(sys.modules[__name__], is_verifier) ]