ext/testlib/loader.py

   1 # Copyright (c) 2017 Mark D. Hill and David A. Wood
   2 # All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met: redistributions of source code must retain the above copyright
   7 # notice, this list of conditions and the following disclaimer;
   8 # redistributions in binary form must reproduce the above copyright
   9 # notice, this list of conditions and the following disclaimer in the
  10 # documentation and/or other materials provided with the distribution;
  11 # neither the name of the copyright holders nor the names of its
  12 # contributors may be used to endorse or promote products derived from
  13 # this software without specific prior written permission.
  14 #
  15 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  16 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  17 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  18 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  19 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  20 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  21 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  22 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  23 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 #
  27 # Authors: Sean Wilson
  28
  29 '''
  30 Contains the :class:`Loader` which is responsible for discovering and loading
  31 tests.
  32
  33 Loading typically follows the following stages.
  34
  35 1. Recurse down a given directory looking for tests which match a given regex.
  36
  37     The default regex used will match any python file (ending in .py) that has
  38     a name starting or ending in test(s). If there are any additional
  39     components of the name they must be connected with '-' or '_'. Lastly,
  40     file names that begin with '.' will be ignored.
  41
  42     The following names would match:
  43
  44     - `tests.py`
  45     - `test.py`
  46     - `test-this.py`
  47     - `tests-that.py`
  48     - `these-test.py`
  49
  50     These would not match:
  51
  52     - `.test.py`    - 'hidden' files are ignored.
  53     - `test`        - Must end in '.py'
  54     - `test-.py`    - Needs a character after the hypen.
  55     - `testthis.py` - Needs a hypen or underscore to separate 'test' and 'this'
  56
  57
  58 2. With all files discovered execute each file gathering its test items we
  59    care about collecting. (`TestCase`, `TestSuite` and `Fixture` objects.)
  60
  61 As a final note, :class:`TestCase` instances which are not put into
  62 a :class:`TestSuite` by the test writer will be placed into
  63 a :class:`TestSuite` named after the module.
  64
  65 .. seealso:: :func:`load_file`
  66 '''
  67
  68 import os
  69 import re
  70 import six
  71 import sys
  72 import traceback
  73
  74 import testlib.log as log
  75 import testlib.suite as suite_mod
  76 import testlib.test_util as test_mod
  77 import testlib.fixture as fixture_mod
  78 import testlib.wrappers as wrappers
  79 import testlib.uid as uid
  80
  81 class DuplicateTestItemException(Exception):
  82     '''
  83     Exception indicates multiple test items with the same UID
  84     were discovered.
  85     '''
  86     pass
  87
  88
  89 # Match filenames that either begin or end with 'test' or tests and use
  90 # - or _ to separate additional name components.
  91 default_filepath_regex = re.compile(
  92             r'(((.+[_])?tests?)|(tests?([-_].+)?))\.py$')
  93
  94 def default_filepath_filter(filepath):
  95     '''The default filter applied to filepaths to marks as test sources.'''
  96     filepath = os.path.basename(filepath)
  97     if default_filepath_regex.match(filepath):
  98         # Make sure doesn't start with .
  99         return not filepath.startswith('.')
 100     return False
 101
 102 def path_as_modulename(filepath):
 103     '''Return the given filepath as a module name.'''
 104     # Remove the file extention (.py)
 105     return os.path.splitext(os.path.basename(filepath))[0]
 106
 107 def path_as_suitename(filepath):
 108     return os.path.split(os.path.dirname(os.path.abspath((filepath))))[-1]
 109
 110 def _assert_files_in_same_dir(files):
 111     if __debug__:
 112         if files:
 113             directory = os.path.dirname(files[0])
 114             for f in files:
 115                 assert(os.path.dirname(f) == directory)
 116
 117 class Loader(object):
 118     '''
 119     Class for discovering tests.
 120
 121     Discovered :class:`TestCase` and :class:`TestSuite` objects are wrapped by
 122     :class:`LoadedTest` and :class:`LoadedSuite` objects respectively.
 123     These objects provided additional methods and metadata about the loaded
 124     objects and are the internal representation used by testlib.
 125
 126     To simply discover and load all tests using the default filter create an
 127     instance and `load_root`.
 128
 129     >>> import os
 130     >>> tl = Loader()
 131     >>> tl.load_root(os.getcwd())
 132
 133     .. note:: If tests are not contained in a TestSuite, they will
 134         automatically be placed into one for the module.
 135
 136     .. warn:: This class is extremely thread-unsafe.
 137        It modifies the sys path and global config.
 138        Use with care.
 139     '''
 140     def __init__(self):
 141         self.suites = []
 142         self.suite_uids = {}
 143         self.filepath_filter = default_filepath_filter
 144
 145         # filepath -> Successful | Failed to load
 146         self._files = {}
 147
 148     @property
 149     def schedule(self):
 150         return wrappers.LoadedLibrary(self.suites)
 151
 152     def load_schedule_for_suites(self, *uids):
 153         files = {uid.UID.uid_to_path(id_) for id_ in uids}
 154         for file_ in files:
 155             self.load_file(file_)
 156
 157         return wrappers.LoadedLibrary(
 158                 [self.suite_uids[id_] for id_ in uids])
 159
 160     def _verify_no_duplicate_suites(self, new_suites):
 161         new_suite_uids = self.suite_uids.copy()
 162         for suite in new_suites:
 163             if suite.uid in new_suite_uids:
 164                 raise DuplicateTestItemException(
 165                         "More than one suite with UID '%s' was defined" %\
 166                                 suite.uid)
 167             new_suite_uids[suite.uid] = suite
 168
 169     def _verify_no_duplicate_tests_in_suites(self, new_suites):
 170         for suite in new_suites:
 171             test_uids = set()
 172             for test in suite:
 173                 if test.uid in test_uids:
 174                      raise DuplicateTestItemException(
 175                             "More than one test with UID '%s' was defined"
 176                             " in suite '%s'"
 177                             % (test.uid, suite.uid))
 178                 test_uids.add(test.uid)
 179
 180     def load_root(self, root):
 181         '''
 182         Load files from the given root directory which match
 183         `self.filepath_filter`.
 184         '''
 185         for directory in self._discover_files(root):
 186             directory = list(directory)
 187             if directory:
 188                 _assert_files_in_same_dir(directory)
 189                 for f in directory:
 190                     self.load_file(f)
 191
 192     def load_file(self, path):
 193         path = os.path.abspath(path)
 194
 195         if path in self._files:
 196             if not self._files[path]:
 197                 raise Exception('Attempted to load a file which already'
 198                         ' failed to load')
 199             else:
 200                 log.test_log.debug('Tried to reload: %s' % path)
 201                 return
 202
 203         # Create a custom dictionary for the loaded module.
 204         newdict = {
 205             '__builtins__':__builtins__,
 206             '__name__': path_as_modulename(path),
 207             '__file__': path,
 208         }
 209
 210         # Add the file's containing directory to the system path. So it can do
 211         # relative imports naturally.
 212         old_path = sys.path[:]
 213         sys.path.insert(0, os.path.dirname(path))
 214         cwd = os.getcwd()
 215         os.chdir(os.path.dirname(path))
 216
 217         new_tests = test_mod.TestCase.collector.create()
 218         new_suites = suite_mod.TestSuite.collector.create()
 219         new_fixtures = fixture_mod.Fixture.collector.create()
 220
 221         try:
 222             exec(open(path).read(), newdict, newdict)
 223         except Exception as e:
 224             log.test_log.debug(traceback.format_exc())
 225             log.test_log.warn(
 226                               'Exception thrown while loading "%s"\n'
 227                               'Ignoring all tests in this file.'
 228                                % (path))
 229             # Clean up
 230             sys.path[:] = old_path
 231             os.chdir(cwd)
 232             test_mod.TestCase.collector.remove(new_tests)
 233             suite_mod.TestSuite.collector.remove(new_suites)
 234             fixture_mod.Fixture.collector.remove(new_fixtures)
 235             return
 236
 237         # Create a module test suite for those not contained in a suite.
 238         orphan_tests = set(new_tests)
 239         for suite in new_suites:
 240             for test in suite:
 241                 # Remove the test if it wasn't already removed.
 242                 # (Suites may contain copies of tests.)
 243                 if test in orphan_tests:
 244                     orphan_tests.remove(test)
 245         if orphan_tests:
 246             orphan_tests = sorted(orphan_tests, key=new_tests.index)
 247             # FIXME Use the config based default to group all uncollected
 248             # tests.
 249             # NOTE: This is automatically collected (we still have the
 250             # collector active.)
 251             suite_mod.TestSuite(tests=orphan_tests,
 252                     name=path_as_suitename(path))
 253
 254         try:
 255             loaded_suites = [wrappers.LoadedSuite(suite, path)
 256                     for suite in new_suites]
 257
 258             self._verify_no_duplicate_suites(loaded_suites)
 259             self._verify_no_duplicate_tests_in_suites(loaded_suites)
 260         except Exception as e:
 261             log.test_log.warn('%s\n'
 262                     'Exception thrown while loading "%s"\n'
 263                     'Ignoring all tests in this file.'
 264                     % (traceback.format_exc(), path))
 265         else:
 266             log.test_log.info('Discovered %d tests and %d suites in %s'
 267                     '' % (len(new_tests), len(loaded_suites), path))
 268
 269             self.suites.extend(loaded_suites)
 270             self.suite_uids.update({suite.uid: suite
 271                     for suite in loaded_suites})
 272         # Clean up
 273         sys.path[:] = old_path
 274         os.chdir(cwd)
 275         test_mod.TestCase.collector.remove(new_tests)
 276         suite_mod.TestSuite.collector.remove(new_suites)
 277         fixture_mod.Fixture.collector.remove(new_fixtures)
 278
 279     def _discover_files(self, root):
 280         '''
 281         Recurse down from the given root directory returning a list of
 282         directories which contain a list of files matching
 283         `self.filepath_filter`.
 284         '''
 285         # Will probably want to order this traversal.
 286         for root, dirnames, filenames in os.walk(root):
 287             dirnames.sort()
 288             if filenames:
 289                 filenames.sort()
 290                 filepaths = [os.path.join(root, filename) \
 291                              for filename in filenames]
 292                 filepaths = filter(self.filepath_filter, filepaths)
 293                 if filepaths:
 294                     yield filepaths