ext/testlib/loader.py

   1 # Copyright (c) 2017 Mark D. Hill and David A. Wood
   2 # All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met: redistributions of source code must retain the above copyright
   7 # notice, this list of conditions and the following disclaimer;
   8 # redistributions in binary form must reproduce the above copyright
   9 # notice, this list of conditions and the following disclaimer in the
  10 # documentation and/or other materials provided with the distribution;
  11 # neither the name of the copyright holders nor the names of its
  12 # contributors may be used to endorse or promote products derived from
  13 # this software without specific prior written permission.
  14 #
  15 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  16 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  17 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  18 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  19 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  20 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  21 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  22 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  23 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 #
  27 # Authors: Sean Wilson
  28
  29 '''
  30 Contains the :class:`Loader` which is responsible for discovering and loading
  31 tests.
  32
  33 Loading typically follows the following stages.
  34
  35 1. Recurse down a given directory looking for tests which match a given regex.
  36
  37     The default regex used will match any python file (ending in .py) that has
  38     a name starting or ending in test(s). If there are any additional
  39     components of the name they must be connected with '-' or '_'. Lastly,
  40     file names that begin with '.' will be ignored.
  41
  42     The following names would match:
  43
  44     - `tests.py`
  45     - `test.py`
  46     - `test-this.py`
  47     - `tests-that.py`
  48     - `these-test.py`
  49
  50     These would not match:
  51
  52     - `.test.py`    - 'hidden' files are ignored.
  53     - `test`        - Must end in '.py'
  54     - `test-.py`    - Needs a character after the hypen.
  55     - `testthis.py` - Needs a hypen or underscore to separate 'test' and 'this'
  56
  57
  58 2. With all files discovered execute each file gathering its test items we
  59    care about collecting. (`TestCase`, `TestSuite` and `Fixture` objects.)
  60
  61 As a final note, :class:`TestCase` instances which are not put into
  62 a :class:`TestSuite` by the test writer will be placed into
  63 a :class:`TestSuite` named after the module.
  64
  65 .. seealso:: :func:`load_file`
  66 '''
  67
  68 import os
  69 import re
  70 import sys
  71 import traceback
  72
  73 import config
  74 import log
  75 import suite as suite_mod
  76 import test as test_mod
  77 import fixture as fixture_mod
  78 import wrappers
  79 import uid
  80
  81 class DuplicateTestItemException(Exception):
  82     '''
  83     Exception indicates multiple test items with the same UID
  84     were discovered.
  85     '''
  86     pass
  87
  88
  89 # Match filenames that either begin or end with 'test' or tests and use
  90 # - or _ to separate additional name components.
  91 default_filepath_regex = re.compile(
  92             r'(((.+[_])?tests?)|(tests?([-_].+)?))\.py$')
  93
  94 def default_filepath_filter(filepath):
  95     '''The default filter applied to filepaths to marks as test sources.'''
  96     filepath = os.path.basename(filepath)
  97     if default_filepath_regex.match(filepath):
  98         # Make sure doesn't start with .
  99         return not filepath.startswith('.')
 100     return False
 101
 102 def path_as_modulename(filepath):
 103     '''Return the given filepath as a module name.'''
 104     # Remove the file extention (.py)
 105     return os.path.splitext(os.path.basename(filepath))[0]
 106
 107 def path_as_suitename(filepath):
 108     return os.path.split(os.path.dirname(os.path.abspath((filepath))))[-1]
 109
 110 def _assert_files_in_same_dir(files):
 111     if __debug__:
 112         if files:
 113             directory = os.path.dirname(files[0])
 114             for f in files:
 115                 assert os.path.dirname(f) == directory
 116
 117 class Loader(object):
 118     '''
 119     Class for discovering tests.
 120
 121     Discovered :class:`TestCase` and :class:`TestSuite` objects are wrapped by
 122     :class:`LoadedTest` and :class:`LoadedSuite` objects respectively.
 123     These objects provided additional methods and metadata about the loaded
 124     objects and are the internal representation used by testlib.
 125
 126     To simply discover and load all tests using the default filter create an
 127     instance and `load_root`.
 128
 129     >>> import os
 130     >>> tl = Loader()
 131     >>> tl.load_root(os.getcwd())
 132
 133     .. note:: If tests are not contained in a TestSuite, they will
 134         automatically be placed into one for the module.
 135
 136     .. warn:: This class is extremely thread-unsafe.
 137        It modifies the sys path and global config.
 138        Use with care.
 139     '''
 140     def __init__(self):
 141         self.suites = []
 142         self.suite_uids = {}
 143         self.filepath_filter = default_filepath_filter
 144
 145         # filepath -> Successful | Failed to load
 146         self._files = {}
 147
 148     @property
 149     def schedule(self):
 150         return wrappers.LoadedLibrary(self.suites)
 151
 152     def load_schedule_for_suites(self, *uids):
 153         files = {uid.UID.uid_to_path(id_) for id_ in uids}
 154         for file_ in files:
 155             self.load_file(file_)
 156
 157         return wrappers.LoadedLibrary(
 158                 [self.suite_uids[id_] for id_ in uids])
 159
 160     def _verify_no_duplicate_suites(self, new_suites):
 161         new_suite_uids = self.suite_uids.copy()
 162         for suite in new_suites:
 163             if suite.uid in new_suite_uids:
 164                 raise DuplicateTestItemException(
 165                         "More than one suite with UID '%s' was defined" %\
 166                                 suite.uid)
 167             new_suite_uids[suite.uid] = suite
 168
 169     def _verify_no_duplicate_tests_in_suites(self, new_suites):
 170         for suite in new_suites:
 171             test_uids = set()
 172             for test in suite:
 173                 if test.uid in test_uids:
 174                      raise DuplicateTestItemException(
 175                             "More than one test with UID '%s' was defined"
 176                             " in suite '%s'"
 177                             % (test.uid, suite.uid))
 178                 test_uids.add(test.uid)
 179
 180     def load_root(self, root):
 181         '''
 182         Load files from the given root directory which match
 183         `self.filepath_filter`.
 184         '''
 185         if __debug__:
 186             self._loaded_a_file = True
 187
 188         for directory in self._discover_files(root):
 189             if directory:
 190                 _assert_files_in_same_dir(directory)
 191                 for f in directory:
 192                     self.load_file(f)
 193
 194     def load_dir(self, directory):
 195         for dir_ in self._discover_files(directory):
 196             _assert_files_in_same_dir(dir_)
 197             for f in dir_:
 198                 self.load_file(f)
 199
 200     def load_file(self, path):
 201         path = os.path.abspath(path)
 202
 203         if path in self._files:
 204             if not self._files[path]:
 205                 raise Exception('Attempted to load a file which already'
 206                         ' failed to load')
 207             else:
 208                 log.test_log.debug('Tried to reload: %s' % path)
 209                 return
 210
 211         # Create a custom dictionary for the loaded module.
 212         newdict = {
 213             '__builtins__':__builtins__,
 214             '__name__': path_as_modulename(path),
 215             '__file__': path,
 216         }
 217
 218         # Add the file's containing directory to the system path. So it can do
 219         # relative imports naturally.
 220         old_path = sys.path[:]
 221         sys.path.insert(0, os.path.dirname(path))
 222         cwd = os.getcwd()
 223         os.chdir(os.path.dirname(path))
 224         config.config.file_under_load = path
 225
 226         new_tests = test_mod.TestCase.collector.create()
 227         new_suites = suite_mod.TestSuite.collector.create()
 228         new_fixtures = fixture_mod.Fixture.collector.create()
 229
 230         def cleanup():
 231             config.config.file_under_load = None
 232             sys.path[:] = old_path
 233             os.chdir(cwd)
 234             test_mod.TestCase.collector.remove(new_tests)
 235             suite_mod.TestSuite.collector.remove(new_suites)
 236             fixture_mod.Fixture.collector.remove(new_fixtures)
 237
 238         try:
 239             execfile(path, newdict, newdict)
 240         except Exception as e:
 241             log.test_log.debug(traceback.format_exc())
 242             log.test_log.warn(
 243                               'Exception thrown while loading "%s"\n'
 244                               'Ignoring all tests in this file.'
 245                                % (path))
 246             cleanup()
 247             return
 248
 249         # Create a module test suite for those not contained in a suite.
 250         orphan_tests = set(new_tests)
 251         for suite in new_suites:
 252             for test in suite:
 253                 # Remove the test if it wasn't already removed.
 254                 # (Suites may contain copies of tests.)
 255                 if test in orphan_tests:
 256                     orphan_tests.remove(test)
 257         if orphan_tests:
 258             orphan_tests = sorted(orphan_tests, key=new_tests.index)
 259             # FIXME Use the config based default to group all uncollected
 260             # tests.
 261             # NOTE: This is automatically collected (we still have the
 262             # collector active.)
 263             suite_mod.TestSuite(tests=orphan_tests,
 264                     name=path_as_suitename(path))
 265
 266         try:
 267             loaded_suites = [wrappers.LoadedSuite(suite, path)
 268                     for suite in new_suites]
 269
 270             self._verify_no_duplicate_suites(loaded_suites)
 271             self._verify_no_duplicate_tests_in_suites(loaded_suites)
 272         except Exception as e:
 273             log.test_log.warn('%s\n'
 274                     'Exception thrown while loading "%s"\n'
 275                     'Ignoring all tests in this file.'
 276                     % (traceback.format_exc(), path))
 277         else:
 278             log.test_log.info('Discovered %d tests and %d suites in %s'
 279                     '' % (len(new_tests), len(loaded_suites), path))
 280
 281             self.suites.extend(loaded_suites)
 282             self.suite_uids.update({suite.uid: suite
 283                     for suite in loaded_suites})
 284         cleanup()
 285
 286     def _discover_files(self, root):
 287         '''
 288         Recurse down from the given root directory returning a list of
 289         directories which contain a list of files matching
 290         `self.filepath_filter`.
 291         '''
 292         # Will probably want to order this traversal.
 293         for root, dirnames, filenames in os.walk(root):
 294             dirnames.sort()
 295             if filenames:
 296                 filenames.sort()
 297                 filepaths = [os.path.join(root, filename) \
 298                              for filename in filenames]
 299                 filepaths = filter(self.filepath_filter, filepaths)
 300                 if filepaths:
 301                     yield filepaths