ext/testlib/main.py

   1 # Copyright (c) 2017 Mark D. Hill and David A. Wood
   2 # All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met: redistributions of source code must retain the above copyright
   7 # notice, this list of conditions and the following disclaimer;
   8 # redistributions in binary form must reproduce the above copyright
   9 # notice, this list of conditions and the following disclaimer in the
  10 # documentation and/or other materials provided with the distribution;
  11 # neither the name of the copyright holders nor the names of its
  12 # contributors may be used to endorse or promote products derived from
  13 # this software without specific prior written permission.
  14 #
  15 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  16 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  17 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  18 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  19 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  20 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  21 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  22 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  23 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 #
  27 # Authors: Sean Wilson
  28
  29 import os
  30 import itertools
  31
  32 import testlib.configuration as configuration
  33 import testlib.handlers as handlers
  34 import testlib.loader as loader_mod
  35 import testlib.log as log
  36 import testlib.query as query
  37 import testlib.result as result
  38 import testlib.runner as runner
  39 import testlib.terminal as terminal
  40 import testlib.uid as uid
  41
  42 def entry_message():
  43     log.test_log.message("Running the new gem5 testing script.")
  44     log.test_log.message("For more information see TESTING.md.")
  45     log.test_log.message("To see details as the testing scripts are"
  46                          " running, use the option"
  47                          " -v, -vv, or -vvv")
  48
  49 class RunLogHandler():
  50     def __init__(self):
  51         term_handler = handlers.TerminalHandler(
  52             verbosity=configuration.config.verbose+log.LogLevel.Info
  53         )
  54         summary_handler = handlers.SummaryHandler()
  55         self.mp_handler = handlers.MultiprocessingHandlerWrapper(
  56                 summary_handler, term_handler)
  57         self.mp_handler.async_process()
  58         log.test_log.add_handler(self.mp_handler)
  59         entry_message()
  60
  61     def schedule_finalized(self, test_schedule):
  62         # Create the result handler object.
  63         self.result_handler = handlers.ResultHandler(
  64                 test_schedule, configuration.config.result_path)
  65         self.mp_handler.add_handler(self.result_handler)
  66
  67     def finish_testing(self):
  68         self.result_handler.close()
  69
  70     def __enter__(self):
  71         return self
  72
  73     def __exit__(self, *args):
  74         self.close()
  75         return False
  76
  77     def close(self):
  78         self.mp_handler.close()
  79
  80     def unsuccessful(self):
  81         '''
  82         Performs an or reduce on all of the results.
  83         Returns true if at least one test is unsuccessful, false when all tests
  84         pass
  85         '''
  86         return self.result_handler.unsuccessful()
  87
  88 def get_config_tags():
  89     return getattr(configuration.config,
  90             configuration.StorePositionalTagsAction.position_kword)
  91
  92 def filter_with_config_tags(loaded_library):
  93     tags = get_config_tags()
  94     final_tags = []
  95     regex_fmt = '^%s$'
  96     cfg = configuration.config
  97
  98     def _append_inc_tag_filter(name):
  99         if hasattr(cfg, name):
 100             tag_opts = getattr(cfg, name)
 101             for tag in tag_opts:
 102                 final_tags.append(configuration.TagRegex(True, regex_fmt % tag))
 103
 104     def _append_rem_tag_filter(name):
 105         if hasattr(cfg, name):
 106             tag_opts = getattr(cfg, name)
 107             for tag in cfg.constants.supported_tags[name]:
 108                 if tag not in tag_opts:
 109                     final_tags.append(configuration.TagRegex(False, regex_fmt % tag))
 110
 111     # Append additional tags for the isa, length, and variant options.
 112     # They apply last (they take priority)
 113     special_tags = (
 114         cfg.constants.isa_tag_type,
 115         cfg.constants.length_tag_type,
 116         cfg.constants.host_isa_tag_type,
 117         cfg.constants.variant_tag_type
 118     )
 119
 120     for tagname in special_tags:
 121         _append_inc_tag_filter(tagname)
 122     for tagname in special_tags:
 123         _append_rem_tag_filter(tagname)
 124
 125     if tags is None:
 126         tags = tuple()
 127
 128     filters = list(itertools.chain(tags, final_tags))
 129     string = 'Filtering suites with tags as follows:\n'
 130     filter_string = '\t\n'.join((str(f) for f in filters))
 131     log.test_log.trace(string + filter_string)
 132
 133     return filter_with_tags(loaded_library, filters)
 134
 135
 136 def filter_with_tags(loaded_library, filters):
 137     '''
 138     Filter logic supports two filter types:
 139     --include-tags <regex>
 140     --exclude-tags <regex>
 141
 142     The logic maintains a `set` of test suites.
 143
 144     If the regex provided with the `--include-tags` flag matches a tag of a
 145     suite, that suite will added to the set.
 146
 147     If the regex provided with the `--exclude-tags` flag matches a tag of a
 148     suite, that suite will removed to the set.
 149
 150     Suites can be added and removed multiple times.
 151
 152     First Flag Special Case Logic:
 153     If include is the first flag, start with an empty set of suites.
 154     If exclude is the first flag, start with the set of all collected suites.
 155
 156
 157     Let's trace out the set as we go through the flags to clarify::
 158
 159         # Say our collection of suites looks like this: set(suite_ARM64,
 160         # suite_X86, suite_Other).
 161         #
 162         # Additionally, we've passed the flags in the following order:
 163         #  --include-tags "ARM64"  --exclude-tags ".*" --include-tags "X86"
 164
 165         # Process --include-tags "ARM64"
 166         set(suite_ARM64)    # Suite begins empty, but adds the ARM64 suite
 167         # Process --exclude-tags ".*"
 168         set()               # Removed all suites which have tags
 169         # Process --include-tags "X86"
 170         set(suite_X86)
 171     '''
 172     if not filters:
 173         return
 174
 175     query_runner = query.QueryRunner(loaded_library)
 176     tags = query_runner.tags()
 177
 178     if not filters[0].include:
 179         suites = set(query_runner.suites())
 180     else:
 181         suites = set()
 182
 183     def exclude(excludes):
 184         return suites - excludes
 185     def include(includes):
 186         return suites | includes
 187
 188     for tag_regex in filters:
 189         matched_tags = (tag for tag in tags if tag_regex.regex.search(tag))
 190         for tag in matched_tags:
 191             matched_suites = set(query_runner.suites_with_tag(tag))
 192             suites = include(matched_suites) if tag_regex.include \
 193                     else exclude(matched_suites)
 194
 195     # Set the library's suites to only those which where accepted by our filter
 196     loaded_library.suites = [suite for suite in loaded_library.suites
 197             if suite in suites]
 198
 199 # TODO Add results command for listing previous results.
 200
 201 def load_tests():
 202     '''
 203     Create a TestLoader and load tests for the directory given by the config.
 204     '''
 205     testloader = loader_mod.Loader()
 206     log.test_log.message(terminal.separator())
 207     log.test_log.message('Loading Tests', bold=True)
 208     testloader.load_root(configuration.config.directory)
 209     return testloader
 210
 211 def do_list():
 212     term_handler = handlers.TerminalHandler(
 213         verbosity=configuration.config.verbose+log.LogLevel.Info,
 214         machine_only=configuration.config.quiet
 215     )
 216     log.test_log.add_handler(term_handler)
 217
 218     entry_message()
 219
 220     test_schedule = load_tests().schedule
 221     filter_with_config_tags(test_schedule)
 222
 223     qrunner = query.QueryRunner(test_schedule)
 224
 225     if configuration.config.suites:
 226         qrunner.list_suites()
 227     elif configuration.config.tests:
 228         qrunner.list_tests()
 229     elif configuration.config.all_tags:
 230         qrunner.list_tags()
 231     else:
 232         qrunner.list_suites()
 233         qrunner.list_tests()
 234         qrunner.list_tags()
 235
 236     return 0
 237
 238 def run_schedule(test_schedule, log_handler):
 239     '''
 240     Test Phases
 241     -----------
 242     * Test Collection
 243     * Fixture Parameterization
 244     * Global Fixture Setup
 245     * Iteratevely run suites:
 246        * Suite Fixture Setup
 247        * Iteratively run tests:
 248           * Test Fixture Setup
 249           * Run Test
 250           * Test Fixture Teardown
 251        * Suite Fixture Teardown
 252     * Global Fixture Teardown
 253     '''
 254
 255     log_handler.schedule_finalized(test_schedule)
 256
 257     log.test_log.message(terminal.separator())
 258     log.test_log.message('Running Tests from {} suites'
 259             .format(len(test_schedule.suites)), bold=True)
 260     log.test_log.message("Results will be stored in {}".format(
 261                 configuration.config.result_path))
 262     log.test_log.message(terminal.separator())
 263
 264     # Build global fixtures and exectute scheduled test suites.
 265     if configuration.config.test_threads > 1:
 266         library_runner = runner.LibraryParallelRunner(test_schedule)
 267         library_runner.set_threads(configuration.config.test_threads)
 268     else:
 269         library_runner = runner.LibraryRunner(test_schedule)
 270     library_runner.run()
 271
 272     failed = log_handler.unsuccessful()
 273
 274     log_handler.finish_testing()
 275
 276     return 1 if failed else 0
 277
 278 def do_run():
 279     # Initialize early parts of the log.
 280     with RunLogHandler() as log_handler:
 281         if configuration.config.uid:
 282             uid_ = uid.UID.from_uid(configuration.config.uid)
 283             if isinstance(uid_, uid.TestUID):
 284                 log.test_log.error('Unable to run a standalone test.\n'
 285                         'Gem5 expects test suites to be the smallest unit '
 286                         ' of test.\n\n'
 287                         'Pass a SuiteUID instead.')
 288                 return
 289             test_schedule = loader_mod.Loader().load_schedule_for_suites(uid_)
 290             if get_config_tags():
 291                 log.test_log.warn(
 292                     "The '--uid' flag was supplied,"
 293                     " '--include-tags' and '--exclude-tags' will be ignored."
 294                 )
 295         else:
 296             test_schedule = load_tests().schedule
 297             # Filter tests based on tags
 298             filter_with_config_tags(test_schedule)
 299         # Execute the tests
 300         return run_schedule(test_schedule, log_handler)
 301
 302 def do_rerun():
 303     # Init early parts of log
 304     with RunLogHandler() as log_handler:
 305         # Load previous results
 306         results = result.InternalSavedResults.load(
 307                 os.path.join(configuration.config.result_path,
 308                 configuration.constants.pickle_filename))
 309
 310         rerun_suites = (suite.uid for suite in results if suite.unsuccessful)
 311
 312         # Use loader to load suites
 313         loader = loader_mod.Loader()
 314         test_schedule = loader.load_schedule_for_suites(*rerun_suites)
 315
 316         # Execute the tests
 317         return run_schedule(test_schedule, log_handler)
 318
 319 def main():
 320     '''
 321     Main entrypoint for the testlib test library.
 322     Returns 0 on success and 1 otherwise so it can be used as a return code
 323     for scripts.
 324     '''
 325     configuration.initialize_config()
 326
 327     # 'do' the given command.
 328     result = globals()['do_'+configuration.config.command]()
 329     log.test_log.close()
 330
 331     return result