ext/testlib/main.py

   1 # Copyright (c) 2017 Mark D. Hill and David A. Wood
   2 # All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met: redistributions of source code must retain the above copyright
   7 # notice, this list of conditions and the following disclaimer;
   8 # redistributions in binary form must reproduce the above copyright
   9 # notice, this list of conditions and the following disclaimer in the
  10 # documentation and/or other materials provided with the distribution;
  11 # neither the name of the copyright holders nor the names of its
  12 # contributors may be used to endorse or promote products derived from
  13 # this software without specific prior written permission.
  14 #
  15 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  16 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  17 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  18 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  19 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  20 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  21 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  22 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  23 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 #
  27 # Authors: Sean Wilson
  28
  29 import os
  30 import itertools
  31
  32 import testlib.configuration as configuration
  33 import testlib.fixture as fixture_mod
  34 import testlib.handlers as handlers
  35 import testlib.loader as loader_mod
  36 import testlib.log as log
  37 import testlib.query as query
  38 import testlib.result as result
  39 import testlib.runner as runner
  40 import testlib.terminal as terminal
  41 import testlib.uid as uid
  42
  43 def entry_message():
  44     log.test_log.message("Running the new gem5 testing script.")
  45     log.test_log.message("For more information see TESTING.md.")
  46     log.test_log.message("To see details as the testing scripts are"
  47                          " running, use the option"
  48                          " -v, -vv, or -vvv")
  49
  50 class RunLogHandler():
  51     def __init__(self):
  52         term_handler = handlers.TerminalHandler(
  53             verbosity=configuration.config.verbose+log.LogLevel.Info
  54         )
  55         summary_handler = handlers.SummaryHandler()
  56         self.mp_handler = handlers.MultiprocessingHandlerWrapper(
  57                 summary_handler, term_handler)
  58         self.mp_handler.async_process()
  59         log.test_log.log_obj.add_handler(self.mp_handler)
  60         entry_message()
  61
  62     def schedule_finalized(self, test_schedule):
  63         # Create the result handler object.
  64         self.result_handler = handlers.ResultHandler(
  65                 test_schedule, configuration.config.result_path)
  66         self.mp_handler.add_handler(self.result_handler)
  67
  68     def finish_testing(self):
  69         self.result_handler.close()
  70
  71     def __enter__(self):
  72         return self
  73
  74     def __exit__(self, *args):
  75         self.close()
  76         return False
  77
  78     def close(self):
  79         self.mp_handler.close()
  80
  81     def unsuccessful(self):
  82         '''
  83         Performs an or reduce on all of the results.
  84         Returns true if at least one test is unsuccessful, false when all tests
  85         pass
  86         '''
  87         return self.result_handler.unsuccessful()
  88
  89 def get_config_tags():
  90     return getattr(configuration.config,
  91             configuration.StorePositionalTagsAction.position_kword)
  92
  93 def filter_with_config_tags(loaded_library):
  94     tags = get_config_tags()
  95     final_tags = []
  96     regex_fmt = '^%s$'
  97     cfg = configuration.config
  98
  99     def _append_inc_tag_filter(name):
 100         if hasattr(cfg, name):
 101             tag_opts = getattr(cfg, name)
 102             for tag in tag_opts:
 103                 final_tags.append(configuration.TagRegex(True, regex_fmt % tag))
 104
 105     def _append_rem_tag_filter(name):
 106         if hasattr(cfg, name):
 107             tag_opts = getattr(cfg, name)
 108             for tag in cfg.constants.supported_tags[name]:
 109                 if tag not in tag_opts:
 110                     final_tags.append(configuration.TagRegex(False, regex_fmt % tag))
 111
 112     # Append additional tags for the isa, length, and variant options.
 113     # They apply last (they take priority)
 114     special_tags = (
 115         cfg.constants.isa_tag_type,
 116         cfg.constants.length_tag_type,
 117         cfg.constants.host_isa_tag_type,
 118         cfg.constants.variant_tag_type
 119     )
 120
 121     for tagname in special_tags:
 122         _append_inc_tag_filter(tagname)
 123     for tagname in special_tags:
 124         _append_rem_tag_filter(tagname)
 125
 126     if tags is None:
 127         tags = tuple()
 128
 129     filters = list(itertools.chain(tags, final_tags))
 130     string = 'Filtering suites with tags as follows:\n'
 131     filter_string = '\t\n'.join((str(f) for f in filters))
 132     log.test_log.trace(string + filter_string)
 133
 134     return filter_with_tags(loaded_library, filters)
 135
 136
 137 def filter_with_tags(loaded_library, filters):
 138     '''
 139     Filter logic supports two filter types:
 140     --include-tags <regex>
 141     --exclude-tags <regex>
 142
 143     The logic maintains a `set` of test suites.
 144
 145     If the regex provided with the `--include-tags` flag matches a tag of a
 146     suite, that suite will added to the set.
 147
 148     If the regex provided with the `--exclude-tags` flag matches a tag of a
 149     suite, that suite will removed to the set.
 150
 151     Suites can be added and removed multiple times.
 152
 153     First Flag Special Case Logic:
 154     If include is the first flag, start with an empty set of suites.
 155     If exclude is the first flag, start with the set of all collected suites.
 156
 157
 158     Let's trace out the set as we go through the flags to clarify::
 159
 160         # Say our collection of suites looks like this: set(suite_ARM64,
 161         # suite_X86, suite_Other).
 162         #
 163         # Additionally, we've passed the flags in the following order:
 164         #  --include-tags "ARM64"  --exclude-tags ".*" --include-tags "X86"
 165
 166         # Process --include-tags "ARM64"
 167         set(suite_ARM64)    # Suite begins empty, but adds the ARM64 suite
 168         # Process --exclude-tags ".*"
 169         set()               # Removed all suites which have tags
 170         # Process --include-tags "X86"
 171         set(suite_X86)
 172     '''
 173     if not filters:
 174         return
 175
 176     query_runner = query.QueryRunner(loaded_library)
 177     tags = query_runner.tags()
 178
 179     if not filters[0].include:
 180         suites = set(query_runner.suites())
 181     else:
 182         suites = set()
 183
 184     def exclude(excludes):
 185         return suites - excludes
 186     def include(includes):
 187         return suites | includes
 188
 189     for tag_regex in filters:
 190         matched_tags = (tag for tag in tags if tag_regex.regex.search(tag))
 191         for tag in matched_tags:
 192             matched_suites = set(query_runner.suites_with_tag(tag))
 193             suites = include(matched_suites) if tag_regex.include \
 194                     else exclude(matched_suites)
 195
 196     # Set the library's suites to only those which where accepted by our filter
 197     loaded_library.suites = [suite for suite in loaded_library.suites
 198             if suite in suites]
 199
 200 # TODO Add results command for listing previous results.
 201
 202 def load_tests():
 203     '''
 204     Create a TestLoader and load tests for the directory given by the config.
 205     '''
 206     testloader = loader_mod.Loader()
 207     log.test_log.message(terminal.separator())
 208     log.test_log.message('Loading Tests', bold=True)
 209     testloader.load_root(configuration.config.directory)
 210     return testloader
 211
 212 def do_list():
 213     term_handler = handlers.TerminalHandler(
 214         verbosity=configuration.config.verbose+log.LogLevel.Info,
 215         machine_only=configuration.config.quiet
 216     )
 217     log.test_log.log_obj.add_handler(term_handler)
 218
 219     entry_message()
 220
 221     test_schedule = load_tests().schedule
 222     filter_with_config_tags(test_schedule)
 223
 224     qrunner = query.QueryRunner(test_schedule)
 225
 226     if configuration.config.suites:
 227         qrunner.list_suites()
 228     elif configuration.config.tests:
 229         qrunner.list_tests()
 230     elif configuration.config.all_tags:
 231         qrunner.list_tags()
 232     else:
 233         qrunner.list_suites()
 234         qrunner.list_tests()
 235         qrunner.list_tags()
 236
 237     return 0
 238
 239 def run_schedule(test_schedule, log_handler):
 240     '''
 241     Test Phases
 242     -----------
 243     * Test Collection
 244     * Fixture Parameterization
 245     * Global Fixture Setup
 246     * Iteratevely run suites:
 247        * Suite Fixture Setup
 248        * Iteratively run tests:
 249           * Test Fixture Setup
 250           * Run Test
 251           * Test Fixture Teardown
 252        * Suite Fixture Teardown
 253     * Global Fixture Teardown
 254     '''
 255
 256     log_handler.schedule_finalized(test_schedule)
 257
 258     log.test_log.message(terminal.separator())
 259     log.test_log.message('Running Tests from {} suites'
 260             .format(len(test_schedule.suites)), bold=True)
 261     log.test_log.message("Results will be stored in {}".format(
 262                 configuration.config.result_path))
 263     log.test_log.message(terminal.separator())
 264
 265     # Build global fixtures and exectute scheduled test suites.
 266     if configuration.config.test_threads > 1:
 267         library_runner = runner.LibraryParallelRunner(test_schedule)
 268         library_runner.set_threads(configuration.config.test_threads)
 269     else:
 270         library_runner = runner.LibraryRunner(test_schedule)
 271     library_runner.run()
 272
 273     failed = log_handler.unsuccessful()
 274
 275     log_handler.finish_testing()
 276
 277     return 1 if failed else 0
 278
 279 def do_run():
 280     # Initialize early parts of the log.
 281     with RunLogHandler() as log_handler:
 282         if configuration.config.uid:
 283             uid_ = uid.UID.from_uid(configuration.config.uid)
 284             if isinstance(uid_, uid.TestUID):
 285                 log.test_log.error('Unable to run a standalone test.\n'
 286                         'Gem5 expects test suites to be the smallest unit '
 287                         ' of test.\n\n'
 288                         'Pass a SuiteUID instead.')
 289                 return
 290             test_schedule = loader_mod.Loader().load_schedule_for_suites(uid_)
 291             if get_config_tags():
 292                 log.test_log.warn(
 293                     "The '--uid' flag was supplied,"
 294                     " '--include-tags' and '--exclude-tags' will be ignored."
 295                 )
 296         else:
 297             test_schedule = load_tests().schedule
 298             # Filter tests based on tags
 299             filter_with_config_tags(test_schedule)
 300         # Execute the tests
 301         return run_schedule(test_schedule, log_handler)
 302
 303 def do_rerun():
 304     # Init early parts of log
 305     with RunLogHandler() as log_handler:
 306         # Load previous results
 307         results = result.InternalSavedResults.load(
 308                 os.path.join(configuration.config.result_path,
 309                 configuration.constants.pickle_filename))
 310
 311         rerun_suites = (suite.uid for suite in results if suite.unsuccessful)
 312
 313         # Use loader to load suites
 314         loader = loader_mod.Loader()
 315         test_schedule = loader.load_schedule_for_suites(*rerun_suites)
 316
 317         # Execute the tests
 318         return run_schedule(test_schedule, log_handler)
 319
 320 def main():
 321     '''
 322     Main entrypoint for the testlib test library.
 323     Returns 0 on success and 1 otherwise so it can be used as a return code
 324     for scripts.
 325     '''
 326     configuration.initialize_config()
 327
 328     # 'do' the given command.
 329     result = globals()['do_'+configuration.config.command]()
 330     log.test_log.close()
 331
 332     return result