ext/testlib/runner.py

   1 # Copyright (c) 2020 ARM Limited
   2 # All rights reserved.
   3 #
   4 # The license below extends only to copyright in the software and shall
   5 # not be construed as granting a license to any other intellectual
   6 # property including but not limited to intellectual property relating
   7 # to a hardware implementation of the functionality of the software
   8 # licensed hereunder.  You may use the software subject to the license
   9 # terms below provided that you ensure that this notice is replicated
  10 # unmodified and in its entirety in all distributions of the software,
  11 # modified or unmodified, in source code or in binary form.
  12 #
  13 # Copyright (c) 2017 Mark D. Hill and David A. Wood
  14 # All rights reserved.
  15 #
  16 # Redistribution and use in source and binary forms, with or without
  17 # modification, are permitted provided that the following conditions are
  18 # met: redistributions of source code must retain the above copyright
  19 # notice, this list of conditions and the following disclaimer;
  20 # redistributions in binary form must reproduce the above copyright
  21 # notice, this list of conditions and the following disclaimer in the
  22 # documentation and/or other materials provided with the distribution;
  23 # neither the name of the copyright holders nor the names of its
  24 # contributors may be used to endorse or promote products derived from
  25 # this software without specific prior written permission.
  26 #
  27 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  28 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  29 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  30 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  31 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  32 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  33 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  34 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  35 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  36 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  37 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  38 #
  39 # Authors: Sean Wilson
  40
  41 import multiprocessing.dummy
  42 import traceback
  43
  44 import testlib.helper as helper
  45 import testlib.log as log
  46
  47 from testlib.state import Status, Result
  48 from testlib.fixture import SkipException
  49
  50 def compute_aggregate_result(iterable):
  51     '''
  52     Status of the test suite by default is:
  53     * Passed if all contained tests passed
  54     * Errored if any contained tests errored
  55     * Failed if no tests errored, but one or more failed.
  56     * Skipped if all contained tests were skipped
  57     '''
  58     failed = []
  59     skipped = []
  60     for testitem in iterable:
  61         result = testitem.result
  62
  63         if result.value == Result.Errored:
  64             return Result(result.value, result.reason)
  65         elif result.value == Result.Failed:
  66             failed.append(result.reason)
  67         elif result.value == result.Skipped:
  68             skipped.append(result.reason)
  69     if failed:
  70         return Result(Result.Failed, failed)
  71     elif skipped:
  72         return Result(Result.Skipped, skipped)
  73     else:
  74         return Result(Result.Passed)
  75
  76 class TestParameters(object):
  77     def __init__(self, test, suite):
  78         self.test = test
  79         self.suite = suite
  80         self.log = log.test_log
  81         self.log.test = test
  82
  83     @helper.cacheresult
  84     def _fixtures(self):
  85         fixtures = {fixture.name:fixture for fixture in self.suite.fixtures}
  86         for fixture in self.test.fixtures:
  87             fixtures[fixture.name] = fixture
  88         return fixtures
  89
  90     @property
  91     def fixtures(self):
  92         return self._fixtures()
  93
  94
  95 class RunnerPattern:
  96     def __init__(self, loaded_testable):
  97         self.testable = loaded_testable
  98         self.builder = FixtureBuilder(self.testable.fixtures)
  99
 100     def handle_error(self, trace):
 101         self.testable.result = Result(Result.Errored, trace)
 102         self.avoid_children(trace)
 103
 104     def handle_skip(self, trace):
 105         self.testable.result = Result(Result.Skipped, trace)
 106         self.avoid_children(trace)
 107
 108     def avoid_children(self, reason):
 109         for testable in self.testable:
 110             testable.result = Result(self.testable.result.value, reason)
 111             testable.status = Status.Avoided
 112
 113     def test(self):
 114         pass
 115
 116     def run(self):
 117         avoided = False
 118         try:
 119             self.testable.status = Status.Building
 120             self.builder.setup(self.testable)
 121         except SkipException:
 122             self.handle_skip(traceback.format_exc())
 123             avoided = True
 124         except BrokenFixtureException:
 125             self.handle_error(traceback.format_exc())
 126             avoided = True
 127         else:
 128             self.testable.status = Status.Running
 129             self.test()
 130         finally:
 131             self.builder.post_test_procedure(self.testable)
 132             self.testable.status = Status.TearingDown
 133             self.builder.teardown(self.testable)
 134
 135         if avoided:
 136             self.testable.status = Status.Avoided
 137         else:
 138             self.testable.status = Status.Complete
 139
 140 class TestRunner(RunnerPattern):
 141     def test(self):
 142         test_params = TestParameters(
 143             self.testable,
 144             self.testable.parent_suite)
 145
 146         try:
 147             # Running the test
 148             test_params.test.test(test_params)
 149         except Exception:
 150             self.testable.result = Result(Result.Failed,
 151                     traceback.format_exc())
 152         else:
 153             self.testable.result = Result(Result.Passed)
 154
 155
 156 class SuiteRunner(RunnerPattern):
 157     def test(self):
 158         for test in self.testable:
 159             test.runner(test).run()
 160         self.testable.result = compute_aggregate_result(
 161                 iter(self.testable))
 162
 163
 164 class LibraryRunner(SuiteRunner):
 165     pass
 166
 167
 168 class LibraryParallelRunner(RunnerPattern):
 169     def set_threads(self, threads):
 170         self.threads = threads
 171
 172     def test(self):
 173         pool = multiprocessing.dummy.Pool(self.threads)
 174         pool.map(lambda suite : suite.runner(suite).run(), self.testable)
 175         self.testable.result = compute_aggregate_result(
 176                 iter(self.testable))
 177
 178
 179 class BrokenFixtureException(Exception):
 180     def __init__(self, fixture, testitem, trace):
 181         self.trace = trace
 182
 183         self.msg = ('%s\n'
 184                    'Exception raised building "%s" raised SkipException'
 185                    ' for "%s".' %
 186                    (trace, fixture.name, testitem.name)
 187         )
 188         super(BrokenFixtureException, self).__init__(self.msg)
 189
 190 class FixtureBuilder(object):
 191     def __init__(self, fixtures):
 192         self.fixtures = fixtures
 193         self.built_fixtures = []
 194
 195     def setup(self, testitem):
 196         for fixture in self.fixtures:
 197             # Mark as built before, so if the build fails
 198             # we still try to tear it down.
 199             self.built_fixtures.append(fixture)
 200             try:
 201                 fixture.setup(testitem)
 202             except SkipException:
 203                 raise
 204             except Exception as e:
 205                 exc = traceback.format_exc()
 206                 msg = 'Exception raised while setting up fixture for %s' %\
 207                         testitem.uid
 208                 log.test_log.warn('%s\n%s' % (exc, msg))
 209
 210                 raise BrokenFixtureException(fixture, testitem,
 211                         traceback.format_exc())
 212
 213     def post_test_procedure(self, testitem):
 214         for fixture in self.built_fixtures:
 215             fixture.post_test_procedure(testitem)
 216
 217     def teardown(self, testitem):
 218         for fixture in self.built_fixtures:
 219             try:
 220                 fixture.teardown(testitem)
 221             except Exception:
 222                 # Log exception but keep cleaning up.
 223                 exc = traceback.format_exc()
 224                 msg = 'Exception raised while tearing down fixture for %s' %\
 225                         testitem.uid
 226                 log.test_log.warn('%s\n%s' % (exc, msg))