ext/testlib/runner.py

   1 # Copyright (c) 2020 ARM Limited
   2 # All rights reserved.
   3 #
   4 # The license below extends only to copyright in the software and shall
   5 # not be construed as granting a license to any other intellectual
   6 # property including but not limited to intellectual property relating
   7 # to a hardware implementation of the functionality of the software
   8 # licensed hereunder.  You may use the software subject to the license
   9 # terms below provided that you ensure that this notice is replicated
  10 # unmodified and in its entirety in all distributions of the software,
  11 # modified or unmodified, in source code or in binary form.
  12 #
  13 # Copyright (c) 2017 Mark D. Hill and David A. Wood
  14 # All rights reserved.
  15 #
  16 # Redistribution and use in source and binary forms, with or without
  17 # modification, are permitted provided that the following conditions are
  18 # met: redistributions of source code must retain the above copyright
  19 # notice, this list of conditions and the following disclaimer;
  20 # redistributions in binary form must reproduce the above copyright
  21 # notice, this list of conditions and the following disclaimer in the
  22 # documentation and/or other materials provided with the distribution;
  23 # neither the name of the copyright holders nor the names of its
  24 # contributors may be used to endorse or promote products derived from
  25 # this software without specific prior written permission.
  26 #
  27 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  28 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  29 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  30 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  31 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  32 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  33 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  34 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  35 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  36 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  37 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  38 #
  39 # Authors: Sean Wilson
  40
  41 import multiprocessing.dummy
  42 import traceback
  43
  44 import testlib.helper as helper
  45 import testlib.log as log
  46
  47 from testlib.state import Status, Result
  48 from testlib.fixture import SkipException
  49
  50 def compute_aggregate_result(iterable):
  51     '''
  52     Status of the test suite by default is:
  53     * Passed if all contained tests passed
  54     * Errored if any contained tests errored
  55     * Failed if no tests errored, but one or more failed.
  56     * Skipped if all contained tests were skipped
  57     '''
  58     failed = []
  59     skipped = []
  60     for testitem in iterable:
  61         result = testitem.result
  62
  63         if result.value == Result.Errored:
  64             return Result(result.value, result.reason)
  65         elif result.value == Result.Failed:
  66             failed.append(result.reason)
  67         elif result.value == result.Skipped:
  68             skipped.append(result.reason)
  69     if failed:
  70         return Result(Result.Failed, failed)
  71     elif skipped:
  72         return Result(Result.Skipped, skipped)
  73     else:
  74         return Result(Result.Passed)
  75
  76 class TestParameters(object):
  77     def __init__(self, test, suite):
  78         self.test = test
  79         self.suite = suite
  80         self.log = log.test_log
  81         self.log.test = test
  82
  83     @helper.cacheresult
  84     def _fixtures(self):
  85         fixtures = {fixture.name:fixture for fixture in self.suite.fixtures}
  86         for fixture in self.test.fixtures:
  87             fixtures[fixture.name] = fixture
  88         return fixtures
  89
  90     @property
  91     def fixtures(self):
  92         return self._fixtures()
  93
  94
  95 class RunnerPattern:
  96     def __init__(self, loaded_testable):
  97         self.testable = loaded_testable
  98         self.builder = FixtureBuilder(self.testable.fixtures)
  99
 100     def handle_error(self, trace):
 101         self.testable.result = Result(Result.Errored, trace)
 102         self.avoid_children(trace)
 103
 104     def handle_skip(self, trace):
 105         self.testable.result = Result(Result.Skipped, trace)
 106         self.avoid_children(trace)
 107
 108     def avoid_children(self, reason):
 109         for testable in self.testable:
 110             testable.result = Result(self.testable.result.value, reason)
 111             testable.status = Status.Avoided
 112
 113     def test(self):
 114         pass
 115
 116     def run(self):
 117         avoided = False
 118         try:
 119             self.testable.status = Status.Building
 120             self.builder.setup(self.testable)
 121         except SkipException:
 122             self.handle_skip(traceback.format_exc())
 123             avoided = True
 124         except BrokenFixtureException:
 125             self.handle_error(traceback.format_exc())
 126             avoided = True
 127         else:
 128             self.testable.status = Status.Running
 129             self.test()
 130         finally:
 131             self.testable.status = Status.TearingDown
 132             self.builder.teardown(self.testable)
 133
 134         if avoided:
 135             self.testable.status = Status.Avoided
 136         else:
 137             self.testable.status = Status.Complete
 138
 139 class TestRunner(RunnerPattern):
 140     def test(self):
 141         test_params = TestParameters(
 142             self.testable,
 143             self.testable.parent_suite)
 144
 145         try:
 146             # Running the test
 147             test_params.test.test(test_params)
 148         except Exception:
 149             self.testable.result = Result(Result.Failed,
 150                     traceback.format_exc())
 151         else:
 152             self.testable.result = Result(Result.Passed)
 153
 154
 155 class SuiteRunner(RunnerPattern):
 156     def test(self):
 157         for test in self.testable:
 158             test.runner(test).run()
 159         self.testable.result = compute_aggregate_result(
 160                 iter(self.testable))
 161
 162
 163 class LibraryRunner(SuiteRunner):
 164     pass
 165
 166
 167 class LibraryParallelRunner(RunnerPattern):
 168     def set_threads(self, threads):
 169         self.threads = threads
 170
 171     def test(self):
 172         pool = multiprocessing.dummy.Pool(self.threads)
 173         pool.map(lambda suite : suite.runner(suite).run(), self.testable)
 174         self.testable.result = compute_aggregate_result(
 175                 iter(self.testable))
 176
 177
 178 class BrokenFixtureException(Exception):
 179     def __init__(self, fixture, testitem, trace):
 180         self.trace = trace
 181
 182         self.msg = ('%s\n'
 183                    'Exception raised building "%s" raised SkipException'
 184                    ' for "%s".' %
 185                    (trace, fixture.name, testitem.name)
 186         )
 187         super(BrokenFixtureException, self).__init__(self.msg)
 188
 189 class FixtureBuilder(object):
 190     def __init__(self, fixtures):
 191         self.fixtures = fixtures
 192         self.built_fixtures = []
 193
 194     def setup(self, testitem):
 195         for fixture in self.fixtures:
 196             # Mark as built before, so if the build fails
 197             # we still try to tear it down.
 198             self.built_fixtures.append(fixture)
 199             try:
 200                 fixture.setup(testitem)
 201             except SkipException:
 202                 raise
 203             except Exception as e:
 204                 exc = traceback.format_exc()
 205                 msg = 'Exception raised while setting up fixture for %s' %\
 206                         testitem.uid
 207                 log.test_log.warn('%s\n%s' % (exc, msg))
 208
 209                 raise BrokenFixtureException(fixture, testitem,
 210                         traceback.format_exc())
 211
 212     def teardown(self, testitem):
 213         for fixture in self.built_fixtures:
 214             try:
 215                 fixture.teardown(testitem)
 216             except Exception:
 217                 # Log exception but keep cleaning up.
 218                 exc = traceback.format_exc()
 219                 msg = 'Exception raised while tearing down fixture for %s' %\
 220                         testitem.uid
 221                 log.test_log.warn('%s\n%s' % (exc, msg))