tests/gem5/suite.py

   1 # Copyright (c) 2020 ARM Limited
   2 # All rights reserved
   3 #
   4 # The license below extends only to copyright in the software and shall
   5 # not be construed as granting a license to any other intellectual
   6 # property including but not limited to intellectual property relating
   7 # to a hardware implementation of the functionality of the software
   8 # licensed hereunder.  You may use the software subject to the license
   9 # terms below provided that you ensure that this notice is replicated
  10 # unmodified and in its entirety in all distributions of the software,
  11 # modified or unmodified, in source code or in binary form.
  12 #
  13 # Copyright (c) 2017 Mark D. Hill and David A. Wood
  14 # All rights reserved.
  15 #
  16 # Redistribution and use in source and binary forms, with or without
  17 # modification, are permitted provided that the following conditions are
  18 # met: redistributions of source code must retain the above copyright
  19 # notice, this list of conditions and the following disclaimer;
  20 # redistributions in binary form must reproduce the above copyright
  21 # notice, this list of conditions and the following disclaimer in the
  22 # documentation and/or other materials provided with the distribution;
  23 # neither the name of the copyright holders nor the names of its
  24 # contributors may be used to endorse or promote products derived from
  25 # this software without specific prior written permission.
  26 #
  27 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  28 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  29 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  30 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  31 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  32 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  33 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  34 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  35 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  36 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  37 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  38
  39 import os
  40 import copy
  41 import subprocess
  42 import sys
  43
  44 from testlib.test_util import TestFunction
  45 from testlib.suite import TestSuite
  46 from testlib.helper import log_call
  47 from testlib.configuration import constants, config
  48 from .fixture import TempdirFixture, Gem5Fixture, VariableFixture
  49
  50 from . import verifier
  51
  52 def gem5_verify_config(name,
  53                        config,
  54                        config_args,
  55                        verifiers,
  56                        gem5_args=tuple(),
  57                        fixtures=[],
  58                        valid_isas=constants.supported_isas,
  59                        valid_variants=constants.supported_variants,
  60                        length=constants.supported_lengths[0],
  61                        valid_hosts=constants.supported_hosts,
  62                        protocol=None):
  63     '''
  64     Helper class to generate common gem5 tests using verifiers.
  65
  66     The generated TestSuite will run gem5 with the provided config and
  67     config_args. After that it will run any provided verifiers to verify
  68     details about the gem5 run.
  69
  70     .. seealso::  For the verifiers see :mod:`testlib.gem5.verifier`
  71
  72     :param name: Name of the test.
  73     :param config: The config to give gem5.
  74     :param config_args: A list of arguments to pass to the given config.
  75
  76     :param verifiers: An iterable with Verifier instances which will be placed
  77         into a suite that will be ran after a gem5 run.
  78
  79     :param gem5_args: An iterable with arguments to give to gem5. (Arguments
  80         that would normally go before the config path.)
  81
  82     :param valid_isas: An iterable with the isas that this test can be ran
  83         for. If None given, will run for all supported_isas.
  84
  85     :param valid_variants: An iterable with the variant levels that
  86         this test can be ran for. (E.g. opt, debug)
  87     '''
  88     fixtures = list(fixtures)
  89     testsuites = []
  90
  91     # Obtain the set of tests to ignore. This is found in the
  92     # ".testignore" file.
  93     __location__ = os.path.realpath(
  94         os.path.join(os.getcwd(), os.path.dirname(__file__)))
  95     _test_ignore_file_loc = os.path.join(__location__,".testignore")
  96     ignore = set()
  97     if os.path.exists(_test_ignore_file_loc):
  98         ignore.update(open(_test_ignore_file_loc).read().splitlines())
  99
 100     for host in valid_hosts:
 101         for opt in valid_variants:
 102             for isa in valid_isas:
 103
 104                 # Create a tempdir fixture to be shared throughout the test.
 105                 tempdir = TempdirFixture()
 106                 gem5_returncode = VariableFixture(
 107                         name=constants.gem5_returncode_fixture_name)
 108
 109                 # Common name of this generated testcase.
 110                 _name = '{given_name}-{isa}-{host}-{opt}'.format(
 111                         given_name=name,
 112                         isa=isa,
 113                         host=host,
 114                         opt=opt)
 115                 if protocol:
 116                     _name += '-'+protocol
 117
 118                 # We check to see if this test suite is to be ignored. If so,
 119                 # we skip it.
 120                 if _name in ignore:
 121                     continue
 122
 123                 # Create the running of gem5 subtest.  NOTE: We specifically
 124                 # create this test before our verifiers so this is listed
 125                 # first.
 126                 tests = []
 127                 gem5_execution = TestFunction(
 128                         _create_test_run_gem5(config, config_args, gem5_args),
 129                         name=_name)
 130                 tests.append(gem5_execution)
 131
 132                 # Create copies of the verifier subtests for this isa and
 133                 # variant.
 134                 for verifier in verifiers:
 135                     tests.append(verifier.instantiate_test(_name))
 136
 137                 # Add the isa and variant to tags list.
 138                 tags = [isa, opt, length, host]
 139
 140                 # Create the gem5 target for the specific architecture and
 141                 # variant.
 142                 _fixtures = copy.copy(fixtures)
 143                 _fixtures.append(Gem5Fixture(isa, opt, protocol))
 144                 _fixtures.append(tempdir)
 145                 _fixtures.append(gem5_returncode)
 146
 147                 # Finally construct the self contained TestSuite out of our
 148                 # tests.
 149                 testsuites.append(TestSuite(
 150                     name=_name,
 151                     fixtures=_fixtures,
 152                     tags=tags,
 153                     tests=tests))
 154     return testsuites
 155
 156 def _create_test_run_gem5(config, config_args, gem5_args):
 157     def test_run_gem5(params):
 158         '''
 159         Simple \'test\' which runs gem5 and saves the result into a tempdir.
 160
 161         NOTE: Requires fixtures: tempdir, gem5
 162         '''
 163         fixtures = params.fixtures
 164
 165         if gem5_args is None:
 166             _gem5_args = tuple()
 167         elif isinstance(gem5_args, str):
 168             # If just a single str, place it in an iterable
 169             _gem5_args = (gem5_args,)
 170         else:
 171             _gem5_args = gem5_args
 172
 173         # FIXME/TODO: I don't like the idea of having to modify this test run
 174         # or always collect results even if not using a verifier. There should
 175         # be some configuration in here that only gathers certain results for
 176         # certain verifiers.
 177         #
 178         # I.E. Only the returncode verifier will use the gem5_returncode
 179         # fixture, but we always require it even if that verifier isn't being
 180         # ran.
 181         returncode = fixtures[constants.gem5_returncode_fixture_name]
 182         tempdir = fixtures[constants.tempdir_fixture_name].path
 183         gem5 = fixtures[constants.gem5_binary_fixture_name].path
 184         command = [
 185             gem5,
 186             '-d',  # Set redirect dir to tempdir.
 187             tempdir,
 188             '-re',# TODO: Change to const. Redirect stdout and stderr
 189         ]
 190         command.extend(_gem5_args)
 191         command.append(config)
 192         # Config_args should set up the program args.
 193         command.extend(config_args)
 194         returncode.value = log_call(params.log, command, stdout=sys.stdout,
 195                                                          stderr=sys.stderr)
 196
 197     return test_run_gem5