tests/gem5/suite.py

   1 # Copyright (c) 2017 Mark D. Hill and David A. Wood
   2 # All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met: redistributions of source code must retain the above copyright
   7 # notice, this list of conditions and the following disclaimer;
   8 # redistributions in binary form must reproduce the above copyright
   9 # notice, this list of conditions and the following disclaimer in the
  10 # documentation and/or other materials provided with the distribution;
  11 # neither the name of the copyright holders nor the names of its
  12 # contributors may be used to endorse or promote products derived from
  13 # this software without specific prior written permission.
  14 #
  15 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  16 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  17 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  18 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  19 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  20 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  21 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  22 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  23 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 #
  27 # Authors: Sean Wilson
  28
  29 import os
  30 import copy
  31 import subprocess
  32
  33 from testlib.test import TestFunction
  34 from testlib.suite import TestSuite
  35 from testlib.helper import log_call
  36 from testlib.config import constants, config
  37 from fixture import TempdirFixture, Gem5Fixture, VariableFixture
  38 import verifier
  39
  40 def gem5_verify_config(name,
  41                        config,
  42                        config_args,
  43                        verifiers,
  44                        gem5_args=tuple(),
  45                        fixtures=[],
  46                        valid_isas=constants.supported_isas,
  47                        valid_variants=constants.supported_variants,
  48                        length=constants.supported_lengths[0]):
  49     '''
  50     Helper class to generate common gem5 tests using verifiers.
  51
  52     The generated TestSuite will run gem5 with the provided config and
  53     config_args. After that it will run any provided verifiers to verify
  54     details about the gem5 run.
  55
  56     .. seealso::  For the verifiers see :mod:`testlib.gem5.verifier`
  57
  58     :param name: Name of the test.
  59     :param config: The config to give gem5.
  60     :param config_args: A list of arguments to pass to the given config.
  61
  62     :param verifiers: An iterable with Verifier instances which will be placed
  63         into a suite that will be ran after a gem5 run.
  64
  65     :param gem5_args: An iterable with arguments to give to gem5. (Arguments
  66         that would normally go before the config path.)
  67
  68     :param valid_isas: An iterable with the isas that this test can be ran
  69         for. If None given, will run for all supported_isas.
  70
  71     :param valid_variants: An iterable with the variant levels that
  72         this test can be ran for. (E.g. opt, debug)
  73     '''
  74     fixtures = list(fixtures)
  75     testsuites = []
  76     for opt in valid_variants:
  77         for isa in valid_isas:
  78
  79             # Create a tempdir fixture to be shared throughout the test.
  80             tempdir = TempdirFixture()
  81             gem5_returncode = VariableFixture(
  82                     name=constants.gem5_returncode_fixture_name)
  83
  84             # Common name of this generated testcase.
  85             _name = '{given_name}-{isa}-{opt}'.format(
  86                     given_name=name,
  87                     isa=isa,
  88                     opt=opt)
  89
  90             # Create the running of gem5 subtest.
  91             # NOTE: We specifically create this test before our verifiers so
  92             # this is listed first.
  93             tests = []
  94             gem5_execution = TestFunction(
  95                     _create_test_run_gem5(config, config_args, gem5_args),
  96                     name=_name)
  97             tests.append(gem5_execution)
  98
  99             # Create copies of the verifier subtests for this isa and
 100             # variant.
 101             for verifier in verifiers:
 102                 tests.append(verifier.instantiate_test(_name))
 103
 104             # Add the isa and variant to tags list.
 105             tags = [isa, opt, length]
 106
 107             # Create the gem5 target for the specific architecture and
 108             # variant.
 109             _fixtures = copy.copy(fixtures)
 110             _fixtures.append(Gem5Fixture(isa, opt))
 111             _fixtures.append(tempdir)
 112             _fixtures.append(gem5_returncode)
 113
 114             # Finally construct the self contained TestSuite out of our
 115             # tests.
 116             testsuites.append(TestSuite(
 117                 name=_name,
 118                 fixtures=_fixtures,
 119                 tags=tags,
 120                 tests=tests))
 121     return testsuites
 122
 123 def _create_test_run_gem5(config, config_args, gem5_args):
 124     def test_run_gem5(params):
 125         '''
 126         Simple \'test\' which runs gem5 and saves the result into a tempdir.
 127
 128         NOTE: Requires fixtures: tempdir, gem5
 129         '''
 130         fixtures = params.fixtures
 131
 132         if gem5_args is None:
 133             _gem5_args = tuple()
 134         elif isinstance(gem5_args, str):
 135             # If just a single str, place it in an iterable
 136             _gem5_args = (gem5_args,)
 137         else:
 138             _gem5_args = gem5_args
 139
 140         # FIXME/TODO: I don't like the idea of having to modify this test run
 141         # or always collect results even if not using a verifier. There should
 142         # be some configuration in here that only gathers certain results for
 143         # certain verifiers.
 144         #
 145         # I.E. Only the returncode verifier will use the gem5_returncode
 146         # fixture, but we always require it even if that verifier isn't being
 147         # ran.
 148         returncode = fixtures[constants.gem5_returncode_fixture_name]
 149         tempdir = fixtures[constants.tempdir_fixture_name].path
 150         gem5 = fixtures[constants.gem5_binary_fixture_name].path
 151         command = [
 152             gem5,
 153             '-d',  # Set redirect dir to tempdir.
 154             tempdir,
 155             '-re',# TODO: Change to const. Redirect stdout and stderr
 156         ]
 157         command.extend(_gem5_args)
 158         command.append(config)
 159         # Config_args should set up the program args.
 160         command.extend(config_args)
 161         returncode.value = log_call(params.log, command)
 162
 163     return test_run_gem5