tests/gem5/verifier.py

   1 # Copyright (c) 2017 Mark D. Hill and David A. Wood
   2 # All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met: redistributions of source code must retain the above copyright
   7 # notice, this list of conditions and the following disclaimer;
   8 # redistributions in binary form must reproduce the above copyright
   9 # notice, this list of conditions and the following disclaimer in the
  10 # documentation and/or other materials provided with the distribution;
  11 # neither the name of the copyright holders nor the names of its
  12 # contributors may be used to endorse or promote products derived from
  13 # this software without specific prior written permission.
  14 #
  15 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  16 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  17 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  18 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  19 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  20 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  21 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  22 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  23 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26
  27 '''
  28 Built in test cases that verify particular details about a gem5 run.
  29 '''
  30 import re
  31
  32 from testlib import test
  33 from testlib.config import constants
  34 from testlib.helper import joinpath, diff_out_file
  35
  36 class Verifier(object):
  37     def __init__(self, fixtures=tuple()):
  38         self.fixtures = fixtures
  39
  40     def _test(self, *args, **kwargs):
  41         # Use a callback wrapper to make stack
  42         # traces easier to understand.
  43         self.test(*args, **kwargs)
  44
  45     def instantiate_test(self, name_pfx):
  46         name = '-'.join([name_pfx, self.__class__.__name__])
  47         return test.TestFunction(self._test,
  48                 name=name, fixtures=self.fixtures)
  49
  50     def failed(self, fixtures):
  51         '''
  52         Called if this verifier fails to cleanup (or not) as needed.
  53         '''
  54         try:
  55             fixtures[constants.tempdir_fixture_name].skip_cleanup()
  56         except KeyError:
  57             pass # No need to do anything if the tempdir fixture doesn't exist
  58
  59
  60 class MatchGoldStandard(Verifier):
  61     '''
  62     Compares a standard output to the test output and passes if they match,
  63     fails if they do not.
  64     '''
  65     def __init__(self, standard_filename, ignore_regex=None,
  66                  test_filename='simout'):
  67         '''
  68         :param standard_filename: The path of the standard file to compare
  69         output to.
  70
  71         :param ignore_regex: A string, compiled regex, or iterable containing
  72         either which will be ignored in 'standard' and test output files when
  73         diffing.
  74         '''
  75         super(MatchGoldStandard, self).__init__()
  76         self.standard_filename = standard_filename
  77         self.test_filename = test_filename
  78
  79         self.ignore_regex = _iterable_regex(ignore_regex)
  80
  81     def test(self, params):
  82         # We need a tempdir fixture from our parent verifier suite.
  83         fixtures = params.fixtures
  84         # Get the file from the tempdir of the test.
  85         tempdir = fixtures[constants.tempdir_fixture_name].path
  86         self.test_filename = joinpath(tempdir, self.test_filename)
  87
  88         diff = diff_out_file(self.standard_filename,
  89                             self.test_filename,
  90                             ignore_regexes=self.ignore_regex,
  91                             logger=params.log)
  92         if diff is not None:
  93             self.failed(fixtures)
  94             test.fail('Stdout did not match:\n%s\nSee %s for full results'
  95                       % (diff, tempdir))
  96
  97     def _generic_instance_warning(self, kwargs):
  98         '''
  99         Method for helper classes to tell users to use this more generic class
 100         if they are going to manually override the test_filename param.
 101         '''
 102         if 'test_filename' in kwargs:
 103             raise ValueError('If you are setting test_filename use the more'
 104                              ' generic %s'
 105                              ' instead' % MatchGoldStandard.__name__)
 106
 107 class DerivedGoldStandard(MatchGoldStandard):
 108     __ignore_regex_sentinel = object()
 109     _file = None
 110     _default_ignore_regex = []
 111
 112     def __init__(self, standard_filename,
 113                  ignore_regex=__ignore_regex_sentinel, **kwargs):
 114
 115         if ignore_regex == self.__ignore_regex_sentinel:
 116             ignore_regex = self._default_ignore_regex
 117
 118         self._generic_instance_warning(kwargs)
 119
 120         super(DerivedGoldStandard, self).__init__(
 121             standard_filename,
 122             test_filename=self._file,
 123             ignore_regex=ignore_regex,
 124             **kwargs)
 125
 126 class MatchStdout(DerivedGoldStandard):
 127     _file = constants.gem5_simulation_stdout
 128     _default_ignore_regex = [
 129             re.compile('^Redirecting (stdout|stderr) to'),
 130             re.compile('^gem5 compiled '),
 131             re.compile('^gem5 started '),
 132             re.compile('^gem5 executing on '),
 133             re.compile('^command line:'),
 134             re.compile("^Couldn't import dot_parser,"),
 135             re.compile("^info: kernel located at:"),
 136             re.compile("^info: Standard input is not a terminal"),
 137             re.compile("^Couldn't unlink "),
 138             re.compile("^Using GPU kernel code file\(s\) "),
 139         ]
 140
 141 class MatchStdoutNoPerf(MatchStdout):
 142     _file = constants.gem5_simulation_stdout
 143     _default_ignore_regex = MatchStdout._default_ignore_regex + [
 144             re.compile('^Exiting @ tick'),
 145         ]
 146
 147 class MatchStderr(DerivedGoldStandard):
 148     _file = constants.gem5_simulation_stderr
 149     _default_ignore_regex = []
 150
 151 class MatchStats(DerivedGoldStandard):
 152     # TODO: Likely will want to change this verifier since we have the weird
 153     # perl script right now. A simple diff probably isn't going to work.
 154     _file = constants.gem5_simulation_stats
 155     _default_ignore_regex = []
 156
 157 class MatchConfigINI(DerivedGoldStandard):
 158     _file = constants.gem5_simulation_config_ini
 159     _default_ignore_regex = (
 160             re.compile("^(executable|readfile|kernel|image_file)="),
 161             re.compile("^(cwd|input|codefile)="),
 162             )
 163
 164 class MatchConfigJSON(DerivedGoldStandard):
 165     _file = constants.gem5_simulation_config_json
 166     _default_ignore_regex = (
 167             re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''),
 168             re.compile(r'''^\s*"(cwd|input|codefile)":'''),
 169             )
 170
 171 class MatchRegex(Verifier):
 172     def __init__(self, regex, match_stderr=True, match_stdout=True):
 173         super(MatchRegex, self).__init__()
 174         self.regex = _iterable_regex(regex)
 175         self.match_stderr = match_stderr
 176         self.match_stdout = match_stdout
 177
 178     def test(self, params):
 179         fixtures = params.fixtures
 180         # Get the file from the tempdir of the test.
 181         tempdir = fixtures[constants.tempdir_fixture_name].path
 182
 183         def parse_file(fname):
 184             with open(fname, 'r') as file_:
 185                 for line in file_:
 186                     for regex in self.regex:
 187                         if re.match(regex, line):
 188                             return True
 189         if self.match_stdout:
 190             if parse_file(joinpath(tempdir,
 191                                    constants.gem5_simulation_stdout)):
 192                 return # Success
 193         if self.match_stderr:
 194             if parse_file(joinpath(tempdir,
 195                                    constants.gem5_simulation_stderr)):
 196                 return # Success
 197         self.failed(fixtures)
 198         test.fail('Could not match regex.')
 199
 200 _re_type = type(re.compile(''))
 201 def _iterable_regex(regex):
 202     if isinstance(regex, _re_type) or isinstance(regex, str):
 203         regex = (regex,)
 204     return regex