src/systemc/tests/verify.py

   1 #!/usr/bin/env python2
   2 #
   3 # Copyright 2018 Google, Inc.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met: redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer;
   9 # redistributions in binary form must reproduce the above copyright
  10 # notice, this list of conditions and the following disclaimer in the
  11 # documentation and/or other materials provided with the distribution;
  12 # neither the name of the copyright holders nor the names of its
  13 # contributors may be used to endorse or promote products derived from
  14 # this software without specific prior written permission.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27 #
  28 # Authors: Gabe Black
  29
  30 from __future__ import print_function
  31
  32 import argparse
  33 import functools
  34 import inspect
  35 import itertools
  36 import json
  37 import logging
  38 import os
  39 import subprocess
  40 import sys
  41
  42 script_path = os.path.abspath(inspect.getfile(inspect.currentframe()))
  43 script_dir = os.path.dirname(script_path)
  44 config_path = os.path.join(script_dir, 'config.py')
  45
  46 systemc_rel_path = 'systemc'
  47 tests_rel_path = os.path.join(systemc_rel_path, 'tests')
  48 json_rel_path = os.path.join(tests_rel_path, 'tests.json')
  49
  50
  51
  52 logging.basicConfig(level=logging.INFO)
  53
  54 def scons(*args):
  55     args = ['scons'] + list(args)
  56     subprocess.check_call(args)
  57
  58
  59
  60 class Test(object):
  61     def __init__(self, target, suffix, build_dir, props):
  62         self.target = target
  63         self.suffix = suffix
  64         self.build_dir = build_dir
  65
  66         for key, val in props.iteritems():
  67             setattr(self, key, val)
  68
  69     def dir(self):
  70         return os.path.join(self.build_dir, tests_rel_path, self.path)
  71
  72     def src_dir(self):
  73         return os.path.join(script_dir, self.path)
  74
  75     def golden_dir(self):
  76         return os.path.join(self.src_dir(), 'golden')
  77
  78     def bin(self):
  79         return '.'.join([self.name, self.suffix])
  80
  81     def full_path(self):
  82         return os.path.join(self.dir(), self.bin())
  83
  84     def m5out_dir(self):
  85         return os.path.join(self.dir(), 'm5out.' + self.suffix)
  86
  87
  88
  89 test_phase_classes = {}
  90
  91 class TestPhaseMeta(type):
  92     def __init__(cls, name, bases, d):
  93         if not d.pop('abstract', False):
  94             test_phase_classes[d['name']] = cls
  95
  96         super(TestPhaseMeta, cls).__init__(name, bases, d)
  97
  98 class TestPhaseBase(object):
  99     __metaclass__ = TestPhaseMeta
 100     abstract = True
 101
 102     def __init__(self, main_args, *args):
 103         self.main_args = main_args
 104         self.args = args
 105
 106     def __lt__(self, other):
 107         return self.number < other.number
 108
 109 class CompilePhase(TestPhaseBase):
 110     name = 'compile'
 111     number = 1
 112
 113     def run(self, tests):
 114         targets = list([test.full_path() for test in tests])
 115         scons_args = list(self.args) + targets
 116         scons(*scons_args)
 117
 118 class RunPhase(TestPhaseBase):
 119     name = 'execute'
 120     number = 2
 121
 122     def run(self, tests):
 123         for test in tests:
 124             if test.compile_only:
 125                 continue
 126             args = [
 127                 test.full_path(),
 128                 '-red', test.m5out_dir(),
 129                 '--listener-mode=off',
 130                 config_path
 131             ]
 132             subprocess.check_call(args)
 133
 134 class VerifyPhase(TestPhaseBase):
 135     name = 'verify'
 136     number = 3
 137
 138     def run(self, tests):
 139         for test in tests:
 140             if test.compile_only:
 141                 continue
 142             logging.info("Would verify %s", test.m5out_dir())
 143
 144
 145
 146 parser = argparse.ArgumentParser(description='SystemC test utility')
 147
 148 parser.add_argument('build_dir', metavar='BUILD_DIR',
 149                     help='The build directory (ie. build/ARM).')
 150
 151 parser.add_argument('--update-json', action='store_true',
 152                     help='Update the json manifest of tests.')
 153
 154 parser.add_argument('--flavor', choices=['debug', 'opt', 'fast'],
 155                     default='opt',
 156                     help='Flavor of binary to test.')
 157
 158 parser.add_argument('--list', action='store_true',
 159                     help='List the available tests')
 160
 161 filter_opts = parser.add_mutually_exclusive_group()
 162 filter_opts.add_argument('--filter', default='True',
 163                          help='Python expression which filters tests based '
 164                          'on their properties')
 165 filter_opts.add_argument('--filter-file', default=None,
 166                          type=argparse.FileType('r'),
 167                          help='Same as --filter, but read from a file')
 168
 169 def collect_phases(args):
 170     phase_groups = [list(g) for k, g in
 171                     itertools.groupby(args, lambda x: x != '--phase') if k]
 172     main_args = parser.parse_args(phase_groups[0][1:])
 173     phases = []
 174     names = []
 175     for group in phase_groups[1:]:
 176         name = group[0]
 177         if name in names:
 178             raise RuntimeException('Phase %s specified more than once' % name)
 179         phase = test_phase_classes[name]
 180         phases.append(phase(main_args, *group[1:]))
 181     phases.sort()
 182     return main_args, phases
 183
 184 main_args, phases = collect_phases(sys.argv)
 185
 186 if len(phases) == 0:
 187     phases = [
 188         CompilePhase(main_args),
 189         RunPhase(main_args),
 190         VerifyPhase(main_args)
 191     ]
 192
 193
 194
 195 json_path = os.path.join(main_args.build_dir, json_rel_path)
 196
 197 if main_args.update_json:
 198     scons(os.path.join(json_path))
 199
 200 with open(json_path) as f:
 201     test_data = json.load(f)
 202
 203     if main_args.filter_file:
 204         f = main_args.filter_file
 205         filt = compile(f.read(), f.name, 'eval')
 206     else:
 207         filt = compile(main_args.filter, '<string>', 'eval')
 208
 209     filtered_tests = {
 210         target: props for (target, props) in
 211                     test_data.iteritems() if eval(filt, dict(props))
 212     }
 213
 214     if main_args.list:
 215         for target, props in sorted(filtered_tests.iteritems()):
 216             print('%s.%s' % (target, main_args.flavor))
 217             for key, val in props.iteritems():
 218                 print('    %s: %s' % (key, val))
 219         print('Total tests: %d' % len(filtered_tests))
 220     else:
 221         tests_to_run = list([
 222             Test(target, main_args.flavor, main_args.build_dir, props) for
 223                 target, props in sorted(filtered_tests.iteritems())
 224         ])
 225
 226         for phase in phases:
 227             phase.run(tests_to_run)