src/systemc/tests/verify.py

   1 #!/usr/bin/env python2
   2 #
   3 # Copyright 2018 Google, Inc.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met: redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer;
   9 # redistributions in binary form must reproduce the above copyright
  10 # notice, this list of conditions and the following disclaimer in the
  11 # documentation and/or other materials provided with the distribution;
  12 # neither the name of the copyright holders nor the names of its
  13 # contributors may be used to endorse or promote products derived from
  14 # this software without specific prior written permission.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27 #
  28 # Authors: Gabe Black
  29
  30 from __future__ import print_function
  31
  32 import argparse
  33 import functools
  34 import inspect
  35 import itertools
  36 import json
  37 import logging
  38 import multiprocessing.pool
  39 import os
  40 import subprocess
  41 import sys
  42
  43 script_path = os.path.abspath(inspect.getfile(inspect.currentframe()))
  44 script_dir = os.path.dirname(script_path)
  45 config_path = os.path.join(script_dir, 'config.py')
  46
  47 systemc_rel_path = 'systemc'
  48 tests_rel_path = os.path.join(systemc_rel_path, 'tests')
  49 json_rel_path = os.path.join(tests_rel_path, 'tests.json')
  50
  51
  52
  53 logging.basicConfig(level=logging.INFO)
  54
  55 def scons(*args):
  56     args = ['scons'] + list(args)
  57     subprocess.check_call(args)
  58
  59
  60
  61 class Test(object):
  62     def __init__(self, target, suffix, build_dir, props):
  63         self.target = target
  64         self.suffix = suffix
  65         self.build_dir = build_dir
  66
  67         for key, val in props.iteritems():
  68             setattr(self, key, val)
  69
  70     def dir(self):
  71         return os.path.join(self.build_dir, tests_rel_path, self.path)
  72
  73     def src_dir(self):
  74         return os.path.join(script_dir, self.path)
  75
  76     def golden_dir(self):
  77         return os.path.join(self.src_dir(), 'golden')
  78
  79     def bin(self):
  80         return '.'.join([self.name, self.suffix])
  81
  82     def full_path(self):
  83         return os.path.join(self.dir(), self.bin())
  84
  85     def m5out_dir(self):
  86         return os.path.join(self.dir(), 'm5out.' + self.suffix)
  87
  88
  89
  90 test_phase_classes = {}
  91
  92 class TestPhaseMeta(type):
  93     def __init__(cls, name, bases, d):
  94         if not d.pop('abstract', False):
  95             test_phase_classes[d['name']] = cls
  96
  97         super(TestPhaseMeta, cls).__init__(name, bases, d)
  98
  99 class TestPhaseBase(object):
 100     __metaclass__ = TestPhaseMeta
 101     abstract = True
 102
 103     def __init__(self, main_args, *args):
 104         self.main_args = main_args
 105         self.args = args
 106
 107     def __lt__(self, other):
 108         return self.number < other.number
 109
 110 class CompilePhase(TestPhaseBase):
 111     name = 'compile'
 112     number = 1
 113
 114     def run(self, tests):
 115         targets = list([test.full_path() for test in tests])
 116         scons_args = list(self.args) + targets
 117         scons(*scons_args)
 118
 119 class RunPhase(TestPhaseBase):
 120     name = 'execute'
 121     number = 2
 122
 123     def run(self, tests):
 124         parser = argparse.ArgumentParser()
 125         parser.add_argument('--timeout', type=int, metavar='SECONDS',
 126                             help='Time limit for each run in seconds.',
 127                             default=0)
 128         parser.add_argument('-j', type=int, default=1,
 129                 help='How many tests to run in parallel.')
 130         args = parser.parse_args(self.args)
 131
 132         timeout_cmd = [
 133             'timeout',
 134             '--kill-after', str(args.timeout * 2),
 135             str(args.timeout)
 136         ]
 137         def run_test(test):
 138             cmd = []
 139             if args.timeout:
 140                 cmd.extend(timeout_cmd)
 141             cmd.extend([
 142                 test.full_path(),
 143                 '-red', test.m5out_dir(),
 144                 '--listener-mode=off',
 145                 config_path
 146             ])
 147             subprocess.check_call(cmd)
 148
 149         runnable = filter(lambda t: not t.compile_only, tests)
 150         if args.j == 1:
 151             map(run_test, runnable)
 152         else:
 153             tp = multiprocessing.pool.ThreadPool(args.j)
 154             map(lambda t: tp.apply_async(run_test, (t,)), runnable)
 155             tp.close()
 156             tp.join()
 157
 158 class VerifyPhase(TestPhaseBase):
 159     name = 'verify'
 160     number = 3
 161
 162     def run(self, tests):
 163         for test in tests:
 164             if test.compile_only:
 165                 continue
 166             logging.info("Would verify %s", test.m5out_dir())
 167
 168
 169
 170 parser = argparse.ArgumentParser(description='SystemC test utility')
 171
 172 parser.add_argument('build_dir', metavar='BUILD_DIR',
 173                     help='The build directory (ie. build/ARM).')
 174
 175 parser.add_argument('--update-json', action='store_true',
 176                     help='Update the json manifest of tests.')
 177
 178 parser.add_argument('--flavor', choices=['debug', 'opt', 'fast'],
 179                     default='opt',
 180                     help='Flavor of binary to test.')
 181
 182 parser.add_argument('--list', action='store_true',
 183                     help='List the available tests')
 184
 185 filter_opts = parser.add_mutually_exclusive_group()
 186 filter_opts.add_argument('--filter', default='True',
 187                          help='Python expression which filters tests based '
 188                          'on their properties')
 189 filter_opts.add_argument('--filter-file', default=None,
 190                          type=argparse.FileType('r'),
 191                          help='Same as --filter, but read from a file')
 192
 193 def collect_phases(args):
 194     phase_groups = [list(g) for k, g in
 195                     itertools.groupby(args, lambda x: x != '--phase') if k]
 196     main_args = parser.parse_args(phase_groups[0][1:])
 197     phases = []
 198     names = []
 199     for group in phase_groups[1:]:
 200         name = group[0]
 201         if name in names:
 202             raise RuntimeException('Phase %s specified more than once' % name)
 203         phase = test_phase_classes[name]
 204         phases.append(phase(main_args, *group[1:]))
 205     phases.sort()
 206     return main_args, phases
 207
 208 main_args, phases = collect_phases(sys.argv)
 209
 210 if len(phases) == 0:
 211     phases = [
 212         CompilePhase(main_args),
 213         RunPhase(main_args),
 214         VerifyPhase(main_args)
 215     ]
 216
 217
 218
 219 json_path = os.path.join(main_args.build_dir, json_rel_path)
 220
 221 if main_args.update_json:
 222     scons(os.path.join(json_path))
 223
 224 with open(json_path) as f:
 225     test_data = json.load(f)
 226
 227     if main_args.filter_file:
 228         f = main_args.filter_file
 229         filt = compile(f.read(), f.name, 'eval')
 230     else:
 231         filt = compile(main_args.filter, '<string>', 'eval')
 232
 233     filtered_tests = {
 234         target: props for (target, props) in
 235                     test_data.iteritems() if eval(filt, dict(props))
 236     }
 237
 238     if main_args.list:
 239         for target, props in sorted(filtered_tests.iteritems()):
 240             print('%s.%s' % (target, main_args.flavor))
 241             for key, val in props.iteritems():
 242                 print('    %s: %s' % (key, val))
 243         print('Total tests: %d' % len(filtered_tests))
 244     else:
 245         tests_to_run = list([
 246             Test(target, main_args.flavor, main_args.build_dir, props) for
 247                 target, props in sorted(filtered_tests.iteritems())
 248         ])
 249
 250         for phase in phases:
 251             phase.run(tests_to_run)