src/systemc/tests/verify.py

   1 #!/usr/bin/env python2
   2 #
   3 # Copyright 2018 Google, Inc.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met: redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer;
   9 # redistributions in binary form must reproduce the above copyright
  10 # notice, this list of conditions and the following disclaimer in the
  11 # documentation and/or other materials provided with the distribution;
  12 # neither the name of the copyright holders nor the names of its
  13 # contributors may be used to endorse or promote products derived from
  14 # this software without specific prior written permission.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27 #
  28 # Authors: Gabe Black
  29
  30 from __future__ import print_function
  31
  32 import argparse
  33 import functools
  34 import inspect
  35 import itertools
  36 import json
  37 import logging
  38 import multiprocessing.pool
  39 import os
  40 import subprocess
  41 import sys
  42
  43 script_path = os.path.abspath(inspect.getfile(inspect.currentframe()))
  44 script_dir = os.path.dirname(script_path)
  45 config_path = os.path.join(script_dir, 'config.py')
  46
  47 systemc_rel_path = 'systemc'
  48 tests_rel_path = os.path.join(systemc_rel_path, 'tests')
  49 json_rel_path = os.path.join(tests_rel_path, 'tests.json')
  50
  51
  52
  53 logging.basicConfig(level=logging.INFO)
  54
  55 def scons(*args):
  56     args = ['scons'] + list(args)
  57     subprocess.check_call(args)
  58
  59
  60
  61 class Test(object):
  62     def __init__(self, target, suffix, build_dir, props):
  63         self.target = target
  64         self.suffix = suffix
  65         self.build_dir = build_dir
  66
  67         for key, val in props.iteritems():
  68             setattr(self, key, val)
  69
  70     def dir(self):
  71         return os.path.join(self.build_dir, tests_rel_path, self.path)
  72
  73     def src_dir(self):
  74         return os.path.join(script_dir, self.path)
  75
  76     def golden_dir(self):
  77         return os.path.join(self.src_dir(), 'golden')
  78
  79     def bin(self):
  80         return '.'.join([self.name, self.suffix])
  81
  82     def full_path(self):
  83         return os.path.join(self.dir(), self.bin())
  84
  85     def m5out_dir(self):
  86         return os.path.join(self.dir(), 'm5out.' + self.suffix)
  87
  88
  89
  90 test_phase_classes = {}
  91
  92 class TestPhaseMeta(type):
  93     def __init__(cls, name, bases, d):
  94         if not d.pop('abstract', False):
  95             test_phase_classes[d['name']] = cls
  96
  97         super(TestPhaseMeta, cls).__init__(name, bases, d)
  98
  99 class TestPhaseBase(object):
 100     __metaclass__ = TestPhaseMeta
 101     abstract = True
 102
 103     def __init__(self, main_args, *args):
 104         self.main_args = main_args
 105         self.args = args
 106
 107     def __lt__(self, other):
 108         return self.number < other.number
 109
 110 class CompilePhase(TestPhaseBase):
 111     name = 'compile'
 112     number = 1
 113
 114     def run(self, tests):
 115         targets = list([test.full_path() for test in tests])
 116         scons_args = list(self.args) + targets
 117         scons(*scons_args)
 118
 119 class RunPhase(TestPhaseBase):
 120     name = 'execute'
 121     number = 2
 122
 123     def run(self, tests):
 124         parser = argparse.ArgumentParser()
 125         parser.add_argument('--timeout', type=int, metavar='SECONDS',
 126                             help='Time limit for each run in seconds.',
 127                             default=0)
 128         parser.add_argument('-j', type=int, default=1,
 129                 help='How many tests to run in parallel.')
 130         args = parser.parse_args(self.args)
 131
 132         timeout_cmd = [
 133             'timeout',
 134             '--kill-after', str(args.timeout * 2),
 135             str(args.timeout)
 136         ]
 137         def run_test(test):
 138             cmd = []
 139             if args.timeout:
 140                 cmd.extend(timeout_cmd)
 141             cmd.extend([
 142                 test.full_path(),
 143                 '-red', test.m5out_dir(),
 144                 '--listener-mode=off',
 145                 config_path
 146             ])
 147             try:
 148                 subprocess.check_call(cmd)
 149             except subprocess.CalledProcessError, error:
 150                 returncode = error.returncode
 151             else:
 152                 returncode = 0
 153             with open(os.path.join(test.m5out_dir(), 'returncode'), 'w') as rc:
 154                 rc.write('%d\n' % returncode)
 155
 156         runnable = filter(lambda t: not t.compile_only, tests)
 157         if args.j == 1:
 158             map(run_test, runnable)
 159         else:
 160             tp = multiprocessing.pool.ThreadPool(args.j)
 161             map(lambda t: tp.apply_async(run_test, (t,)), runnable)
 162             tp.close()
 163             tp.join()
 164
 165 class VerifyPhase(TestPhaseBase):
 166     name = 'verify'
 167     number = 3
 168
 169     def run(self, tests):
 170         for test in tests:
 171             if test.compile_only:
 172                 continue
 173             logging.info("Would verify %s", test.m5out_dir())
 174
 175
 176
 177 parser = argparse.ArgumentParser(description='SystemC test utility')
 178
 179 parser.add_argument('build_dir', metavar='BUILD_DIR',
 180                     help='The build directory (ie. build/ARM).')
 181
 182 parser.add_argument('--update-json', action='store_true',
 183                     help='Update the json manifest of tests.')
 184
 185 parser.add_argument('--flavor', choices=['debug', 'opt', 'fast'],
 186                     default='opt',
 187                     help='Flavor of binary to test.')
 188
 189 parser.add_argument('--list', action='store_true',
 190                     help='List the available tests')
 191
 192 filter_opts = parser.add_mutually_exclusive_group()
 193 filter_opts.add_argument('--filter', default='True',
 194                          help='Python expression which filters tests based '
 195                          'on their properties')
 196 filter_opts.add_argument('--filter-file', default=None,
 197                          type=argparse.FileType('r'),
 198                          help='Same as --filter, but read from a file')
 199
 200 def collect_phases(args):
 201     phase_groups = [list(g) for k, g in
 202                     itertools.groupby(args, lambda x: x != '--phase') if k]
 203     main_args = parser.parse_args(phase_groups[0][1:])
 204     phases = []
 205     names = []
 206     for group in phase_groups[1:]:
 207         name = group[0]
 208         if name in names:
 209             raise RuntimeException('Phase %s specified more than once' % name)
 210         phase = test_phase_classes[name]
 211         phases.append(phase(main_args, *group[1:]))
 212     phases.sort()
 213     return main_args, phases
 214
 215 main_args, phases = collect_phases(sys.argv)
 216
 217 if len(phases) == 0:
 218     phases = [
 219         CompilePhase(main_args),
 220         RunPhase(main_args),
 221         VerifyPhase(main_args)
 222     ]
 223
 224
 225
 226 json_path = os.path.join(main_args.build_dir, json_rel_path)
 227
 228 if main_args.update_json:
 229     scons(os.path.join(json_path))
 230
 231 with open(json_path) as f:
 232     test_data = json.load(f)
 233
 234     if main_args.filter_file:
 235         f = main_args.filter_file
 236         filt = compile(f.read(), f.name, 'eval')
 237     else:
 238         filt = compile(main_args.filter, '<string>', 'eval')
 239
 240     filtered_tests = {
 241         target: props for (target, props) in
 242                     test_data.iteritems() if eval(filt, dict(props))
 243     }
 244
 245     if main_args.list:
 246         for target, props in sorted(filtered_tests.iteritems()):
 247             print('%s.%s' % (target, main_args.flavor))
 248             for key, val in props.iteritems():
 249                 print('    %s: %s' % (key, val))
 250         print('Total tests: %d' % len(filtered_tests))
 251     else:
 252         tests_to_run = list([
 253             Test(target, main_args.flavor, main_args.build_dir, props) for
 254                 target, props in sorted(filtered_tests.iteritems())
 255         ])
 256
 257         for phase in phases:
 258             phase.run(tests_to_run)