tests/SConscript

   1 # -*- mode:python -*-
   2 #
   3 # Copyright (c) 2016 ARM Limited
   4 # All rights reserved
   5 #
   6 # The license below extends only to copyright in the software and shall
   7 # not be construed as granting a license to any other intellectual
   8 # property including but not limited to intellectual property relating
   9 # to a hardware implementation of the functionality of the software
  10 # licensed hereunder.  You may use the software subject to the license
  11 # terms below provided that you ensure that this notice is replicated
  12 # unmodified and in its entirety in all distributions of the software,
  13 # modified or unmodified, in source code or in binary form.
  14 #
  15 # Copyright (c) 2004-2006 The Regents of The University of Michigan
  16 # All rights reserved.
  17 #
  18 # Redistribution and use in source and binary forms, with or without
  19 # modification, are permitted provided that the following conditions are
  20 # met: redistributions of source code must retain the above copyright
  21 # notice, this list of conditions and the following disclaimer;
  22 # redistributions in binary form must reproduce the above copyright
  23 # notice, this list of conditions and the following disclaimer in the
  24 # documentation and/or other materials provided with the distribution;
  25 # neither the name of the copyright holders nor the names of its
  26 # contributors may be used to endorse or promote products derived from
  27 # this software without specific prior written permission.
  28 #
  29 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  30 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  31 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  32 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  33 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  34 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  35 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  36 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  37 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  38 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  39 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  40
  41 from __future__ import print_function
  42
  43 from SCons.Script.SConscript import SConsEnvironment
  44 import os
  45 import pickle
  46 import sys
  47
  48 sys.path.insert(0, Dir(".").srcnode().abspath)
  49 import testing.tests as tests
  50 import testing.results as results
  51 from gem5_scons.util import get_termcap
  52
  53 Import('env')
  54
  55 # get the termcap from the environment
  56 termcap = get_termcap()
  57
  58 # Dict that accumulates lists of tests by category (quick, medium, long)
  59 env.Tests = {}
  60 gpu_isa = env['TARGET_GPU_ISA'] if env['BUILD_GPU'] else None
  61 for cat in tests.all_categories:
  62     env.Tests[cat] = tuple(
  63         tests.get_tests(env["TARGET_ISA"],
  64                         categories=(cat, ),
  65                         ruby_protocol=env["PROTOCOL"],
  66                         gpu_isa=gpu_isa))
  67
  68 def color_message(color, msg):
  69     return color + msg + termcap.Normal
  70
  71 def run_test(target, source, env):
  72     """Run a test and produce results as a pickle file.
  73
  74     Targets are as follows:
  75     target[0] : Pickle file
  76
  77     Sources are:
  78     source[0] : gem5 binary
  79     source[1] : tests/run.py script
  80     source[2:] : reference files
  81
  82     """
  83     tgt_dir = os.path.dirname(str(target[0]))
  84     config = tests.ClassicConfig(*tgt_dir.split('/')[-6:])
  85     test = tests.ClassicTest(source[0].abspath, tgt_dir, config,
  86                              timeout=5*60*60,
  87                              skip_diff_out=True)
  88
  89     for ref in test.ref_files():
  90         out_file = os.path.join(tgt_dir, ref)
  91         if os.path.exists(out_file):
  92             env.Execute(Delete(out_file))
  93
  94     with open(target[0].abspath, "wb") as fout:
  95         formatter = results.Pickle(fout=fout)
  96         formatter.dump_suites([ test.run() ])
  97
  98     return 0
  99
 100 def run_test_string(target, source, env):
 101     return env.subst("Running test in ${TARGETS[0].dir}.",
 102                      target=target, source=source)
 103
 104 testAction = env.Action(run_test, run_test_string)
 105
 106 def print_test(target, source, env):
 107     """Run a test and produce results as a pickle file.
 108
 109     Targets are as follows:
 110     target[*] : Dummy targets
 111
 112     Sources are:
 113     source[0] : Pickle file
 114
 115     """
 116     with open(source[0].abspath, "rb") as fin:
 117         result = pickle.load(fin)
 118
 119     assert len(result) == 1
 120     result = result[0]
 121
 122     formatter = None
 123     if result.skipped():
 124         status = color_message(termcap.Cyan, "skipped.")
 125     elif result.changed():
 126         status = color_message(termcap.Yellow, "CHANGED!")
 127         formatter = results.Text()
 128     elif result:
 129         status = color_message(termcap.Green, "passed.")
 130     else:
 131         status = color_message(termcap.Red, "FAILED!")
 132         formatter = results.Text()
 133
 134     if formatter:
 135         formatter.dump_suites([result])
 136
 137     print("***** %s: %s" % (source[0].dir, status))
 138     return 0
 139
 140 printAction = env.Action(print_test, strfunction=None)
 141
 142 def update_test(target, source, env):
 143     """Update test reference data
 144
 145     Targets are as follows:
 146     target[0] : Dummy file
 147
 148     Sources are:
 149     source[0] : Pickle file
 150     """
 151
 152     src_dir = os.path.dirname(str(source[0]))
 153     config = tests.ClassicConfig(*src_dir.split('/')[-6:])
 154     test = tests.ClassicTest(source[0].abspath, src_dir, config)
 155     ref_dir = test.ref_dir
 156
 157     with open(source[0].abspath, "rb") as fin:
 158         result = pickle.load(fin)
 159
 160     assert len(result) == 1
 161     result = result[0]
 162
 163     if result.skipped():
 164         print("*** %s: %s: Test skipped, not updating." %
 165               (source[0].dir, color_message(termcap.Yellow, "WARNING")))
 166         return 0
 167     elif result:
 168         print("*** %s: %s: Test successful, not updating." %
 169               (source[0].dir, color_message(termcap.Green, "skipped")))
 170         return 0
 171     elif result.failed_run():
 172         print("*** %s: %s: Test failed, not updating." %
 173               (source[0].dir, color_message(termcap.Red, "ERROR")))
 174         return 1
 175
 176     print("** Updating %s" % test)
 177     test.update_ref()
 178
 179     return 0
 180
 181 def update_test_string(target, source, env):
 182     return env.subst("Updating ${SOURCES[0].dir}",
 183                      target=target, source=source)
 184
 185 updateAction = env.Action(update_test, update_test_string)
 186
 187 def test_builder(test_tuple):
 188     """Define a test."""
 189
 190     out_dir = "/".join(test_tuple)
 191     binary = env.M5Binary.abspath
 192     test = tests.ClassicTest(binary, out_dir, test_tuple)
 193
 194     def tgt(name):
 195         return os.path.join(out_dir, name)
 196
 197     def ref(name):
 198         return os.path.join(test.ref_dir, name)
 199
 200     pickle_file = tgt("status.pickle")
 201     targets = [
 202         pickle_file,
 203     ]
 204
 205     sources = [
 206         env.M5Binary,
 207         "run.py",
 208     ] + [ ref(f) for f in test.ref_files() ]
 209
 210     env.Command(targets, sources, testAction)
 211
 212     # phony target to echo status
 213     if GetOption('update_ref'):
 214         p = env.Command(tgt("_update"), [pickle_file], updateAction)
 215     else:
 216         p = env.Command(tgt("_print"), [pickle_file], printAction)
 217
 218     env.AlwaysBuild(p)
 219
 220 def list_tests(target, source, env):
 221     """Create a list of tests
 222
 223     Targets are as follows:
 224     target[0] : List file (e.g., tests/opt/all.list,  tests/opt/quick.list)
 225
 226     Sources are: -
 227
 228     """
 229
 230     tgt_name = os.path.basename(str(target[0]))
 231     base, ext = os.path.splitext(tgt_name)
 232     categories = tests.all_categories if base == "all" else (base, )
 233
 234     with open(target[0].abspath, "w") as fout:
 235         for cat in categories:
 236             for test in env.Tests[cat]:
 237                 print("/".join(test), file=fout)
 238
 239     return 0
 240
 241 testListAction = env.Action(list_tests, strfunction=None)
 242
 243 env.Command("all.list", tuple(), testListAction)
 244 for cat, test_list in env.Tests.items():
 245     env.Command("%s.list" % cat, tuple(), testListAction)
 246     for test in test_list:
 247         test_builder(test)