tests/SConscript

   1 # -*- mode:python -*-
   2 #
   3 # Copyright (c) 2016 ARM Limited
   4 # All rights reserved
   5 #
   6 # The license below extends only to copyright in the software and shall
   7 # not be construed as granting a license to any other intellectual
   8 # property including but not limited to intellectual property relating
   9 # to a hardware implementation of the functionality of the software
  10 # licensed hereunder.  You may use the software subject to the license
  11 # terms below provided that you ensure that this notice is replicated
  12 # unmodified and in its entirety in all distributions of the software,
  13 # modified or unmodified, in source code or in binary form.
  14 #
  15 # Copyright (c) 2004-2006 The Regents of The University of Michigan
  16 # All rights reserved.
  17 #
  18 # Redistribution and use in source and binary forms, with or without
  19 # modification, are permitted provided that the following conditions are
  20 # met: redistributions of source code must retain the above copyright
  21 # notice, this list of conditions and the following disclaimer;
  22 # redistributions in binary form must reproduce the above copyright
  23 # notice, this list of conditions and the following disclaimer in the
  24 # documentation and/or other materials provided with the distribution;
  25 # neither the name of the copyright holders nor the names of its
  26 # contributors may be used to endorse or promote products derived from
  27 # this software without specific prior written permission.
  28 #
  29 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  30 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  31 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  32 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  33 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  34 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  35 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  36 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  37 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  38 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  39 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  40 #
  41 # Authors: Steve Reinhardt
  42 #          Kevin Lim
  43 #          Andreas Sandberg
  44
  45 from __future__ import print_function
  46
  47 from SCons.Script.SConscript import SConsEnvironment
  48 import os
  49 import pickle
  50 import sys
  51
  52 sys.path.insert(0, Dir(".").srcnode().abspath)
  53 import testing.tests as tests
  54 import testing.results as results
  55 from gem5_scons.util import get_termcap
  56
  57 Import('env')
  58
  59 # get the termcap from the environment
  60 termcap = get_termcap()
  61
  62 # Dict that accumulates lists of tests by category (quick, medium, long)
  63 env.Tests = {}
  64 gpu_isa = env['TARGET_GPU_ISA'] if env['BUILD_GPU'] else None
  65 for cat in tests.all_categories:
  66     env.Tests[cat] = tuple(
  67         tests.get_tests(env["TARGET_ISA"],
  68                         categories=(cat, ),
  69                         ruby_protocol=env["PROTOCOL"],
  70                         gpu_isa=gpu_isa))
  71
  72 def color_message(color, msg):
  73     return color + msg + termcap.Normal
  74
  75 def run_test(target, source, env):
  76     """Run a test and produce results as a pickle file.
  77
  78     Targets are as follows:
  79     target[0] : Pickle file
  80
  81     Sources are:
  82     source[0] : gem5 binary
  83     source[1] : tests/run.py script
  84     source[2:] : reference files
  85
  86     """
  87     tgt_dir = os.path.dirname(str(target[0]))
  88     config = tests.ClassicConfig(*tgt_dir.split('/')[-6:])
  89     test = tests.ClassicTest(source[0].abspath, tgt_dir, config,
  90                              timeout=5*60*60,
  91                              skip_diff_out=True)
  92
  93     for ref in test.ref_files():
  94         out_file = os.path.join(tgt_dir, ref)
  95         if os.path.exists(out_file):
  96             env.Execute(Delete(out_file))
  97
  98     with open(target[0].abspath, "wb") as fout:
  99         formatter = results.Pickle(fout=fout)
 100         formatter.dump_suites([ test.run() ])
 101
 102     return 0
 103
 104 def run_test_string(target, source, env):
 105     return env.subst("Running test in ${TARGETS[0].dir}.",
 106                      target=target, source=source)
 107
 108 testAction = env.Action(run_test, run_test_string)
 109
 110 def print_test(target, source, env):
 111     """Run a test and produce results as a pickle file.
 112
 113     Targets are as follows:
 114     target[*] : Dummy targets
 115
 116     Sources are:
 117     source[0] : Pickle file
 118
 119     """
 120     with open(source[0].abspath, "rb") as fin:
 121         result = pickle.load(fin)
 122
 123     assert len(result) == 1
 124     result = result[0]
 125
 126     formatter = None
 127     if result.skipped():
 128         status = color_message(termcap.Cyan, "skipped.")
 129     elif result.changed():
 130         status = color_message(termcap.Yellow, "CHANGED!")
 131         formatter = results.Text()
 132     elif result:
 133         status = color_message(termcap.Green, "passed.")
 134     else:
 135         status = color_message(termcap.Red, "FAILED!")
 136         formatter = results.Text()
 137
 138     if formatter:
 139         formatter.dump_suites([result])
 140
 141     print("***** %s: %s" % (source[0].dir, status))
 142     return 0
 143
 144 printAction = env.Action(print_test, strfunction=None)
 145
 146 def update_test(target, source, env):
 147     """Update test reference data
 148
 149     Targets are as follows:
 150     target[0] : Dummy file
 151
 152     Sources are:
 153     source[0] : Pickle file
 154     """
 155
 156     src_dir = os.path.dirname(str(source[0]))
 157     config = tests.ClassicConfig(*src_dir.split('/')[-6:])
 158     test = tests.ClassicTest(source[0].abspath, src_dir, config)
 159     ref_dir = test.ref_dir
 160
 161     with open(source[0].abspath, "rb") as fin:
 162         result = pickle.load(fin)
 163
 164     assert len(result) == 1
 165     result = result[0]
 166
 167     if result.skipped():
 168         print("*** %s: %s: Test skipped, not updating." %
 169               (source[0].dir, color_message(termcap.Yellow, "WARNING")))
 170         return 0
 171     elif result:
 172         print("*** %s: %s: Test successful, not updating." %
 173               (source[0].dir, color_message(termcap.Green, "skipped")))
 174         return 0
 175     elif result.failed_run():
 176         print("*** %s: %s: Test failed, not updating." %
 177               (source[0].dir, color_message(termcap.Red, "ERROR")))
 178         return 1
 179
 180     print("** Updating %s" % test)
 181     test.update_ref()
 182
 183     return 0
 184
 185 def update_test_string(target, source, env):
 186     return env.subst("Updating ${SOURCES[0].dir}",
 187                      target=target, source=source)
 188
 189 updateAction = env.Action(update_test, update_test_string)
 190
 191 def test_builder(test_tuple):
 192     """Define a test."""
 193
 194     out_dir = "/".join(test_tuple)
 195     binary = env.M5Binary.abspath
 196     test = tests.ClassicTest(binary, out_dir, test_tuple)
 197
 198     def tgt(name):
 199         return os.path.join(out_dir, name)
 200
 201     def ref(name):
 202         return os.path.join(test.ref_dir, name)
 203
 204     pickle_file = tgt("status.pickle")
 205     targets = [
 206         pickle_file,
 207     ]
 208
 209     sources = [
 210         env.M5Binary,
 211         "run.py",
 212     ] + [ ref(f) for f in test.ref_files() ]
 213
 214     env.Command(targets, sources, testAction)
 215
 216     # phony target to echo status
 217     if GetOption('update_ref'):
 218         p = env.Command(tgt("_update"), [pickle_file], updateAction)
 219     else:
 220         p = env.Command(tgt("_print"), [pickle_file], printAction)
 221
 222     env.AlwaysBuild(p)
 223
 224 def list_tests(target, source, env):
 225     """Create a list of tests
 226
 227     Targets are as follows:
 228     target[0] : List file (e.g., tests/opt/all.list,  tests/opt/quick.list)
 229
 230     Sources are: -
 231
 232     """
 233
 234     tgt_name = os.path.basename(str(target[0]))
 235     base, ext = os.path.splitext(tgt_name)
 236     categories = tests.all_categories if base == "all" else (base, )
 237
 238     with open(target[0].abspath, "w") as fout:
 239         for cat in categories:
 240             for test in env.Tests[cat]:
 241                 print("/".join(test), file=fout)
 242
 243     return 0
 244
 245 testListAction = env.Action(list_tests, strfunction=None)
 246
 247 env.Command("all.list", tuple(), testListAction)
 248 for cat, test_list in env.Tests.items():
 249     env.Command("%s.list" % cat, tuple(), testListAction)
 250     for test in test_list:
 251         test_builder(test)