tests/SConscript

   1 # -*- mode:python -*-
   2 #
   3 # Copyright (c) 2016 ARM Limited
   4 # All rights reserved
   5 #
   6 # The license below extends only to copyright in the software and shall
   7 # not be construed as granting a license to any other intellectual
   8 # property including but not limited to intellectual property relating
   9 # to a hardware implementation of the functionality of the software
  10 # licensed hereunder.  You may use the software subject to the license
  11 # terms below provided that you ensure that this notice is replicated
  12 # unmodified and in its entirety in all distributions of the software,
  13 # modified or unmodified, in source code or in binary form.
  14 #
  15 # Copyright (c) 2004-2006 The Regents of The University of Michigan
  16 # All rights reserved.
  17 #
  18 # Redistribution and use in source and binary forms, with or without
  19 # modification, are permitted provided that the following conditions are
  20 # met: redistributions of source code must retain the above copyright
  21 # notice, this list of conditions and the following disclaimer;
  22 # redistributions in binary form must reproduce the above copyright
  23 # notice, this list of conditions and the following disclaimer in the
  24 # documentation and/or other materials provided with the distribution;
  25 # neither the name of the copyright holders nor the names of its
  26 # contributors may be used to endorse or promote products derived from
  27 # this software without specific prior written permission.
  28 #
  29 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  30 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  31 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  32 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  33 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  34 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  35 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  36 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  37 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  38 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  39 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  40 #
  41 # Authors: Steve Reinhardt
  42 #          Kevin Lim
  43 #          Andreas Sandberg
  44
  45 from SCons.Script.SConscript import SConsEnvironment
  46 import os
  47 import pickle
  48 import sys
  49
  50 sys.path.insert(0, Dir(".").srcnode().abspath)
  51 import testing.tests as tests
  52 import testing.results as results
  53 from gem5_scons.util import get_termcap
  54
  55 Import('env')
  56
  57 # get the termcap from the environment
  58 termcap = get_termcap()
  59
  60 # Dict that accumulates lists of tests by category (quick, medium, long)
  61 env.Tests = {}
  62 gpu_isa = env['TARGET_GPU_ISA'] if env['BUILD_GPU'] else None
  63 for cat in tests.all_categories:
  64     env.Tests[cat] = tuple(
  65         tests.get_tests(env["TARGET_ISA"],
  66                         categories=(cat, ),
  67                         ruby_protocol=env["PROTOCOL"],
  68                         gpu_isa=gpu_isa))
  69
  70 def color_message(color, msg):
  71     return color + msg + termcap.Normal
  72
  73 def run_test(target, source, env):
  74     """Run a test and produce results as a pickle file.
  75
  76     Targets are as follows:
  77     target[0] : Pickle file
  78
  79     Sources are:
  80     source[0] : gem5 binary
  81     source[1] : tests/run.py script
  82     source[2:] : reference files
  83
  84     """
  85     tgt_dir = os.path.dirname(str(target[0]))
  86     config = tests.ClassicConfig(*tgt_dir.split('/')[-6:])
  87     test = tests.ClassicTest(source[0].abspath, tgt_dir, config,
  88                              timeout=5*60*60,
  89                              skip_diff_out=True)
  90
  91     for ref in test.ref_files():
  92         out_file = os.path.join(tgt_dir, ref)
  93         if os.path.exists(out_file):
  94             env.Execute(Delete(out_file))
  95
  96     with open(target[0].abspath, "wb") as fout:
  97         formatter = results.Pickle(fout=fout)
  98         formatter.dump_suites([ test.run() ])
  99
 100     return 0
 101
 102 def run_test_string(target, source, env):
 103     return env.subst("Running test in ${TARGETS[0].dir}.",
 104                      target=target, source=source)
 105
 106 testAction = env.Action(run_test, run_test_string)
 107
 108 def print_test(target, source, env):
 109     """Run a test and produce results as a pickle file.
 110
 111     Targets are as follows:
 112     target[*] : Dummy targets
 113
 114     Sources are:
 115     source[0] : Pickle file
 116
 117     """
 118     with open(source[0].abspath, "rb") as fin:
 119         result = pickle.load(fin)
 120
 121     assert len(result) == 1
 122     result = result[0]
 123
 124     formatter = None
 125     if result.skipped():
 126         status = color_message(termcap.Cyan, "skipped.")
 127     elif result.changed():
 128         status = color_message(termcap.Yellow, "CHANGED!")
 129         formatter = results.Text()
 130     elif result:
 131         status = color_message(termcap.Green, "passed.")
 132     else:
 133         status = color_message(termcap.Red, "FAILED!")
 134         formatter = results.Text()
 135
 136     if formatter:
 137         formatter.dump_suites([result])
 138
 139     print "***** %s: %s" % (source[0].dir, status)
 140     return 0
 141
 142 printAction = env.Action(print_test, strfunction=None)
 143
 144 def update_test(target, source, env):
 145     """Update test reference data
 146
 147     Targets are as follows:
 148     target[0] : Dummy file
 149
 150     Sources are:
 151     source[0] : Pickle file
 152     """
 153
 154     src_dir = os.path.dirname(str(source[0]))
 155     config = tests.ClassicConfig(*src_dir.split('/')[-6:])
 156     test = tests.ClassicTest(source[0].abspath, src_dir, config)
 157     ref_dir = test.ref_dir
 158
 159     with open(source[0].abspath, "rb") as fin:
 160         result = pickle.load(fin)
 161
 162     assert len(result) == 1
 163     result = result[0]
 164
 165     if result.skipped():
 166         print "*** %s: %s: Test skipped, not updating." % (
 167             source[0].dir, color_message(termcap.Yellow, "WARNING"), )
 168         return 0
 169     elif result:
 170         print "*** %s: %s: Test successful, not updating." % (
 171             source[0].dir, color_message(termcap.Green, "skipped"), )
 172         return 0
 173     elif result.failed_run():
 174         print "*** %s: %s: Test failed, not updating." % (
 175             source[0].dir, color_message(termcap.Red, "ERROR"), )
 176         return 1
 177
 178     print "** Updating %s" % (test, )
 179     test.update_ref()
 180
 181     return 0
 182
 183 def update_test_string(target, source, env):
 184     return env.subst("Updating ${SOURCES[0].dir}",
 185                      target=target, source=source)
 186
 187 updateAction = env.Action(update_test, update_test_string)
 188
 189 def test_builder(test_tuple):
 190     """Define a test."""
 191
 192     out_dir = "/".join(test_tuple)
 193     binary = env.M5Binary.abspath
 194     test = tests.ClassicTest(binary, out_dir, test_tuple)
 195
 196     def tgt(name):
 197         return os.path.join(out_dir, name)
 198
 199     def ref(name):
 200         return os.path.join(test.ref_dir, name)
 201
 202     pickle_file = tgt("status.pickle")
 203     targets = [
 204         pickle_file,
 205     ]
 206
 207     sources = [
 208         env.M5Binary,
 209         "run.py",
 210     ] + [ ref(f) for f in test.ref_files() ]
 211
 212     env.Command(targets, sources, testAction)
 213
 214     # phony target to echo status
 215     if GetOption('update_ref'):
 216         p = env.Command(tgt("_update"), [pickle_file], updateAction)
 217     else:
 218         p = env.Command(tgt("_print"), [pickle_file], printAction)
 219
 220     env.AlwaysBuild(p)
 221
 222 def list_tests(target, source, env):
 223     """Create a list of tests
 224
 225     Targets are as follows:
 226     target[0] : List file (e.g., tests/opt/all.list,  tests/opt/quick.list)
 227
 228     Sources are: -
 229
 230     """
 231
 232     tgt_name = os.path.basename(str(target[0]))
 233     base, ext = os.path.splitext(tgt_name)
 234     categories = tests.all_categories if base == "all" else (base, )
 235
 236     with open(target[0].abspath, "w") as fout:
 237         for cat in categories:
 238             for test in env.Tests[cat]:
 239                 print >> fout,"/".join(test)
 240
 241     return 0
 242
 243 testListAction = env.Action(list_tests, strfunction=None)
 244
 245 env.Command("all.list", tuple(), testListAction)
 246 for cat, test_list in env.Tests.items():
 247     env.Command("%s.list" % cat, tuple(), testListAction)
 248     for test in test_list:
 249         test_builder(test)