tests/SConscript

   1 # -*- mode:python -*-
   2 #
   3 # Copyright (c) 2016 ARM Limited
   4 # All rights reserved
   5 #
   6 # The license below extends only to copyright in the software and shall
   7 # not be construed as granting a license to any other intellectual
   8 # property including but not limited to intellectual property relating
   9 # to a hardware implementation of the functionality of the software
  10 # licensed hereunder.  You may use the software subject to the license
  11 # terms below provided that you ensure that this notice is replicated
  12 # unmodified and in its entirety in all distributions of the software,
  13 # modified or unmodified, in source code or in binary form.
  14 #
  15 # Copyright (c) 2004-2006 The Regents of The University of Michigan
  16 # All rights reserved.
  17 #
  18 # Redistribution and use in source and binary forms, with or without
  19 # modification, are permitted provided that the following conditions are
  20 # met: redistributions of source code must retain the above copyright
  21 # notice, this list of conditions and the following disclaimer;
  22 # redistributions in binary form must reproduce the above copyright
  23 # notice, this list of conditions and the following disclaimer in the
  24 # documentation and/or other materials provided with the distribution;
  25 # neither the name of the copyright holders nor the names of its
  26 # contributors may be used to endorse or promote products derived from
  27 # this software without specific prior written permission.
  28 #
  29 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  30 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  31 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  32 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  33 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  34 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  35 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  36 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  37 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  38 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  39 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  40 #
  41 # Authors: Steve Reinhardt
  42 #          Kevin Lim
  43 #          Andreas Sandberg
  44
  45 from SCons.Script.SConscript import SConsEnvironment
  46 import os
  47 import pickle
  48 import sys
  49
  50 sys.path.insert(0, Dir(".").srcnode().abspath)
  51 import testing.tests as tests
  52 import testing.results as results
  53
  54 Import('env')
  55
  56 # get the termcap from the environment
  57 termcap = env['TERMCAP']
  58
  59 # Dict that accumulates lists of tests by category (quick, medium, long)
  60 env.Tests = {}
  61 gpu_isa = env['TARGET_GPU_ISA'] if env['BUILD_GPU'] else None
  62 for cat in tests.all_categories:
  63     env.Tests[cat] = tuple(
  64         tests.get_tests(env["TARGET_ISA"],
  65                         categories=(cat, ),
  66                         ruby_protocol=env["PROTOCOL"],
  67                         gpu_isa=gpu_isa))
  68
  69 def color_message(color, msg):
  70     return color + msg + termcap.Normal
  71
  72 def run_test(target, source, env):
  73     """Run a test and produce results as a pickle file.
  74
  75     Targets are as follows:
  76     target[0] : Pickle file
  77
  78     Sources are:
  79     source[0] : gem5 binary
  80     source[1] : tests/run.py script
  81     source[2:] : reference files
  82
  83     """
  84     tgt_dir = os.path.dirname(str(target[0]))
  85     config = tests.ClassicConfig(*tgt_dir.split('/')[-6:])
  86     test = tests.ClassicTest(source[0].abspath, tgt_dir, config,
  87                              timeout=5*60*60,
  88                              skip_diff_out=True)
  89
  90     for ref in test.ref_files():
  91         out_file = os.path.join(tgt_dir, ref)
  92         if os.path.exists(out_file):
  93             env.Execute(Delete(out_file))
  94
  95     with open(target[0].abspath, "wb") as fout:
  96         formatter = results.Pickle(fout=fout)
  97         formatter.dump_suites([ test.run() ])
  98
  99     return 0
 100
 101 def run_test_string(target, source, env):
 102     return env.subst("Running test in ${TARGETS[0].dir}.",
 103                      target=target, source=source)
 104
 105 testAction = env.Action(run_test, run_test_string)
 106
 107 def print_test(target, source, env):
 108     """Run a test and produce results as a pickle file.
 109
 110     Targets are as follows:
 111     target[*] : Dummy targets
 112
 113     Sources are:
 114     source[0] : Pickle file
 115
 116     """
 117     with open(source[0].abspath, "rb") as fin:
 118         result = pickle.load(fin)
 119
 120     assert len(result) == 1
 121     result = result[0]
 122
 123     formatter = None
 124     if result.skipped():
 125         status = color_message(termcap.Cyan, "skipped.")
 126     elif result.changed():
 127         status = color_message(termcap.Yellow, "CHANGED!")
 128         formatter = results.Text()
 129     elif result:
 130         status = color_message(termcap.Green, "passed.")
 131     else:
 132         status = color_message(termcap.Red, "FAILED!")
 133         formatter = results.Text()
 134
 135     if formatter:
 136         formatter.dump_suites([result])
 137
 138     print "***** %s: %s" % (source[0].dir, status)
 139     return 0
 140
 141 printAction = env.Action(print_test, strfunction=None)
 142
 143 def update_test(target, source, env):
 144     """Update test reference data
 145
 146     Targets are as follows:
 147     target[0] : Dummy file
 148
 149     Sources are:
 150     source[0] : Pickle file
 151     """
 152
 153     src_dir = os.path.dirname(str(source[0]))
 154     config = tests.ClassicConfig(*src_dir.split('/')[-6:])
 155     test = tests.ClassicTest(source[0].abspath, src_dir, config)
 156     ref_dir = test.ref_dir
 157
 158     with open(source[0].abspath, "rb") as fin:
 159         result = pickle.load(fin)
 160
 161     assert len(result) == 1
 162     result = result[0]
 163
 164     if result.skipped():
 165         print "*** %s: %s: Test skipped, not updating." % (
 166             source[0].dir, color_message(termcap.Yellow, "WARNING"), )
 167         return 0
 168     elif result:
 169         print "*** %s: %s: Test successful, not updating." % (
 170             source[0].dir, color_message(termcap.Green, "skipped"), )
 171         return 0
 172     elif result.failed_run():
 173         print "*** %s: %s: Test failed, not updating." % (
 174             source[0].dir, color_message(termcap.Red, "ERROR"), )
 175         return 1
 176
 177     print "** Updating %s" % (test, )
 178     test.update_ref()
 179
 180     return 0
 181
 182 def update_test_string(target, source, env):
 183     return env.subst("Updating ${SOURCES[0].dir}",
 184                      target=target, source=source)
 185
 186 updateAction = env.Action(update_test, update_test_string)
 187
 188 def test_builder(test_tuple):
 189     """Define a test."""
 190
 191     out_dir = "/".join(test_tuple)
 192     binary = env.M5Binary.abspath
 193     test = tests.ClassicTest(binary, out_dir, test_tuple)
 194
 195     def tgt(name):
 196         return os.path.join(out_dir, name)
 197
 198     def ref(name):
 199         return os.path.join(test.ref_dir, name)
 200
 201     pickle_file = tgt("status.pickle")
 202     targets = [
 203         pickle_file,
 204     ]
 205
 206     sources = [
 207         env.M5Binary,
 208         "run.py",
 209     ] + [ ref(f) for f in test.ref_files() ]
 210
 211     env.Command(targets, sources, testAction)
 212
 213     # phony target to echo status
 214     if GetOption('update_ref'):
 215         p = env.Command(tgt("_update"), [pickle_file], updateAction)
 216     else:
 217         p = env.Command(tgt("_print"), [pickle_file], printAction)
 218
 219     env.AlwaysBuild(p)
 220
 221 def list_tests(target, source, env):
 222     """Create a list of tests
 223
 224     Targets are as follows:
 225     target[0] : List file (e.g., tests/opt/all.list,  tests/opt/quick.list)
 226
 227     Sources are: -
 228
 229     """
 230
 231     tgt_name = os.path.basename(str(target[0]))
 232     base, ext = os.path.splitext(tgt_name)
 233     categories = tests.all_categories if base == "all" else (base, )
 234
 235     with open(target[0].abspath, "w") as fout:
 236         for cat in categories:
 237             for test in env.Tests[cat]:
 238                 print >> fout,"/".join(test)
 239
 240     return 0
 241
 242 testListAction = env.Action(list_tests, strfunction=None)
 243
 244 env.Command("all.list", tuple(), testListAction)
 245 for cat, test_list in env.Tests.items():
 246     env.Command("%s.list" % cat, tuple(), testListAction)
 247     for test in test_list:
 248         test_builder(test)