X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=tests%2FSConscript;h=8e219b30e374cf725792c43586c28709321f2044;hb=e2e0ae576a3f0425cd7092d9115debefa64d7476;hp=8c9029be609c0c4fa0ea2cfd6bd5b5b74515dc7d;hpb=a983c4968cebeb9c2b67957934ffd26eb914b525;p=gem5.git diff --git a/tests/SConscript b/tests/SConscript index 8c9029be6..8e219b30e 100644 --- a/tests/SConscript +++ b/tests/SConscript @@ -29,7 +29,7 @@ # Authors: Steve Reinhardt # Kevin Lim -import os +import os, signal import sys import glob from SCons.Script.SConscript import SConsEnvironment @@ -44,57 +44,115 @@ env.Tests = {} def contents(node): return file(str(node)).read() -def check_test(target, source, env): +# functions to parse return value from scons Execute()... not the same +# as wait() etc., so python built-in os funcs don't work. +def signaled(status): + return (status & 0x80) != 0; + +def signum(status): + return (status & 0x7f); + +# List of signals that indicate that we should retry the test rather +# than consider it failed. +retry_signals = (signal.SIGTERM, signal.SIGKILL, signal.SIGINT, + signal.SIGQUIT, signal.SIGHUP) + +# regular expressions of lines to ignore when diffing outputs +output_ignore_regexes = ( + '^command line:', # for stdout file + '^M5 compiled ', # for stderr file + '^M5 started ', # for stderr file + '^M5 executing on ', # for stderr file + '^Simulation complete at', # for stderr file + '^Listening for', # for stderr file + 'listening for remote gdb', # for stderr file + ) + +output_ignore_args = ' '.join(["-I '"+s+"'" for s in output_ignore_regexes]) + +output_ignore_args += ' --exclude=stats.txt --exclude=outdiff' + +def run_test(target, source, env): """Check output from running test. Targets are as follows: - target[0] : outdiff - target[1] : statsdiff - target[2] : status + target[0] : status + + Sources are: + source[0] : M5 binary + source[1] : tests/run.py script + source[2] : reference stats file """ # make sure target files are all gone for t in target: if os.path.exists(t.abspath): - Execute(Delete(t.abspath)) - # Run diff on output & ref directories to find differences. - # Exclude m5stats.txt since we will use diff-out on that. - Execute(env.subst('diff -ubr ${SOURCES[0].dir} ${SOURCES[1].dir} ' + - '-I "^command line:" ' + # for stdout file - '-I "^M5 compiled " ' + # for stderr file - '-I "^M5 started " ' + # for stderr file - '-I "^M5 executing on " ' + # for stderr file - '-I "^Simulation complete at" ' + # for stderr file - '-I "^Listening for" ' + # for stderr file - '-I "listening for remote gdb" ' + # for stderr file - '--exclude=m5stats.txt --exclude=SCCS ' + - '--exclude=${TARGETS[0].file} ' + - '> ${TARGETS[0]}', target=target, source=source), None) - print "===== Output differences =====" - print contents(target[0]) - # Run diff-out on m5stats.txt file - status = Execute(env.subst('$DIFFOUT $SOURCES > ${TARGETS[1]}', - target=target, source=source), - strfunction=None) - print "===== Statistics differences =====" - print contents(target[1]) - # Generate status file contents based on exit status of diff-out + env.Execute(Delete(t.abspath)) + + tgt_dir = os.path.dirname(str(target[0])) + + # Base command for running test. We mess around with indirectly + # referring to files via SOURCES and TARGETS so that scons can mess + # with paths all it wants to and we still get the right files. + cmd = '${SOURCES[0]} -d %s -re ${SOURCES[1]} %s' % (tgt_dir, tgt_dir) + + # Prefix test run with batch job submission command if appropriate. + # Batch command also supports timeout arg (in seconds, not minutes). + timeout = 15 * 60 # used to be a param, probably should be again + if env['BATCH']: + cmd = '%s -t %d %s' % (env['BATCH_CMD'], timeout, cmd) + + status = env.Execute(env.subst(cmd, target=target, source=source)) + if status == 0: + # M5 terminated normally. + # Run diff on output & ref directories to find differences. + # Exclude the stats file since we will use diff-out on that. + outdiff = os.path.join(tgt_dir, 'outdiff') + diffcmd = 'diff -ubr %s ${SOURCES[2].dir} %s > %s' \ + % (output_ignore_args, tgt_dir, outdiff) + env.Execute(env.subst(diffcmd, target=target, source=source)) + print "===== Output differences =====" + print contents(outdiff) + # Run diff-out on stats.txt file + statsdiff = os.path.join(tgt_dir, 'statsdiff') + diffcmd = '$DIFFOUT ${SOURCES[2]} %s > %s' \ + % (os.path.join(tgt_dir, 'stats.txt'), statsdiff) + diffcmd = env.subst(diffcmd, target=target, source=source) + status = env.Execute(diffcmd, strfunction=None) + print "===== Statistics differences =====" + print contents(statsdiff) + + else: # m5 exit status != 0 + # M5 did not terminate properly, so no need to check the output + if signaled(status): + print 'M5 terminated with signal', signum(status) + if signum(status) in retry_signals: + # Consider the test incomplete; don't create a 'status' output. + # Hand the return status to scons and let scons decide what + # to do about it (typically terminate unless run with -k). + return status + else: + print 'M5 exited with non-zero status', status + # complete but failed execution (call to exit() with non-zero + # status, SIGABORT due to assertion failure, etc.)... fall through + # and generate FAILED status as if output comparison had failed + + # Generate status file contents based on exit status of m5 or diff-out if status == 0: status_str = "passed." else: status_str = "FAILED!" - f = file(str(target[2]), 'w') - print >>f, env.subst('${TARGETS[2].dir}', target=target, source=source), \ - status_str + f = file(str(target[0]), 'w') + print >>f, tgt_dir, status_str f.close() # done return 0 -def check_test_string(target, source, env): - return env.subst("Comparing outputs in ${TARGETS[0].dir}.", +def run_test_string(target, source, env): + return env.subst("Running test in ${TARGETS[0].dir}.", target=target, source=source) -testAction = env.Action(check_test, check_test_string) +testAction = env.Action(run_test, run_test_string) def print_test(target, source, env): print '***** ' + contents(source[0]) @@ -102,19 +160,44 @@ def print_test(target, source, env): printAction = env.Action(print_test, strfunction = None) +# Static vars for update_test: +# - long-winded message about ignored sources +ignore_msg = ''' +Note: The following file(s) will not be copied. New non-standard + output files must be copied manually once before update_ref will + recognize them as outputs. Otherwise they are assumed to be + inputs and are ignored. +''' +# - reference files always needed +needed_files = set(['simout', 'simerr', 'stats.txt', 'config.ini']) +# - source files we always want to ignore +known_ignores = set(['status', 'outdiff', 'statsdiff']) + def update_test(target, source, env): """Update reference test outputs. - Target is phony. First two sources are the ref & new m5stats.txt + Target is phony. First two sources are the ref & new stats.txt file files, respectively. We actually copy everything in the respective directories except the status & diff output files. """ dest_dir = str(source[0].get_dir()) src_dir = str(source[1].get_dir()) - dest_files = os.listdir(dest_dir) - src_files = os.listdir(src_dir) - for f in ('stdout', 'stderr', 'm5stats.txt', 'config.ini', 'config.out'): + dest_files = set(os.listdir(dest_dir)) + src_files = set(os.listdir(src_dir)) + # Copy all of the required files plus any existing dest files. + wanted_files = needed_files | dest_files + missing_files = wanted_files - src_files + if len(missing_files) > 0: + print " WARNING: the following file(s) are missing " \ + "and will not be updated:" + print " ", " ,".join(missing_files) + copy_files = wanted_files - missing_files + warn_ignored_files = (src_files - copy_files) - known_ignores + if len(warn_ignored_files) > 0: + print ignore_msg, + print " ", ", ".join(warn_ignored_files) + for f in copy_files: if f in dest_files: print " Replacing file", f dest_files.remove(f) @@ -122,13 +205,7 @@ def update_test(target, source, env): print " Creating new file", f copyAction = Copy(os.path.join(dest_dir, f), os.path.join(src_dir, f)) copyAction.strfunction = None - Execute(copyAction) - # warn about any files in dest not overwritten (other than SCCS dir) - if 'SCCS' in dest_files: - dest_files.remove('SCCS') - if dest_files: - print "Warning: file(s) in", dest_dir, "not updated:", - print ', '.join(dest_files) + env.Execute(copyAction) return 0 def update_test_string(target, source, env): @@ -151,35 +228,12 @@ def test_builder(env, ref_dir): def tgt(f): return os.path.join(tgt_dir, f) - ref_stats = os.path.join(ref_dir, 'm5stats.txt') - new_stats = tgt('m5stats.txt') + ref_stats = os.path.join(ref_dir, 'stats.txt') + new_stats = tgt('stats.txt') status_file = tgt('status') - # Base command for running test. We mess around with indirectly - # referring to files via SOURCES and TARGETS so that scons can - # mess with paths all it wants to and we still get the right - # files. - base_cmd = '${SOURCES[0]} -d $TARGET.dir ${SOURCES[1]} %s' % tgt_dir - # stdout and stderr files - cmd_stdout = '${TARGETS[0]}' - cmd_stderr = '${TARGETS[1]}' - - # Prefix test run with batch job submission command if appropriate. - # Output redirection is also different for batch runs. - # Batch command also supports timeout arg (in seconds, not minutes). - timeout = 15 # used to be a param, probably should be again - if env['BATCH']: - cmd = [env['BATCH_CMD'], '-t', str(timeout * 60), - '-o', cmd_stdout, '-e', cmd_stderr, base_cmd] - else: - cmd = [base_cmd, '>', cmd_stdout, '2>', cmd_stderr] - - env.Command([tgt('stdout'), tgt('stderr'), new_stats], - [env.M5Binary, 'run.py'], ' '.join(cmd)) - - # order of targets is important... see check_test - env.Command([tgt('outdiff'), tgt('statsdiff'), status_file], - [ref_stats, new_stats], + env.Command([status_file], + [env.M5Binary, 'run.py', ref_stats], testAction) # phony target to echo status @@ -197,20 +251,30 @@ def test_builder(env, ref_dir): configs = [] if env['FULL_SYSTEM']: if env['TARGET_ISA'] == 'alpha': - if not env['ALPHA_TLASER']: - configs += ['tsunami-simple-atomic', - 'tsunami-simple-timing', - 'tsunami-simple-atomic-dual', - 'tsunami-simple-timing-dual', - 'twosys-tsunami-simple-atomic'] + configs += ['tsunami-simple-atomic', + 'tsunami-simple-timing', + 'tsunami-simple-atomic-dual', + 'tsunami-simple-timing-dual', + 'twosys-tsunami-simple-atomic', + 'tsunami-o3', 'tsunami-o3-dual'] + if env['TARGET_ISA'] == 'sparc': + configs += ['t1000-simple-atomic', + 't1000-simple-timing'] else: - configs += ['simple-atomic', 'simple-timing', 'o3-timing'] + configs += ['simple-atomic', 'simple-timing', 'o3-timing', 'memtest', + 'simple-atomic-mp', 'simple-timing-mp', 'o3-timing-mp', + 'inorder-timing'] + +if env['RUBY']: + # Hack for Ruby + configs += [c + '-ruby' for c in configs] cwd = os.getcwd() os.chdir(str(Dir('.').srcdir)) for config in configs: dirs = glob.glob('*/*/ref/%s/*/%s' % (env['TARGET_ISA'], config)) for d in dirs: - test_builder(env, d) + if not os.path.exists(os.path.join(d, 'skip')): + test_builder(env, d) os.chdir(cwd)