X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=tests%2FSConscript;h=5c1eb5f7b94d7dbb9d4bd56cbc55d5c3714311f4;hb=fc6d1f33990a240f136b39c5a4a8df6efa99af71;hp=80131057df2fed574ea8fd4392153dde0926414c;hpb=bb9d2c345791331916df482f23173e6b5f5bf804;p=gem5.git diff --git a/tests/SConscript b/tests/SConscript index 80131057d..5c1eb5f7b 100644 --- a/tests/SConscript +++ b/tests/SConscript @@ -29,8 +29,8 @@ # Authors: Steve Reinhardt # Kevin Lim -import os -import sys +import os, signal +import sys, time import glob from SCons.Script.SConscript import SConsEnvironment @@ -38,85 +38,216 @@ Import('env') env['DIFFOUT'] = File('diff-out') +# get the termcap from the environment +termcap = env['TERMCAP'] + # Dict that accumulates lists of tests by category (quick, medium, long) env.Tests = {} def contents(node): return file(str(node)).read() -def check_test(target, source, env): +# functions to parse return value from scons Execute()... not the same +# as wait() etc., so python built-in os funcs don't work. +def signaled(status): + return (status & 0x80) != 0; + +def signum(status): + return (status & 0x7f); + +# List of signals that indicate that we should retry the test rather +# than consider it failed. +retry_signals = (signal.SIGTERM, signal.SIGKILL, signal.SIGINT, + signal.SIGQUIT, signal.SIGHUP) + +# regular expressions of lines to ignore when diffing outputs +output_ignore_regexes = ( + '^command line:', # for stdout file + '^gem5 compiled ', # for stderr file + '^gem5 started ', # for stderr file + '^gem5 executing on ', # for stderr file + '^Simulation complete at', # for stderr file + '^Listening for', # for stderr file + 'listening for remote gdb', # for stderr file + ) + +output_ignore_args = ' '.join(["-I '"+s+"'" for s in output_ignore_regexes]) + +output_ignore_args += ' --exclude=stats.txt --exclude=outdiff' + +def run_test(target, source, env): """Check output from running test. Targets are as follows: - target[0] : outdiff - target[1] : statsdiff - target[2] : status + target[0] : status + + Sources are: + source[0] : gem5 binary + source[1] : tests/run.py script + source[2] : reference stats file """ # make sure target files are all gone for t in target: if os.path.exists(t.abspath): - Execute(Delete(t.abspath)) - # Run diff on output & ref directories to find differences. - # Exclude m5stats.txt since we will use diff-out on that. - Execute(env.subst('diff -ubr ${SOURCES[0].dir} ${SOURCES[1].dir} ' + - '-I "^command line:" ' + # for stdout file - '-I "^M5 compiled on" ' + # for stderr file - '-I "^M5 simulation started" ' + # for stderr file - '-I "^Simulation complete at" ' + # for stderr file - '-I "^Listening for" ' + # for stderr file - '--exclude=m5stats.txt --exclude=SCCS ' + - '--exclude=${TARGETS[0].file} ' + - '> ${TARGETS[0]}', target=target, source=source), None) - print "===== Output differences =====" - print contents(target[0]) - # Run diff-out on m5stats.txt file - status = Execute(env.subst('$DIFFOUT $SOURCES > ${TARGETS[1]}', - target=target, source=source), - strfunction=None) - print "===== Statistics differences =====" - print contents(target[1]) - # Generate status file contents based on exit status of diff-out + env.Execute(Delete(t.abspath)) + + tgt_dir = os.path.dirname(str(target[0])) + + # Base command for running test. We mess around with indirectly + # referring to files via SOURCES and TARGETS so that scons can mess + # with paths all it wants to and we still get the right files. + cmd = '${SOURCES[0]} -d %s -re ${SOURCES[1]} %s' % (tgt_dir, tgt_dir) + + # Prefix test run with batch job submission command if appropriate. + # Batch command also supports timeout arg (in seconds, not minutes). + timeout = 15 * 60 # used to be a param, probably should be again + if env['BATCH']: + cmd = '%s -t %d %s' % (env['BATCH_CMD'], timeout, cmd) + + # Create a default value for the status string, changed as needed + # based on the status. + status_str = "passed." + + pre_exec_time = time.time() + status = env.Execute(env.subst(cmd, target=target, source=source)) if status == 0: - status_str = "passed." - else: + # gem5 terminated normally. + # Run diff on output & ref directories to find differences. + # Exclude the stats file since we will use diff-out on that. + + # NFS file systems can be annoying and not have updated yet + # wait until we see the file modified + statsdiff = os.path.join(tgt_dir, 'statsdiff') + m_time = 0 + nap = 0 + while m_time < pre_exec_time and nap < 10: + try: + m_time = os.stat(statsdiff).st_mtime + except OSError: + pass + time.sleep(1) + nap += 1 + + outdiff = os.path.join(tgt_dir, 'outdiff') + # tack 'true' on the end so scons doesn't report diff's + # non-zero exit code as a build error + diffcmd = 'diff -ubrs %s ${SOURCES[2].dir} %s > %s; true' \ + % (output_ignore_args, tgt_dir, outdiff) + env.Execute(env.subst(diffcmd, target=target, source=source)) + print "===== Output differences =====" + print contents(outdiff) + # Run diff-out on stats.txt file + diffcmd = '$DIFFOUT ${SOURCES[2]} %s > %s' \ + % (os.path.join(tgt_dir, 'stats.txt'), statsdiff) + diffcmd = env.subst(diffcmd, target=target, source=source) + diff_status = env.Execute(diffcmd, strfunction=None) + # If there is a difference, change the status string to say so + if diff_status != 0: + status_str = "CHANGED!" + print "===== Statistics differences =====" + print contents(statsdiff) + + else: # gem5 exit status != 0 + # Consider it a failed test unless the exit status is 2 status_str = "FAILED!" - f = file(str(target[2]), 'w') - print >>f, env.subst('${TARGETS[2].dir}', target=target, source=source), \ - status_str + # gem5 did not terminate properly, so no need to check the output + if signaled(status): + print 'gem5 terminated with signal', signum(status) + if signum(status) in retry_signals: + # Consider the test incomplete; don't create a 'status' output. + # Hand the return status to scons and let scons decide what + # to do about it (typically terminate unless run with -k). + return status + elif status == 2: + # The test was skipped, change the status string to say so + status_str = "skipped." + else: + print 'gem5 exited with non-zero status', status + # complete but failed execution (call to exit() with non-zero + # status, SIGABORT due to assertion failure, etc.)... fall through + # and generate FAILED status as if output comparison had failed + + # Generate status file contents based on exit status of gem5 and diff-out + f = file(str(target[0]), 'w') + print >>f, tgt_dir, status_str f.close() # done return 0 -def check_test_string(target, source, env): - return env.subst("Comparing outputs in ${TARGETS[0].dir}.", +def run_test_string(target, source, env): + return env.subst("Running test in ${TARGETS[0].dir}.", target=target, source=source) -testAction = env.Action(check_test, check_test_string) +testAction = env.Action(run_test, run_test_string) def print_test(target, source, env): - print '***** ' + contents(source[0]) + # print the status with colours to make it easier to see what + # passed and what failed + line = contents(source[0]) + + # split the line to words and get the last one + words = line.split() + status = words[-1] + + # if the test failed make it red, if it passed make it green, and + # skip the punctuation + if status == "FAILED!": + status = termcap.Red + status[:-1] + termcap.Normal + status[-1] + elif status == "CHANGED!": + status = termcap.Yellow + status[:-1] + termcap.Normal + status[-1] + elif status == "passed.": + status = termcap.Green + status[:-1] + termcap.Normal + status[-1] + elif status == "skipped.": + status = termcap.Cyan + status[:-1] + termcap.Normal + status[-1] + + # put it back in the list and join with space + words[-1] = status + line = " ".join(words) + + print '***** ' + line return 0 printAction = env.Action(print_test, strfunction = None) +# Static vars for update_test: +# - long-winded message about ignored sources +ignore_msg = ''' +Note: The following file(s) will not be copied. New non-standard + output files must be copied manually once before --update-ref will + recognize them as outputs. Otherwise they are assumed to be + inputs and are ignored. +''' +# - reference files always needed +needed_files = set(['simout', 'simerr', 'stats.txt', 'config.ini']) +# - source files we always want to ignore +known_ignores = set(['status', 'outdiff', 'statsdiff']) + def update_test(target, source, env): """Update reference test outputs. - Target is phony. First two sources are the ref & new m5stats.txt + Target is phony. First two sources are the ref & new stats.txt file files, respectively. We actually copy everything in the respective directories except the status & diff output files. """ dest_dir = str(source[0].get_dir()) src_dir = str(source[1].get_dir()) - dest_files = os.listdir(dest_dir) - src_files = os.listdir(src_dir) - # Exclude status & diff outputs - for f in ('outdiff', 'statsdiff', 'status'): - if f in src_files: - src_files.remove(f) - for f in src_files: + dest_files = set(os.listdir(dest_dir)) + src_files = set(os.listdir(src_dir)) + # Copy all of the required files plus any existing dest files. + wanted_files = needed_files | dest_files + missing_files = wanted_files - src_files + if len(missing_files) > 0: + print " WARNING: the following file(s) are missing " \ + "and will not be updated:" + print " ", " ,".join(missing_files) + copy_files = wanted_files - missing_files + warn_ignored_files = (src_files - copy_files) - known_ignores + if len(warn_ignored_files) > 0: + print ignore_msg, + print " ", ", ".join(warn_ignored_files) + for f in copy_files: if f in dest_files: print " Replacing file", f dest_files.remove(f) @@ -124,13 +255,7 @@ def update_test(target, source, env): print " Creating new file", f copyAction = Copy(os.path.join(dest_dir, f), os.path.join(src_dir, f)) copyAction.strfunction = None - Execute(copyAction) - # warn about any files in dest not overwritten (other than SCCS dir) - if 'SCCS' in dest_files: - dest_files.remove('SCCS') - if dest_files: - print "Warning: file(s) in", dest_dir, "not updated:", - print ', '.join(dest_files) + env.Execute(copyAction) return 0 def update_test_string(target, source, env): @@ -142,49 +267,27 @@ updateAction = env.Action(update_test, update_test_string) def test_builder(env, ref_dir): """Define a test.""" - (category, name, _ref, isa, opsys, config) = ref_dir.split('/') + (category, mode, name, _ref, isa, opsys, config) = ref_dir.split('/') assert(_ref == 'ref') # target path (where test output goes) is the same except without # the 'ref' component - tgt_dir = os.path.join(category, name, isa, opsys, config) + tgt_dir = os.path.join(category, mode, name, isa, opsys, config) # prepend file name with tgt_dir def tgt(f): return os.path.join(tgt_dir, f) - ref_stats = os.path.join(ref_dir, 'm5stats.txt') - new_stats = tgt('m5stats.txt') + ref_stats = os.path.join(ref_dir, 'stats.txt') + new_stats = tgt('stats.txt') status_file = tgt('status') - # Base command for running test. We mess around with indirectly - # referring to files via SOURCES and TARGETS so that scons can - # mess with paths all it wants to and we still get the right - # files. - base_cmd = '${SOURCES[0]} -d $TARGET.dir ${SOURCES[1]} %s' % tgt_dir - # stdout and stderr files - cmd_stdout = '${TARGETS[0]}' - cmd_stderr = '${TARGETS[1]}' - - # Prefix test run with batch job submission command if appropriate. - # Output redirection is also different for batch runs. - # Batch command also supports timeout arg (in seconds, not minutes). - if env['BATCH']: - cmd = [env['BATCH_CMD'], '-t', str(timeout * 60), - '-o', cmd_stdout, '-e', cmd_stderr, base_cmd] - else: - cmd = [base_cmd, '>', cmd_stdout, '2>', cmd_stderr] - - env.Command([tgt('stdout'), tgt('stderr'), new_stats], - [env.M5Binary, 'run.py'], ' '.join(cmd)) - - # order of targets is important... see check_test - env.Command([tgt('outdiff'), tgt('statsdiff'), status_file], - [ref_stats, new_stats], + env.Command([status_file, new_stats], + [env.M5Binary, 'run.py', ref_stats], testAction) # phony target to echo status - if env['update_ref']: + if GetOption('update_ref'): p = env.Command(tgt('_update'), [ref_stats, new_stats, status_file], updateAction) @@ -194,10 +297,56 @@ def test_builder(env, ref_dir): env.AlwaysBuild(p) +# Figure out applicable configs based on build type +configs = [] +if env['TARGET_ISA'] == 'alpha': + configs += ['tsunami-simple-atomic', + 'tsunami-simple-timing', + 'tsunami-simple-atomic-dual', + 'tsunami-simple-timing-dual', + 'twosys-tsunami-simple-atomic', + 'tsunami-o3', 'tsunami-o3-dual', + 'tsunami-inorder', + 'tsunami-switcheroo-full'] +if env['TARGET_ISA'] == 'sparc': + configs += ['t1000-simple-atomic', + 't1000-simple-timing'] +if env['TARGET_ISA'] == 'arm': + configs += ['simple-atomic-dummychecker', + 'o3-timing-checker', + 'realview-simple-atomic', + 'realview-simple-atomic-dual', + 'realview-simple-timing', + 'realview-simple-timing-dual', + 'realview-o3', + 'realview-o3-checker', + 'realview-o3-dual', + 'realview-switcheroo-atomic', + 'realview-switcheroo-timing', + 'realview-switcheroo-o3', + 'realview-switcheroo-full'] +if env['TARGET_ISA'] == 'x86': + configs += ['pc-simple-atomic', + 'pc-simple-timing', + 'pc-o3-timing', + 'pc-switcheroo-full'] + +configs += ['simple-atomic', 'simple-timing', 'o3-timing', 'memtest', + 'simple-atomic-mp', 'simple-timing-mp', 'o3-timing-mp', + 'inorder-timing', 'rubytest', 'tgen-simple-mem', + 'tgen-simple-dram'] + +if env['PROTOCOL'] != 'None': + if env['PROTOCOL'] == 'MI_example': + configs += [c + "-ruby" for c in configs] + else: + configs = [c + "-ruby-" + env['PROTOCOL'] for c in configs] + cwd = os.getcwd() os.chdir(str(Dir('.').srcdir)) -for config in ['simple-atomic']: - dirs = glob.glob('*/*/ref/%s/*/%s' % (env['TARGET_ISA'], config)) +for config in configs: + dirs = glob.glob('*/*/*/ref/%s/*/%s' % (env['TARGET_ISA'], config)) for d in dirs: - test_builder(env, d) + if not os.path.exists(os.path.join(d, 'skip')): + test_builder(env, d) os.chdir(cwd)