From: Gabe Black Date: Tue, 4 Aug 2020 05:39:38 +0000 (-0700) Subject: tests: Get rid of the now unused diff-out script. X-Git-Tag: v20.1.0.0~347 X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=31921a8ac26fa15b52a40f124db8557f1db8b0b6;p=gem5.git tests: Get rid of the now unused diff-out script. This script had been used to compare the output of gem5 regression tests to a golden reference, but all the tests that used it have been deleted. Change-Id: Ib65e4271ce8081dd5994b412ac2240869ab02d44 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/32120 Tested-by: kokoro Reviewed-by: Jason Lowe-Power Maintainer: Bobby R. Bruce --- diff --git a/tests/diff-out b/tests/diff-out deleted file mode 100755 index 6b0b23984..000000000 --- a/tests/diff-out +++ /dev/null @@ -1,374 +0,0 @@ -#!/usr/bin/perl -# Copyright (c) 2001-2005 The Regents of The University of Michigan -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer; -# redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution; -# neither the name of the copyright holders nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# -# This script diffs two SimpleScalar statistics output files. -# - -use Getopt::Std; - -getopts('adn:t:h'); - -if ($#ARGV < 1) -{ - print "\nError: need two file arguments ( ).\n"; - print " Options: -d = Ignore distributions\n"; - print " -a = Sort errors alphabetically (default: by percentage)\n"; - print " -h = Diff header info separately from stats\n"; - print " -n = Print top errors (default 20, 0 for all)\n"; - print " -t = Ignore errors below percent (default 0)\n\n"; - exit; -} - -open(REF, "<$ARGV[0]") or die "Error: can't open $ARGV[0].\n"; -open(NEW, "<$ARGV[1]") or die "Error: can't open $ARGV[1].\n"; - - -# -# Things that really should be adjustable via the command line -# - -# Ignorable error (in percent) -$err_thresh = defined($opt_t) ? $opt_t : 0; - -# Number of stats to print before omitting -$omit_count = defined($opt_n) ? $opt_n : 20; - - -# -# First copy everything up to the simulation statistics to a pair of -# temporary files, stripping out date-related items, and do a plain -# diff. Any differences in the arguments are not necessarily an issue; -# any differences in the program output should be caught by the EIO -# mechanism if an EIO file is used. -# - -# copy_header takes input filehandle and output filename - -sub copy_header -{ - my ($inhandle, $outname) = @_; - - open(OUTPUT, ">$outname") or die "Error: can't open $outname.\n"; - - while (<$inhandle>) - { - # strip out lines that can vary - next if /^(command line:|M5 compiled on |M5 simulation started |M5 executing on )/; - last if /Begin Simulation Statistics/; - print OUTPUT; - } - close OUTPUT; -} - -if ($opt_h) { - - # Diff header separately from stats - - $refheader = "/tmp/smt-test.refheader.$$"; - $newheader = "/tmp/smt-test.newheader.$$"; - - copy_header(\*REF, $refheader); - copy_header(\*NEW, $newheader); - - print "\n===== Header and program output differences =====\n\n"; - - print `diff $refheader $newheader`; - - print "\n===== Statistics differences =====\n\n"; -} - -# -# Now parse statistics -# - -# -# This function takes an open filehandle and returns a reference to -# a hash containing all the statistics variables and their values. -# -sub parse_file -{ - $stathandle = shift; - - $in_dist = undef; - $hashref = { }; # initialize hash for values - - while (<$stathandle>) - { - next if /^\s*$/; # skip blank lines - last if /End Simulation Statistics/; - - s/ *#.*//; # strip comments - - if (/^Memory usage: (\d+) KBytes/) { - $stat = 'memory usage'; - $value = $1; - } - elsif ($in_dist) { - if (/(.*)\.end_dist/) { - # end line of distribution: clear $in_dist flag - $in_dist = undef; - next; - } - if ($opt_d) { - next; # bail out if we are ignoring dists... - } elsif (/(.*)\.(min|max)_value/) { - # treat these like normal stats - ($stat, $value) = /^(\S+)\s+(.*)/; - } else { - ($stat, $value) = - /^(\S+(?:.*\S)?)\s+(\d+)\s+\d+\.\d+%/; - $stat = $in_dist . '::' . $stat; - } - } - else { - if (/(.*)\.start_dist/) { - # start line of distribution: set $in_dist flag - # and save distribution name for future reference - $in_dist = $1; - $stat = $1; - $value = 0; - } - else { - ($stat, $value) = /^(\S+)\s+(.*)/; - } - } - - $$hashref{$stat} = $value; - } - - close($stathandle); - return $hashref; -} - - -# -# pct_diff($old, $new) returns percent difference from $old to $new. -# -sub pct_diff -{ - my ($old, $new) = @_; - return ($old == 0) ? (($new == 0) ? 0 : 9999) : 100 * ($new - $old) / $old; -} - - -# -# Statistics to ignore: these relate to simulator performance, not -# correctness, so don't fail on changes here. -# -%ignore = ( - 'host_seconds' => 1, - 'host_tick_rate' => 1, - 'host_inst_rate' => 1, - 'host_op_rate' => 1, - 'host_mem_usage' => 1 -); - -# -# List of key statistics (always displayed) -# ==> list stats here WITHOUT trailing thread ID -# -@key_stat_list = ( - 'ipc', - 'committedInsts', - 'committedOps', - 'sim_insts', - 'sim_ops', - 'sim_ticks', - 'host_inst_rate', - 'host_mem_usage' -); - -$key_stat_pattern = join('|', @key_stat_list); - -# initialize first statistics from each file - -$max_err_mag = 0; - -$refhash = parse_file(\*REF); -$newhash = parse_file(\*NEW); - -# The string sim-smt prints on a divide by zero -$divbyzero = ''; - -foreach $stat (sort keys %$refhash) -{ - $refvalue = $$refhash{$stat}; - $newvalue = $$newhash{$stat}; - - if (!defined($newvalue)) { - # stat missing from new file - push @missing_stats, $stat; - next; - } - - if ($stat =~ /($key_stat_pattern)/o) { - # key statistics: always record & display changes in these - push @key_stats, [$stat, $refvalue, $newvalue]; - } - - if ($ignore{$stat} or $refvalue eq $newvalue) { - # stat is in "ignore" list, or hasn't changed - } - else { - if ($refvalue eq $divbyzero || $newvalue eq $divbyzero) { - # one or the other was a divide by zero: - # no point in trying to quantify error - print "$stat: $refvalue --> $newvalue\n"; - } - else { - $reldiff = pct_diff($refvalue, $newvalue); - $diffmag = abs($reldiff); - - if ($diffmag > $err_thresh) { - push @errs, - [$stat, $refvalue, $newvalue, $reldiff]; - } - - if ($diffmag > $max_err_mag) { - $max_err_mag = $diffmag; - } - } - } - - # remove from new hash so we can detect added stats - delete $$newhash{$stat}; -} - - -# -# All done. Print comparison summary. -# - -printf("Maximum error magnitude: %+f%%\n\n", $max_err_mag); - -printf(" %-30s %10s %10s %10s %7s\n", ' ', 'Reference', 'New Value', 'Abs Diff', 'Pct Chg'); - -printf("Key statistics:\n\n"); - -foreach $key_stat (@key_stats) -{ - ($statname, $refvalue, $newvalue, $reldiff) = @$key_stat; - - # deduce format from reference value - $pointpos = rindex($refvalue, '.'); - $digits = ($pointpos < 0) ? 0 :(length($refvalue) - $pointpos - 1); - $fmt = "%10.${digits}f"; - - # print differing values with absolute and relative error - printf(" %-30s $fmt $fmt $fmt %+7.2f%%\n", - $statname, $refvalue, $newvalue, - $newvalue - $refvalue, pct_diff($refvalue, $newvalue)); -} - -printf("\nDifferences > %d%%:\n\n", $err_thresh); - -if ($opt_a) { - # leave stats sorted alphabetically, doesn't make sense to cut them off - $omit_count = 0; -} else { - # sort differences by percent change - @errs = sort { abs($$b[3]) <=> abs($$a[3]) } @errs; -} - -$num_errs = 0; - -foreach $err (@errs) -{ - ($statname, $refvalue, $newvalue, $reldiff) = @$err; - - # deduce format from reference value - $pointpos1 = rindex($refvalue, '.'); - $digits1 = ($pointpos1 < 0) ? 0 :(length($refvalue) - $pointpos1 - 1); - $pointpos2 = rindex($newvalue, '.'); - $digits2 = ($pointpos2 < 0) ? 0 :(length($newvalue) - $pointpos2 - 1); - $digits = ($digits1 > $digits2) ? $digits1 : $digits2; - $fmt = "%10.${digits}f"; - - # print differing values with absolute and relative error - printf(" %-30s $fmt $fmt $fmt %+7.2f%%\n", - $statname, $refvalue, $newvalue, $newvalue - $refvalue, $reldiff); - - # only print top N errors - if ($omit_count > 0 && ++$num_errs >= $omit_count) - { - print "[... showing top $omit_count errors only, additional errors omitted ...]\n"; - last; - } -} - -# -# Report missing stats -# -# get count -$missing_stats = scalar(@missing_stats); - -if ($missing_stats) -{ - print "\nMissing $missing_stats reference statistics:\n\n"; - foreach $stat (@missing_stats) - { -# print "\t$stat\n"; - printf " %-50s ", $stat; - print "$$refhash{$stat}\n"; - } -} - -# -# Any stats left in newhash are added since the reference file -# - -@added_stats = keys %$newhash; - -# get count -$added_stats = scalar(@added_stats); - -if ($added_stats) -{ - print "\nFound $added_stats new statistics:\n\n"; - foreach $stat (sort @added_stats) - { -# print "\t$stat\n"; - printf " %-50s ", $stat; - print "$$newhash{$stat}\n"; - } -} - -cleanup(); -# Exit codes: -# 0 if all stats are found (with no extras) & no stats error -# 1 if there are additional stats, but no stat errors -# 2 otherwise -$no_hard_errors = $missing_stats == 0 && $max_err_mag == 0.0; -$status = $no_hard_errors ? ($added_stats == 0 ? 0 : 1) : 2; -exit $status; - -sub cleanup -{ - unlink($refheader) if ($refheader); - unlink($newheader) if ($newheader); -} diff --git a/tests/legacy-configs/run.py b/tests/legacy-configs/run.py deleted file mode 100644 index eaf43404e..000000000 --- a/tests/legacy-configs/run.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright (c) 2017 Mark D. Hill and David A. Wood -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer; -# redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution; -# neither the name of the copyright holders nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -''' -New version of the run.py script. For this, all dependencies should be -handled outside of the script. - -.. warning:: This script is NOT the recommended way to handle configurations - for new tests. This exists for legacy support only. New Tests should - either use configs from the normal gem5 configs or create their own for - a test. -''' -import argparse -import sys -import os -from os.path import abspath, join as joinpath, dirname - -import m5 - -# Add the normal gem5 config path to system path. -# This requirement should be removed if possible from all legacy scripts, but -# I've left it here for now. -sys.path.insert(0, abspath(joinpath(dirname(__file__), '../../configs'))) - -# set default maxtick... script can override -# -1 means run forever -maxtick = m5.MaxTick - -def run_test(root): - """Default run_test implementations. Scripts can override it.""" - - # instantiate configuration - m5.instantiate() - - # simulate until program terminates - exit_event = m5.simulate(maxtick) - print 'Exiting @ tick', m5.curTick(), 'because', exit_event.getCause() - -test_progs = os.environ.get('M5_TEST_PROGS', '/dist/m5/regression/test-progs') - -# Since we're in batch mode, dont allow tcp socket connections -m5.disableAllListeners() - -parser = argparse.ArgumentParser() -parser.add_argument('--cmd', - action='store', - type=str, - help='Command to pass to the test system') -parser.add_argument('--executable', - action='store', - type=str, - help='Executable to pass to the test system') -parser.add_argument('--config', - action='append', - type=str, - help='A config file to initialize the system with.'\ - + ' If more than one given, loads them in order given.') -args = parser.parse_args() - -executable = args.executable - -for config in args.config: - exec(compile(open(config).read(), config, 'exec')) - -# Initialize all CPUs in a system -def initCPUs(sys): - def initCPU(cpu): - # We might actually have a MemTest object or something similar - # here that just pretends to be a CPU. - try: - cpu.createThreads() - except: - pass - - # The CPU attribute doesn't exist in some cases, e.g. the Ruby testers. - if not hasattr(sys, "cpu"): - return - - # The CPU can either be a list of CPUs or a single object. - if isinstance(sys.cpu, list): - [ initCPU(cpu) for cpu in sys.cpu ] - else: - initCPU(sys.cpu) - -# TODO: Might want to automatically place the cmd and executable on the -# cpu[0].workload, although I think most legacy configs do this automatically -# or somewhere in their `test.py` config. - - -# We might be creating a single system or a dual system. Try -# initializing the CPUs in all known system attributes. -for sysattr in [ "system", "testsys", "drivesys" ]: - if hasattr(root, sysattr): - initCPUs(getattr(root, sysattr)) - -run_test(root)