tests/diff-out

   1 #!/usr/bin/perl
   2 # Copyright (c) 2001-2005 The Regents of The University of Michigan
   3 # All rights reserved.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met: redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer;
   9 # redistributions in binary form must reproduce the above copyright
  10 # notice, this list of conditions and the following disclaimer in the
  11 # documentation and/or other materials provided with the distribution;
  12 # neither the name of the copyright holders nor the names of its
  13 # contributors may be used to endorse or promote products derived from
  14 # this software without specific prior written permission.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27 #
  28 # Authors: Steve Reinhardt
  29
  30 #
  31 # This script diffs two SimpleScalar statistics output files.
  32 #
  33
  34 use Getopt::Std;
  35
  36 getopts('adn:t:h');
  37
  38 if ($#ARGV < 1)
  39 {
  40     print "\nError: need two file arguments (<reference> <new>).\n";
  41     print "   Options: -d = Ignore distributions\n";
  42     print "            -a = Sort errors alphabetically (default: by percentage)\n";
  43     print "            -h = Diff header info separately from stats\n";
  44     print "            -n <num> = Print top <num> errors (default 20, 0 for all)\n";
  45     print "            -t <num> = Ignore errors below <num> percent (default 0)\n\n";
  46     exit;
  47 }
  48
  49 open(REF, "<$ARGV[0]") or die "Error: can't open $ARGV[0].\n";
  50 open(NEW, "<$ARGV[1]") or die "Error: can't open $ARGV[1].\n";
  51
  52
  53 #
  54 # Things that really should be adjustable via the command line
  55 #
  56
  57 # Ignorable error (in percent)
  58 $err_thresh = defined($opt_t) ? $opt_t : 0;
  59
  60 # Number of stats to print before omitting
  61 $omit_count = defined($opt_n) ? $opt_n : 20;
  62
  63
  64 #
  65 # First copy everything up to the simulation statistics to a pair of
  66 # temporary files, stripping out date-related items, and do a plain
  67 # diff.  Any differences in the arguments are not necessarily an issue;
  68 # any differences in the program output should be caught by the EIO
  69 # mechanism if an EIO file is used.
  70 #
  71
  72 # copy_header takes input filehandle and output filename
  73
  74 sub copy_header
  75 {
  76     my ($inhandle, $outname) = @_;
  77
  78     open(OUTPUT, ">$outname") or die "Error: can't open $outname.\n";
  79
  80     while (<$inhandle>)
  81     {
  82         # strip out lines that can vary
  83         next if /^(command line:|M5 compiled on |M5 simulation started |M5 executing on )/;
  84         last if /Begin Simulation Statistics/;
  85         print OUTPUT;
  86     }
  87     close OUTPUT;
  88 }
  89
  90 if ($opt_h) {
  91
  92     # Diff header separately from stats
  93
  94     $refheader = "/tmp/smt-test.refheader.$$";
  95     $newheader = "/tmp/smt-test.newheader.$$";
  96
  97     copy_header(\*REF, $refheader);
  98     copy_header(\*NEW, $newheader);
  99
 100     print "\n===== Header and program output differences =====\n\n";
 101
 102     print `diff $refheader $newheader`;
 103
 104     print "\n===== Statistics differences =====\n\n";
 105 }
 106
 107 #
 108 # Now parse statistics
 109 #
 110
 111 #
 112 # This function takes an open filehandle and returns a reference to
 113 # a hash containing all the statistics variables and their values.
 114 #
 115 sub parse_file
 116 {
 117     $stathandle = shift;
 118
 119     $in_dist = undef;
 120     $hashref = { };     # initialize hash for values
 121
 122     while (<$stathandle>)
 123     {
 124         next if /^\s*$/;        # skip blank lines
 125         last if /End Simulation Statistics/;
 126
 127         s/ *#.*//;              # strip comments
 128
 129         if (/^Memory usage: (\d+) KBytes/) {
 130             $stat = 'memory usage';
 131             $value = $1;
 132         }
 133         elsif ($in_dist) {
 134             if (/(.*)\.end_dist/) {
 135                 # end line of distribution: clear $in_dist flag
 136                 $in_dist = undef;
 137                 next;
 138             }
 139             if ($opt_d) {
 140                 next;           #  bail out if we are ignoring dists...
 141             } elsif (/(.*)\.(min|max)_value/) {
 142                 # treat these like normal stats
 143                 ($stat, $value) = /^(\S+)\s+(.*)/;
 144             } else {
 145                 ($stat, $value) =
 146                   /^(\S+(?:.*\S)?)\s+(\d+)\s+\d+\.\d+%/;
 147                 $stat = $in_dist . '::' . $stat;
 148             }
 149         }
 150         else {
 151             if (/(.*)\.start_dist/) {
 152                 # start line of distribution: set $in_dist flag
 153                 # and save distribution name for future reference
 154                 $in_dist = $1;
 155                 $stat = $1;
 156                 $value = 0;
 157             }
 158             else {
 159                 ($stat, $value) = /^(\S+)\s+(.*)/;
 160             }
 161         }
 162
 163         $$hashref{$stat} = $value;
 164     }
 165
 166     close($stathandle);
 167     return $hashref;
 168 }
 169
 170
 171 #
 172 # pct_diff($old, $new) returns percent difference from $old to $new.
 173 #
 174 sub pct_diff
 175 {
 176     my ($old, $new) = @_;
 177     return ($old == 0) ? (($new == 0) ? 0 : 9999) : 100 * ($new - $old) / $old;
 178 }
 179
 180
 181 #
 182 # Statistics to ignore: these relate to simulator performance, not
 183 # correctness, so don't fail on changes here.
 184 #
 185 %ignore = (
 186   'host_seconds' => 1,
 187   'host_tick_rate' => 1,
 188   'host_inst_rate' => 1,
 189   'host_op_rate' => 1,
 190   'host_mem_usage' => 1
 191 );
 192
 193 #
 194 # List of key statistics (always displayed)
 195 #  ==> list stats here WITHOUT trailing thread ID
 196 #
 197 @key_stat_list = (
 198   'ipc',
 199   'committedInsts',
 200   'committedOps',
 201   'sim_insts',
 202   'sim_ops',
 203   'sim_ticks',
 204   'host_inst_rate',
 205   'host_mem_usage'
 206 );
 207
 208 $key_stat_pattern = join('|', @key_stat_list);
 209
 210 # initialize first statistics from each file
 211
 212 $max_err_mag = 0;
 213
 214 $refhash = parse_file(\*REF);
 215 $newhash = parse_file(\*NEW);
 216
 217 # The string sim-smt prints on a divide by zero
 218 $divbyzero = '<err: divide by zero>';
 219
 220 foreach $stat (sort keys %$refhash)
 221 {
 222     $refvalue = $$refhash{$stat};
 223     $newvalue = $$newhash{$stat};
 224
 225     if (!defined($newvalue)) {
 226         # stat missing from new file
 227         push @missing_stats, $stat;
 228         next;
 229     }
 230
 231     if ($stat =~ /($key_stat_pattern)/o) {
 232         # key statistics: always record & display changes in these
 233         push @key_stats, [$stat, $refvalue, $newvalue];
 234     }
 235
 236     if ($ignore{$stat} or $refvalue eq $newvalue) {
 237         # stat is in "ignore" list, or hasn't changed
 238     }
 239     else {
 240         if ($refvalue eq $divbyzero || $newvalue eq $divbyzero) {
 241             # one or the other was a divide by zero:
 242             # no point in trying to quantify error
 243             print "$stat: $refvalue --> $newvalue\n";
 244         }
 245         else {
 246             $reldiff = pct_diff($refvalue, $newvalue);
 247             $diffmag = abs($reldiff);
 248
 249             if ($diffmag > $err_thresh) {
 250                 push @errs,
 251                 [$stat, $refvalue, $newvalue, $reldiff];
 252             }
 253
 254             if ($diffmag > $max_err_mag) {
 255                 $max_err_mag = $diffmag;
 256             }
 257         }
 258     }
 259
 260     # remove from new hash so we can detect added stats
 261     delete $$newhash{$stat};
 262 }
 263
 264
 265 #
 266 # All done.  Print comparison summary.
 267 #
 268
 269 printf("Maximum error magnitude: %+f%%\n\n", $max_err_mag);
 270
 271 printf("  %-30s %10s %10s %10s   %7s\n", ' ', 'Reference', 'New Value', 'Abs Diff', 'Pct Chg');
 272
 273 printf("Key statistics:\n\n");
 274
 275 foreach $key_stat (@key_stats)
 276 {
 277     ($statname, $refvalue, $newvalue, $reldiff) = @$key_stat;
 278
 279     # deduce format from reference value
 280     $pointpos = rindex($refvalue, '.');
 281     $digits = ($pointpos < 0) ? 0 :(length($refvalue) - $pointpos - 1);
 282     $fmt = "%10.${digits}f";
 283
 284     # print differing values with absolute and relative error
 285     printf("  %-30s $fmt $fmt $fmt  %+7.2f%%\n",
 286            $statname, $refvalue, $newvalue,
 287            $newvalue - $refvalue, pct_diff($refvalue, $newvalue));
 288 }
 289
 290 printf("\nDifferences > %d%%:\n\n", $err_thresh);
 291
 292 if ($opt_a) {
 293     # leave stats sorted alphabetically, doesn't make sense to cut them off
 294     $omit_count = 0;
 295 } else {
 296     # sort differences by percent change
 297     @errs = sort { abs($$b[3]) <=> abs($$a[3]) } @errs;
 298 }
 299
 300 $num_errs = 0;
 301
 302 foreach $err (@errs)
 303 {
 304     ($statname, $refvalue, $newvalue, $reldiff) = @$err;
 305
 306     # deduce format from reference value
 307     $pointpos1 = rindex($refvalue, '.');
 308     $digits1 = ($pointpos1 < 0) ? 0 :(length($refvalue) - $pointpos1 - 1);
 309     $pointpos2 = rindex($newvalue, '.');
 310     $digits2 = ($pointpos2 < 0) ? 0 :(length($newvalue) - $pointpos2 - 1);
 311     $digits = ($digits1 > $digits2) ? $digits1 : $digits2;
 312     $fmt = "%10.${digits}f";
 313
 314     # print differing values with absolute and relative error
 315     printf("  %-30s $fmt $fmt $fmt  %+7.2f%%\n",
 316            $statname, $refvalue, $newvalue, $newvalue - $refvalue, $reldiff);
 317
 318     # only print top N errors
 319     if ($omit_count > 0 && ++$num_errs >= $omit_count)
 320     {
 321         print "[... showing top $omit_count errors only, additional errors omitted ...]\n";
 322         last;
 323     }
 324 }
 325
 326 #
 327 # Report missing stats
 328 #
 329 # get count
 330 $missing_stats = scalar(@missing_stats);
 331
 332 if ($missing_stats)
 333 {
 334     print "\nMissing $missing_stats reference statistics:\n\n";
 335     foreach $stat (@missing_stats)
 336     {
 337 #       print "\t$stat\n";
 338         printf "  %-50s    ", $stat;
 339         print  "$$refhash{$stat}\n";
 340     }
 341 }
 342
 343 #
 344 # Any stats left in newhash are added since the reference file
 345 #
 346
 347 @added_stats = keys %$newhash;
 348
 349 # get count
 350 $added_stats = scalar(@added_stats);
 351
 352 if ($added_stats)
 353 {
 354     print "\nFound $added_stats new statistics:\n\n";
 355     foreach $stat (sort @added_stats)
 356     {
 357 #       print "\t$stat\n";
 358         printf "  %-50s    ", $stat;
 359         print  "$$newhash{$stat}\n";
 360     }
 361 }
 362
 363 cleanup();
 364 # Exit codes:
 365 # 0 if all stats are found (with no extras) & no stats error
 366 # 1 if there are additional stats, but no stat errors
 367 # 2 otherwise
 368 $no_hard_errors = $missing_stats == 0 && $max_err_mag == 0.0;
 369 $status = $no_hard_errors ? ($added_stats == 0 ? 0 : 1) : 2;
 370 exit $status;
 371
 372 sub cleanup
 373 {
 374     unlink($refheader) if ($refheader);
 375     unlink($newheader) if ($newheader);
 376 }