tests/diff-out

   1 #!/usr/bin/perl
   2 # Copyright (c) 2001-2005 The Regents of The University of Michigan
   3 # All rights reserved.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met: redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer;
   9 # redistributions in binary form must reproduce the above copyright
  10 # notice, this list of conditions and the following disclaimer in the
  11 # documentation and/or other materials provided with the distribution;
  12 # neither the name of the copyright holders nor the names of its
  13 # contributors may be used to endorse or promote products derived from
  14 # this software without specific prior written permission.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27 #
  28 # Authors: Steve Reinhardt
  29
  30 #
  31 # This script diffs two SimpleScalar statistics output files.
  32 #
  33
  34 use Getopt::Std;
  35
  36 getopts('adn:t:h');
  37
  38 if ($#ARGV < 1)
  39 {
  40     print "\nError: need two file arguments (<reference> <new>).\n";
  41     print "   Options: -d = Ignore distributions\n";
  42     print "            -a = Sort errors alphabetically (default: by percentage)\n";
  43     print "            -h = Diff header info separately from stats\n";
  44     print "            -n <num> = Print top <num> errors (default 20, 0 for all)\n";
  45     print "            -t <num> = Ignore errors below <num> percent (default 0)\n\n";
  46     exit;
  47 }
  48
  49 open(REF, "<$ARGV[0]") or die "Error: can't open $ARGV[0].\n";
  50 open(NEW, "<$ARGV[1]") or die "Error: can't open $ARGV[1].\n";
  51
  52
  53 #
  54 # Things that really should be adjustable via the command line
  55 #
  56
  57 # Ignorable error (in percent)
  58 $err_thresh = defined($opt_t) ? $opt_t : 0;
  59
  60 # Number of stats to print before omitting
  61 $omit_count = defined($opt_n) ? $opt_n : 20;
  62
  63
  64 #
  65 # First copy everything up to the simulation statistics to a pair of
  66 # temporary files, stripping out date-related items, and do a plain
  67 # diff.  Any differences in the arguments are not necessarily an issue;
  68 # any differences in the program output should be caught by the EIO
  69 # mechanism if an EIO file is used.
  70 #
  71
  72 # copy_header takes input filehandle and output filename
  73
  74 sub copy_header
  75 {
  76     my ($inhandle, $outname) = @_;
  77
  78     open(OUTPUT, ">$outname") or die "Error: can't open $outname.\n";
  79
  80     while (<$inhandle>)
  81     {
  82         # strip out lines that can vary
  83         next if /^(command line:|M5 compiled on |M5 simulation started |M5 executing on )/;
  84         last if /Begin Simulation Statistics/;
  85         print OUTPUT;
  86     }
  87     close OUTPUT;
  88 }
  89
  90 if ($opt_h) {
  91
  92     # Diff header separately from stats
  93
  94     $refheader = "/tmp/smt-test.refheader.$$";
  95     $newheader = "/tmp/smt-test.newheader.$$";
  96
  97     copy_header(\*REF, $refheader);
  98     copy_header(\*NEW, $newheader);
  99
 100     print "\n===== Header and program output differences =====\n\n";
 101
 102     print `diff $refheader $newheader`;
 103
 104     print "\n===== Statistics differences =====\n\n";
 105 }
 106
 107 #
 108 # Now parse statistics
 109 #
 110
 111 #
 112 # This function takes an open filehandle and returns a reference to
 113 # a hash containing all the statistics variables and their values.
 114 #
 115 sub parse_file
 116 {
 117     $stathandle = shift;
 118
 119     $in_dist = undef;
 120     $hashref = { };     # initialize hash for values
 121
 122     while (<$stathandle>)
 123     {
 124         next if /^\s*$/;        # skip blank lines
 125         last if /End Simulation Statistics/;
 126
 127         s/ *#.*//;              # strip comments
 128
 129         if (/^Memory usage: (\d+) KBytes/) {
 130             $stat = 'memory usage';
 131             $value = $1;
 132         }
 133         elsif ($in_dist) {
 134             if (/(.*)\.end_dist/) {
 135                 # end line of distribution: clear $in_dist flag
 136                 $in_dist = undef;
 137                 next;
 138             }
 139             if ($opt_d) {
 140                 next;           #  bail out if we are ignoring dists...
 141             } elsif (/(.*)\.(min|max)_value/) {
 142                 # treat these like normal stats
 143                 ($stat, $value) = /^(\S+)\s+(.*)/;
 144             } else {
 145                 ($stat, $value) =
 146                   /^(\S+(?:.*\S)?)\s+(\d+)\s+\d+\.\d+%/;
 147                 $stat = $in_dist . '::' . $stat;
 148             }
 149         }
 150         else {
 151             if (/(.*)\.start_dist/) {
 152                 # start line of distribution: set $in_dist flag
 153                 # and save distribution name for future reference
 154                 $in_dist = $1;
 155                 $stat = $1;
 156                 $value = 0;
 157             }
 158             else {
 159                 ($stat, $value) = /^(\S+)\s+(.*)/;
 160             }
 161         }
 162
 163         $$hashref{$stat} = $value;
 164     }
 165
 166     close($stathandle);
 167     return $hashref;
 168 }
 169
 170
 171 #
 172 # pct_diff($old, $new) returns percent difference from $old to $new.
 173 #
 174 sub pct_diff
 175 {
 176     my ($old, $new) = @_;
 177     return ($old == 0) ? (($new == 0) ? 0 : 9999) : 100 * ($new - $old) / $old;
 178 }
 179
 180
 181 #
 182 # Statistics to ignore: these relate to simulator performance, not
 183 # correctness, so don't fail on changes here.
 184 #
 185 %ignore = (
 186   'host_seconds' => 1,
 187   'host_tick_rate' => 1,
 188   'host_inst_rate' => 1,
 189   'host_mem_usage' => 1
 190 );
 191
 192 #
 193 # List of key statistics (always displayed)
 194 #  ==> list stats here WITHOUT trailing thread ID
 195 #
 196 @key_stat_list = (
 197   'COM:IPC',
 198   'ISSUE:MSIPC',
 199   'COM:count',
 200   'host_inst_rate',
 201   'sim_insts',
 202   'sim_ticks',
 203   'host_mem_usage'
 204 );
 205
 206 $key_stat_pattern = join('|', @key_stat_list);
 207
 208 # initialize first statistics from each file
 209
 210 $max_err_mag = 0;
 211
 212 $refhash = parse_file(\*REF);
 213 $newhash = parse_file(\*NEW);
 214
 215 # The string sim-smt prints on a divide by zero
 216 $divbyzero = '<err: divide by zero>';
 217
 218 foreach $stat (sort keys %$refhash)
 219 {
 220     $refvalue = $$refhash{$stat};
 221     $newvalue = $$newhash{$stat};
 222
 223     if (!defined($newvalue)) {
 224         # stat missing from new file
 225         push @missing_stats, $stat;
 226         next;
 227     }
 228
 229     if ($stat =~ /($key_stat_pattern)/o) {
 230         # key statistics: always record & display changes in these
 231         push @key_stats, [$stat, $refvalue, $newvalue];
 232     }
 233
 234     if ($ignore{$stat} or $refvalue eq $newvalue) {
 235         # stat is in "ignore" list, or hasn't changed
 236     }
 237     else {
 238         if ($refvalue eq $divbyzero || $newvalue eq $divbyzero) {
 239             # one or the other was a divide by zero:
 240             # no point in trying to quantify error
 241             print "$stat: $refvalue --> $newvalue\n";
 242         }
 243         else {
 244             $reldiff = pct_diff($refvalue, $newvalue);
 245             $diffmag = abs($reldiff);
 246
 247             if ($diffmag > $err_thresh) {
 248                 push @errs,
 249                 [$stat, $refvalue, $newvalue, $reldiff];
 250             }
 251
 252             if ($diffmag > $max_err_mag) {
 253                 $max_err_mag = $diffmag;
 254             }
 255         }
 256     }
 257
 258     # remove from new hash so we can detect added stats
 259     delete $$newhash{$stat};
 260 }
 261
 262
 263 #
 264 # All done.  Print comparison summary.
 265 #
 266
 267 printf("Maximum error magnitude: %+f%%\n\n", $max_err_mag);
 268
 269 printf("  %-30s %10s %10s %10s   %7s\n", ' ', 'Reference', 'New Value', 'Abs Diff', 'Pct Chg');
 270
 271 printf("Key statistics:\n\n");
 272
 273 foreach $key_stat (@key_stats)
 274 {
 275     ($statname, $refvalue, $newvalue, $reldiff) = @$key_stat;
 276
 277     # deduce format from reference value
 278     $pointpos = rindex($refvalue, '.');
 279     $digits = ($pointpos < 0) ? 0 :(length($refvalue) - $pointpos - 1);
 280     $fmt = "%10.${digits}f";
 281
 282     # print differing values with absolute and relative error
 283     printf("  %-30s $fmt $fmt $fmt  %+7.2f%%\n",
 284            $statname, $refvalue, $newvalue,
 285            $newvalue - $refvalue, pct_diff($refvalue, $newvalue));
 286 }
 287
 288 printf("\nDifferences > %d%%:\n\n", $err_thresh);
 289
 290 if ($opt_a) {
 291     # leave stats sorted alphabetically, doesn't make sense to cut them off
 292     $omit_count = 0;
 293 } else {
 294     # sort differences by percent change
 295     @errs = sort { abs($$b[3]) <=> abs($$a[3]) } @errs;
 296 }
 297
 298 $num_errs = 0;
 299
 300 foreach $err (@errs)
 301 {
 302     ($statname, $refvalue, $newvalue, $reldiff) = @$err;
 303
 304     # deduce format from reference value
 305     $pointpos1 = rindex($refvalue, '.');
 306     $digits1 = ($pointpos1 < 0) ? 0 :(length($refvalue) - $pointpos1 - 1);
 307     $pointpos2 = rindex($newvalue, '.');
 308     $digits2 = ($pointpos2 < 0) ? 0 :(length($newvalue) - $pointpos2 - 1);
 309     $digits = ($digits1 > $digits2) ? $digits1 : $digits2;
 310     $fmt = "%10.${digits}f";
 311
 312     # print differing values with absolute and relative error
 313     printf("  %-30s $fmt $fmt $fmt  %+7.2f%%\n",
 314            $statname, $refvalue, $newvalue, $newvalue - $refvalue, $reldiff);
 315
 316     # only print top N errors
 317     if ($omit_count > 0 && ++$num_errs >= $omit_count)
 318     {
 319         print "[... showing top $omit_count errors only, additional errors omitted ...]\n";
 320         last;
 321     }
 322 }
 323
 324 #
 325 # Report missing stats
 326 #
 327 # get count
 328 $missing_stats = scalar(@missing_stats);
 329
 330 if ($missing_stats)
 331 {
 332     print "\nMissing $missing_stats reference statistics:\n\n";
 333     foreach $stat (@missing_stats)
 334     {
 335 #       print "\t$stat\n";
 336         printf "  %-50s    ", $stat;
 337         print  "$$refhash{$stat}\n";
 338     }
 339 }
 340
 341 #
 342 # Any stats left in newhash are added since the reference file
 343 #
 344
 345 @added_stats = keys %$newhash;
 346
 347 # get count
 348 $added_stats = scalar(@added_stats);
 349
 350 if ($added_stats)
 351 {
 352     print "\nFound $added_stats new statistics:\n\n";
 353     foreach $stat (sort @added_stats)
 354     {
 355 #       print "\t$stat\n";
 356         printf "  %-50s    ", $stat;
 357         print  "$$newhash{$stat}\n";
 358     }
 359 }
 360
 361 cleanup();
 362 # Exit code is 0 if all stats are found (with no extras) & no stats error, 1 otherwise
 363 $status = ($missing_stats == 0 && $added_stats == 0 && $max_err_mag == 0.0) ? 0 : 1;
 364 exit $status;
 365
 366 sub cleanup
 367 {
 368     unlink($refheader) if ($refheader);
 369     unlink($newheader) if ($newheader);
 370 }