tests/diff-out

   1 #!/usr/bin/perl
   2 # Copyright (c) 2001-2005 The Regents of The University of Michigan
   3 # All rights reserved.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met: redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer;
   9 # redistributions in binary form must reproduce the above copyright
  10 # notice, this list of conditions and the following disclaimer in the
  11 # documentation and/or other materials provided with the distribution;
  12 # neither the name of the copyright holders nor the names of its
  13 # contributors may be used to endorse or promote products derived from
  14 # this software without specific prior written permission.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27
  28 #
  29 # This script diffs two SimpleScalar statistics output files.
  30 #
  31
  32 use Getopt::Std;
  33
  34 getopts('adn:t:h');
  35
  36 if ($#ARGV < 1)
  37 {
  38     print "\nError: need two file arguments (<reference> <new>).\n";
  39     print "   Options: -d = Ignore distributions\n";
  40     print "            -a = Sort errors alphabetically (default: by percentage)\n";
  41     print "            -h = Diff header info separately from stats\n";
  42     print "            -n <num> = Print top <num> errors (default 20, 0 for all)\n";
  43     print "            -t <num> = Ignore errors below <num> percent (default 0)\n\n";
  44     exit;
  45 }
  46
  47 open(REF, "<$ARGV[0]") or die "Error: can't open $ARGV[0].\n";
  48 open(NEW, "<$ARGV[1]") or die "Error: can't open $ARGV[1].\n";
  49
  50
  51 #
  52 # Things that really should be adjustable via the command line
  53 #
  54
  55 # Ignorable error (in percent)
  56 $err_thresh = defined($opt_t) ? $opt_t : 0;
  57
  58 # Number of stats to print before omitting
  59 $omit_count = defined($opt_n) ? $opt_n : 20;
  60
  61
  62 #
  63 # First copy everything up to the simulation statistics to a pair of
  64 # temporary files, stripping out date-related items, and do a plain
  65 # diff.  Any differences in the arguments are not necessarily an issue;
  66 # any differences in the program output should be caught by the EIO
  67 # mechanism if an EIO file is used.
  68 #
  69
  70 # copy_header takes input filehandle and output filename
  71
  72 sub copy_header
  73 {
  74     my ($inhandle, $outname) = @_;
  75
  76     open(OUTPUT, ">$outname") or die "Error: can't open $outname.\n";
  77
  78     while (<$inhandle>)
  79     {
  80         # strip out lines that can vary
  81         next if /^(command line:|M5 compiled on |M5 simulation started |M5 executing on )/;
  82         last if /Begin Simulation Statistics/;
  83         print OUTPUT;
  84     }
  85     close OUTPUT;
  86 }
  87
  88 if ($opt_h) {
  89
  90     # Diff header separately from stats
  91
  92     $refheader = "/tmp/smt-test.refheader.$$";
  93     $newheader = "/tmp/smt-test.newheader.$$";
  94
  95     copy_header(\*REF, $refheader);
  96     copy_header(\*NEW, $newheader);
  97
  98     print "\n===== Header and program output differences =====\n\n";
  99
 100     print `diff $refheader $newheader`;
 101
 102     print "\n===== Statistics differences =====\n\n";
 103 }
 104
 105 #
 106 # Now parse statistics
 107 #
 108
 109 #
 110 # This function takes an open filehandle and returns a reference to
 111 # a hash containing all the statistics variables and their values.
 112 #
 113 sub parse_file
 114 {
 115     $stathandle = shift;
 116
 117     $in_dist = undef;
 118     $hashref = { };     # initialize hash for values
 119
 120     while (<$stathandle>)
 121     {
 122         next if /^\s*$/;        # skip blank lines
 123         last if /End Simulation Statistics/;
 124
 125         s/ *#.*//;              # strip comments
 126
 127         if (/^Memory usage: (\d+) KBytes/) {
 128             $stat = 'memory usage';
 129             $value = $1;
 130         }
 131         elsif ($in_dist) {
 132             if (/(.*)\.end_dist/) {
 133                 # end line of distribution: clear $in_dist flag
 134                 $in_dist = undef;
 135                 next;
 136             }
 137             if ($opt_d) {
 138                 next;           #  bail out if we are ignoring dists...
 139             } elsif (/(.*)\.(min|max)_value/) {
 140                 # treat these like normal stats
 141                 ($stat, $value) = /^(\S+)\s+(.*)/;
 142             } else {
 143                 ($stat, $value) =
 144                   /^(\S+(?:.*\S)?)\s+(\d+)\s+\d+\.\d+%/;
 145                 $stat = $in_dist . '::' . $stat;
 146             }
 147         }
 148         else {
 149             if (/(.*)\.start_dist/) {
 150                 # start line of distribution: set $in_dist flag
 151                 # and save distribution name for future reference
 152                 $in_dist = $1;
 153                 $stat = $1;
 154                 $value = 0;
 155             }
 156             else {
 157                 ($stat, $value) = /^(\S+)\s+(.*)/;
 158             }
 159         }
 160
 161         $$hashref{$stat} = $value;
 162     }
 163
 164     close($stathandle);
 165     return $hashref;
 166 }
 167
 168
 169 #
 170 # pct_diff($old, $new) returns percent difference from $old to $new.
 171 #
 172 sub pct_diff
 173 {
 174     my ($old, $new) = @_;
 175     return ($old == 0) ? (($new == 0) ? 0 : 9999) : 100 * ($new - $old) / $old;
 176 }
 177
 178
 179 #
 180 # Statistics to ignore: these relate to simulator performance, not
 181 # correctness, so don't fail on changes here.
 182 #
 183 %ignore = (
 184   'host_seconds' => 1,
 185   'host_tick_rate' => 1,
 186   'host_inst_rate' => 1,
 187   'host_op_rate' => 1,
 188   'host_mem_usage' => 1
 189 );
 190
 191 #
 192 # List of key statistics (always displayed)
 193 #  ==> list stats here WITHOUT trailing thread ID
 194 #
 195 @key_stat_list = (
 196   'ipc',
 197   'committedInsts',
 198   'committedOps',
 199   'sim_insts',
 200   'sim_ops',
 201   'sim_ticks',
 202   'host_inst_rate',
 203   'host_mem_usage'
 204 );
 205
 206 $key_stat_pattern = join('|', @key_stat_list);
 207
 208 # initialize first statistics from each file
 209
 210 $max_err_mag = 0;
 211
 212 $refhash = parse_file(\*REF);
 213 $newhash = parse_file(\*NEW);
 214
 215 # The string sim-smt prints on a divide by zero
 216 $divbyzero = '<err: divide by zero>';
 217
 218 foreach $stat (sort keys %$refhash)
 219 {
 220     $refvalue = $$refhash{$stat};
 221     $newvalue = $$newhash{$stat};
 222
 223     if (!defined($newvalue)) {
 224         # stat missing from new file
 225         push @missing_stats, $stat;
 226         next;
 227     }
 228
 229     if ($stat =~ /($key_stat_pattern)/o) {
 230         # key statistics: always record & display changes in these
 231         push @key_stats, [$stat, $refvalue, $newvalue];
 232     }
 233
 234     if ($ignore{$stat} or $refvalue eq $newvalue) {
 235         # stat is in "ignore" list, or hasn't changed
 236     }
 237     else {
 238         if ($refvalue eq $divbyzero || $newvalue eq $divbyzero) {
 239             # one or the other was a divide by zero:
 240             # no point in trying to quantify error
 241             print "$stat: $refvalue --> $newvalue\n";
 242         }
 243         else {
 244             $reldiff = pct_diff($refvalue, $newvalue);
 245             $diffmag = abs($reldiff);
 246
 247             if ($diffmag > $err_thresh) {
 248                 push @errs,
 249                 [$stat, $refvalue, $newvalue, $reldiff];
 250             }
 251
 252             if ($diffmag > $max_err_mag) {
 253                 $max_err_mag = $diffmag;
 254             }
 255         }
 256     }
 257
 258     # remove from new hash so we can detect added stats
 259     delete $$newhash{$stat};
 260 }
 261
 262
 263 #
 264 # All done.  Print comparison summary.
 265 #
 266
 267 printf("Maximum error magnitude: %+f%%\n\n", $max_err_mag);
 268
 269 printf("  %-30s %10s %10s %10s   %7s\n", ' ', 'Reference', 'New Value', 'Abs Diff', 'Pct Chg');
 270
 271 printf("Key statistics:\n\n");
 272
 273 foreach $key_stat (@key_stats)
 274 {
 275     ($statname, $refvalue, $newvalue, $reldiff) = @$key_stat;
 276
 277     # deduce format from reference value
 278     $pointpos = rindex($refvalue, '.');
 279     $digits = ($pointpos < 0) ? 0 :(length($refvalue) - $pointpos - 1);
 280     $fmt = "%10.${digits}f";
 281
 282     # print differing values with absolute and relative error
 283     printf("  %-30s $fmt $fmt $fmt  %+7.2f%%\n",
 284            $statname, $refvalue, $newvalue,
 285            $newvalue - $refvalue, pct_diff($refvalue, $newvalue));
 286 }
 287
 288 printf("\nDifferences > %d%%:\n\n", $err_thresh);
 289
 290 if ($opt_a) {
 291     # leave stats sorted alphabetically, doesn't make sense to cut them off
 292     $omit_count = 0;
 293 } else {
 294     # sort differences by percent change
 295     @errs = sort { abs($$b[3]) <=> abs($$a[3]) } @errs;
 296 }
 297
 298 $num_errs = 0;
 299
 300 foreach $err (@errs)
 301 {
 302     ($statname, $refvalue, $newvalue, $reldiff) = @$err;
 303
 304     # deduce format from reference value
 305     $pointpos1 = rindex($refvalue, '.');
 306     $digits1 = ($pointpos1 < 0) ? 0 :(length($refvalue) - $pointpos1 - 1);
 307     $pointpos2 = rindex($newvalue, '.');
 308     $digits2 = ($pointpos2 < 0) ? 0 :(length($newvalue) - $pointpos2 - 1);
 309     $digits = ($digits1 > $digits2) ? $digits1 : $digits2;
 310     $fmt = "%10.${digits}f";
 311
 312     # print differing values with absolute and relative error
 313     printf("  %-30s $fmt $fmt $fmt  %+7.2f%%\n",
 314            $statname, $refvalue, $newvalue, $newvalue - $refvalue, $reldiff);
 315
 316     # only print top N errors
 317     if ($omit_count > 0 && ++$num_errs >= $omit_count)
 318     {
 319         print "[... showing top $omit_count errors only, additional errors omitted ...]\n";
 320         last;
 321     }
 322 }
 323
 324 #
 325 # Report missing stats
 326 #
 327 # get count
 328 $missing_stats = scalar(@missing_stats);
 329
 330 if ($missing_stats)
 331 {
 332     print "\nMissing $missing_stats reference statistics:\n\n";
 333     foreach $stat (@missing_stats)
 334     {
 335 #       print "\t$stat\n";
 336         printf "  %-50s    ", $stat;
 337         print  "$$refhash{$stat}\n";
 338     }
 339 }
 340
 341 #
 342 # Any stats left in newhash are added since the reference file
 343 #
 344
 345 @added_stats = keys %$newhash;
 346
 347 # get count
 348 $added_stats = scalar(@added_stats);
 349
 350 if ($added_stats)
 351 {
 352     print "\nFound $added_stats new statistics:\n\n";
 353     foreach $stat (sort @added_stats)
 354     {
 355 #       print "\t$stat\n";
 356         printf "  %-50s    ", $stat;
 357         print  "$$newhash{$stat}\n";
 358     }
 359 }
 360
 361 cleanup();
 362 # Exit codes:
 363 # 0 if all stats are found (with no extras) & no stats error
 364 # 1 if there are additional stats, but no stat errors
 365 # 2 otherwise
 366 $no_hard_errors = $missing_stats == 0 && $max_err_mag == 0.0;
 367 $status = $no_hard_errors ? ($added_stats == 0 ? 0 : 1) : 2;
 368 exit $status;
 369
 370 sub cleanup
 371 {
 372     unlink($refheader) if ($refheader);
 373     unlink($newheader) if ($newheader);
 374 }