tests/testing/units.py

   1 #!/usr/bin/env python2.7
   2 #
   3 # Copyright (c) 2016 ARM Limited
   4 # All rights reserved
   5 #
   6 # The license below extends only to copyright in the software and shall
   7 # not be construed as granting a license to any other intellectual
   8 # property including but not limited to intellectual property relating
   9 # to a hardware implementation of the functionality of the software
  10 # licensed hereunder.  You may use the software subject to the license
  11 # terms below provided that you ensure that this notice is replicated
  12 # unmodified and in its entirety in all distributions of the software,
  13 # modified or unmodified, in source code or in binary form.
  14 #
  15 # Redistribution and use in source and binary forms, with or without
  16 # modification, are permitted provided that the following conditions are
  17 # met: redistributions of source code must retain the above copyright
  18 # notice, this list of conditions and the following disclaimer;
  19 # redistributions in binary form must reproduce the above copyright
  20 # notice, this list of conditions and the following disclaimer in the
  21 # documentation and/or other materials provided with the distribution;
  22 # neither the name of the copyright holders nor the names of its
  23 # contributors may be used to endorse or promote products derived from
  24 # this software without specific prior written permission.
  25 #
  26 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  27 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  28 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  29 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  30 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  31 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  32 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  33 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  34 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  35 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  36 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  37
  38 from abc import ABCMeta, abstractmethod
  39 from datetime import datetime
  40 import difflib
  41 import functools
  42 import os
  43 import re
  44 import subprocess
  45 import sys
  46 import traceback
  47
  48 from results import UnitResult
  49 from helpers import *
  50
  51 _test_base = os.path.join(os.path.dirname(__file__), "..")
  52
  53 class TestUnit(object):
  54     """Base class for all test units.
  55
  56     A test unit is a part of a larger test case. Test cases usually
  57     contain two types of units, run units (run gem5) and verify units
  58     (diff output files). All unit implementations inherit from this
  59     class.
  60
  61     A unit implementation overrides the _run() method. The test runner
  62     calls the run() method, which wraps _run() to protect against
  63     exceptions.
  64
  65     """
  66
  67     __metaclass__ = ABCMeta
  68
  69     def __init__(self, name, ref_dir, test_dir, skip=False):
  70         self.name = name
  71         self.ref_dir = ref_dir
  72         self.test_dir = test_dir
  73         self.force_skip = skip
  74         self.start_time = None
  75         self.stop_time = None
  76
  77     def result(self, state, **kwargs):
  78         if self.start_time is not None and "runtime" not in kwargs:
  79             self.stop_time = datetime.utcnow()
  80             delta = self.stop_time - self.start_time
  81             kwargs["runtime"] = delta.total_seconds()
  82
  83         return UnitResult(self.name, state, **kwargs)
  84
  85     def ok(self, **kwargs):
  86         return self.result(UnitResult.STATE_OK, **kwargs)
  87
  88     def skip(self, **kwargs):
  89         return self.result(UnitResult.STATE_SKIPPED, **kwargs)
  90
  91     def error(self, message, **kwargs):
  92         return self.result(UnitResult.STATE_ERROR, message=message, **kwargs)
  93
  94     def failure(self, message, **kwargs):
  95         return self.result(UnitResult.STATE_FAILURE, message=message, **kwargs)
  96
  97     def ref_file(self, fname):
  98         return os.path.join(self.ref_dir, fname)
  99
 100     def out_file(self, fname):
 101         return os.path.join(self.test_dir, fname)
 102
 103     def _read_output(self, fname, default=""):
 104         try:
 105             with open(self.out_file(fname), "r") as f:
 106                 return f.read()
 107         except IOError:
 108             return default
 109
 110     def run(self):
 111         self.start_time = datetime.utcnow()
 112         try:
 113             if self.force_skip:
 114                 return self.skip()
 115             else:
 116                 return self._run()
 117         except:
 118             return self.error("Python exception:\n%s" % traceback.format_exc())
 119
 120     @abstractmethod
 121     def _run(self):
 122         pass
 123
 124 class RunGem5(TestUnit):
 125     """Test unit representing a gem5 run.
 126
 127     Possible failure modes:
 128        - gem5 failed to run -> STATE_ERROR
 129        - timeout -> STATE_ERROR
 130        - non-zero exit code -> STATE_ERROR
 131
 132     Possible non-failure results:
 133        - exit code == 0 -> STATE_OK
 134        - exit code == 2 -> STATE_SKIPPED
 135     """
 136
 137     def __init__(self, gem5, gem5_args, timeout=0, **kwargs):
 138         super(RunGem5, self).__init__("gem5", **kwargs)
 139         self.gem5 = gem5
 140         self.args = gem5_args
 141         self.timeout = timeout
 142
 143     def _run(self):
 144         gem5_cmd = [
 145             self.gem5,
 146             "-d", self.test_dir,
 147             "--stats-file", "text://stats.txt?desc=False",
 148             "-re",
 149         ] + self.args
 150
 151         try:
 152             with ProcessHelper(gem5_cmd, stdout=subprocess.PIPE,
 153                                stderr=subprocess.PIPE) as p:
 154                 status, gem5_stdout, gem5_stderr = p.call(timeout=self.timeout)
 155         except CallTimeoutException as te:
 156             return self.error("Timeout", stdout=te.stdout, stderr=te.stderr)
 157         except OSError as ose:
 158             return self.error("Failed to launch gem5: %s" % ose)
 159
 160         stderr = "\n".join([
 161             "*** gem5 stderr ***",
 162             gem5_stderr,
 163             "",
 164             "*** m5out/simerr ***",
 165             self._read_output("simerr"),
 166         ])
 167
 168         stdout = "\n".join([
 169             "*** gem5 stdout ***",
 170             gem5_stdout,
 171             "",
 172             "*** m5out/simout ***",
 173             self._read_output("simout"),
 174         ])
 175
 176         # Signal
 177         if status < 0:
 178             return self.error("gem5 terminated by signal %i" % (-status, ),
 179                               stdout=stdout, stderr=stderr)
 180         elif status == 2:
 181             return self.skip(stdout=stdout, stderr=stderr)
 182         elif status > 0:
 183             return self.error("gem5 exited with non-zero status: %i" % status,
 184                               stdout=stdout, stderr=stderr)
 185         else:
 186             return self.ok(stdout=stdout, stderr=stderr)
 187
 188 class DiffOutFile(TestUnit):
 189     """Test unit comparing and output file and a reference file."""
 190
 191     # regular expressions of lines to ignore when diffing outputs
 192     diff_ignore_regexes = {
 193         "simout" : [
 194             re.compile('^Redirecting (stdout|stderr) to'),
 195             re.compile('^gem5 compiled '),
 196             re.compile('^gem5 started '),
 197             re.compile('^gem5 executing on '),
 198             re.compile('^command line:'),
 199             re.compile("^Couldn't import dot_parser,"),
 200             re.compile("^info: kernel located at:"),
 201             re.compile("^Couldn't unlink "),
 202             re.compile("^Using GPU kernel code file\(s\) "),
 203         ],
 204         "simerr" : [
 205             #re.compile('^Simulation complete at'),
 206         ],
 207         "config.ini" : [
 208             re.compile("^(executable|readfile|kernel|image_file)="),
 209             re.compile("^(cwd|input|codefile)="),
 210         ],
 211         "config.json" : [
 212             re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''),
 213             re.compile(r'''^\s*"(cwd|input|codefile)":'''),
 214         ],
 215     }
 216
 217     def __init__(self, fname, **kwargs):
 218         super(DiffOutFile, self).__init__("diff[%s]" % fname,
 219                                           **kwargs)
 220
 221         self.fname = fname
 222         self.line_filters = DiffOutFile.diff_ignore_regexes.get(fname, tuple())
 223
 224     def _filter_file(self, fname):
 225         def match_line(l):
 226             for r in self.line_filters:
 227                 if r.match(l):
 228                     return True
 229             return False
 230
 231         with open(fname, "r") as f:
 232             for l in f:
 233                 if not match_line(l):
 234                     yield l
 235
 236
 237     def _run(self):
 238         fname = self.fname
 239         ref = self.ref_file(fname)
 240         out = self.out_file(fname)
 241
 242         if not os.path.exists(ref):
 243             return self.error("%s doesn't exist in reference directory" \
 244                               % fname)
 245
 246         if not os.path.exists(out):
 247             return self.error("%s doesn't exist in output directory" % fname)
 248
 249         diff = difflib.unified_diff(
 250             tuple(self._filter_file(ref)),
 251             tuple(self._filter_file(out)),
 252             fromfile="ref/%s" % fname, tofile="out/%s" % fname)
 253
 254         diff = list(diff)
 255         if diff:
 256             return self.error("ref/%s and out/%s differ" % (fname, fname),
 257                               stderr="".join(diff))
 258         else:
 259             return self.ok(stdout="-- ref/%s and out/%s are identical --" \
 260                            % (fname, fname))
 261
 262 class DiffStatFile(TestUnit):
 263     """Test unit comparing two gem5 stat files."""
 264
 265     def __init__(self, **kwargs):
 266         super(DiffStatFile, self).__init__("stat_diff", **kwargs)
 267
 268         self.stat_diff = os.path.join(_test_base, "diff-out")
 269
 270     def _run(self):
 271         STATUS_OK = 0
 272         STATUS_NEW_STATS = 1
 273         STATUS_FAILED = 2
 274
 275         stats = "stats.txt"
 276
 277         cmd = [
 278             self.stat_diff,
 279             self.ref_file(stats), self.out_file(stats),
 280         ]
 281         with ProcessHelper(cmd,
 282                            stdout=subprocess.PIPE,
 283                            stderr=subprocess.PIPE) as p:
 284             status, stdout, stderr = p.call()
 285
 286         if status in (STATUS_OK, STATUS_NEW_STATS):
 287             return self.ok(stdout=stdout, stderr=stderr)
 288         elif status == STATUS_FAILED:
 289             return self.failure("Statistics mismatch",
 290                                 stdout=stdout, stderr=stderr)
 291         else:
 292             return self.error("diff-out returned an error: %i" % status,
 293                               stdout=stdout, stderr=stderr)