--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright (c) 2016 ARM Limited
+# All rights reserved
+#
+# The license below extends only to copyright in the software and shall
+# not be construed as granting a license to any other intellectual
+# property including but not limited to intellectual property relating
+# to a hardware implementation of the functionality of the software
+# licensed hereunder. You may use the software subject to the license
+# terms below provided that you ensure that this notice is replicated
+# unmodified and in its entirety in all distributions of the software,
+# modified or unmodified, in source code or in binary form.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Andreas Sandberg
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright (c) 2016 ARM Limited
+# All rights reserved
+#
+# The license below extends only to copyright in the software and shall
+# not be construed as granting a license to any other intellectual
+# property including but not limited to intellectual property relating
+# to a hardware implementation of the functionality of the software
+# licensed hereunder. You may use the software subject to the license
+# terms below provided that you ensure that this notice is replicated
+# unmodified and in its entirety in all distributions of the software,
+# modified or unmodified, in source code or in binary form.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Andreas Sandberg
+
+import subprocess
+from threading import Timer
+import time
+
+class CallTimeoutException(Exception):
+ """Exception that indicates that a process call timed out"""
+
+ def __init__(self, status, stdout, stderr):
+ self.status = status
+ self.stdout = stdout
+ self.stderr = stderr
+
+class ProcessHelper(subprocess.Popen):
+ """Helper class to run child processes.
+
+ This class wraps a subprocess.Popen class and adds support for
+ using it in a with block. When the process goes out of scope, it's
+ automatically terminated.
+
+ with ProcessHelper(["/bin/ls"], stdout=subprocess.PIPE) as p:
+ return p.call()
+ """
+ def __init__(self, *args, **kwargs):
+ super(ProcessHelper, self).__init__(*args, **kwargs)
+
+ def _terminate_nicely(self, timeout=5):
+ def on_timeout():
+ self.kill()
+
+ if self.returncode is not None:
+ return self.returncode
+
+ timer = Timer(timeout, on_timeout)
+ self.terminate()
+ status = self.wait()
+ timer.cancel()
+
+ return status
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ if self.returncode is None:
+ self._terminate_nicely()
+
+ def call(self, timeout=0):
+ self._timeout = False
+ def on_timeout():
+ self._timeout = True
+ self._terminate_nicely()
+
+ status, stdout, stderr = None, None, None
+ timer = Timer(timeout, on_timeout)
+ if timeout:
+ timer.start()
+
+ stdout, stderr = self.communicate()
+ status = self.wait()
+
+ timer.cancel()
+
+ if self._timeout:
+ self._terminate_nicely()
+ raise CallTimeoutException(self.returncode, stdout, stderr)
+ else:
+ return status, stdout, stderr
+
+if __name__ == "__main__":
+ # Run internal self tests to ensure that the helpers are working
+ # properly. The expected output when running this script is
+ # "SUCCESS!".
+
+ cmd_foo = [ "/bin/echo", "-n", "foo" ]
+ cmd_sleep = [ "/bin/sleep", "10" ]
+
+ # Test that things don't break if the process hasn't been started
+ with ProcessHelper(cmd_foo) as p:
+ pass
+
+ with ProcessHelper(cmd_foo, stdout=subprocess.PIPE) as p:
+ status, stdout, stderr = p.call()
+ assert stdout == "foo"
+ assert status == 0
+
+ try:
+ with ProcessHelper(cmd_sleep) as p:
+ status, stdout, stderr = p.call(timeout=1)
+ assert False, "Timeout not triggered"
+ except CallTimeoutException:
+ pass
+
+ print "SUCCESS!"
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright (c) 2016 ARM Limited
+# All rights reserved
+#
+# The license below extends only to copyright in the software and shall
+# not be construed as granting a license to any other intellectual
+# property including but not limited to intellectual property relating
+# to a hardware implementation of the functionality of the software
+# licensed hereunder. You may use the software subject to the license
+# terms below provided that you ensure that this notice is replicated
+# unmodified and in its entirety in all distributions of the software,
+# modified or unmodified, in source code or in binary form.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Andreas Sandberg
+
+from abc import ABCMeta, abstractmethod
+import inspect
+import pickle
+import string
+import sys
+
+import xml.etree.cElementTree as ET
+
+class UnitResult(object):
+ """Results of a single test unit.
+
+ A test result can be one of:
+ - STATE_OK: Test ran successfully.
+ - STATE_SKIPPED: The test was skipped.
+ - STATE_ERROR: The test failed to run.
+ - STATE_FAILED: Test ran, but failed.
+
+ The difference between STATE_ERROR and STATE_FAILED is very
+ subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed
+ to start or crashed, while STATE_FAILED would mean that a test
+ failed (e.g., statistics mismatch).
+
+ """
+
+ STATE_OK = 0
+ STATE_SKIPPED = 1
+ STATE_ERROR = 2
+ STATE_FAILURE = 3
+
+ state_names = {
+ STATE_OK : "OK",
+ STATE_SKIPPED : "SKIPPED",
+ STATE_ERROR : "ERROR",
+ STATE_FAILURE : "FAILURE",
+ }
+
+ def __init__(self, name, state, message="", stderr="", stdout="",
+ runtime=0.0):
+ self.name = name
+ self.state = state
+ self.message = message
+ self.stdout = stdout
+ self.stderr = stderr
+ self.runtime = runtime
+
+ def skipped(self):
+ return self.state == UnitResult.STATE_SKIPPED
+
+ def success(self):
+ return self.state == UnitResult.STATE_OK
+
+ def state_name(self):
+ return UnitResult.state_names[self.state]
+
+ def __nonzero__(self):
+ return self.success() or self.skipped()
+
+ def __str__(self):
+ state_name = self.state_name()
+
+ status = "%s: %s" % (state_name, self.message) if self.message else \
+ state_name
+
+ return "%s: %s" % (self.name, status)
+
+class TestResult(object):
+ """Results for from a single test consisting of one or more units."""
+
+ def __init__(self, name, results=[]):
+ self.name = name
+ self.results = results
+
+ def success(self):
+ return all([ r.success() for r in self.results])
+
+ def skipped(self):
+ return all([ r.skipped() for r in self.results])
+
+ def failed(self):
+ return any([ not r for r in self.results])
+
+ def runtime(self):
+ return sum([ r.runtime for r in self.results ])
+
+ def __nonzero__(self):
+ return all([r for r in self.results])
+
+class ResultFormatter(object):
+ __metaclass__ = ABCMeta
+
+ def __init__(self, fout=sys.stdout, verbose=False):
+ self.verbose = verbose
+ self.fout = fout
+
+ @abstractmethod
+ def dump_suites(self, suites):
+ pass
+
+class Pickle(ResultFormatter):
+ """Save test results as a binary using Python's pickle
+ functionality.
+
+ """
+
+ def __init__(self, **kwargs):
+ super(Pickle, self).__init__(**kwargs)
+
+ def dump_suites(self, suites):
+ pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL)
+
+class Text(ResultFormatter):
+ """Output test results as text."""
+
+ def __init__(self, **kwargs):
+ super(Text, self).__init__(**kwargs)
+
+ def dump_suites(self, suites):
+ fout = self.fout
+ for suite in suites:
+ print >> fout, "--- %s ---" % suite.name
+
+ for t in suite.results:
+ print >> fout, "*** %s" % t
+
+ if t and not self.verbose:
+ continue
+
+ if t.message:
+ print >> fout, t.message
+
+ if t.stderr:
+ print >> fout, t.stderr
+ if t.stdout:
+ print >> fout, t.stdout
+
+class TextSummary(ResultFormatter):
+ """Output test results as a text summary"""
+
+ def __init__(self, **kwargs):
+ super(TextSummary, self).__init__(**kwargs)
+
+ def dump_suites(self, suites):
+ fout = self.fout
+ for suite in suites:
+ status = "SKIPPED" if suite.skipped() else \
+ ("OK" if suite else "FAILED")
+ print >> fout, "%s: %s" % (suite.name, status)
+
+class JUnit(ResultFormatter):
+ """Output test results as JUnit XML"""
+
+ def __init__(self, translate_names=True, **kwargs):
+ super(JUnit, self).__init__(**kwargs)
+
+ if translate_names:
+ self.name_table = string.maketrans(
+ "/.",
+ ".-",
+ )
+ else:
+ self.name_table = string.maketrans("", "")
+
+ def convert_unit(self, x_suite, test):
+ x_test = ET.SubElement(x_suite, "testcase",
+ name=test.name,
+ time="%f" % test.runtime)
+
+ x_state = None
+ if test.state == UnitResult.STATE_OK:
+ pass
+ elif test.state == UnitResult.STATE_SKIPPED:
+ x_state = ET.SubElement(x_test, "skipped")
+ elif test.state == UnitResult.STATE_FAILURE:
+ x_state = ET.SubElement(x_test, "failure")
+ elif test.state == UnitResult.STATE_ERROR:
+ x_state = ET.SubElement(x_test, "error")
+ else:
+ assert False, "Unknown test state"
+
+ if x_state is not None:
+ if test.message:
+ x_state.set("message", test.message)
+
+ msg = []
+ if test.stderr:
+ msg.append("*** Standard Errror: ***")
+ msg.append(test.stderr)
+ if test.stdout:
+ msg.append("*** Standard Out: ***")
+ msg.append(test.stdout)
+
+ x_state.text = "\n".join(msg)
+
+ return x_test
+
+ def convert_suite(self, x_suites, suite):
+ x_suite = ET.SubElement(x_suites, "testsuite",
+ name=suite.name.translate(self.name_table),
+ time="%f" % suite.runtime())
+ errors = 0
+ failures = 0
+ skipped = 0
+
+ for test in suite.results:
+ if test.state != UnitResult.STATE_OK:
+ if test.state == UnitResult.STATE_SKIPPED:
+ skipped += 1
+ elif test.state == UnitResult.STATE_ERROR:
+ errors += 1
+ elif test.state == UnitResult.STATE_FAILURE:
+ failures += 1
+
+ x_test = self.convert_unit(x_suite, test)
+
+ x_suite.set("errors", str(errors))
+ x_suite.set("failures", str(failures))
+ x_suite.set("skipped", str(skipped))
+ x_suite.set("tests", str(len(suite.results)))
+
+ return x_suite
+
+ def convert_suites(self, suites):
+ x_root = ET.Element("testsuites")
+
+ for suite in suites:
+ self.convert_suite(x_root, suite)
+
+ return x_root
+
+ def dump_suites(self, suites):
+ et = ET.ElementTree(self.convert_suites(suites))
+ et.write(self.fout, encoding="UTF-8")
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright (c) 2016 ARM Limited
+# All rights reserved
+#
+# The license below extends only to copyright in the software and shall
+# not be construed as granting a license to any other intellectual
+# property including but not limited to intellectual property relating
+# to a hardware implementation of the functionality of the software
+# licensed hereunder. You may use the software subject to the license
+# terms below provided that you ensure that this notice is replicated
+# unmodified and in its entirety in all distributions of the software,
+# modified or unmodified, in source code or in binary form.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Andreas Sandberg
+
+from abc import ABCMeta, abstractmethod
+import os
+from collections import namedtuple
+from units import *
+from results import TestResult
+import shutil
+
+_test_base = os.path.join(os.path.dirname(__file__), "..")
+
+ClassicConfig = namedtuple("ClassicConfig", (
+ "category",
+ "mode",
+ "workload",
+ "isa",
+ "os",
+ "config",
+))
+
+# There are currently two "classes" of test
+# configurations. Architecture-specific ones and generic ones
+# (typically SE mode tests). In both cases, the configuration name
+# matches a file in tests/configs/ that will be picked up by the test
+# runner (run.py).
+#
+# Architecture specific configurations are listed in the arch_configs
+# dictionary. This is indexed by a (cpu architecture, gpu
+# architecture) tuple. GPU architecture is optional and may be None.
+#
+# Generic configurations are listed in the generic_configs tuple.
+#
+# When discovering available test cases, this script look uses the
+# test list as a list of /candidate/ configurations. A configuration
+# is only used if a test has a reference output for that
+# configuration. In addition to the base configurations from
+# arch_configs and generic_configs, a Ruby configuration may be
+# appended to the base name (this is probed /in addition/ to the
+# original name. See get_tests() for details.
+#
+arch_configs = {
+ ("alpha", None) : (
+ 'tsunami-simple-atomic',
+ 'tsunami-simple-timing',
+ 'tsunami-simple-atomic-dual',
+ 'tsunami-simple-timing-dual',
+ 'twosys-tsunami-simple-atomic',
+ 'tsunami-o3', 'tsunami-o3-dual',
+ 'tsunami-minor', 'tsunami-minor-dual',
+ 'tsunami-switcheroo-full',
+ ),
+
+ ("arm", None) : (
+ 'simple-atomic-dummychecker',
+ 'o3-timing-checker',
+ 'realview-simple-atomic',
+ 'realview-simple-atomic-dual',
+ 'realview-simple-atomic-checkpoint',
+ 'realview-simple-timing',
+ 'realview-simple-timing-dual',
+ 'realview-o3',
+ 'realview-o3-checker',
+ 'realview-o3-dual',
+ 'realview-minor',
+ 'realview-minor-dual',
+ 'realview-switcheroo-atomic',
+ 'realview-switcheroo-timing',
+ 'realview-switcheroo-o3',
+ 'realview-switcheroo-full',
+ 'realview64-simple-atomic',
+ 'realview64-simple-atomic-checkpoint',
+ 'realview64-simple-atomic-dual',
+ 'realview64-simple-timing',
+ 'realview64-simple-timing-dual',
+ 'realview64-o3',
+ 'realview64-o3-checker',
+ 'realview64-o3-dual',
+ 'realview64-minor',
+ 'realview64-minor-dual',
+ 'realview64-switcheroo-atomic',
+ 'realview64-switcheroo-timing',
+ 'realview64-switcheroo-o3',
+ 'realview64-switcheroo-full',
+ ),
+
+ ("sparc", None) : (
+ 't1000-simple-atomic',
+ 't1000-simple-x86',
+ ),
+
+ ("timing", None) : (
+ 'pc-simple-atomic',
+ 'pc-simple-timing',
+ 'pc-o3-timing',
+ 'pc-switcheroo-full',
+ ),
+
+ ("x86", "hsail") : (
+ 'gpu',
+ ),
+}
+
+generic_configs = (
+ 'simple-atomic',
+ 'simple-atomic-mp',
+ 'simple-timing',
+ 'simple-timing-mp',
+
+ 'minor-timing',
+ 'minor-timing-mp',
+
+ 'o3-timing',
+ 'o3-timing-mt',
+ 'o3-timing-mp',
+
+ 'rubytest',
+ 'memcheck',
+ 'memtest',
+ 'memtest-filter',
+ 'tgen-simple-mem',
+ 'tgen-dram-ctrl',
+
+ 'learning-gem5-p1-simple',
+ 'learning-gem5-p1-two-level',
+)
+
+all_categories = ("quick", "long")
+all_modes = ("fs", "se")
+
+class Test(object):
+ """Test case base class.
+
+ Test cases consists of one or more test units that are run in two
+ phases. A run phase (units produced by run_units() and a verify
+ phase (units from verify_units()). The verify phase is skipped if
+ the run phase fails.
+
+ """
+
+ __metaclass__ = ABCMeta
+
+ def __init__(self, name):
+ self.test_name = name
+
+ @abstractmethod
+ def ref_files(self):
+ """Get a list of reference files used by this test case"""
+ pass
+
+ @abstractmethod
+ def run_units(self):
+ """Units (typically RunGem5 instances) that describe the run phase of
+ this test.
+
+ """
+ pass
+
+ @abstractmethod
+ def verify_units(self):
+ """Verify the output from the run phase (see run_units())."""
+ pass
+
+ @abstractmethod
+ def update_ref(self):
+ """Update reference files with files from a test run"""
+ pass
+
+ def run(self):
+ """Run this test case and return a list of results"""
+
+ run_results = [ u.run() for u in self.run_units() ]
+ run_ok = all([not r.skipped() and r for r in run_results ])
+
+ verify_results = [
+ u.run() if run_ok else u.skip()
+ for u in self.verify_units()
+ ]
+
+ return TestResult(self.test_name, run_results + verify_results)
+
+ def __str__(self):
+ return self.test_name
+
+class ClassicTest(Test):
+ diff_ignore_files = [
+ # Stat files use a special stat differ, so don't include them
+ # here.
+ "stats.txt",
+ ]
+
+ def __init__(self, gem5, output_dir, config_tuple,
+ timeout=None,
+ skip=False, skip_diff_out=False, skip_diff_stat=False):
+
+ super(ClassicTest, self).__init__("/".join(config_tuple))
+
+ ct = config_tuple
+
+ self.gem5 = os.path.abspath(gem5)
+ self.script = os.path.join(_test_base, "run.py")
+ self.config_tuple = ct
+ self.timeout = timeout
+
+ self.output_dir = output_dir
+ self.ref_dir = os.path.join(_test_base,
+ ct.category, ct.mode, ct.workload,
+ "ref", ct.isa, ct.os, ct.config)
+ self.skip_run = skip
+ self.skip_diff_out = skip or skip_diff_out
+ self.skip_diff_stat = skip or skip_diff_stat
+
+ def ref_files(self):
+ ref_dir = os.path.abspath(self.ref_dir)
+ for root, dirs, files in os.walk(ref_dir, topdown=False):
+ for f in files:
+ fpath = os.path.join(root[len(ref_dir) + 1:], f)
+ if fpath not in ClassicTest.diff_ignore_files:
+ yield fpath
+
+ def run_units(self):
+ args = [
+ self.script,
+ "/".join(self.config_tuple),
+ ]
+
+ return [
+ RunGem5(self.gem5, args,
+ ref_dir=self.ref_dir, test_dir=self.output_dir,
+ skip=self.skip_run),
+ ]
+
+ def verify_units(self):
+ return [
+ DiffStatFile(ref_dir=self.ref_dir, test_dir=self.output_dir,
+ skip=self.skip_diff_stat)
+ ] + [
+ DiffOutFile(f,
+ ref_dir=self.ref_dir, test_dir=self.output_dir,
+ skip=self.skip_diff_out)
+ for f in self.ref_files()
+ if f not in ClassicTest.diff_ignore_files
+ ]
+
+ def update_ref(self):
+ for fname in self.ref_files():
+ shutil.copy(
+ os.path.join(self.output_dir, fname),
+ os.path.join(self.ref_dir, fname))
+
+def parse_test_filter(test_filter):
+ wildcards = ("", "*")
+
+ _filter = list(test_filter.split("/"))
+ if len(_filter) > 3:
+ raise RuntimeError("Illegal test filter string")
+ _filter += [ "", ] * (3 - len(_filter))
+
+ isa, cat, mode = _filter
+
+ if isa in wildcards:
+ raise RuntimeError("No ISA specified")
+
+ cat = all_categories if cat in wildcards else (cat, )
+ mode = all_modes if mode in wildcards else (mode, )
+
+ return isa, cat, mode
+
+def get_tests(isa,
+ categories=all_categories, modes=all_modes,
+ ruby_protocol=None, gpu_isa=None):
+
+ # Generate a list of candidate configs
+ configs = list(arch_configs.get((isa, gpu_isa), []))
+
+ if (isa, gpu_isa) == ("x86", "hsail"):
+ if ruby_protocol == "GPU_RfO":
+ configs += ['gpu-randomtest']
+ else:
+ configs += generic_configs
+
+ if ruby_protocol == 'MI_example':
+ configs += [ "%s-ruby" % (c, ) for c in configs ]
+ elif ruby_protocol is not None:
+ configs += [ "%s-ruby-%s" % (c, ruby_protocol) for c in configs ]
+
+ # /(quick|long)/(fs|se)/workload/ref/arch/guest/config/
+ for conf_script in configs:
+ for cat in categories:
+ for mode in modes:
+ mode_dir = os.path.join(_test_base, cat, mode)
+ if not os.path.exists(mode_dir):
+ continue
+
+ for workload in os.listdir(mode_dir):
+ isa_dir = os.path.join(mode_dir, workload, "ref", isa)
+ if not os.path.isdir(isa_dir):
+ continue
+
+ for _os in os.listdir(isa_dir):
+ test_dir = os.path.join(isa_dir, _os, conf_script)
+ if not os.path.exists(test_dir) or \
+ os.path.exists(os.path.join(test_dir, "skip")):
+ continue
+
+ yield ClassicConfig(cat, mode, workload, isa, _os,
+ conf_script)
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright (c) 2016 ARM Limited
+# All rights reserved
+#
+# The license below extends only to copyright in the software and shall
+# not be construed as granting a license to any other intellectual
+# property including but not limited to intellectual property relating
+# to a hardware implementation of the functionality of the software
+# licensed hereunder. You may use the software subject to the license
+# terms below provided that you ensure that this notice is replicated
+# unmodified and in its entirety in all distributions of the software,
+# modified or unmodified, in source code or in binary form.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Andreas Sandberg
+
+from abc import ABCMeta, abstractmethod
+from datetime import datetime
+import difflib
+import functools
+import os
+import re
+import subprocess
+import sys
+import traceback
+
+from results import UnitResult
+from helpers import *
+
+_test_base = os.path.join(os.path.dirname(__file__), "..")
+
+class TestUnit(object):
+ """Base class for all test units.
+
+ A test unit is a part of a larger test case. Test cases usually
+ contain two types of units, run units (run gem5) and verify units
+ (diff output files). All unit implementations inherit from this
+ class.
+
+ A unit implementation overrides the _run() method. The test runner
+ calls the run() method, which wraps _run() to protect against
+ exceptions.
+
+ """
+
+ __metaclass__ = ABCMeta
+
+ def __init__(self, name, ref_dir, test_dir, skip=False):
+ self.name = name
+ self.ref_dir = ref_dir
+ self.test_dir = test_dir
+ self.force_skip = skip
+ self.start_time = None
+ self.stop_time = None
+
+ def result(self, state, **kwargs):
+ if self.start_time is not None and "runtime" not in kwargs:
+ self.stop_time = datetime.utcnow()
+ delta = self.stop_time - self.start_time
+ kwargs["runtime"] = delta.total_seconds()
+
+ return UnitResult(self.name, state, **kwargs)
+
+ def ok(self, **kwargs):
+ return self.result(UnitResult.STATE_OK, **kwargs)
+
+ def skip(self, **kwargs):
+ return self.result(UnitResult.STATE_SKIPPED, **kwargs)
+
+ def error(self, message, **kwargs):
+ return self.result(UnitResult.STATE_ERROR, message=message, **kwargs)
+
+ def failure(self, message, **kwargs):
+ return self.result(UnitResult.STATE_FAILURE, message=message, **kwargs)
+
+ def ref_file(self, fname):
+ return os.path.join(self.ref_dir, fname)
+
+ def out_file(self, fname):
+ return os.path.join(self.test_dir, fname)
+
+ def _read_output(self, fname, default=""):
+ try:
+ with open(self.out_file(fname), "r") as f:
+ return f.read()
+ except IOError:
+ return default
+
+ def run(self):
+ self.start_time = datetime.utcnow()
+ try:
+ if self.force_skip:
+ return self.skip()
+ else:
+ return self._run()
+ except:
+ return self.error("Python exception:\n%s" % traceback.format_exc())
+
+ @abstractmethod
+ def _run(self):
+ pass
+
+class RunGem5(TestUnit):
+ """Test unit representing a gem5 run.
+
+ Possible failure modes:
+ - gem5 failed to run -> STATE_ERROR
+ - timeout -> STATE_ERROR
+ - non-zero exit code -> STATE_ERROR
+
+ Possible non-failure results:
+ - exit code == 0 -> STATE_OK
+ - exit code == 2 -> STATE_SKIPPED
+ """
+
+ def __init__(self, gem5, gem5_args, timeout=0, **kwargs):
+ super(RunGem5, self).__init__("gem5", **kwargs)
+ self.gem5 = gem5
+ self.args = gem5_args
+ self.timeout = timeout
+
+ def _run(self):
+ gem5_cmd = [
+ self.gem5,
+ "-d", self.test_dir,
+ "-re",
+ ] + self.args
+
+ try:
+ with ProcessHelper(gem5_cmd, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE) as p:
+ status, gem5_stdout, gem5_stderr = p.call(timeout=self.timeout)
+ except CallTimeoutException as te:
+ return self.error("Timeout", stdout=te.stdout, stderr=te.stderr)
+ except OSError as ose:
+ return self.error("Failed to launch gem5: %s" % ose)
+
+ stderr = "\n".join([
+ "*** gem5 stderr ***",
+ gem5_stderr,
+ "",
+ "*** m5out/simerr ***",
+ self._read_output("simerr"),
+ ])
+
+ stdout = "\n".join([
+ "*** gem5 stdout ***",
+ gem5_stdout,
+ "",
+ "*** m5out/simout ***",
+ self._read_output("simout"),
+ ])
+
+ # Signal
+ if status < 0:
+ return self.error("gem5 terminated by signal %i" % (-status, ),
+ stdout=stdout, stderr=stderr)
+ elif status == 2:
+ return self.skip(stdout=stdout, stderr=stderr)
+ elif status > 0:
+ return self.error("gem5 exited with non-zero status: %i" % status,
+ stdout=stdout, stderr=stderr)
+ else:
+ return self.ok(stdout=stdout, stderr=stderr)
+
+class DiffOutFile(TestUnit):
+ """Test unit comparing and output file and a reference file."""
+
+ # regular expressions of lines to ignore when diffing outputs
+ diff_ignore_regexes = {
+ "simout" : [
+ re.compile('^Redirecting (stdout|stderr) to'),
+ re.compile('^gem5 compiled '),
+ re.compile('^gem5 started '),
+ re.compile('^gem5 executing on '),
+ re.compile('^command line:'),
+ re.compile("^Couldn't import dot_parser,"),
+ re.compile("^info: kernel located at:"),
+ re.compile("^Couldn't unlink "),
+ re.compile("^Using GPU kernel code file\(s\) "),
+ ],
+ "simerr" : [
+ #re.compile('^Simulation complete at'),
+ ],
+ "config.ini" : [
+ re.compile("^(executable|readfile|kernel|image_file)="),
+ re.compile("^(cwd|input|codefile)="),
+ ],
+ "config.json" : [
+ re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''),
+ re.compile(r'''^\s*"(cwd|input|codefile)":'''),
+ ],
+ }
+
+ def __init__(self, fname, **kwargs):
+ super(DiffOutFile, self).__init__("diff[%s]" % fname,
+ **kwargs)
+
+ self.fname = fname
+ self.line_filters = DiffOutFile.diff_ignore_regexes.get(fname, tuple())
+
+ def _filter_file(self, fname):
+ def match_line(l):
+ for r in self.line_filters:
+ if r.match(l):
+ return True
+ return False
+
+ with open(fname, "r") as f:
+ for l in f:
+ if not match_line(l):
+ yield l
+
+
+ def _run(self):
+ fname = self.fname
+ ref = self.ref_file(fname)
+ out = self.out_file(fname)
+
+ if not os.path.exists(ref):
+ return self.error("%s doesn't exist in reference directory" \
+ % fname)
+
+ if not os.path.exists(out):
+ return self.error("%s doesn't exist in output directory" % fname)
+
+ diff = difflib.unified_diff(
+ tuple(self._filter_file(ref)),
+ tuple(self._filter_file(out)),
+ fromfile="ref/%s" % fname, tofile="out/%s" % fname)
+
+ diff = list(diff)
+ if diff:
+ return self.error("ref/%s and out/%s differ" % (fname, fname),
+ stderr="".join(diff))
+ else:
+ return self.ok(stdout="-- ref/%s and out/%s are identical --" \
+ % (fname, fname))
+
+class DiffStatFile(TestUnit):
+ """Test unit comparing two gem5 stat files."""
+
+ def __init__(self, **kwargs):
+ super(DiffStatFile, self).__init__("stat_diff", **kwargs)
+
+ self.stat_diff = os.path.join(_test_base, "diff-out")
+
+ def _run(self):
+ stats = "stats.txt"
+
+ cmd = [
+ self.stat_diff,
+ self.ref_file(stats), self.out_file(stats),
+ ]
+ with ProcessHelper(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE) as p:
+ status, stdout, stderr = p.call()
+
+ if status == 0:
+ return self.ok(stdout=stdout, stderr=stderr)
+ if status == 1:
+ return self.failure("Statistics mismatch",
+ stdout=stdout, stderr=stderr)
+ else:
+ return self.error("diff-out returned an error: %i" % status,
+ stdout=stdout, stderr=stderr)
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright (c) 2016 ARM Limited
+# All rights reserved
+#
+# The license below extends only to copyright in the software and shall
+# not be construed as granting a license to any other intellectual
+# property including but not limited to intellectual property relating
+# to a hardware implementation of the functionality of the software
+# licensed hereunder. You may use the software subject to the license
+# terms below provided that you ensure that this notice is replicated
+# unmodified and in its entirety in all distributions of the software,
+# modified or unmodified, in source code or in binary form.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Andreas Sandberg
+
+import argparse
+import sys
+import os
+import pickle
+
+from testing.tests import *
+import testing.results
+
+class ParagraphHelpFormatter(argparse.HelpFormatter):
+ def _fill_text(self, text, width, indent):
+ return "\n\n".join([
+ super(ParagraphHelpFormatter, self)._fill_text(p, width, indent) \
+ for p in text.split("\n\n") ])
+
+formatters = {
+ "junit" : testing.results.JUnit,
+ "text" : testing.results.Text,
+ "summary" : testing.results.TextSummary,
+ "pickle" : testing.results.Pickle,
+}
+
+
+def _add_format_args(parser):
+ parser.add_argument("--format", choices=formatters, default="text",
+ help="Output format")
+
+ parser.add_argument("--no-junit-xlate-names", action="store_true",
+ help="Don't translate test names to " \
+ "package-like names")
+
+ parser.add_argument("--output", "-o",
+ type=argparse.FileType('w'), default=sys.stdout,
+ help="Test result output file")
+
+
+def _create_formatter(args):
+ formatter = formatters[args.format]
+ kwargs = {
+ "fout" : args.output,
+ "verbose" : args.verbose
+ }
+
+ if issubclass(formatter, testing.results.JUnit):
+ kwargs.update({
+ "translate_names" : not args.no_junit_xlate_names,
+ })
+
+ return formatter(**kwargs)
+
+
+def _list_tests_args(subparsers):
+ parser = subparsers.add_parser(
+ "list",
+ formatter_class=ParagraphHelpFormatter,
+ help="List available tests",
+ description="List available tests",
+ epilog="""
+ Generate a list of available tests using a list filter.
+
+ The filter is a string consisting of the target ISA optionally
+ followed by the test category and mode separated by
+ slashes. The test names emitted by this command can be fed
+ into the run command.
+
+ For example, to list all quick arm tests, run the following:
+ tests.py list arm/quick
+
+ Non-mandatory parts of the filter string (anything other than
+ the ISA) can be left out or replaced with the wildcard
+ character. For example, all full-system tests can be listed
+ with this command: tests.py list arm/*/fs""")
+
+ parser.add_argument("--ruby-protocol", type=str, default=None,
+ help="Ruby protocol")
+
+ parser.add_argument("--gpu-isa", type=str, default=None,
+ help="GPU ISA")
+
+ parser.add_argument("list_filter", metavar="ISA[/category/mode]",
+ action="append", type=str,
+ help="List available test cases")
+
+def _list_tests(args):
+ for isa, categories, modes in \
+ ( parse_test_filter(f) for f in args.list_filter ):
+
+ for test in get_tests(isa, categories=categories, modes=modes,
+ ruby_protocol=args.ruby_protocol,
+ gpu_isa=args.gpu_isa):
+ print "/".join(test)
+ sys.exit(0)
+
+def _run_tests_args(subparsers):
+ parser = subparsers.add_parser(
+ "run",
+ formatter_class=ParagraphHelpFormatter,
+ help='Run one or more tests',
+ description="Run one or more tests.",
+ epilog="""
+ Run one or more tests described by a gem5 test tuple.
+
+ The test tuple consists of a test category (quick or long), a
+ test mode (fs or se), a workload name, an isa, an operating
+ system, and a config name separate by slashes. For example:
+ quick/se/00.hello/arm/linux/simple-timing
+
+ Available tests can be listed using the 'list' sub-command
+ (e.g., "tests.py list arm/quick" or one of the scons test list
+ targets (e.g., "scons build/ARM/tests/opt/quick.list").
+
+ The test results can be stored in multiple different output
+ formats. See the help for the show command for more details
+ about output formatting.""")
+
+ parser.add_argument("gem5", type=str,
+ help="gem5 binary")
+
+ parser.add_argument("test", type=str, nargs="*",
+ help="List of tests to execute")
+
+ parser.add_argument("--directory", "-d",
+ type=str, default="m5tests",
+ help="Test work directory")
+
+ parser.add_argument("--timeout", "-t",
+ type=int, default="0", metavar="MINUTES",
+ help="Timeout, 0 to disable")
+
+ parser.add_argument("--skip-diff-out", action="store_true",
+ help="Skip output diffing stage")
+
+ parser.add_argument("--skip-diff-stat", action="store_true",
+ help="Skip stat diffing stage")
+
+ _add_format_args(parser)
+
+def _run_tests(args):
+ formatter = _create_formatter(args)
+
+ out_base = os.path.abspath(args.directory)
+ if not os.path.exists(out_base):
+ os.mkdir(out_base)
+ tests = []
+ for test_name in args.test:
+ config = ClassicConfig(*test_name.split("/"))
+ out_dir = os.path.join(out_base, "/".join(config))
+ tests.append(
+ ClassicTest(args.gem5, out_dir, config,
+ timeout=args.timeout,
+ skip_diff_stat=args.skip_diff_stat,
+ skip_diff_out=args.skip_diff_out))
+
+ all_results = []
+ print "Running %i tests" % len(tests)
+ for testno, test in enumerate(tests):
+ print "%i: Running '%s'..." % (testno, test)
+
+ all_results.append(test.run())
+
+ formatter.dump_suites(all_results)
+
+def _show_args(subparsers):
+ parser = subparsers.add_parser(
+ "show",
+ formatter_class=ParagraphHelpFormatter,
+ help='Display pickled test results',
+ description='Display pickled test results',
+ epilog="""
+ Reformat the pickled output from one or more test runs. This
+ command is typically used with the output from a single test
+ run, but it can also be used to merge the outputs from
+ multiple runs.
+
+ The 'text' format is a verbose output format that provides
+ information about individual test units and the output from
+ failed tests. It's mainly useful for debugging test failures.
+
+ The 'summary' format provides outputs the results of one test
+ per line with the test's overall status (OK, SKIPPED, or
+ FAILED).
+
+ The 'junit' format is primarily intended for use with CI
+ systems. It provides an XML representation of test
+ status. Similar to the text format, it includes detailed
+ information about test failures. Since many JUnit parser make
+ assume that test names look like Java packet strings, the
+ JUnit formatter automatically to something the looks like a
+ Java class path ('.'->'-', '/'->'.').
+
+ The 'pickle' format stores the raw results in a format that
+ can be reformatted using this command. It's typically used
+ with the show command to merge multiple test results into one
+ pickle file.""")
+
+ _add_format_args(parser)
+
+ parser.add_argument("result", type=argparse.FileType("rb"), nargs="*",
+ help="Pickled test results")
+
+def _show(args):
+ formatter = _create_formatter(args)
+ suites = sum([ pickle.load(f) for f in args.result ], [])
+ formatter.dump_suites(suites)
+
+_commands = {
+ "list" : (_list_tests, _list_tests_args),
+ "run" : (_run_tests, _run_tests_args),
+ "show" : (_show, _show_args),
+}
+
+def main():
+ parser = argparse.ArgumentParser(
+ formatter_class=ParagraphHelpFormatter,
+ description="""gem5 testing multi tool.""",
+ epilog="""
+ This tool provides an interface to gem5's test framework that
+ doesn't depend on gem5's build system. It supports test
+ listing, running, and output formatting.
+
+ The list sub-command (e.g., "test.py list arm/quick") produces
+ a list of tests tuples that can be used by the run command
+ (e.g., "tests.py run gem5.opt
+ quick/se/00.hello/arm/linux/simple-timing").
+
+ The run command supports several output formats. One of them,
+ pickle, contains the raw output from the tests and can be
+ re-formatted using the show command (e.g., "tests.py show
+ --format summary *.pickle"). Such pickle files are also
+ generated by the build system when scons is used to run
+ regressions.
+
+ See the usage strings for the individual sub-commands for
+ details.""")
+
+ parser.add_argument("--verbose", action="store_true",
+ help="Produce more verbose output")
+
+ subparsers = parser.add_subparsers(dest="command")
+
+ for key, (impl, cmd_parser) in _commands.items():
+ cmd_parser(subparsers)
+
+ args = parser.parse_args()
+ impl, cmd_parser = _commands[args.command]
+ impl(args)
+
+if __name__ == "__main__":
+ main()
projects / gem5.git / commitdiff