This was used by the now deleted tests/tests.py script.
Change-Id: I18481b02a78432b88e6cd9226a4c046bc6433743
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/32119
Tested-by: kokoro <noreply+kokoro@google.com>
Reviewed-by: Jason Lowe-Power <power.jg@gmail.com>
Maintainer: Bobby R. Bruce <bbruce@ucdavis.edu>
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright (c) 2016 ARM Limited
-# All rights reserved
-#
-# The license below extends only to copyright in the software and shall
-# not be construed as granting a license to any other intellectual
-# property including but not limited to intellectual property relating
-# to a hardware implementation of the functionality of the software
-# licensed hereunder. You may use the software subject to the license
-# terms below provided that you ensure that this notice is replicated
-# unmodified and in its entirety in all distributions of the software,
-# modified or unmodified, in source code or in binary form.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met: redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer;
-# redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution;
-# neither the name of the copyright holders nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright (c) 2016 ARM Limited
-# All rights reserved
-#
-# The license below extends only to copyright in the software and shall
-# not be construed as granting a license to any other intellectual
-# property including but not limited to intellectual property relating
-# to a hardware implementation of the functionality of the software
-# licensed hereunder. You may use the software subject to the license
-# terms below provided that you ensure that this notice is replicated
-# unmodified and in its entirety in all distributions of the software,
-# modified or unmodified, in source code or in binary form.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met: redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer;
-# redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution;
-# neither the name of the copyright holders nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import print_function
-
-import subprocess
-from threading import Timer
-import time
-import re
-
-class CallTimeoutException(Exception):
- """Exception that indicates that a process call timed out"""
-
- def __init__(self, status, stdout, stderr):
- self.status = status
- self.stdout = stdout
- self.stderr = stderr
-
-class ProcessHelper(subprocess.Popen):
- """Helper class to run child processes.
-
- This class wraps a subprocess.Popen class and adds support for
- using it in a with block. When the process goes out of scope, it's
- automatically terminated.
-
- with ProcessHelper(["/bin/ls"], stdout=subprocess.PIPE) as p:
- return p.call()
- """
- def __init__(self, *args, **kwargs):
- super(ProcessHelper, self).__init__(*args, **kwargs)
-
- def _terminate_nicely(self, timeout=5):
- def on_timeout():
- self.kill()
-
- if self.returncode is not None:
- return self.returncode
-
- timer = Timer(timeout, on_timeout)
- self.terminate()
- status = self.wait()
- timer.cancel()
-
- return status
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_value, traceback):
- if self.returncode is None:
- self._terminate_nicely()
-
- def call(self, timeout=0):
- self._timeout = False
- def on_timeout():
- self._timeout = True
- self._terminate_nicely()
-
- status, stdout, stderr = None, None, None
- timer = Timer(timeout, on_timeout)
- if timeout:
- timer.start()
-
- stdout, stderr = self.communicate()
- status = self.wait()
-
- timer.cancel()
-
- if self._timeout:
- self._terminate_nicely()
- raise CallTimeoutException(self.returncode, stdout, stderr)
- else:
- return status, stdout, stderr
-
-class FileIgnoreList(object):
- """Helper class to implement file ignore lists.
-
- This class implements ignore lists using plain string matching and
- regular expressions. In the simplest use case, rules are created
- statically upon initialization:
-
- ignore_list = FileIgnoreList(name=("ignore_me.txt", ), rex=(r".*~", )
-
- Ignores can be queried using in the same ways as normal Python
- containers:
-
- if file_name in ignore_list:
- print "Ignoring %s" % file_name
-
-
- New rules can be added at runtime by extending the list in the
- rules attribute:
-
- ignore_list.rules.append(FileIgnoreList.simple("bar.txt"))
- """
-
- @staticmethod
- def simple(r):
- return lambda f: f == r
-
- @staticmethod
- def rex(r):
- re_obj = r if hasattr(r, "search") else re.compile(r)
- return lambda name: re_obj.search(name)
-
- def __init__(self, names=(), rex=()):
- self.rules = [ FileIgnoreList.simple(n) for n in names ] + \
- [ FileIgnoreList.rex(r) for r in rex ]
-
- def __contains__(self, name):
- for rule in self.rules:
- if rule(name):
- return True
- return False
-
-if __name__ == "__main__":
- # Run internal self tests to ensure that the helpers are working
- # properly. The expected output when running this script is
- # "SUCCESS!".
-
- cmd_foo = [ "/bin/echo", "-n", "foo" ]
- cmd_sleep = [ "/bin/sleep", "10" ]
-
- # Test that things don't break if the process hasn't been started
- with ProcessHelper(cmd_foo) as p:
- pass
-
- with ProcessHelper(cmd_foo, stdout=subprocess.PIPE) as p:
- status, stdout, stderr = p.call()
- assert stdout == "foo"
- assert status == 0
-
- try:
- with ProcessHelper(cmd_sleep) as p:
- status, stdout, stderr = p.call(timeout=1)
- assert False, "Timeout not triggered"
- except CallTimeoutException:
- pass
-
- ignore_list = FileIgnoreList(
- names=("ignore.txt", "foo/test.txt"),
- rex=(r"~$", re.compile("^#")))
-
- assert "ignore.txt" in ignore_list
- assert "bar.txt" not in ignore_list
- assert "foo/test.txt" in ignore_list
- assert "test.txt" not in ignore_list
- assert "file1.c~" in ignore_list
- assert "file1.c" not in ignore_list
- assert "#foo" in ignore_list
- assert "foo#" not in ignore_list
-
- ignore_list.rules.append(FileIgnoreList.simple("bar.txt"))
- assert "bar.txt" in ignore_list
-
- print("SUCCESS!")
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright (c) 2016 ARM Limited
-# All rights reserved
-#
-# The license below extends only to copyright in the software and shall
-# not be construed as granting a license to any other intellectual
-# property including but not limited to intellectual property relating
-# to a hardware implementation of the functionality of the software
-# licensed hereunder. You may use the software subject to the license
-# terms below provided that you ensure that this notice is replicated
-# unmodified and in its entirety in all distributions of the software,
-# modified or unmodified, in source code or in binary form.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met: redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer;
-# redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution;
-# neither the name of the copyright holders nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import print_function
-
-from abc import ABCMeta, abstractmethod
-import inspect
-import pickle
-from six import add_metaclass
-import string
-import sys
-
-import xml.etree.cElementTree as ET
-
-class UnitResult(object):
- """Results of a single test unit.
-
- A test result can be one of:
- - STATE_OK: Test ran successfully.
- - STATE_SKIPPED: The test was skipped.
- - STATE_ERROR: The test failed to run.
- - STATE_FAILED: Test ran, but failed.
-
- The difference between STATE_ERROR and STATE_FAILED is very
- subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed
- to start or crashed, while STATE_FAILED would mean that a test
- failed (e.g., statistics mismatch).
-
- """
-
- STATE_OK = 0
- STATE_SKIPPED = 1
- STATE_ERROR = 2
- STATE_FAILURE = 3
-
- state_names = {
- STATE_OK : "OK",
- STATE_SKIPPED : "SKIPPED",
- STATE_ERROR : "ERROR",
- STATE_FAILURE : "FAILURE",
- }
-
- def __init__(self, name, state, message="", stderr="", stdout="",
- runtime=0.0):
- self.name = name
- self.state = state
- self.message = message
- self.stdout = stdout
- self.stderr = stderr
- self.runtime = runtime
-
- def skipped(self):
- return self.state == UnitResult.STATE_SKIPPED
-
- def success(self):
- return self.state == UnitResult.STATE_OK
-
- def state_name(self):
- return UnitResult.state_names[self.state]
-
- def __nonzero__(self):
- return self.success() or self.skipped()
-
- def __str__(self):
- state_name = self.state_name()
-
- status = "%s: %s" % (state_name, self.message) if self.message else \
- state_name
-
- return "%s: %s" % (self.name, status)
-
-class TestResult(object):
- """Results for from a single test consisting of one or more units."""
-
- def __init__(self, name, run_results=[], verify_results=[]):
- self.name = name
- self.results = run_results + verify_results
- self.run_results = run_results
- self.verify_results = verify_results
-
- def success(self):
- return self.success_run() and self.success_verify()
-
- def success_run(self):
- return all([ r.success() for r in self.run_results ])
-
- def success_verify(self):
- return all([ r.success() for r in self.verify_results ])
-
- def failed(self):
- return self.failed_run() or self.failed_verify()
-
- def failed_run(self):
- return any([ not r for r in self.run_results ])
-
- def failed_verify(self):
- return any([ not r for r in self.verify_results ])
-
- def skipped(self):
- return all([ r.skipped() for r in self.run_results ])
-
- def changed(self):
- return self.success_run() and self.failed_verify()
-
- def runtime(self):
- return sum([ r.runtime for r in self.results ])
-
- def __nonzero__(self):
- return all([ r for r in self.results ])
-
-@add_metaclass(ABCMeta)
-class ResultFormatter(object):
-
- def __init__(self, fout=sys.stdout, verbose=False):
- self.verbose = verbose
- self.fout = fout
-
- @abstractmethod
- def dump_suites(self, suites):
- pass
-
-class Pickle(ResultFormatter):
- """Save test results as a binary using Python's pickle
- functionality.
-
- """
-
- def __init__(self, **kwargs):
- super(Pickle, self).__init__(**kwargs)
-
- def dump_suites(self, suites):
- pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL)
-
-class Text(ResultFormatter):
- """Output test results as text."""
-
- def __init__(self, **kwargs):
- super(Text, self).__init__(**kwargs)
-
- def dump_suites(self, suites):
- fout = self.fout
- for suite in suites:
- print("--- %s ---" % suite.name, file=fout)
-
- for t in suite.results:
- print("*** %s" % t, file=fout)
-
- if t and not self.verbose:
- continue
-
- if t.message:
- print(t.message, file=fout)
-
- if t.stderr:
- print(t.stderr, file=fout)
- if t.stdout:
- print(t.stdout, file=fout)
-
-class TextSummary(ResultFormatter):
- """Output test results as a text summary"""
-
- def __init__(self, **kwargs):
- super(TextSummary, self).__init__(**kwargs)
-
- def test_status(self, suite):
- if suite.skipped():
- return "SKIPPED"
- elif suite.changed():
- return "CHANGED"
- elif suite:
- return "OK"
- else:
- return "FAILED"
-
- def dump_suites(self, suites):
- fout = self.fout
- for suite in suites:
- status = self.test_status(suite)
- print("%s: %s" % (suite.name, status), file=fout)
-
-class JUnit(ResultFormatter):
- """Output test results as JUnit XML"""
-
- def __init__(self, translate_names=True, **kwargs):
- super(JUnit, self).__init__(**kwargs)
-
- if translate_names:
- self.name_table = string.maketrans(
- "/.",
- ".-",
- )
- else:
- self.name_table = string.maketrans("", "")
-
- def convert_unit(self, x_suite, test):
- x_test = ET.SubElement(x_suite, "testcase",
- name=test.name,
- time="%f" % test.runtime)
-
- x_state = None
- if test.state == UnitResult.STATE_OK:
- pass
- elif test.state == UnitResult.STATE_SKIPPED:
- x_state = ET.SubElement(x_test, "skipped")
- elif test.state == UnitResult.STATE_FAILURE:
- x_state = ET.SubElement(x_test, "failure")
- elif test.state == UnitResult.STATE_ERROR:
- x_state = ET.SubElement(x_test, "error")
- else:
- assert False, "Unknown test state"
-
- if x_state is not None:
- if test.message:
- x_state.set("message", test.message)
-
- msg = []
- if test.stderr:
- msg.append("*** Standard Errror: ***")
- msg.append(test.stderr)
- if test.stdout:
- msg.append("*** Standard Out: ***")
- msg.append(test.stdout)
-
- x_state.text = "\n".join(msg)
-
- return x_test
-
- def convert_suite(self, x_suites, suite):
- x_suite = ET.SubElement(x_suites, "testsuite",
- name=suite.name.translate(self.name_table),
- time="%f" % suite.runtime())
- errors = 0
- failures = 0
- skipped = 0
-
- for test in suite.results:
- if test.state != UnitResult.STATE_OK:
- if test.state == UnitResult.STATE_SKIPPED:
- skipped += 1
- elif test.state == UnitResult.STATE_ERROR:
- errors += 1
- elif test.state == UnitResult.STATE_FAILURE:
- failures += 1
-
- x_test = self.convert_unit(x_suite, test)
-
- x_suite.set("errors", str(errors))
- x_suite.set("failures", str(failures))
- x_suite.set("skipped", str(skipped))
- x_suite.set("tests", str(len(suite.results)))
-
- return x_suite
-
- def convert_suites(self, suites):
- x_root = ET.Element("testsuites")
-
- for suite in suites:
- self.convert_suite(x_root, suite)
-
- return x_root
-
- def dump_suites(self, suites):
- et = ET.ElementTree(self.convert_suites(suites))
- et.write(self.fout, encoding="UTF-8")
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright (c) 2016-2017 ARM Limited
-# All rights reserved
-#
-# The license below extends only to copyright in the software and shall
-# not be construed as granting a license to any other intellectual
-# property including but not limited to intellectual property relating
-# to a hardware implementation of the functionality of the software
-# licensed hereunder. You may use the software subject to the license
-# terms below provided that you ensure that this notice is replicated
-# unmodified and in its entirety in all distributions of the software,
-# modified or unmodified, in source code or in binary form.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met: redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer;
-# redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution;
-# neither the name of the copyright holders nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-from abc import ABCMeta, abstractmethod
-import os
-from collections import namedtuple
-
-from six import add_metaclass
-
-import sys
-sys.path.append(os.path.dirname(__file__))
-
-from units import *
-from helpers import FileIgnoreList
-from results import TestResult
-import shutil
-
-_test_base = os.path.join(os.path.dirname(__file__), "..")
-
-ClassicConfig = namedtuple("ClassicConfig", (
- "category",
- "mode",
- "workload",
- "isa",
- "os",
- "config",
-))
-
-# There are currently two "classes" of test
-# configurations. Architecture-specific ones and generic ones
-# (typically SE mode tests). In both cases, the configuration name
-# matches a file in tests/configs/ that will be picked up by the test
-# runner (run.py).
-#
-# Architecture specific configurations are listed in the arch_configs
-# dictionary. This is indexed by a (cpu architecture, gpu
-# architecture) tuple. GPU architecture is optional and may be None.
-#
-# Generic configurations are listed in the generic_configs tuple.
-#
-# When discovering available test cases, this script look uses the
-# test list as a list of /candidate/ configurations. A configuration
-# is only used if a test has a reference output for that
-# configuration. In addition to the base configurations from
-# arch_configs and generic_configs, a Ruby configuration may be
-# appended to the base name (this is probed /in addition/ to the
-# original name. See get_tests() for details.
-#
-arch_configs = {
- ("arm", None) : (
- 'simple-atomic-dummychecker',
- 'o3-timing-checker',
- 'realview-simple-atomic',
- 'realview-simple-atomic-dual',
- 'realview-simple-atomic-checkpoint',
- 'realview-simple-timing',
- 'realview-simple-timing-dual',
- 'realview-o3',
- 'realview-o3-checker',
- 'realview-o3-dual',
- 'realview-minor',
- 'realview-minor-dual',
- 'realview-switcheroo-atomic',
- 'realview-switcheroo-timing',
- 'realview-switcheroo-noncaching-timing',
- 'realview-switcheroo-o3',
- 'realview-switcheroo-full',
- 'realview64-simple-atomic',
- 'realview64-simple-atomic-checkpoint',
- 'realview64-simple-atomic-dual',
- 'realview64-simple-timing',
- 'realview64-simple-timing-dual',
- 'realview64-o3',
- 'realview64-o3-checker',
- 'realview64-o3-dual',
- 'realview64-minor',
- 'realview64-minor-dual',
- 'realview64-switcheroo-atomic',
- 'realview64-switcheroo-timing',
- 'realview64-switcheroo-o3',
- 'realview64-switcheroo-full',
- ),
-
- ("sparc", None) : (
- 't1000-simple-atomic',
- 't1000-simple-x86',
- ),
-
- ("x86", None) : (
- 'pc-simple-atomic',
- 'pc-simple-timing',
- 'pc-o3-timing',
- 'pc-switcheroo-full',
- ),
-
- ("x86", "hsail") : (
- 'gpu',
- ),
-}
-
-generic_configs = (
- 'simple-atomic',
- 'simple-atomic-mp',
- 'simple-timing',
- 'simple-timing-mp',
-
- 'minor-timing',
- 'minor-timing-mp',
-
- 'o3-timing',
- 'o3-timing-mt',
- 'o3-timing-mp',
-
- 'rubytest',
- 'memcheck',
- 'memtest',
- 'memtest-filter',
- 'tgen-simple-mem',
- 'tgen-dram-ctrl',
- 'dram-lowp',
-
- 'learning-gem5-p1-simple',
- 'learning-gem5-p1-two-level',
-)
-
-default_ruby_protocol = {
- "arm" : "MOESI_CMP_directory",
-}
-
-def get_default_protocol(arch):
- return default_ruby_protocol.get(arch, 'MI_example')
-
-all_categories = ("quick", "long")
-all_modes = ("fs", "se")
-
-@add_metaclass(ABCMeta)
-class Test(object):
- """Test case base class.
-
- Test cases consists of one or more test units that are run in two
- phases. A run phase (units produced by run_units() and a verify
- phase (units from verify_units()). The verify phase is skipped if
- the run phase fails.
-
- """
-
- def __init__(self, name):
- self.test_name = name
-
- @abstractmethod
- def ref_files(self):
- """Get a list of reference files used by this test case"""
- pass
-
- @abstractmethod
- def run_units(self):
- """Units (typically RunGem5 instances) that describe the run phase of
- this test.
-
- """
- pass
-
- @abstractmethod
- def verify_units(self):
- """Verify the output from the run phase (see run_units())."""
- pass
-
- @abstractmethod
- def update_ref(self):
- """Update reference files with files from a test run"""
- pass
-
- def run(self):
- """Run this test case and return a list of results"""
-
- run_results = [ u.run() for u in self.run_units() ]
- run_ok = all([not r.skipped() and r for r in run_results ])
-
- verify_results = [
- u.run() if run_ok else u.skip()
- for u in self.verify_units()
- ]
-
- return TestResult(self.test_name,
- run_results=run_results,
- verify_results=verify_results)
-
- def __str__(self):
- return self.test_name
-
-class ClassicTest(Test):
- # The diff ignore list contains all files that shouldn't be diffed
- # using DiffOutFile. These files typically use special-purpose
- # diff tools (e.g., DiffStatFile).
- diff_ignore_files = FileIgnoreList(
- names=(
- # Stat files use a special stat differ
- "stats.txt",
- ), rex=(
- ))
-
- # These files should never be included in the list of
- # reference files. This list should include temporary files
- # and other files that we don't care about.
- ref_ignore_files = FileIgnoreList(
- names=(
- "EMPTY",
- ), rex=(
- # Mercurial sometimes leaves backups when applying MQ patches
- r"\.orig$",
- r"\.rej$",
- ))
-
- def __init__(self, gem5, output_dir, config_tuple,
- timeout=None,
- skip=False, skip_diff_out=False, skip_diff_stat=False):
-
- super(ClassicTest, self).__init__("/".join(config_tuple))
-
- ct = config_tuple
-
- self.gem5 = os.path.abspath(gem5)
- self.script = os.path.join(_test_base, "run.py")
- self.config_tuple = ct
- self.timeout = timeout
-
- self.output_dir = output_dir
- self.ref_dir = os.path.join(_test_base,
- ct.category, ct.mode, ct.workload,
- "ref", ct.isa, ct.os, ct.config)
- self.skip_run = skip
- self.skip_diff_out = skip or skip_diff_out
- self.skip_diff_stat = skip or skip_diff_stat
-
- def ref_files(self):
- ref_dir = os.path.abspath(self.ref_dir)
- for root, dirs, files in os.walk(ref_dir, topdown=False):
- for f in files:
- fpath = os.path.join(root[len(ref_dir) + 1:], f)
- if fpath not in ClassicTest.ref_ignore_files:
- yield fpath
-
- def run_units(self):
- args = [
- self.script,
- "/".join(self.config_tuple),
- ]
-
- return [
- RunGem5(self.gem5, args,
- ref_dir=self.ref_dir, test_dir=self.output_dir,
- skip=self.skip_run),
- ]
-
- def verify_units(self):
- ref_files = set(self.ref_files())
- units = []
- if "stats.txt" in ref_files:
- units.append(
- DiffStatFile(ref_dir=self.ref_dir, test_dir=self.output_dir,
- skip=self.skip_diff_stat))
- units += [
- DiffOutFile(f,
- ref_dir=self.ref_dir, test_dir=self.output_dir,
- skip=self.skip_diff_out)
- for f in ref_files if f not in ClassicTest.diff_ignore_files
- ]
-
- return units
-
- def update_ref(self):
- for fname in self.ref_files():
- shutil.copy(
- os.path.join(self.output_dir, fname),
- os.path.join(self.ref_dir, fname))
-
-def parse_test_filter(test_filter):
- wildcards = ("", "*")
-
- _filter = list(test_filter.split("/"))
- if len(_filter) > 3:
- raise RuntimeError("Illegal test filter string")
- _filter += [ "", ] * (3 - len(_filter))
-
- isa, cat, mode = _filter
-
- if isa in wildcards:
- raise RuntimeError("No ISA specified")
-
- cat = all_categories if cat in wildcards else (cat, )
- mode = all_modes if mode in wildcards else (mode, )
-
- return isa, cat, mode
-
-def get_tests(isa,
- categories=all_categories, modes=all_modes,
- ruby_protocol=None, gpu_isa=None):
-
- # Generate a list of candidate configs
- configs = list(arch_configs.get((isa, gpu_isa), []))
-
- if (isa, gpu_isa) == ("x86", "hsail"):
- if ruby_protocol == "GPU_RfO":
- configs += ['gpu-randomtest']
- else:
- configs += generic_configs
-
- if ruby_protocol == get_default_protocol(isa):
- if ruby_protocol == 'MI_example':
- configs += [ "%s-ruby" % (c, ) for c in configs ]
- else:
- configs += [ "%s-ruby-%s" % (c, ruby_protocol) for c in configs ]
- elif ruby_protocol is not None:
- # Override generic ISA configs when using Ruby (excluding
- # MI_example which is included in all ISAs by default). This
- # reduces the number of generic tests we re-run for when
- # compiling Ruby targets.
- configs = [ "%s-ruby-%s" % (c, ruby_protocol) for c in configs ]
-
- # /(quick|long)/(fs|se)/workload/ref/arch/guest/config/
- for conf_script in configs:
- for cat in categories:
- for mode in modes:
- mode_dir = os.path.join(_test_base, cat, mode)
- if not os.path.exists(mode_dir):
- continue
-
- for workload in os.listdir(mode_dir):
- isa_dir = os.path.join(mode_dir, workload, "ref", isa)
- if not os.path.isdir(isa_dir):
- continue
-
- for _os in os.listdir(isa_dir):
- test_dir = os.path.join(isa_dir, _os, conf_script)
- if not os.path.exists(test_dir) or \
- os.path.exists(os.path.join(test_dir, "skip")):
- continue
-
- yield ClassicConfig(cat, mode, workload, isa, _os,
- conf_script)
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright (c) 2016 ARM Limited
-# All rights reserved
-#
-# The license below extends only to copyright in the software and shall
-# not be construed as granting a license to any other intellectual
-# property including but not limited to intellectual property relating
-# to a hardware implementation of the functionality of the software
-# licensed hereunder. You may use the software subject to the license
-# terms below provided that you ensure that this notice is replicated
-# unmodified and in its entirety in all distributions of the software,
-# modified or unmodified, in source code or in binary form.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met: redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer;
-# redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution;
-# neither the name of the copyright holders nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-from abc import ABCMeta, abstractmethod
-from datetime import datetime
-import difflib
-import functools
-import os
-import re
-from six import add_metaclass
-import subprocess
-import sys
-import traceback
-
-sys.path.append(os.path.dirname(__file__))
-from results import UnitResult
-from helpers import *
-
-_test_base = os.path.join(os.path.dirname(__file__), "..")
-
-@add_metaclass(ABCMeta)
-class TestUnit(object):
- """Base class for all test units.
-
- A test unit is a part of a larger test case. Test cases usually
- contain two types of units, run units (run gem5) and verify units
- (diff output files). All unit implementations inherit from this
- class.
-
- A unit implementation overrides the _run() method. The test runner
- calls the run() method, which wraps _run() to protect against
- exceptions.
-
- """
-
- def __init__(self, name, ref_dir, test_dir, skip=False):
- self.name = name
- self.ref_dir = ref_dir
- self.test_dir = test_dir
- self.force_skip = skip
- self.start_time = None
- self.stop_time = None
-
- def result(self, state, **kwargs):
- if self.start_time is not None and "runtime" not in kwargs:
- self.stop_time = datetime.utcnow()
- delta = self.stop_time - self.start_time
- kwargs["runtime"] = delta.total_seconds()
-
- return UnitResult(self.name, state, **kwargs)
-
- def ok(self, **kwargs):
- return self.result(UnitResult.STATE_OK, **kwargs)
-
- def skip(self, **kwargs):
- return self.result(UnitResult.STATE_SKIPPED, **kwargs)
-
- def error(self, message, **kwargs):
- return self.result(UnitResult.STATE_ERROR, message=message, **kwargs)
-
- def failure(self, message, **kwargs):
- return self.result(UnitResult.STATE_FAILURE, message=message, **kwargs)
-
- def ref_file(self, fname):
- return os.path.join(self.ref_dir, fname)
-
- def out_file(self, fname):
- return os.path.join(self.test_dir, fname)
-
- def _read_output(self, fname, default=""):
- try:
- with open(self.out_file(fname), "r") as f:
- return f.read()
- except IOError:
- return default
-
- def run(self):
- self.start_time = datetime.utcnow()
- try:
- if self.force_skip:
- return self.skip()
- else:
- return self._run()
- except:
- return self.error("Python exception:\n%s" % traceback.format_exc())
-
- @abstractmethod
- def _run(self):
- pass
-
-class RunGem5(TestUnit):
- """Test unit representing a gem5 run.
-
- Possible failure modes:
- - gem5 failed to run -> STATE_ERROR
- - timeout -> STATE_ERROR
- - non-zero exit code -> STATE_ERROR
-
- Possible non-failure results:
- - exit code == 0 -> STATE_OK
- - exit code == 2 -> STATE_SKIPPED
- """
-
- def __init__(self, gem5, gem5_args, timeout=0, **kwargs):
- super(RunGem5, self).__init__("gem5", **kwargs)
- self.gem5 = gem5
- self.args = gem5_args
- self.timeout = timeout
-
- def _run(self):
- gem5_cmd = [
- self.gem5,
- "-d", self.test_dir,
- "--stats-file", "text://stats.txt?desc=False",
- "-re",
- ] + self.args
-
- try:
- with ProcessHelper(gem5_cmd, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE) as p:
- status, gem5_stdout, gem5_stderr = p.call(timeout=self.timeout)
- except CallTimeoutException as te:
- return self.error("Timeout", stdout=te.stdout, stderr=te.stderr)
- except OSError as ose:
- return self.error("Failed to launch gem5: %s" % ose)
-
- stderr = "\n".join([
- "*** gem5 stderr ***",
- gem5_stderr,
- "",
- "*** m5out/simerr ***",
- self._read_output("simerr"),
- ])
-
- stdout = "\n".join([
- "*** gem5 stdout ***",
- gem5_stdout,
- "",
- "*** m5out/simout ***",
- self._read_output("simout"),
- ])
-
- # Signal
- if status < 0:
- return self.error("gem5 terminated by signal %i" % (-status, ),
- stdout=stdout, stderr=stderr)
- elif status == 2:
- return self.skip(stdout=stdout, stderr=stderr)
- elif status > 0:
- return self.error("gem5 exited with non-zero status: %i" % status,
- stdout=stdout, stderr=stderr)
- else:
- return self.ok(stdout=stdout, stderr=stderr)
-
-class DiffOutFile(TestUnit):
- """Test unit comparing and output file and a reference file."""
-
- # regular expressions of lines to ignore when diffing outputs
- diff_ignore_regexes = {
- "simout" : [
- re.compile('^Redirecting (stdout|stderr) to'),
- re.compile('^gem5 compiled '),
- re.compile('^gem5 started '),
- re.compile('^gem5 executing on '),
- re.compile('^command line:'),
- re.compile("^Couldn't import dot_parser,"),
- re.compile("^info: kernel located at:"),
- re.compile("^Couldn't unlink "),
- re.compile("^Using GPU kernel code file\(s\) "),
- ],
- "simerr" : [
- #re.compile('^Simulation complete at'),
- ],
- "config.ini" : [
- re.compile("^(executable|readfile|kernel|image_file)="),
- re.compile("^(cwd|input|codefile)="),
- ],
- "config.json" : [
- re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''),
- re.compile(r'''^\s*"(cwd|input|codefile)":'''),
- ],
- }
-
- def __init__(self, fname, **kwargs):
- super(DiffOutFile, self).__init__("diff[%s]" % fname,
- **kwargs)
-
- self.fname = fname
- self.line_filters = DiffOutFile.diff_ignore_regexes.get(fname, tuple())
-
- def _filter_file(self, fname):
- def match_line(l):
- for r in self.line_filters:
- if r.match(l):
- return True
- return False
-
- with open(fname, "r") as f:
- for l in f:
- if not match_line(l):
- yield l
-
-
- def _run(self):
- fname = self.fname
- ref = self.ref_file(fname)
- out = self.out_file(fname)
-
- if not os.path.exists(ref):
- return self.error("%s doesn't exist in reference directory" \
- % fname)
-
- if not os.path.exists(out):
- return self.error("%s doesn't exist in output directory" % fname)
-
- diff = difflib.unified_diff(
- tuple(self._filter_file(ref)),
- tuple(self._filter_file(out)),
- fromfile="ref/%s" % fname, tofile="out/%s" % fname)
-
- diff = list(diff)
- if diff:
- return self.error("ref/%s and out/%s differ" % (fname, fname),
- stderr="".join(diff))
- else:
- return self.ok(stdout="-- ref/%s and out/%s are identical --" \
- % (fname, fname))
-
-class DiffStatFile(TestUnit):
- """Test unit comparing two gem5 stat files."""
-
- def __init__(self, **kwargs):
- super(DiffStatFile, self).__init__("stat_diff", **kwargs)
-
- self.stat_diff = os.path.join(_test_base, "diff-out")
-
- def _run(self):
- STATUS_OK = 0
- STATUS_NEW_STATS = 1
- STATUS_FAILED = 2
-
- stats = "stats.txt"
-
- cmd = [
- self.stat_diff,
- self.ref_file(stats), self.out_file(stats),
- ]
- with ProcessHelper(cmd,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE) as p:
- status, stdout, stderr = p.call()
-
- if status in (STATUS_OK, STATUS_NEW_STATS):
- return self.ok(stdout=stdout, stderr=stderr)
- elif status == STATUS_FAILED:
- return self.failure("Statistics mismatch",
- stdout=stdout, stderr=stderr)
- else:
- return self.error("diff-out returned an error: %i" % status,
- stdout=stdout, stderr=stderr)
projects / gem5.git / commitdiff