From: Gabe Black Date: Tue, 4 Aug 2020 05:37:31 +0000 (-0700) Subject: tests: Get rid of the tests/testing python package. X-Git-Tag: v20.1.0.0~348 X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=204cf65ec5427314c64b7b067339be44fa804e11;p=gem5.git tests: Get rid of the tests/testing python package. This was used by the now deleted tests/tests.py script. Change-Id: I18481b02a78432b88e6cd9226a4c046bc6433743 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/32119 Tested-by: kokoro Reviewed-by: Jason Lowe-Power Maintainer: Bobby R. Bruce --- diff --git a/tests/testing/__init__.py b/tests/testing/__init__.py deleted file mode 100644 index 5dcc684da..000000000 --- a/tests/testing/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) 2016 ARM Limited -# All rights reserved -# -# The license below extends only to copyright in the software and shall -# not be construed as granting a license to any other intellectual -# property including but not limited to intellectual property relating -# to a hardware implementation of the functionality of the software -# licensed hereunder. You may use the software subject to the license -# terms below provided that you ensure that this notice is replicated -# unmodified and in its entirety in all distributions of the software, -# modified or unmodified, in source code or in binary form. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer; -# redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution; -# neither the name of the copyright holders nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tests/testing/helpers.py b/tests/testing/helpers.py deleted file mode 100755 index f46915c8f..000000000 --- a/tests/testing/helpers.py +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) 2016 ARM Limited -# All rights reserved -# -# The license below extends only to copyright in the software and shall -# not be construed as granting a license to any other intellectual -# property including but not limited to intellectual property relating -# to a hardware implementation of the functionality of the software -# licensed hereunder. You may use the software subject to the license -# terms below provided that you ensure that this notice is replicated -# unmodified and in its entirety in all distributions of the software, -# modified or unmodified, in source code or in binary form. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer; -# redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution; -# neither the name of the copyright holders nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from __future__ import print_function - -import subprocess -from threading import Timer -import time -import re - -class CallTimeoutException(Exception): - """Exception that indicates that a process call timed out""" - - def __init__(self, status, stdout, stderr): - self.status = status - self.stdout = stdout - self.stderr = stderr - -class ProcessHelper(subprocess.Popen): - """Helper class to run child processes. - - This class wraps a subprocess.Popen class and adds support for - using it in a with block. When the process goes out of scope, it's - automatically terminated. - - with ProcessHelper(["/bin/ls"], stdout=subprocess.PIPE) as p: - return p.call() - """ - def __init__(self, *args, **kwargs): - super(ProcessHelper, self).__init__(*args, **kwargs) - - def _terminate_nicely(self, timeout=5): - def on_timeout(): - self.kill() - - if self.returncode is not None: - return self.returncode - - timer = Timer(timeout, on_timeout) - self.terminate() - status = self.wait() - timer.cancel() - - return status - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - if self.returncode is None: - self._terminate_nicely() - - def call(self, timeout=0): - self._timeout = False - def on_timeout(): - self._timeout = True - self._terminate_nicely() - - status, stdout, stderr = None, None, None - timer = Timer(timeout, on_timeout) - if timeout: - timer.start() - - stdout, stderr = self.communicate() - status = self.wait() - - timer.cancel() - - if self._timeout: - self._terminate_nicely() - raise CallTimeoutException(self.returncode, stdout, stderr) - else: - return status, stdout, stderr - -class FileIgnoreList(object): - """Helper class to implement file ignore lists. - - This class implements ignore lists using plain string matching and - regular expressions. In the simplest use case, rules are created - statically upon initialization: - - ignore_list = FileIgnoreList(name=("ignore_me.txt", ), rex=(r".*~", ) - - Ignores can be queried using in the same ways as normal Python - containers: - - if file_name in ignore_list: - print "Ignoring %s" % file_name - - - New rules can be added at runtime by extending the list in the - rules attribute: - - ignore_list.rules.append(FileIgnoreList.simple("bar.txt")) - """ - - @staticmethod - def simple(r): - return lambda f: f == r - - @staticmethod - def rex(r): - re_obj = r if hasattr(r, "search") else re.compile(r) - return lambda name: re_obj.search(name) - - def __init__(self, names=(), rex=()): - self.rules = [ FileIgnoreList.simple(n) for n in names ] + \ - [ FileIgnoreList.rex(r) for r in rex ] - - def __contains__(self, name): - for rule in self.rules: - if rule(name): - return True - return False - -if __name__ == "__main__": - # Run internal self tests to ensure that the helpers are working - # properly. The expected output when running this script is - # "SUCCESS!". - - cmd_foo = [ "/bin/echo", "-n", "foo" ] - cmd_sleep = [ "/bin/sleep", "10" ] - - # Test that things don't break if the process hasn't been started - with ProcessHelper(cmd_foo) as p: - pass - - with ProcessHelper(cmd_foo, stdout=subprocess.PIPE) as p: - status, stdout, stderr = p.call() - assert stdout == "foo" - assert status == 0 - - try: - with ProcessHelper(cmd_sleep) as p: - status, stdout, stderr = p.call(timeout=1) - assert False, "Timeout not triggered" - except CallTimeoutException: - pass - - ignore_list = FileIgnoreList( - names=("ignore.txt", "foo/test.txt"), - rex=(r"~$", re.compile("^#"))) - - assert "ignore.txt" in ignore_list - assert "bar.txt" not in ignore_list - assert "foo/test.txt" in ignore_list - assert "test.txt" not in ignore_list - assert "file1.c~" in ignore_list - assert "file1.c" not in ignore_list - assert "#foo" in ignore_list - assert "foo#" not in ignore_list - - ignore_list.rules.append(FileIgnoreList.simple("bar.txt")) - assert "bar.txt" in ignore_list - - print("SUCCESS!") diff --git a/tests/testing/results.py b/tests/testing/results.py deleted file mode 100644 index eb88d4c23..000000000 --- a/tests/testing/results.py +++ /dev/null @@ -1,298 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) 2016 ARM Limited -# All rights reserved -# -# The license below extends only to copyright in the software and shall -# not be construed as granting a license to any other intellectual -# property including but not limited to intellectual property relating -# to a hardware implementation of the functionality of the software -# licensed hereunder. You may use the software subject to the license -# terms below provided that you ensure that this notice is replicated -# unmodified and in its entirety in all distributions of the software, -# modified or unmodified, in source code or in binary form. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer; -# redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution; -# neither the name of the copyright holders nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from __future__ import print_function - -from abc import ABCMeta, abstractmethod -import inspect -import pickle -from six import add_metaclass -import string -import sys - -import xml.etree.cElementTree as ET - -class UnitResult(object): - """Results of a single test unit. - - A test result can be one of: - - STATE_OK: Test ran successfully. - - STATE_SKIPPED: The test was skipped. - - STATE_ERROR: The test failed to run. - - STATE_FAILED: Test ran, but failed. - - The difference between STATE_ERROR and STATE_FAILED is very - subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed - to start or crashed, while STATE_FAILED would mean that a test - failed (e.g., statistics mismatch). - - """ - - STATE_OK = 0 - STATE_SKIPPED = 1 - STATE_ERROR = 2 - STATE_FAILURE = 3 - - state_names = { - STATE_OK : "OK", - STATE_SKIPPED : "SKIPPED", - STATE_ERROR : "ERROR", - STATE_FAILURE : "FAILURE", - } - - def __init__(self, name, state, message="", stderr="", stdout="", - runtime=0.0): - self.name = name - self.state = state - self.message = message - self.stdout = stdout - self.stderr = stderr - self.runtime = runtime - - def skipped(self): - return self.state == UnitResult.STATE_SKIPPED - - def success(self): - return self.state == UnitResult.STATE_OK - - def state_name(self): - return UnitResult.state_names[self.state] - - def __nonzero__(self): - return self.success() or self.skipped() - - def __str__(self): - state_name = self.state_name() - - status = "%s: %s" % (state_name, self.message) if self.message else \ - state_name - - return "%s: %s" % (self.name, status) - -class TestResult(object): - """Results for from a single test consisting of one or more units.""" - - def __init__(self, name, run_results=[], verify_results=[]): - self.name = name - self.results = run_results + verify_results - self.run_results = run_results - self.verify_results = verify_results - - def success(self): - return self.success_run() and self.success_verify() - - def success_run(self): - return all([ r.success() for r in self.run_results ]) - - def success_verify(self): - return all([ r.success() for r in self.verify_results ]) - - def failed(self): - return self.failed_run() or self.failed_verify() - - def failed_run(self): - return any([ not r for r in self.run_results ]) - - def failed_verify(self): - return any([ not r for r in self.verify_results ]) - - def skipped(self): - return all([ r.skipped() for r in self.run_results ]) - - def changed(self): - return self.success_run() and self.failed_verify() - - def runtime(self): - return sum([ r.runtime for r in self.results ]) - - def __nonzero__(self): - return all([ r for r in self.results ]) - -@add_metaclass(ABCMeta) -class ResultFormatter(object): - - def __init__(self, fout=sys.stdout, verbose=False): - self.verbose = verbose - self.fout = fout - - @abstractmethod - def dump_suites(self, suites): - pass - -class Pickle(ResultFormatter): - """Save test results as a binary using Python's pickle - functionality. - - """ - - def __init__(self, **kwargs): - super(Pickle, self).__init__(**kwargs) - - def dump_suites(self, suites): - pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL) - -class Text(ResultFormatter): - """Output test results as text.""" - - def __init__(self, **kwargs): - super(Text, self).__init__(**kwargs) - - def dump_suites(self, suites): - fout = self.fout - for suite in suites: - print("--- %s ---" % suite.name, file=fout) - - for t in suite.results: - print("*** %s" % t, file=fout) - - if t and not self.verbose: - continue - - if t.message: - print(t.message, file=fout) - - if t.stderr: - print(t.stderr, file=fout) - if t.stdout: - print(t.stdout, file=fout) - -class TextSummary(ResultFormatter): - """Output test results as a text summary""" - - def __init__(self, **kwargs): - super(TextSummary, self).__init__(**kwargs) - - def test_status(self, suite): - if suite.skipped(): - return "SKIPPED" - elif suite.changed(): - return "CHANGED" - elif suite: - return "OK" - else: - return "FAILED" - - def dump_suites(self, suites): - fout = self.fout - for suite in suites: - status = self.test_status(suite) - print("%s: %s" % (suite.name, status), file=fout) - -class JUnit(ResultFormatter): - """Output test results as JUnit XML""" - - def __init__(self, translate_names=True, **kwargs): - super(JUnit, self).__init__(**kwargs) - - if translate_names: - self.name_table = string.maketrans( - "/.", - ".-", - ) - else: - self.name_table = string.maketrans("", "") - - def convert_unit(self, x_suite, test): - x_test = ET.SubElement(x_suite, "testcase", - name=test.name, - time="%f" % test.runtime) - - x_state = None - if test.state == UnitResult.STATE_OK: - pass - elif test.state == UnitResult.STATE_SKIPPED: - x_state = ET.SubElement(x_test, "skipped") - elif test.state == UnitResult.STATE_FAILURE: - x_state = ET.SubElement(x_test, "failure") - elif test.state == UnitResult.STATE_ERROR: - x_state = ET.SubElement(x_test, "error") - else: - assert False, "Unknown test state" - - if x_state is not None: - if test.message: - x_state.set("message", test.message) - - msg = [] - if test.stderr: - msg.append("*** Standard Errror: ***") - msg.append(test.stderr) - if test.stdout: - msg.append("*** Standard Out: ***") - msg.append(test.stdout) - - x_state.text = "\n".join(msg) - - return x_test - - def convert_suite(self, x_suites, suite): - x_suite = ET.SubElement(x_suites, "testsuite", - name=suite.name.translate(self.name_table), - time="%f" % suite.runtime()) - errors = 0 - failures = 0 - skipped = 0 - - for test in suite.results: - if test.state != UnitResult.STATE_OK: - if test.state == UnitResult.STATE_SKIPPED: - skipped += 1 - elif test.state == UnitResult.STATE_ERROR: - errors += 1 - elif test.state == UnitResult.STATE_FAILURE: - failures += 1 - - x_test = self.convert_unit(x_suite, test) - - x_suite.set("errors", str(errors)) - x_suite.set("failures", str(failures)) - x_suite.set("skipped", str(skipped)) - x_suite.set("tests", str(len(suite.results))) - - return x_suite - - def convert_suites(self, suites): - x_root = ET.Element("testsuites") - - for suite in suites: - self.convert_suite(x_root, suite) - - return x_root - - def dump_suites(self, suites): - et = ET.ElementTree(self.convert_suites(suites)) - et.write(self.fout, encoding="UTF-8") diff --git a/tests/testing/tests.py b/tests/testing/tests.py deleted file mode 100755 index 26f431c8d..000000000 --- a/tests/testing/tests.py +++ /dev/null @@ -1,374 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) 2016-2017 ARM Limited -# All rights reserved -# -# The license below extends only to copyright in the software and shall -# not be construed as granting a license to any other intellectual -# property including but not limited to intellectual property relating -# to a hardware implementation of the functionality of the software -# licensed hereunder. You may use the software subject to the license -# terms below provided that you ensure that this notice is replicated -# unmodified and in its entirety in all distributions of the software, -# modified or unmodified, in source code or in binary form. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer; -# redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution; -# neither the name of the copyright holders nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from abc import ABCMeta, abstractmethod -import os -from collections import namedtuple - -from six import add_metaclass - -import sys -sys.path.append(os.path.dirname(__file__)) - -from units import * -from helpers import FileIgnoreList -from results import TestResult -import shutil - -_test_base = os.path.join(os.path.dirname(__file__), "..") - -ClassicConfig = namedtuple("ClassicConfig", ( - "category", - "mode", - "workload", - "isa", - "os", - "config", -)) - -# There are currently two "classes" of test -# configurations. Architecture-specific ones and generic ones -# (typically SE mode tests). In both cases, the configuration name -# matches a file in tests/configs/ that will be picked up by the test -# runner (run.py). -# -# Architecture specific configurations are listed in the arch_configs -# dictionary. This is indexed by a (cpu architecture, gpu -# architecture) tuple. GPU architecture is optional and may be None. -# -# Generic configurations are listed in the generic_configs tuple. -# -# When discovering available test cases, this script look uses the -# test list as a list of /candidate/ configurations. A configuration -# is only used if a test has a reference output for that -# configuration. In addition to the base configurations from -# arch_configs and generic_configs, a Ruby configuration may be -# appended to the base name (this is probed /in addition/ to the -# original name. See get_tests() for details. -# -arch_configs = { - ("arm", None) : ( - 'simple-atomic-dummychecker', - 'o3-timing-checker', - 'realview-simple-atomic', - 'realview-simple-atomic-dual', - 'realview-simple-atomic-checkpoint', - 'realview-simple-timing', - 'realview-simple-timing-dual', - 'realview-o3', - 'realview-o3-checker', - 'realview-o3-dual', - 'realview-minor', - 'realview-minor-dual', - 'realview-switcheroo-atomic', - 'realview-switcheroo-timing', - 'realview-switcheroo-noncaching-timing', - 'realview-switcheroo-o3', - 'realview-switcheroo-full', - 'realview64-simple-atomic', - 'realview64-simple-atomic-checkpoint', - 'realview64-simple-atomic-dual', - 'realview64-simple-timing', - 'realview64-simple-timing-dual', - 'realview64-o3', - 'realview64-o3-checker', - 'realview64-o3-dual', - 'realview64-minor', - 'realview64-minor-dual', - 'realview64-switcheroo-atomic', - 'realview64-switcheroo-timing', - 'realview64-switcheroo-o3', - 'realview64-switcheroo-full', - ), - - ("sparc", None) : ( - 't1000-simple-atomic', - 't1000-simple-x86', - ), - - ("x86", None) : ( - 'pc-simple-atomic', - 'pc-simple-timing', - 'pc-o3-timing', - 'pc-switcheroo-full', - ), - - ("x86", "hsail") : ( - 'gpu', - ), -} - -generic_configs = ( - 'simple-atomic', - 'simple-atomic-mp', - 'simple-timing', - 'simple-timing-mp', - - 'minor-timing', - 'minor-timing-mp', - - 'o3-timing', - 'o3-timing-mt', - 'o3-timing-mp', - - 'rubytest', - 'memcheck', - 'memtest', - 'memtest-filter', - 'tgen-simple-mem', - 'tgen-dram-ctrl', - 'dram-lowp', - - 'learning-gem5-p1-simple', - 'learning-gem5-p1-two-level', -) - -default_ruby_protocol = { - "arm" : "MOESI_CMP_directory", -} - -def get_default_protocol(arch): - return default_ruby_protocol.get(arch, 'MI_example') - -all_categories = ("quick", "long") -all_modes = ("fs", "se") - -@add_metaclass(ABCMeta) -class Test(object): - """Test case base class. - - Test cases consists of one or more test units that are run in two - phases. A run phase (units produced by run_units() and a verify - phase (units from verify_units()). The verify phase is skipped if - the run phase fails. - - """ - - def __init__(self, name): - self.test_name = name - - @abstractmethod - def ref_files(self): - """Get a list of reference files used by this test case""" - pass - - @abstractmethod - def run_units(self): - """Units (typically RunGem5 instances) that describe the run phase of - this test. - - """ - pass - - @abstractmethod - def verify_units(self): - """Verify the output from the run phase (see run_units()).""" - pass - - @abstractmethod - def update_ref(self): - """Update reference files with files from a test run""" - pass - - def run(self): - """Run this test case and return a list of results""" - - run_results = [ u.run() for u in self.run_units() ] - run_ok = all([not r.skipped() and r for r in run_results ]) - - verify_results = [ - u.run() if run_ok else u.skip() - for u in self.verify_units() - ] - - return TestResult(self.test_name, - run_results=run_results, - verify_results=verify_results) - - def __str__(self): - return self.test_name - -class ClassicTest(Test): - # The diff ignore list contains all files that shouldn't be diffed - # using DiffOutFile. These files typically use special-purpose - # diff tools (e.g., DiffStatFile). - diff_ignore_files = FileIgnoreList( - names=( - # Stat files use a special stat differ - "stats.txt", - ), rex=( - )) - - # These files should never be included in the list of - # reference files. This list should include temporary files - # and other files that we don't care about. - ref_ignore_files = FileIgnoreList( - names=( - "EMPTY", - ), rex=( - # Mercurial sometimes leaves backups when applying MQ patches - r"\.orig$", - r"\.rej$", - )) - - def __init__(self, gem5, output_dir, config_tuple, - timeout=None, - skip=False, skip_diff_out=False, skip_diff_stat=False): - - super(ClassicTest, self).__init__("/".join(config_tuple)) - - ct = config_tuple - - self.gem5 = os.path.abspath(gem5) - self.script = os.path.join(_test_base, "run.py") - self.config_tuple = ct - self.timeout = timeout - - self.output_dir = output_dir - self.ref_dir = os.path.join(_test_base, - ct.category, ct.mode, ct.workload, - "ref", ct.isa, ct.os, ct.config) - self.skip_run = skip - self.skip_diff_out = skip or skip_diff_out - self.skip_diff_stat = skip or skip_diff_stat - - def ref_files(self): - ref_dir = os.path.abspath(self.ref_dir) - for root, dirs, files in os.walk(ref_dir, topdown=False): - for f in files: - fpath = os.path.join(root[len(ref_dir) + 1:], f) - if fpath not in ClassicTest.ref_ignore_files: - yield fpath - - def run_units(self): - args = [ - self.script, - "/".join(self.config_tuple), - ] - - return [ - RunGem5(self.gem5, args, - ref_dir=self.ref_dir, test_dir=self.output_dir, - skip=self.skip_run), - ] - - def verify_units(self): - ref_files = set(self.ref_files()) - units = [] - if "stats.txt" in ref_files: - units.append( - DiffStatFile(ref_dir=self.ref_dir, test_dir=self.output_dir, - skip=self.skip_diff_stat)) - units += [ - DiffOutFile(f, - ref_dir=self.ref_dir, test_dir=self.output_dir, - skip=self.skip_diff_out) - for f in ref_files if f not in ClassicTest.diff_ignore_files - ] - - return units - - def update_ref(self): - for fname in self.ref_files(): - shutil.copy( - os.path.join(self.output_dir, fname), - os.path.join(self.ref_dir, fname)) - -def parse_test_filter(test_filter): - wildcards = ("", "*") - - _filter = list(test_filter.split("/")) - if len(_filter) > 3: - raise RuntimeError("Illegal test filter string") - _filter += [ "", ] * (3 - len(_filter)) - - isa, cat, mode = _filter - - if isa in wildcards: - raise RuntimeError("No ISA specified") - - cat = all_categories if cat in wildcards else (cat, ) - mode = all_modes if mode in wildcards else (mode, ) - - return isa, cat, mode - -def get_tests(isa, - categories=all_categories, modes=all_modes, - ruby_protocol=None, gpu_isa=None): - - # Generate a list of candidate configs - configs = list(arch_configs.get((isa, gpu_isa), [])) - - if (isa, gpu_isa) == ("x86", "hsail"): - if ruby_protocol == "GPU_RfO": - configs += ['gpu-randomtest'] - else: - configs += generic_configs - - if ruby_protocol == get_default_protocol(isa): - if ruby_protocol == 'MI_example': - configs += [ "%s-ruby" % (c, ) for c in configs ] - else: - configs += [ "%s-ruby-%s" % (c, ruby_protocol) for c in configs ] - elif ruby_protocol is not None: - # Override generic ISA configs when using Ruby (excluding - # MI_example which is included in all ISAs by default). This - # reduces the number of generic tests we re-run for when - # compiling Ruby targets. - configs = [ "%s-ruby-%s" % (c, ruby_protocol) for c in configs ] - - # /(quick|long)/(fs|se)/workload/ref/arch/guest/config/ - for conf_script in configs: - for cat in categories: - for mode in modes: - mode_dir = os.path.join(_test_base, cat, mode) - if not os.path.exists(mode_dir): - continue - - for workload in os.listdir(mode_dir): - isa_dir = os.path.join(mode_dir, workload, "ref", isa) - if not os.path.isdir(isa_dir): - continue - - for _os in os.listdir(isa_dir): - test_dir = os.path.join(isa_dir, _os, conf_script) - if not os.path.exists(test_dir) or \ - os.path.exists(os.path.join(test_dir, "skip")): - continue - - yield ClassicConfig(cat, mode, workload, isa, _os, - conf_script) diff --git a/tests/testing/units.py b/tests/testing/units.py deleted file mode 100644 index d1fa6b227..000000000 --- a/tests/testing/units.py +++ /dev/null @@ -1,294 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) 2016 ARM Limited -# All rights reserved -# -# The license below extends only to copyright in the software and shall -# not be construed as granting a license to any other intellectual -# property including but not limited to intellectual property relating -# to a hardware implementation of the functionality of the software -# licensed hereunder. You may use the software subject to the license -# terms below provided that you ensure that this notice is replicated -# unmodified and in its entirety in all distributions of the software, -# modified or unmodified, in source code or in binary form. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer; -# redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution; -# neither the name of the copyright holders nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from abc import ABCMeta, abstractmethod -from datetime import datetime -import difflib -import functools -import os -import re -from six import add_metaclass -import subprocess -import sys -import traceback - -sys.path.append(os.path.dirname(__file__)) -from results import UnitResult -from helpers import * - -_test_base = os.path.join(os.path.dirname(__file__), "..") - -@add_metaclass(ABCMeta) -class TestUnit(object): - """Base class for all test units. - - A test unit is a part of a larger test case. Test cases usually - contain two types of units, run units (run gem5) and verify units - (diff output files). All unit implementations inherit from this - class. - - A unit implementation overrides the _run() method. The test runner - calls the run() method, which wraps _run() to protect against - exceptions. - - """ - - def __init__(self, name, ref_dir, test_dir, skip=False): - self.name = name - self.ref_dir = ref_dir - self.test_dir = test_dir - self.force_skip = skip - self.start_time = None - self.stop_time = None - - def result(self, state, **kwargs): - if self.start_time is not None and "runtime" not in kwargs: - self.stop_time = datetime.utcnow() - delta = self.stop_time - self.start_time - kwargs["runtime"] = delta.total_seconds() - - return UnitResult(self.name, state, **kwargs) - - def ok(self, **kwargs): - return self.result(UnitResult.STATE_OK, **kwargs) - - def skip(self, **kwargs): - return self.result(UnitResult.STATE_SKIPPED, **kwargs) - - def error(self, message, **kwargs): - return self.result(UnitResult.STATE_ERROR, message=message, **kwargs) - - def failure(self, message, **kwargs): - return self.result(UnitResult.STATE_FAILURE, message=message, **kwargs) - - def ref_file(self, fname): - return os.path.join(self.ref_dir, fname) - - def out_file(self, fname): - return os.path.join(self.test_dir, fname) - - def _read_output(self, fname, default=""): - try: - with open(self.out_file(fname), "r") as f: - return f.read() - except IOError: - return default - - def run(self): - self.start_time = datetime.utcnow() - try: - if self.force_skip: - return self.skip() - else: - return self._run() - except: - return self.error("Python exception:\n%s" % traceback.format_exc()) - - @abstractmethod - def _run(self): - pass - -class RunGem5(TestUnit): - """Test unit representing a gem5 run. - - Possible failure modes: - - gem5 failed to run -> STATE_ERROR - - timeout -> STATE_ERROR - - non-zero exit code -> STATE_ERROR - - Possible non-failure results: - - exit code == 0 -> STATE_OK - - exit code == 2 -> STATE_SKIPPED - """ - - def __init__(self, gem5, gem5_args, timeout=0, **kwargs): - super(RunGem5, self).__init__("gem5", **kwargs) - self.gem5 = gem5 - self.args = gem5_args - self.timeout = timeout - - def _run(self): - gem5_cmd = [ - self.gem5, - "-d", self.test_dir, - "--stats-file", "text://stats.txt?desc=False", - "-re", - ] + self.args - - try: - with ProcessHelper(gem5_cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) as p: - status, gem5_stdout, gem5_stderr = p.call(timeout=self.timeout) - except CallTimeoutException as te: - return self.error("Timeout", stdout=te.stdout, stderr=te.stderr) - except OSError as ose: - return self.error("Failed to launch gem5: %s" % ose) - - stderr = "\n".join([ - "*** gem5 stderr ***", - gem5_stderr, - "", - "*** m5out/simerr ***", - self._read_output("simerr"), - ]) - - stdout = "\n".join([ - "*** gem5 stdout ***", - gem5_stdout, - "", - "*** m5out/simout ***", - self._read_output("simout"), - ]) - - # Signal - if status < 0: - return self.error("gem5 terminated by signal %i" % (-status, ), - stdout=stdout, stderr=stderr) - elif status == 2: - return self.skip(stdout=stdout, stderr=stderr) - elif status > 0: - return self.error("gem5 exited with non-zero status: %i" % status, - stdout=stdout, stderr=stderr) - else: - return self.ok(stdout=stdout, stderr=stderr) - -class DiffOutFile(TestUnit): - """Test unit comparing and output file and a reference file.""" - - # regular expressions of lines to ignore when diffing outputs - diff_ignore_regexes = { - "simout" : [ - re.compile('^Redirecting (stdout|stderr) to'), - re.compile('^gem5 compiled '), - re.compile('^gem5 started '), - re.compile('^gem5 executing on '), - re.compile('^command line:'), - re.compile("^Couldn't import dot_parser,"), - re.compile("^info: kernel located at:"), - re.compile("^Couldn't unlink "), - re.compile("^Using GPU kernel code file\(s\) "), - ], - "simerr" : [ - #re.compile('^Simulation complete at'), - ], - "config.ini" : [ - re.compile("^(executable|readfile|kernel|image_file)="), - re.compile("^(cwd|input|codefile)="), - ], - "config.json" : [ - re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''), - re.compile(r'''^\s*"(cwd|input|codefile)":'''), - ], - } - - def __init__(self, fname, **kwargs): - super(DiffOutFile, self).__init__("diff[%s]" % fname, - **kwargs) - - self.fname = fname - self.line_filters = DiffOutFile.diff_ignore_regexes.get(fname, tuple()) - - def _filter_file(self, fname): - def match_line(l): - for r in self.line_filters: - if r.match(l): - return True - return False - - with open(fname, "r") as f: - for l in f: - if not match_line(l): - yield l - - - def _run(self): - fname = self.fname - ref = self.ref_file(fname) - out = self.out_file(fname) - - if not os.path.exists(ref): - return self.error("%s doesn't exist in reference directory" \ - % fname) - - if not os.path.exists(out): - return self.error("%s doesn't exist in output directory" % fname) - - diff = difflib.unified_diff( - tuple(self._filter_file(ref)), - tuple(self._filter_file(out)), - fromfile="ref/%s" % fname, tofile="out/%s" % fname) - - diff = list(diff) - if diff: - return self.error("ref/%s and out/%s differ" % (fname, fname), - stderr="".join(diff)) - else: - return self.ok(stdout="-- ref/%s and out/%s are identical --" \ - % (fname, fname)) - -class DiffStatFile(TestUnit): - """Test unit comparing two gem5 stat files.""" - - def __init__(self, **kwargs): - super(DiffStatFile, self).__init__("stat_diff", **kwargs) - - self.stat_diff = os.path.join(_test_base, "diff-out") - - def _run(self): - STATUS_OK = 0 - STATUS_NEW_STATS = 1 - STATUS_FAILED = 2 - - stats = "stats.txt" - - cmd = [ - self.stat_diff, - self.ref_file(stats), self.out_file(stats), - ] - with ProcessHelper(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) as p: - status, stdout, stderr = p.call() - - if status in (STATUS_OK, STATUS_NEW_STATS): - return self.ok(stdout=stdout, stderr=stderr) - elif status == STATUS_FAILED: - return self.failure("Statistics mismatch", - stdout=stdout, stderr=stderr) - else: - return self.error("diff-out returned an error: %i" % status, - stdout=stdout, stderr=stderr)