+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright (c) 2016 ARM Limited
-# All rights reserved
-#
-# The license below extends only to copyright in the software and shall
-# not be construed as granting a license to any other intellectual
-# property including but not limited to intellectual property relating
-# to a hardware implementation of the functionality of the software
-# licensed hereunder. You may use the software subject to the license
-# terms below provided that you ensure that this notice is replicated
-# unmodified and in its entirety in all distributions of the software,
-# modified or unmodified, in source code or in binary form.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met: redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer;
-# redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution;
-# neither the name of the copyright holders nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import print_function
-
-import argparse
-import sys
-import os
-import pickle
-
-from testing.tests import *
-import testing.results
-
-class ParagraphHelpFormatter(argparse.HelpFormatter):
- def _fill_text(self, text, width, indent):
- return "\n\n".join([
- super(ParagraphHelpFormatter, self)._fill_text(p, width, indent) \
- for p in text.split("\n\n") ])
-
-formatters = {
- "junit" : testing.results.JUnit,
- "text" : testing.results.Text,
- "summary" : testing.results.TextSummary,
- "pickle" : testing.results.Pickle,
-}
-
-
-def _add_format_args(parser):
- parser.add_argument("--format", choices=formatters, default="text",
- help="Output format")
-
- parser.add_argument("--no-junit-xlate-names", action="store_true",
- help="Don't translate test names to " \
- "package-like names")
-
- parser.add_argument("--output", "-o",
- type=argparse.FileType('w'), default=sys.stdout,
- help="Test result output file")
-
-
-def _create_formatter(args):
- formatter = formatters[args.format]
- kwargs = {
- "fout" : args.output,
- "verbose" : args.verbose
- }
-
- if issubclass(formatter, testing.results.JUnit):
- kwargs.update({
- "translate_names" : not args.no_junit_xlate_names,
- })
-
- return formatter(**kwargs)
-
-
-def _list_tests_args(subparsers):
- parser = subparsers.add_parser(
- "list",
- formatter_class=ParagraphHelpFormatter,
- help="List available tests",
- description="List available tests",
- epilog="""
- Generate a list of available tests using a list filter.
-
- The filter is a string consisting of the target ISA optionally
- followed by the test category and mode separated by
- slashes. The test names emitted by this command can be fed
- into the run command.
-
- For example, to list all quick arm tests, run the following:
- tests.py list arm/quick
-
- Non-mandatory parts of the filter string (anything other than
- the ISA) can be left out or replaced with the wildcard
- character. For example, all full-system tests can be listed
- with this command: tests.py list arm/*/fs""")
-
- parser.add_argument("--ruby-protocol", type=str, default=None,
- help="Ruby protocol")
-
- parser.add_argument("--gpu-isa", type=str, default=None,
- help="GPU ISA")
-
- parser.add_argument("list_filter", metavar="ISA[/category/mode]",
- action="append", type=str,
- help="List available test cases")
-
-def _list_tests(args):
- for isa, categories, modes in \
- ( parse_test_filter(f) for f in args.list_filter ):
-
- for test in get_tests(isa, categories=categories, modes=modes,
- ruby_protocol=args.ruby_protocol,
- gpu_isa=args.gpu_isa):
- print("/".join(test))
- sys.exit(0)
-
-def _run_tests_args(subparsers):
- parser = subparsers.add_parser(
- "run",
- formatter_class=ParagraphHelpFormatter,
- help='Run one or more tests',
- description="Run one or more tests.",
- epilog="""
- Run one or more tests described by a gem5 test tuple.
-
- The test tuple consists of a test category (quick or long), a
- test mode (fs or se), a workload name, an isa, an operating
- system, and a config name separate by slashes. For example:
- quick/se/00.hello/arm/linux/simple-timing
-
- Available tests can be listed using the 'list' sub-command
- (e.g., "tests.py list arm/quick" or one of the scons test list
- targets (e.g., "scons build/ARM/tests/opt/quick.list").
-
- The test results can be stored in multiple different output
- formats. See the help for the show command for more details
- about output formatting.""")
-
- parser.add_argument("gem5", type=str,
- help="gem5 binary")
-
- parser.add_argument("test", type=str, nargs="*",
- help="List of tests to execute")
-
- parser.add_argument("--directory", "-d",
- type=str, default="m5tests",
- help="Test work directory")
-
- parser.add_argument("--timeout", "-t",
- type=int, default="0", metavar="MINUTES",
- help="Timeout, 0 to disable")
-
- parser.add_argument("--skip-diff-out", action="store_true",
- help="Skip output diffing stage")
-
- parser.add_argument("--skip-diff-stat", action="store_true",
- help="Skip stat diffing stage")
-
- _add_format_args(parser)
-
-def _run_tests(args):
- if not os.path.isfile(args.gem5) or not os.access(args.gem5, os.X_OK):
- print("gem5 binary '%s' not an executable file" % args.gem5,
- file=sys.stderr)
- sys.exit(2)
-
- formatter = _create_formatter(args)
-
- out_base = os.path.abspath(args.directory)
- if not os.path.exists(out_base):
- os.mkdir(out_base)
- tests = []
- for test_name in args.test:
- config = ClassicConfig(*test_name.split("/"))
- out_dir = os.path.join(out_base, "/".join(config))
- tests.append(
- ClassicTest(args.gem5, out_dir, config,
- timeout=args.timeout,
- skip_diff_stat=args.skip_diff_stat,
- skip_diff_out=args.skip_diff_out))
-
- all_results = []
- print("Running %i tests" % len(tests))
- for testno, test in enumerate(tests):
- print("%i: Running '%s'..." % (testno, test))
-
- all_results.append(test.run())
-
- formatter.dump_suites(all_results)
-
-def _show_args(subparsers):
- parser = subparsers.add_parser(
- "show",
- formatter_class=ParagraphHelpFormatter,
- help='Display pickled test results',
- description='Display pickled test results',
- epilog="""
- Reformat the pickled output from one or more test runs. This
- command is typically used with the output from a single test
- run, but it can also be used to merge the outputs from
- multiple runs.
-
- The 'text' format is a verbose output format that provides
- information about individual test units and the output from
- failed tests. It's mainly useful for debugging test failures.
-
- The 'summary' format provides outputs the results of one test
- per line with the test's overall status (OK, SKIPPED, or
- FAILED).
-
- The 'junit' format is primarily intended for use with CI
- systems. It provides an XML representation of test
- status. Similar to the text format, it includes detailed
- information about test failures. Since many JUnit parser make
- assume that test names look like Java packet strings, the
- JUnit formatter automatically to something the looks like a
- Java class path ('.'->'-', '/'->'.').
-
- The 'pickle' format stores the raw results in a format that
- can be reformatted using this command. It's typically used
- with the show command to merge multiple test results into one
- pickle file.""")
-
- _add_format_args(parser)
-
- parser.add_argument("result", type=argparse.FileType("rb"), nargs="*",
- help="Pickled test results")
-
-def _show(args):
- def _load(f):
- # Load the pickled status file, sometimes e.g., when a
- # regression is still running the status file might be
- # incomplete.
- try:
- return pickle.load(f)
- except EOFError:
- print('Could not read file %s' % f.name, file=sys.stderr)
- return []
-
- formatter = _create_formatter(args)
- suites = sum([ _load(f) for f in args.result ], [])
- formatter.dump_suites(suites)
-
-def _test_args(subparsers):
- parser = subparsers.add_parser(
- "test",
- formatter_class=ParagraphHelpFormatter,
- help='Probe test results and set exit code',
- epilog="""
-
- Load one or more pickled test file and return an exit code
- corresponding to the test outcome. The following exit codes
- can be returned:
-
- 0: All tests were successful or skipped.
-
- 1: General fault in the script such as incorrect parameters or
- failing to parse a pickle file.
-
- 2: At least one test failed to run. This is what the summary
- formatter usually shows as a 'FAILED'.
-
- 3: All tests ran correctly, but at least one failed to
- verify its output. When displaying test output using the
- summary formatter, such a test would show up as 'CHANGED'.
- """)
-
- parser.add_argument("result", type=argparse.FileType("rb"), nargs="*",
- help="Pickled test results")
-
-def _test(args):
- try:
- suites = sum([ pickle.load(f) for f in args.result ], [])
- except EOFError:
- print('Could not read all files', file=sys.stderr)
- sys.exit(2)
-
- if all(s for s in suites):
- sys.exit(0)
- elif any([ s.failed_run() for s in suites ]):
- sys.exit(2)
- elif any([ s.changed() for s in suites ]):
- sys.exit(3)
- else:
- assert False, "Unexpected return status from test"
-
-_commands = {
- "list" : (_list_tests, _list_tests_args),
- "run" : (_run_tests, _run_tests_args),
- "show" : (_show, _show_args),
- "test" : (_test, _test_args),
-}
-
-def main():
- parser = argparse.ArgumentParser(
- formatter_class=ParagraphHelpFormatter,
- description="""gem5 testing multi tool.""",
- epilog="""
- This tool provides an interface to gem5's test framework that
- doesn't depend on gem5's build system. It supports test
- listing, running, and output formatting.
-
- The list sub-command (e.g., "test.py list arm/quick") produces
- a list of tests tuples that can be used by the run command
- (e.g., "tests.py run gem5.opt
- quick/se/00.hello/arm/linux/simple-timing").
-
- The run command supports several output formats. One of them,
- pickle, contains the raw output from the tests and can be
- re-formatted using the show command (e.g., "tests.py show
- --format summary *.pickle"). Such pickle files are also
- generated by the build system when scons is used to run
- regressions.
-
- See the usage strings for the individual sub-commands for
- details.""")
-
- parser.add_argument("--verbose", action="store_true",
- help="Produce more verbose output")
-
- subparsers = parser.add_subparsers(dest="command")
-
- for key, (impl, cmd_parser) in _commands.items():
- cmd_parser(subparsers)
-
- args = parser.parse_args()
- impl, cmd_parser = _commands[args.command]
- impl(args)
-
-if __name__ == "__main__":
- main()
projects / gem5.git / commitdiff