tests/tests.py

   1 #!/usr/bin/env python2.7
   2 #
   3 # Copyright (c) 2016 ARM Limited
   4 # All rights reserved
   5 #
   6 # The license below extends only to copyright in the software and shall
   7 # not be construed as granting a license to any other intellectual
   8 # property including but not limited to intellectual property relating
   9 # to a hardware implementation of the functionality of the software
  10 # licensed hereunder.  You may use the software subject to the license
  11 # terms below provided that you ensure that this notice is replicated
  12 # unmodified and in its entirety in all distributions of the software,
  13 # modified or unmodified, in source code or in binary form.
  14 #
  15 # Redistribution and use in source and binary forms, with or without
  16 # modification, are permitted provided that the following conditions are
  17 # met: redistributions of source code must retain the above copyright
  18 # notice, this list of conditions and the following disclaimer;
  19 # redistributions in binary form must reproduce the above copyright
  20 # notice, this list of conditions and the following disclaimer in the
  21 # documentation and/or other materials provided with the distribution;
  22 # neither the name of the copyright holders nor the names of its
  23 # contributors may be used to endorse or promote products derived from
  24 # this software without specific prior written permission.
  25 #
  26 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  27 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  28 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  29 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  30 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  31 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  32 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  33 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  34 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  35 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  36 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  37
  38 from __future__ import print_function
  39
  40 import argparse
  41 import sys
  42 import os
  43 import pickle
  44
  45 from testing.tests import *
  46 import testing.results
  47
  48 class ParagraphHelpFormatter(argparse.HelpFormatter):
  49     def _fill_text(self, text, width, indent):
  50         return "\n\n".join([
  51             super(ParagraphHelpFormatter, self)._fill_text(p, width, indent) \
  52             for p in text.split("\n\n") ])
  53
  54 formatters = {
  55     "junit" : testing.results.JUnit,
  56     "text" : testing.results.Text,
  57     "summary" : testing.results.TextSummary,
  58     "pickle" : testing.results.Pickle,
  59 }
  60
  61
  62 def _add_format_args(parser):
  63     parser.add_argument("--format", choices=formatters, default="text",
  64                         help="Output format")
  65
  66     parser.add_argument("--no-junit-xlate-names", action="store_true",
  67                         help="Don't translate test names to " \
  68                         "package-like names")
  69
  70     parser.add_argument("--output", "-o",
  71                         type=argparse.FileType('w'), default=sys.stdout,
  72                         help="Test result output file")
  73
  74
  75 def _create_formatter(args):
  76     formatter = formatters[args.format]
  77     kwargs = {
  78         "fout" : args.output,
  79         "verbose" : args.verbose
  80     }
  81
  82     if issubclass(formatter, testing.results.JUnit):
  83         kwargs.update({
  84             "translate_names" : not args.no_junit_xlate_names,
  85         })
  86
  87     return formatter(**kwargs)
  88
  89
  90 def _list_tests_args(subparsers):
  91     parser = subparsers.add_parser(
  92         "list",
  93         formatter_class=ParagraphHelpFormatter,
  94         help="List available tests",
  95         description="List available tests",
  96         epilog="""
  97         Generate a list of available tests using a list filter.
  98
  99         The filter is a string consisting of the target ISA optionally
 100         followed by the test category and mode separated by
 101         slashes. The test names emitted by this command can be fed
 102         into the run command.
 103
 104         For example, to list all quick arm tests, run the following:
 105         tests.py list arm/quick
 106
 107         Non-mandatory parts of the filter string (anything other than
 108         the ISA) can be left out or replaced with the wildcard
 109         character. For example, all full-system tests can be listed
 110         with this command: tests.py list arm/*/fs""")
 111
 112     parser.add_argument("--ruby-protocol", type=str, default=None,
 113                         help="Ruby protocol")
 114
 115     parser.add_argument("--gpu-isa", type=str, default=None,
 116                         help="GPU ISA")
 117
 118     parser.add_argument("list_filter", metavar="ISA[/category/mode]",
 119                         action="append", type=str,
 120                         help="List available test cases")
 121
 122 def _list_tests(args):
 123     for isa, categories, modes in \
 124         ( parse_test_filter(f) for f in args.list_filter ):
 125
 126         for test in get_tests(isa, categories=categories, modes=modes,
 127                               ruby_protocol=args.ruby_protocol,
 128                               gpu_isa=args.gpu_isa):
 129             print("/".join(test))
 130     sys.exit(0)
 131
 132 def _run_tests_args(subparsers):
 133     parser = subparsers.add_parser(
 134         "run",
 135         formatter_class=ParagraphHelpFormatter,
 136         help='Run one or more tests',
 137         description="Run one or more tests.",
 138         epilog="""
 139         Run one or more tests described by a gem5 test tuple.
 140
 141         The test tuple consists of a test category (quick or long), a
 142         test mode (fs or se), a workload name, an isa, an operating
 143         system, and a config name separate by slashes. For example:
 144         quick/se/00.hello/arm/linux/simple-timing
 145
 146         Available tests can be listed using the 'list' sub-command
 147         (e.g., "tests.py list arm/quick" or one of the scons test list
 148         targets (e.g., "scons build/ARM/tests/opt/quick.list").
 149
 150         The test results can be stored in multiple different output
 151         formats. See the help for the show command for more details
 152         about output formatting.""")
 153
 154     parser.add_argument("gem5", type=str,
 155                         help="gem5 binary")
 156
 157     parser.add_argument("test", type=str, nargs="*",
 158                         help="List of tests to execute")
 159
 160     parser.add_argument("--directory", "-d",
 161                         type=str, default="m5tests",
 162                         help="Test work directory")
 163
 164     parser.add_argument("--timeout", "-t",
 165                         type=int, default="0", metavar="MINUTES",
 166                         help="Timeout, 0 to disable")
 167
 168     parser.add_argument("--skip-diff-out", action="store_true",
 169                         help="Skip output diffing stage")
 170
 171     parser.add_argument("--skip-diff-stat", action="store_true",
 172                         help="Skip stat diffing stage")
 173
 174     _add_format_args(parser)
 175
 176 def _run_tests(args):
 177     if not os.path.isfile(args.gem5) or not os.access(args.gem5, os.X_OK):
 178         print("gem5 binary '%s' not an executable file" % args.gem5,
 179             file=sys.stderr)
 180         sys.exit(2)
 181
 182     formatter = _create_formatter(args)
 183
 184     out_base = os.path.abspath(args.directory)
 185     if not os.path.exists(out_base):
 186         os.mkdir(out_base)
 187     tests = []
 188     for test_name in args.test:
 189         config = ClassicConfig(*test_name.split("/"))
 190         out_dir = os.path.join(out_base, "/".join(config))
 191         tests.append(
 192             ClassicTest(args.gem5, out_dir, config,
 193                         timeout=args.timeout,
 194                         skip_diff_stat=args.skip_diff_stat,
 195                         skip_diff_out=args.skip_diff_out))
 196
 197     all_results = []
 198     print("Running %i tests" % len(tests))
 199     for testno, test in enumerate(tests):
 200         print("%i: Running '%s'..." % (testno, test))
 201
 202         all_results.append(test.run())
 203
 204     formatter.dump_suites(all_results)
 205
 206 def _show_args(subparsers):
 207     parser = subparsers.add_parser(
 208         "show",
 209         formatter_class=ParagraphHelpFormatter,
 210         help='Display pickled test results',
 211         description='Display pickled test results',
 212         epilog="""
 213         Reformat the pickled output from one or more test runs. This
 214         command is typically used with the output from a single test
 215         run, but it can also be used to merge the outputs from
 216         multiple runs.
 217
 218         The 'text' format is a verbose output format that provides
 219         information about individual test units and the output from
 220         failed tests. It's mainly useful for debugging test failures.
 221
 222         The 'summary' format provides outputs the results of one test
 223         per line with the test's overall status (OK, SKIPPED, or
 224         FAILED).
 225
 226         The 'junit' format is primarily intended for use with CI
 227         systems. It provides an XML representation of test
 228         status. Similar to the text format, it includes detailed
 229         information about test failures. Since many JUnit parser make
 230         assume that test names look like Java packet strings, the
 231         JUnit formatter automatically to something the looks like a
 232         Java class path ('.'->'-', '/'->'.').
 233
 234         The 'pickle' format stores the raw results in a format that
 235         can be reformatted using this command. It's typically used
 236         with the show command to merge multiple test results into one
 237         pickle file.""")
 238
 239     _add_format_args(parser)
 240
 241     parser.add_argument("result", type=argparse.FileType("rb"), nargs="*",
 242                         help="Pickled test results")
 243
 244 def _show(args):
 245     def _load(f):
 246         # Load the pickled status file, sometimes e.g., when a
 247         # regression is still running the status file might be
 248         # incomplete.
 249         try:
 250             return pickle.load(f)
 251         except EOFError:
 252             print('Could not read file %s' % f.name, file=sys.stderr)
 253             return []
 254
 255     formatter = _create_formatter(args)
 256     suites = sum([ _load(f) for f in args.result ], [])
 257     formatter.dump_suites(suites)
 258
 259 def _test_args(subparsers):
 260     parser = subparsers.add_parser(
 261         "test",
 262         formatter_class=ParagraphHelpFormatter,
 263         help='Probe test results and set exit code',
 264         epilog="""
 265
 266         Load one or more pickled test file and return an exit code
 267         corresponding to the test outcome. The following exit codes
 268         can be returned:
 269
 270         0: All tests were successful or skipped.
 271
 272         1: General fault in the script such as incorrect parameters or
 273         failing to parse a pickle file.
 274
 275         2: At least one test failed to run. This is what the summary
 276         formatter usually shows as a 'FAILED'.
 277
 278         3: All tests ran correctly, but at least one failed to
 279         verify its output. When displaying test output using the
 280         summary formatter, such a test would show up as 'CHANGED'.
 281         """)
 282
 283     parser.add_argument("result", type=argparse.FileType("rb"), nargs="*",
 284                         help="Pickled test results")
 285
 286 def _test(args):
 287     try:
 288         suites = sum([ pickle.load(f) for f in args.result ], [])
 289     except EOFError:
 290         print('Could not read all files', file=sys.stderr)
 291         sys.exit(2)
 292
 293     if all(s for s in suites):
 294         sys.exit(0)
 295     elif any([ s.failed_run() for s in suites ]):
 296         sys.exit(2)
 297     elif any([ s.changed() for s in suites ]):
 298         sys.exit(3)
 299     else:
 300         assert False, "Unexpected return status from test"
 301
 302 _commands = {
 303     "list" : (_list_tests, _list_tests_args),
 304     "run" : (_run_tests, _run_tests_args),
 305     "show" : (_show, _show_args),
 306     "test" : (_test, _test_args),
 307 }
 308
 309 def main():
 310     parser = argparse.ArgumentParser(
 311         formatter_class=ParagraphHelpFormatter,
 312         description="""gem5 testing multi tool.""",
 313         epilog="""
 314         This tool provides an interface to gem5's test framework that
 315         doesn't depend on gem5's build system. It supports test
 316         listing, running, and output formatting.
 317
 318         The list sub-command (e.g., "test.py list arm/quick") produces
 319         a list of tests tuples that can be used by the run command
 320         (e.g., "tests.py run gem5.opt
 321         quick/se/00.hello/arm/linux/simple-timing").
 322
 323         The run command supports several output formats. One of them,
 324         pickle, contains the raw output from the tests and can be
 325         re-formatted using the show command (e.g., "tests.py show
 326         --format summary *.pickle"). Such pickle files are also
 327         generated by the build system when scons is used to run
 328         regressions.
 329
 330         See the usage strings for the individual sub-commands for
 331         details.""")
 332
 333     parser.add_argument("--verbose", action="store_true",
 334                         help="Produce more verbose output")
 335
 336     subparsers = parser.add_subparsers(dest="command")
 337
 338     for key, (impl, cmd_parser) in _commands.items():
 339         cmd_parser(subparsers)
 340
 341     args = parser.parse_args()
 342     impl, cmd_parser = _commands[args.command]
 343     impl(args)
 344
 345 if __name__ == "__main__":
 346     main()