3 # Copyright (c) 2016 ARM Limited
6 # The license below extends only to copyright in the software and shall
7 # not be construed as granting a license to any other intellectual
8 # property including but not limited to intellectual property relating
9 # to a hardware implementation of the functionality of the software
10 # licensed hereunder. You may use the software subject to the license
11 # terms below provided that you ensure that this notice is replicated
12 # unmodified and in its entirety in all distributions of the software,
13 # modified or unmodified, in source code or in binary form.
15 # Redistribution and use in source and binary forms, with or without
16 # modification, are permitted provided that the following conditions are
17 # met: redistributions of source code must retain the above copyright
18 # notice, this list of conditions and the following disclaimer;
19 # redistributions in binary form must reproduce the above copyright
20 # notice, this list of conditions and the following disclaimer in the
21 # documentation and/or other materials provided with the distribution;
22 # neither the name of the copyright holders nor the names of its
23 # contributors may be used to endorse or promote products derived from
24 # this software without specific prior written permission.
26 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 # Authors: Andreas Sandberg
45 from testing
.tests
import *
46 import testing
.results
48 class ParagraphHelpFormatter(argparse
.HelpFormatter
):
49 def _fill_text(self
, text
, width
, indent
):
51 super(ParagraphHelpFormatter
, self
)._fill
_text
(p
, width
, indent
) \
52 for p
in text
.split("\n\n") ])
55 "junit" : testing
.results
.JUnit
,
56 "text" : testing
.results
.Text
,
57 "summary" : testing
.results
.TextSummary
,
58 "pickle" : testing
.results
.Pickle
,
62 def _add_format_args(parser
):
63 parser
.add_argument("--format", choices
=formatters
, default
="text",
66 parser
.add_argument("--no-junit-xlate-names", action
="store_true",
67 help="Don't translate test names to " \
70 parser
.add_argument("--output", "-o",
71 type=argparse
.FileType('w'), default
=sys
.stdout
,
72 help="Test result output file")
75 def _create_formatter(args
):
76 formatter
= formatters
[args
.format
]
79 "verbose" : args
.verbose
82 if issubclass(formatter
, testing
.results
.JUnit
):
84 "translate_names" : not args
.no_junit_xlate_names
,
87 return formatter(**kwargs
)
90 def _list_tests_args(subparsers
):
91 parser
= subparsers
.add_parser(
93 formatter_class
=ParagraphHelpFormatter
,
94 help="List available tests",
95 description
="List available tests",
97 Generate a list of available tests using a list filter.
99 The filter is a string consisting of the target ISA optionally
100 followed by the test category and mode separated by
101 slashes. The test names emitted by this command can be fed
102 into the run command.
104 For example, to list all quick arm tests, run the following:
105 tests.py list arm/quick
107 Non-mandatory parts of the filter string (anything other than
108 the ISA) can be left out or replaced with the wildcard
109 character. For example, all full-system tests can be listed
110 with this command: tests.py list arm/*/fs""")
112 parser
.add_argument("--ruby-protocol", type=str, default
=None,
113 help="Ruby protocol")
115 parser
.add_argument("--gpu-isa", type=str, default
=None,
118 parser
.add_argument("list_filter", metavar
="ISA[/category/mode]",
119 action
="append", type=str,
120 help="List available test cases")
122 def _list_tests(args
):
123 for isa
, categories
, modes
in \
124 ( parse_test_filter(f
) for f
in args
.list_filter
):
126 for test
in get_tests(isa
, categories
=categories
, modes
=modes
,
127 ruby_protocol
=args
.ruby_protocol
,
128 gpu_isa
=args
.gpu_isa
):
132 def _run_tests_args(subparsers
):
133 parser
= subparsers
.add_parser(
135 formatter_class
=ParagraphHelpFormatter
,
136 help='Run one or more tests',
137 description
="Run one or more tests.",
139 Run one or more tests described by a gem5 test tuple.
141 The test tuple consists of a test category (quick or long), a
142 test mode (fs or se), a workload name, an isa, an operating
143 system, and a config name separate by slashes. For example:
144 quick/se/00.hello/arm/linux/simple-timing
146 Available tests can be listed using the 'list' sub-command
147 (e.g., "tests.py list arm/quick" or one of the scons test list
148 targets (e.g., "scons build/ARM/tests/opt/quick.list").
150 The test results can be stored in multiple different output
151 formats. See the help for the show command for more details
152 about output formatting.""")
154 parser
.add_argument("gem5", type=str,
157 parser
.add_argument("test", type=str, nargs
="*",
158 help="List of tests to execute")
160 parser
.add_argument("--directory", "-d",
161 type=str, default
="m5tests",
162 help="Test work directory")
164 parser
.add_argument("--timeout", "-t",
165 type=int, default
="0", metavar
="MINUTES",
166 help="Timeout, 0 to disable")
168 parser
.add_argument("--skip-diff-out", action
="store_true",
169 help="Skip output diffing stage")
171 parser
.add_argument("--skip-diff-stat", action
="store_true",
172 help="Skip stat diffing stage")
174 _add_format_args(parser
)
176 def _run_tests(args
):
177 if not os
.path
.isfile(args
.gem5
) or not os
.access(args
.gem5
, os
.X_OK
):
178 print >> sys
.stderr
, \
179 "gem5 binary '%s' not an executable file" % args
.gem5
182 formatter
= _create_formatter(args
)
184 out_base
= os
.path
.abspath(args
.directory
)
185 if not os
.path
.exists(out_base
):
188 for test_name
in args
.test
:
189 config
= ClassicConfig(*test_name
.split("/"))
190 out_dir
= os
.path
.join(out_base
, "/".join(config
))
192 ClassicTest(args
.gem5
, out_dir
, config
,
193 timeout
=args
.timeout
,
194 skip_diff_stat
=args
.skip_diff_stat
,
195 skip_diff_out
=args
.skip_diff_out
))
198 print "Running %i tests" % len(tests
)
199 for testno
, test
in enumerate(tests
):
200 print "%i: Running '%s'..." % (testno
, test
)
202 all_results
.append(test
.run())
204 formatter
.dump_suites(all_results
)
206 def _show_args(subparsers
):
207 parser
= subparsers
.add_parser(
209 formatter_class
=ParagraphHelpFormatter
,
210 help='Display pickled test results',
211 description
='Display pickled test results',
213 Reformat the pickled output from one or more test runs. This
214 command is typically used with the output from a single test
215 run, but it can also be used to merge the outputs from
218 The 'text' format is a verbose output format that provides
219 information about individual test units and the output from
220 failed tests. It's mainly useful for debugging test failures.
222 The 'summary' format provides outputs the results of one test
223 per line with the test's overall status (OK, SKIPPED, or
226 The 'junit' format is primarily intended for use with CI
227 systems. It provides an XML representation of test
228 status. Similar to the text format, it includes detailed
229 information about test failures. Since many JUnit parser make
230 assume that test names look like Java packet strings, the
231 JUnit formatter automatically to something the looks like a
232 Java class path ('.'->'-', '/'->'.').
234 The 'pickle' format stores the raw results in a format that
235 can be reformatted using this command. It's typically used
236 with the show command to merge multiple test results into one
239 _add_format_args(parser
)
241 parser
.add_argument("result", type=argparse
.FileType("rb"), nargs
="*",
242 help="Pickled test results")
245 formatter
= _create_formatter(args
)
246 suites
= sum([ pickle
.load(f
) for f
in args
.result
], [])
247 formatter
.dump_suites(suites
)
249 def _test_args(subparsers
):
250 parser
= subparsers
.add_parser(
252 formatter_class
=ParagraphHelpFormatter
,
253 help='Probe test results and set exit code',
256 Load one or more pickled test file and return an exit code
257 corresponding to the test outcome. The following exit codes
260 0: All tests were successful or skipped.
262 1: General fault in the script such as incorrect parameters or
263 failing to parse a pickle file.
265 2: At least one test failed to run. This is what the summary
266 formatter usually shows as a 'FAILED'.
268 3: All tests ran correctly, but at least one failed to
269 verify its output. When displaying test output using the
270 summary formatter, such a test would show up as 'CHANGED'.
273 _add_format_args(parser
)
275 parser
.add_argument("result", type=argparse
.FileType("rb"), nargs
="*",
276 help="Pickled test results")
279 suites
= sum([ pickle
.load(f
) for f
in args
.result
], [])
281 if all(s
for s
in suites
):
283 elif any([ s
.failed_run() for s
in suites
]):
285 elif any([ s
.changed() for s
in suites
]):
288 assert False, "Unexpected return status from test"
291 "list" : (_list_tests
, _list_tests_args
),
292 "run" : (_run_tests
, _run_tests_args
),
293 "show" : (_show
, _show_args
),
294 "test" : (_test
, _test_args
),
298 parser
= argparse
.ArgumentParser(
299 formatter_class
=ParagraphHelpFormatter
,
300 description
="""gem5 testing multi tool.""",
302 This tool provides an interface to gem5's test framework that
303 doesn't depend on gem5's build system. It supports test
304 listing, running, and output formatting.
306 The list sub-command (e.g., "test.py list arm/quick") produces
307 a list of tests tuples that can be used by the run command
308 (e.g., "tests.py run gem5.opt
309 quick/se/00.hello/arm/linux/simple-timing").
311 The run command supports several output formats. One of them,
312 pickle, contains the raw output from the tests and can be
313 re-formatted using the show command (e.g., "tests.py show
314 --format summary *.pickle"). Such pickle files are also
315 generated by the build system when scons is used to run
318 See the usage strings for the individual sub-commands for
321 parser
.add_argument("--verbose", action
="store_true",
322 help="Produce more verbose output")
324 subparsers
= parser
.add_subparsers(dest
="command")
326 for key
, (impl
, cmd_parser
) in _commands
.items():
327 cmd_parser(subparsers
)
329 args
= parser
.parse_args()
330 impl
, cmd_parser
= _commands
[args
.command
]
333 if __name__
== "__main__":