1 #!/usr/bin/env python2.7
3 # Copyright (c) 2016 ARM Limited
6 # The license below extends only to copyright in the software and shall
7 # not be construed as granting a license to any other intellectual
8 # property including but not limited to intellectual property relating
9 # to a hardware implementation of the functionality of the software
10 # licensed hereunder. You may use the software subject to the license
11 # terms below provided that you ensure that this notice is replicated
12 # unmodified and in its entirety in all distributions of the software,
13 # modified or unmodified, in source code or in binary form.
15 # Redistribution and use in source and binary forms, with or without
16 # modification, are permitted provided that the following conditions are
17 # met: redistributions of source code must retain the above copyright
18 # notice, this list of conditions and the following disclaimer;
19 # redistributions in binary form must reproduce the above copyright
20 # notice, this list of conditions and the following disclaimer in the
21 # documentation and/or other materials provided with the distribution;
22 # neither the name of the copyright holders nor the names of its
23 # contributors may be used to endorse or promote products derived from
24 # this software without specific prior written permission.
26 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 # Authors: Andreas Sandberg
40 from __future__
import print_function
42 from abc
import ABCMeta
, abstractmethod
48 import xml
.etree
.cElementTree
as ET
50 class UnitResult(object):
51 """Results of a single test unit.
53 A test result can be one of:
54 - STATE_OK: Test ran successfully.
55 - STATE_SKIPPED: The test was skipped.
56 - STATE_ERROR: The test failed to run.
57 - STATE_FAILED: Test ran, but failed.
59 The difference between STATE_ERROR and STATE_FAILED is very
60 subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed
61 to start or crashed, while STATE_FAILED would mean that a test
62 failed (e.g., statistics mismatch).
73 STATE_SKIPPED
: "SKIPPED",
74 STATE_ERROR
: "ERROR",
75 STATE_FAILURE
: "FAILURE",
78 def __init__(self
, name
, state
, message
="", stderr
="", stdout
="",
82 self
.message
= message
85 self
.runtime
= runtime
88 return self
.state
== UnitResult
.STATE_SKIPPED
91 return self
.state
== UnitResult
.STATE_OK
94 return UnitResult
.state_names
[self
.state
]
96 def __nonzero__(self
):
97 return self
.success() or self
.skipped()
100 state_name
= self
.state_name()
102 status
= "%s: %s" % (state_name
, self
.message
) if self
.message
else \
105 return "%s: %s" % (self
.name
, status
)
107 class TestResult(object):
108 """Results for from a single test consisting of one or more units."""
110 def __init__(self
, name
, run_results
=[], verify_results
=[]):
112 self
.results
= run_results
+ verify_results
113 self
.run_results
= run_results
114 self
.verify_results
= verify_results
117 return self
.success_run() and self
.success_verify()
119 def success_run(self
):
120 return all([ r
.success() for r
in self
.run_results
])
122 def success_verify(self
):
123 return all([ r
.success() for r
in self
.verify_results
])
126 return self
.failed_run() or self
.failed_verify()
128 def failed_run(self
):
129 return any([ not r
for r
in self
.run_results
])
131 def failed_verify(self
):
132 return any([ not r
for r
in self
.verify_results
])
135 return all([ r
.skipped() for r
in self
.run_results
])
138 return self
.success_run() and self
.failed_verify()
141 return sum([ r
.runtime
for r
in self
.results
])
143 def __nonzero__(self
):
144 return all([ r
for r
in self
.results
])
146 class ResultFormatter(object):
147 __metaclass__
= ABCMeta
149 def __init__(self
, fout
=sys
.stdout
, verbose
=False):
150 self
.verbose
= verbose
154 def dump_suites(self
, suites
):
157 class Pickle(ResultFormatter
):
158 """Save test results as a binary using Python's pickle
163 def __init__(self
, **kwargs
):
164 super(Pickle
, self
).__init
__(**kwargs
)
166 def dump_suites(self
, suites
):
167 pickle
.dump(suites
, self
.fout
, pickle
.HIGHEST_PROTOCOL
)
169 class Text(ResultFormatter
):
170 """Output test results as text."""
172 def __init__(self
, **kwargs
):
173 super(Text
, self
).__init
__(**kwargs
)
175 def dump_suites(self
, suites
):
178 print("--- %s ---" % suite
.name
, file=fout
)
180 for t
in suite
.results
:
181 print("*** %s" % t
, file=fout
)
183 if t
and not self
.verbose
:
187 print(t
.message
, file=fout
)
190 print(t
.stderr
, file=fout
)
192 print(t
.stdout
, file=fout
)
194 class TextSummary(ResultFormatter
):
195 """Output test results as a text summary"""
197 def __init__(self
, **kwargs
):
198 super(TextSummary
, self
).__init
__(**kwargs
)
200 def test_status(self
, suite
):
203 elif suite
.changed():
210 def dump_suites(self
, suites
):
213 status
= self
.test_status(suite
)
214 print("%s: %s" % (suite
.name
, status
), file=fout
)
216 class JUnit(ResultFormatter
):
217 """Output test results as JUnit XML"""
219 def __init__(self
, translate_names
=True, **kwargs
):
220 super(JUnit
, self
).__init
__(**kwargs
)
223 self
.name_table
= string
.maketrans(
228 self
.name_table
= string
.maketrans("", "")
230 def convert_unit(self
, x_suite
, test
):
231 x_test
= ET
.SubElement(x_suite
, "testcase",
233 time
="%f" % test
.runtime
)
236 if test
.state
== UnitResult
.STATE_OK
:
238 elif test
.state
== UnitResult
.STATE_SKIPPED
:
239 x_state
= ET
.SubElement(x_test
, "skipped")
240 elif test
.state
== UnitResult
.STATE_FAILURE
:
241 x_state
= ET
.SubElement(x_test
, "failure")
242 elif test
.state
== UnitResult
.STATE_ERROR
:
243 x_state
= ET
.SubElement(x_test
, "error")
245 assert False, "Unknown test state"
247 if x_state
is not None:
249 x_state
.set("message", test
.message
)
253 msg
.append("*** Standard Errror: ***")
254 msg
.append(test
.stderr
)
256 msg
.append("*** Standard Out: ***")
257 msg
.append(test
.stdout
)
259 x_state
.text
= "\n".join(msg
)
263 def convert_suite(self
, x_suites
, suite
):
264 x_suite
= ET
.SubElement(x_suites
, "testsuite",
265 name
=suite
.name
.translate(self
.name_table
),
266 time
="%f" % suite
.runtime())
271 for test
in suite
.results
:
272 if test
.state
!= UnitResult
.STATE_OK
:
273 if test
.state
== UnitResult
.STATE_SKIPPED
:
275 elif test
.state
== UnitResult
.STATE_ERROR
:
277 elif test
.state
== UnitResult
.STATE_FAILURE
:
280 x_test
= self
.convert_unit(x_suite
, test
)
282 x_suite
.set("errors", str(errors
))
283 x_suite
.set("failures", str(failures
))
284 x_suite
.set("skipped", str(skipped
))
285 x_suite
.set("tests", str(len(suite
.results
)))
289 def convert_suites(self
, suites
):
290 x_root
= ET
.Element("testsuites")
293 self
.convert_suite(x_root
, suite
)
297 def dump_suites(self
, suites
):
298 et
= ET
.ElementTree(self
.convert_suites(suites
))
299 et
.write(self
.fout
, encoding
="UTF-8")