1 #!/usr/bin/env python2.7
3 # Copyright (c) 2016-2017 ARM Limited
6 # The license below extends only to copyright in the software and shall
7 # not be construed as granting a license to any other intellectual
8 # property including but not limited to intellectual property relating
9 # to a hardware implementation of the functionality of the software
10 # licensed hereunder. You may use the software subject to the license
11 # terms below provided that you ensure that this notice is replicated
12 # unmodified and in its entirety in all distributions of the software,
13 # modified or unmodified, in source code or in binary form.
15 # Redistribution and use in source and binary forms, with or without
16 # modification, are permitted provided that the following conditions are
17 # met: redistributions of source code must retain the above copyright
18 # notice, this list of conditions and the following disclaimer;
19 # redistributions in binary form must reproduce the above copyright
20 # notice, this list of conditions and the following disclaimer in the
21 # documentation and/or other materials provided with the distribution;
22 # neither the name of the copyright holders nor the names of its
23 # contributors may be used to endorse or promote products derived from
24 # this software without specific prior written permission.
26 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 # Authors: Andreas Sandberg
40 from abc
import ABCMeta
, abstractmethod
42 from collections
import namedtuple
44 from results
import TestResult
47 _test_base
= os
.path
.join(os
.path
.dirname(__file__
), "..")
49 ClassicConfig
= namedtuple("ClassicConfig", (
58 # There are currently two "classes" of test
59 # configurations. Architecture-specific ones and generic ones
60 # (typically SE mode tests). In both cases, the configuration name
61 # matches a file in tests/configs/ that will be picked up by the test
64 # Architecture specific configurations are listed in the arch_configs
65 # dictionary. This is indexed by a (cpu architecture, gpu
66 # architecture) tuple. GPU architecture is optional and may be None.
68 # Generic configurations are listed in the generic_configs tuple.
70 # When discovering available test cases, this script look uses the
71 # test list as a list of /candidate/ configurations. A configuration
72 # is only used if a test has a reference output for that
73 # configuration. In addition to the base configurations from
74 # arch_configs and generic_configs, a Ruby configuration may be
75 # appended to the base name (this is probed /in addition/ to the
76 # original name. See get_tests() for details.
80 'tsunami-simple-atomic',
81 'tsunami-simple-timing',
82 'tsunami-simple-atomic-dual',
83 'tsunami-simple-timing-dual',
84 'twosys-tsunami-simple-atomic',
85 'tsunami-o3', 'tsunami-o3-dual',
86 'tsunami-minor', 'tsunami-minor-dual',
87 'tsunami-switcheroo-full',
91 'simple-atomic-dummychecker',
93 'realview-simple-atomic',
94 'realview-simple-atomic-dual',
95 'realview-simple-atomic-checkpoint',
96 'realview-simple-timing',
97 'realview-simple-timing-dual',
99 'realview-o3-checker',
102 'realview-minor-dual',
103 'realview-switcheroo-atomic',
104 'realview-switcheroo-timing',
105 'realview-switcheroo-noncaching-timing',
106 'realview-switcheroo-o3',
107 'realview-switcheroo-full',
108 'realview64-simple-atomic',
109 'realview64-simple-atomic-checkpoint',
110 'realview64-simple-atomic-dual',
111 'realview64-simple-timing',
112 'realview64-simple-timing-dual',
114 'realview64-o3-checker',
115 'realview64-o3-dual',
117 'realview64-minor-dual',
118 'realview64-switcheroo-atomic',
119 'realview64-switcheroo-timing',
120 'realview64-switcheroo-o3',
121 'realview64-switcheroo-full',
125 't1000-simple-atomic',
133 'pc-switcheroo-full',
162 'learning-gem5-p1-simple',
163 'learning-gem5-p1-two-level',
166 default_ruby_protocol
= {
167 "arm" : "MOESI_CMP_directory",
170 def get_default_protocol(arch
):
171 return default_ruby_protocol
.get(arch
, 'MI_example')
173 all_categories
= ("quick", "long")
174 all_modes
= ("fs", "se")
177 """Test case base class.
179 Test cases consists of one or more test units that are run in two
180 phases. A run phase (units produced by run_units() and a verify
181 phase (units from verify_units()). The verify phase is skipped if
186 __metaclass__
= ABCMeta
188 def __init__(self
, name
):
189 self
.test_name
= name
193 """Get a list of reference files used by this test case"""
198 """Units (typically RunGem5 instances) that describe the run phase of
205 def verify_units(self
):
206 """Verify the output from the run phase (see run_units())."""
210 def update_ref(self
):
211 """Update reference files with files from a test run"""
215 """Run this test case and return a list of results"""
217 run_results
= [ u
.run() for u
in self
.run_units() ]
218 run_ok
= all([not r
.skipped() and r
for r
in run_results
])
221 u
.run() if run_ok
else u
.skip()
222 for u
in self
.verify_units()
225 return TestResult(self
.test_name
,
226 run_results
=run_results
,
227 verify_results
=verify_results
)
230 return self
.test_name
232 class ClassicTest(Test
):
233 # The diff ignore list contains all files that shouldn't be diffed
234 # using DiffOutFile. These files typically use special-purpose
235 # diff tools (e.g., DiffStatFile).
236 diff_ignore_files
= FileIgnoreList(
238 # Stat files use a special stat differ
243 # These files should never be included in the list of
244 # reference files. This list should include temporary files
245 # and other files that we don't care about.
246 ref_ignore_files
= FileIgnoreList(
250 # Mercurial sometimes leaves backups when applying MQ patches
255 def __init__(self
, gem5
, output_dir
, config_tuple
,
257 skip
=False, skip_diff_out
=False, skip_diff_stat
=False):
259 super(ClassicTest
, self
).__init
__("/".join(config_tuple
))
263 self
.gem5
= os
.path
.abspath(gem5
)
264 self
.script
= os
.path
.join(_test_base
, "run.py")
265 self
.config_tuple
= ct
266 self
.timeout
= timeout
268 self
.output_dir
= output_dir
269 self
.ref_dir
= os
.path
.join(_test_base
,
270 ct
.category
, ct
.mode
, ct
.workload
,
271 "ref", ct
.isa
, ct
.os
, ct
.config
)
273 self
.skip_diff_out
= skip
or skip_diff_out
274 self
.skip_diff_stat
= skip
or skip_diff_stat
277 ref_dir
= os
.path
.abspath(self
.ref_dir
)
278 for root
, dirs
, files
in os
.walk(ref_dir
, topdown
=False):
280 fpath
= os
.path
.join(root
[len(ref_dir
) + 1:], f
)
281 if fpath
not in ClassicTest
.ref_ignore_files
:
287 "/".join(self
.config_tuple
),
291 RunGem5(self
.gem5
, args
,
292 ref_dir
=self
.ref_dir
, test_dir
=self
.output_dir
,
296 def verify_units(self
):
297 ref_files
= set(self
.ref_files())
299 if "stats.txt" in ref_files
:
301 DiffStatFile(ref_dir
=self
.ref_dir
, test_dir
=self
.output_dir
,
302 skip
=self
.skip_diff_stat
))
305 ref_dir
=self
.ref_dir
, test_dir
=self
.output_dir
,
306 skip
=self
.skip_diff_out
)
307 for f
in ref_files
if f
not in ClassicTest
.diff_ignore_files
312 def update_ref(self
):
313 for fname
in self
.ref_files():
315 os
.path
.join(self
.output_dir
, fname
),
316 os
.path
.join(self
.ref_dir
, fname
))
318 def parse_test_filter(test_filter
):
319 wildcards
= ("", "*")
321 _filter
= list(test_filter
.split("/"))
323 raise RuntimeError("Illegal test filter string")
324 _filter
+= [ "", ] * (3 - len(_filter
))
326 isa
, cat
, mode
= _filter
329 raise RuntimeError("No ISA specified")
331 cat
= all_categories
if cat
in wildcards
else (cat
, )
332 mode
= all_modes
if mode
in wildcards
else (mode
, )
334 return isa
, cat
, mode
337 categories
=all_categories
, modes
=all_modes
,
338 ruby_protocol
=None, gpu_isa
=None):
340 # Generate a list of candidate configs
341 configs
= list(arch_configs
.get((isa
, gpu_isa
), []))
343 if (isa
, gpu_isa
) == ("x86", "hsail"):
344 if ruby_protocol
== "GPU_RfO":
345 configs
+= ['gpu-randomtest']
347 configs
+= generic_configs
349 if ruby_protocol
== get_default_protocol(isa
):
350 if ruby_protocol
== 'MI_example':
351 configs
+= [ "%s-ruby" % (c
, ) for c
in configs
]
353 configs
+= [ "%s-ruby-%s" % (c
, ruby_protocol
) for c
in configs
]
354 elif ruby_protocol
is not None:
355 # Override generic ISA configs when using Ruby (excluding
356 # MI_example which is included in all ISAs by default). This
357 # reduces the number of generic tests we re-run for when
358 # compiling Ruby targets.
359 configs
= [ "%s-ruby-%s" % (c
, ruby_protocol
) for c
in configs
]
361 # /(quick|long)/(fs|se)/workload/ref/arch/guest/config/
362 for conf_script
in configs
:
363 for cat
in categories
:
365 mode_dir
= os
.path
.join(_test_base
, cat
, mode
)
366 if not os
.path
.exists(mode_dir
):
369 for workload
in os
.listdir(mode_dir
):
370 isa_dir
= os
.path
.join(mode_dir
, workload
, "ref", isa
)
371 if not os
.path
.isdir(isa_dir
):
374 for _os
in os
.listdir(isa_dir
):
375 test_dir
= os
.path
.join(isa_dir
, _os
, conf_script
)
376 if not os
.path
.exists(test_dir
) or \
377 os
.path
.exists(os
.path
.join(test_dir
, "skip")):
380 yield ClassicConfig(cat
, mode
, workload
, isa
, _os
,