cebb98f3a82c82a3b09e64a559366238bbbfb8e5
[gem5.git] / ext / testlib / configuration.py
1 # Copyright (c) 2020 ARM Limited
2 # All rights reserved
3 #
4 # The license below extends only to copyright in the software and shall
5 # not be construed as granting a license to any other intellectual
6 # property including but not limited to intellectual property relating
7 # to a hardware implementation of the functionality of the software
8 # licensed hereunder. You may use the software subject to the license
9 # terms below provided that you ensure that this notice is replicated
10 # unmodified and in its entirety in all distributions of the software,
11 # modified or unmodified, in source code or in binary form.
12 #
13 # Copyright (c) 2017 Mark D. Hill and David A. Wood
14 # All rights reserved.
15 #
16 # Redistribution and use in source and binary forms, with or without
17 # modification, are permitted provided that the following conditions are
18 # met: redistributions of source code must retain the above copyright
19 # notice, this list of conditions and the following disclaimer;
20 # redistributions in binary form must reproduce the above copyright
21 # notice, this list of conditions and the following disclaimer in the
22 # documentation and/or other materials provided with the distribution;
23 # neither the name of the copyright holders nor the names of its
24 # contributors may be used to endorse or promote products derived from
25 # this software without specific prior written permission.
26 #
27 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
28 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
29 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
30 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
31 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
32 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
33 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
34 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
35 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
36 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
37 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 #
39 # Authors: Sean Wilson
40
41 '''
42 Global configuration module which exposes two types of configuration
43 variables:
44
45 1. config
46 2. constants (Also attached to the config variable as an attribute)
47
48 The main motivation for this module is to have a centralized location for
49 defaults and configuration by command line and files for the test framework.
50
51 A secondary goal is to reduce programming errors by providing common constant
52 strings and values as python attributes to simplify detection of typos.
53 A simple typo in a string can take a lot of debugging to uncover the issue,
54 attribute errors are easier to notice and most autocompletion systems detect
55 them.
56
57 The config variable is initialzed by calling :func:`initialize_config`.
58 Before this point only ``constants`` will be availaible. This is to ensure
59 that library function writers never accidentally get stale config attributes.
60
61 Program arguments/flag arguments are available from the config as attributes.
62 If an attribute was not set by the command line or the optional config file,
63 then it will fallback to the `_defaults` value, if still the value is not
64 found an AttributeError will be raised.
65
66 :func define_defaults:
67 Provided by the config if the attribute is not found in the config or
68 commandline. For instance, if we are using the list command fixtures might
69 not be able to count on the build_dir being provided since we aren't going
70 to build anything.
71
72 :var constants:
73 Values not directly exposed by the config, but are attached to the object
74 for centralized access. I.E. you can reach them with
75 :code:`config.constants.attribute`. These should be used for setting
76 common string names used across the test framework.
77 :code:`_defaults.build_dir = None` Once this module has been imported
78 constants should not be modified and their base attributes are frozen.
79 '''
80 import abc
81 import argparse
82 import copy
83 import os
84 import re
85
86 from six import add_metaclass
87 from pickle import HIGHEST_PROTOCOL as highest_pickle_protocol
88
89 from testlib.helper import absdirpath, AttrDict, FrozenAttrDict
90
91 class UninitialzedAttributeException(Exception):
92 '''
93 Signals that an attribute in the config file was not initialized.
94 '''
95 pass
96
97 class UninitializedConfigException(Exception):
98 '''
99 Signals that the config was not initialized before trying to access an
100 attribute.
101 '''
102 pass
103
104 class TagRegex(object):
105 def __init__(self, include, regex):
106 self.include = include
107 self.regex = re.compile(regex)
108
109 def __str__(self):
110 type_ = 'Include' if self.include else 'Remove'
111 return '%10s: %s' % (type_, self.regex.pattern)
112
113 class _Config(object):
114 _initialized = False
115
116 __shared_dict = {}
117
118 constants = AttrDict()
119 _defaults = AttrDict()
120 _config = {}
121
122 _cli_args = {}
123 _post_processors = {}
124
125 def __init__(self):
126 # This object will act as if it were a singleton.
127 self.__dict__ = self.__shared_dict
128
129 def _init(self, parser):
130 self._parse_commandline_args(parser)
131 self._run_post_processors()
132 self._initialized = True
133
134 def _add_post_processor(self, attr, post_processor):
135 '''
136 :param attr: Attribute to pass to and recieve from the
137 :func:`post_processor`.
138
139 :param post_processor: A callback functions called in a chain to
140 perform additional setup for a config argument. Should return a
141 tuple containing the new value for the config attr.
142 '''
143 if attr not in self._post_processors:
144 self._post_processors[attr] = []
145 self._post_processors[attr].append(post_processor)
146
147 def _set(self, name, value):
148 self._config[name] = value
149
150 def _parse_commandline_args(self, parser):
151 args = parser.parse_args()
152
153 self._config_file_args = {}
154
155 for attr in dir(args):
156 # Ignore non-argument attributes.
157 if not attr.startswith('_'):
158 self._config_file_args[attr] = getattr(args, attr)
159 self._config.update(self._config_file_args)
160
161 def _run_post_processors(self):
162 for attr, callbacks in self._post_processors.items():
163 newval = self._lookup_val(attr)
164 for callback in callbacks:
165 newval = callback(newval)
166 if newval is not None:
167 newval = newval[0]
168 self._set(attr, newval)
169
170
171 def _lookup_val(self, attr):
172 '''
173 Get the attribute from the config or fallback to defaults.
174
175 :returns: If the value is not stored return None. Otherwise a tuple
176 containing the value.
177 '''
178 if attr in self._config:
179 return (self._config[attr],)
180 elif hasattr(self._defaults, attr):
181 return (getattr(self._defaults, attr),)
182
183 def __getattr__(self, attr):
184 if attr in dir(super(_Config, self)):
185 return getattr(super(_Config, self), attr)
186 elif not self._initialized:
187 raise UninitializedConfigException(
188 'Cannot directly access elements from the config before it is'
189 ' initialized')
190 else:
191 val = self._lookup_val(attr)
192 if val is not None:
193 return val[0]
194 else:
195 raise UninitialzedAttributeException(
196 '%s was not initialzed in the config.' % attr)
197
198 def get_tags(self):
199 d = {typ: set(self.__getattr__(typ))
200 for typ in self.constants.supported_tags}
201 if any(map(lambda vals: bool(vals), d.values())):
202 return d
203 else:
204 return {}
205
206 def define_defaults(defaults):
207 '''
208 Defaults are provided by the config if the attribute is not found in the
209 config or commandline. For instance, if we are using the list command
210 fixtures might not be able to count on the build_dir being provided since
211 we aren't going to build anything.
212 '''
213 defaults.base_dir = os.path.abspath(os.path.join(absdirpath(__file__),
214 os.pardir,
215 os.pardir))
216 defaults.result_path = os.path.join(os.getcwd(), '.testing-results')
217 defaults.resource_url = 'http://dist.gem5.org/dist/develop'
218
219 def define_constants(constants):
220 '''
221 'constants' are values not directly exposed by the config, but are attached
222 to the object for centralized access. These should be used for setting
223 common string names used across the test framework. A simple typo in
224 a string can take a lot of debugging to uncover the issue, attribute errors
225 are easier to notice and most autocompletion systems detect them.
226 '''
227 constants.system_out_name = 'system-out'
228 constants.system_err_name = 'system-err'
229
230 constants.isa_tag_type = 'isa'
231 constants.x86_tag = 'X86'
232 constants.sparc_tag = 'SPARC'
233 constants.riscv_tag = 'RISCV'
234 constants.arm_tag = 'ARM'
235 constants.mips_tag = 'MIPS'
236 constants.power_tag = 'POWER'
237 constants.null_tag = 'NULL'
238
239 constants.variant_tag_type = 'variant'
240 constants.opt_tag = 'opt'
241 constants.debug_tag = 'debug'
242 constants.fast_tag = 'fast'
243
244 constants.length_tag_type = 'length'
245 constants.quick_tag = 'quick'
246 constants.long_tag = 'long'
247
248 constants.host_isa_tag_type = 'host'
249 constants.host_x86_64_tag = 'x86_64'
250 constants.host_i386_tag = 'i386'
251 constants.host_arm_tag = 'aarch64'
252
253 constants.supported_tags = {
254 constants.isa_tag_type : (
255 constants.x86_tag,
256 constants.sparc_tag,
257 constants.riscv_tag,
258 constants.arm_tag,
259 constants.mips_tag,
260 constants.power_tag,
261 constants.null_tag,
262 ),
263 constants.variant_tag_type: (
264 constants.opt_tag,
265 constants.debug_tag,
266 constants.fast_tag,
267 ),
268 constants.length_tag_type: (
269 constants.quick_tag,
270 constants.long_tag,
271 ),
272 constants.host_isa_tag_type: (
273 constants.host_x86_64_tag,
274 constants.host_i386_tag,
275 constants.host_arm_tag,
276 ),
277 }
278
279 # Binding target ISA with host ISA. This is useful for the
280 # case where host ISA and target ISA need to coincide
281 constants.target_host = {
282 constants.arm_tag : (constants.host_arm_tag,),
283 constants.x86_tag : (constants.host_x86_64_tag, constants.host_i386_tag),
284 constants.sparc_tag : (constants.host_x86_64_tag, constants.host_i386_tag),
285 constants.riscv_tag : (constants.host_x86_64_tag, constants.host_i386_tag),
286 constants.mips_tag : (constants.host_x86_64_tag, constants.host_i386_tag),
287 constants.power_tag : (constants.host_x86_64_tag, constants.host_i386_tag),
288 constants.null_tag : (None,)
289 }
290
291 constants.supported_isas = constants.supported_tags['isa']
292 constants.supported_variants = constants.supported_tags['variant']
293 constants.supported_lengths = constants.supported_tags['length']
294 constants.supported_hosts = constants.supported_tags['host']
295
296 constants.tempdir_fixture_name = 'tempdir'
297 constants.gem5_simulation_stderr = 'simerr'
298 constants.gem5_simulation_stdout = 'simout'
299 constants.gem5_simulation_stats = 'stats.txt'
300 constants.gem5_simulation_config_ini = 'config.ini'
301 constants.gem5_simulation_config_json = 'config.json'
302 constants.gem5_returncode_fixture_name = 'gem5-returncode'
303 constants.gem5_binary_fixture_name = 'gem5'
304 constants.xml_filename = 'results.xml'
305 constants.pickle_filename = 'results.pickle'
306 constants.pickle_protocol = highest_pickle_protocol
307
308 # The root directory which all test names will be based off of.
309 constants.testing_base = absdirpath(os.path.join(absdirpath(__file__),
310 os.pardir))
311
312 def define_post_processors(config):
313 '''
314 post_processors are used to do final configuration of variables. This is
315 useful if there is a dynamically set default, or some function that needs
316 to be applied after parsing in order to set a configration value.
317
318 Post processors must accept a single argument that will either be a tuple
319 containing the already set config value or ``None`` if the config value
320 has not been set to anything. They must return the modified value in the
321 same format.
322 '''
323
324 def set_default_build_dir(build_dir):
325 '''
326 Post-processor to set the default build_dir based on the base_dir.
327
328 .. seealso :func:`~_Config._add_post_processor`
329 '''
330 if not build_dir or build_dir[0] is None:
331 base_dir = config._lookup_val('base_dir')[0]
332 build_dir = (os.path.join(base_dir, 'build'),)
333 return build_dir
334
335 def fix_verbosity_hack(verbose):
336 return (verbose[0].val,)
337
338 def threads_as_int(threads):
339 if threads is not None:
340 return (int(threads[0]),)
341
342 def test_threads_as_int(test_threads):
343 if test_threads is not None:
344 return (int(test_threads[0]),)
345
346 def default_isa(isa):
347 if not isa[0]:
348 return [constants.supported_tags[constants.isa_tag_type]]
349 else:
350 return isa
351
352 def default_variant(variant):
353 if not variant[0]:
354 # Default variant is only opt. No need to run tests with multiple
355 # different compilation targets
356 return [[constants.opt_tag]]
357 else:
358 return variant
359
360 def default_length(length):
361 if not length[0]:
362 return [[constants.quick_tag]]
363 else:
364 return length
365
366 def default_host(host):
367 if not host[0]:
368 try:
369 import platform
370 host_machine = platform.machine()
371 if host_machine not in constants.supported_hosts:
372 raise ValueError("Invalid host machine")
373 return [[host_machine]]
374 except:
375 return [[constants.host_x86_64_tag]]
376 else:
377 return host
378
379 def compile_tag_regex(positional_tags):
380 if not positional_tags:
381 return positional_tags
382 else:
383 new_positional_tags_list = []
384 positional_tags = positional_tags[0]
385
386 for flag, regex in positional_tags:
387 if flag == 'exclude_tags':
388 tag_regex = TagRegex(False, regex)
389 elif flag == 'include_tags':
390 tag_regex = TagRegex(True, regex)
391 else:
392 raise ValueError('Unsupported flag.')
393 new_positional_tags_list.append(tag_regex)
394
395 return (new_positional_tags_list,)
396
397 config._add_post_processor('build_dir', set_default_build_dir)
398 config._add_post_processor('verbose', fix_verbosity_hack)
399 config._add_post_processor('isa', default_isa)
400 config._add_post_processor('variant', default_variant)
401 config._add_post_processor('length', default_length)
402 config._add_post_processor('host', default_host)
403 config._add_post_processor('threads', threads_as_int)
404 config._add_post_processor('test_threads', test_threads_as_int)
405 config._add_post_processor(StorePositionalTagsAction.position_kword,
406 compile_tag_regex)
407 class Argument(object):
408 '''
409 Class represents a cli argument/flag for a argparse parser.
410
411 :attr name: The long name of this object that will be stored in the arg
412 output by the final parser.
413 '''
414 def __init__(self, *flags, **kwargs):
415 self.flags = flags
416 self.kwargs = kwargs
417
418 if len(flags) == 0:
419 raise ValueError("Need at least one argument.")
420 elif 'dest' in kwargs:
421 self.name = kwargs['dest']
422 elif len(flags) > 1 or flags[0].startswith('-'):
423 for flag in flags:
424 if not flag.startswith('-'):
425 raise ValueError("invalid option string %s: must start"
426 "with a character '-'" % flag)
427
428 if flag.startswith('--'):
429 if not hasattr(self, 'name'):
430 self.name = flag.lstrip('-')
431
432 if not hasattr(self, 'name'):
433 self.name = flags[0].lstrip('-')
434 self.name = self.name.replace('-', '_')
435
436 def add_to(self, parser):
437 '''Add this argument to the given parser.'''
438 parser.add_argument(*self.flags, **self.kwargs)
439
440 def copy(self):
441 '''Copy this argument so you might modify any of its kwargs.'''
442 return copy.deepcopy(self)
443
444
445 class _StickyInt:
446 '''
447 A class that is used to cheat the verbosity count incrementer by
448 pretending to be an int. This makes the int stay on the heap and eat other
449 real numbers when they are added to it.
450
451 We use this so we can allow the verbose flag to be provided before or after
452 the subcommand. This likely has no utility outside of this use case.
453 '''
454 def __init__(self, val=0):
455 self.val = val
456 self.type = int
457 def __add__(self, other):
458 self.val += other
459 return self
460
461 common_args = NotImplemented
462
463 class StorePositionAction(argparse.Action):
464 '''Base class for classes wishing to create namespaces where
465 arguments are stored in the order provided via the command line.
466 '''
467 position_kword = 'positional'
468
469 def __call__(self, parser, namespace, values, option_string=None):
470 if not self.position_kword in namespace:
471 setattr(namespace, self.position_kword, [])
472 previous = getattr(namespace, self.position_kword)
473 previous.append((self.dest, values))
474 setattr(namespace, self.position_kword, previous)
475
476 class StorePositionalTagsAction(StorePositionAction):
477 position_kword = 'tag_filters'
478
479 def define_common_args(config):
480 '''
481 Common args are arguments which are likely to be simular between different
482 subcommands, so they are available to all by placing their definitions
483 here.
484 '''
485 global common_args
486
487 # A list of common arguments/flags used across cli parsers.
488 common_args = [
489 Argument(
490 'directory',
491 nargs='?',
492 default=os.getcwd(),
493 help='Directory to start searching for tests in'),
494 Argument(
495 '--exclude-tags',
496 action=StorePositionalTagsAction,
497 help='A tag comparison used to select tests.'),
498 Argument(
499 '--include-tags',
500 action=StorePositionalTagsAction,
501 help='A tag comparison used to select tests.'),
502 Argument(
503 '--isa',
504 action='append',
505 default=[],
506 help="Only tests that are valid with one of these ISAs. "
507 "Comma separated."),
508 Argument(
509 '--variant',
510 action='append',
511 default=[],
512 help="Only tests that are valid with one of these binary variants"
513 "(e.g., opt, debug). Comma separated."),
514 Argument(
515 '--length',
516 action='append',
517 default=[],
518 help="Only tests that are one of these lengths. Comma separated."),
519 Argument(
520 '--host',
521 action='append',
522 default=[],
523 help="Only tests that are meant to runnable on the selected host"),
524 Argument(
525 '--uid',
526 action='store',
527 default=None,
528 help='UID of a specific test item to run.'),
529 Argument(
530 '--build-dir',
531 action='store',
532 help='Build directory for SCons'),
533 Argument(
534 '--base-dir',
535 action='store',
536 default=config._defaults.base_dir,
537 help='Directory to change to in order to exec scons.'),
538 Argument(
539 '-j', '--threads',
540 action='store',
541 default=1,
542 help='Number of threads to run SCons with.'),
543 Argument(
544 '-t', '--test-threads',
545 action='store',
546 default=1,
547 help='Number of threads to spawn to run concurrent tests with.'),
548 Argument(
549 '-v',
550 action='count',
551 dest='verbose',
552 default=_StickyInt(),
553 help='Increase verbosity'),
554 Argument(
555 '--config-path',
556 action='store',
557 default=os.getcwd(),
558 help='Path to read a testing.ini config in'
559 ),
560 Argument(
561 '--skip-build',
562 action='store_true',
563 default=False,
564 help='Skip the building component of SCons targets.'
565 ),
566 Argument(
567 '--result-path',
568 action='store',
569 help='The path to store results in.'
570 ),
571 Argument(
572 '--bin-path',
573 action='store',
574 default=None,
575 help='Path where binaries are stored (downloaded if not present)'
576 ),
577 Argument(
578 '--resource-url',
579 action='store',
580 default=config._defaults.resource_url,
581 help='The URL where the resources reside.'
582 ),
583
584 ]
585
586 # NOTE: There is a limitation which arises due to this format. If you have
587 # multiple arguments with the same name only the final one in the list
588 # will be saved.
589 #
590 # e.g. if you have a -v argument which increments verbosity level and
591 # a separate --verbose flag which 'store's verbosity level. the final
592 # one in the list will be saved.
593 common_args = AttrDict({arg.name:arg for arg in common_args})
594
595 @add_metaclass(abc.ABCMeta)
596 class ArgParser(object):
597
598 def __init__(self, parser):
599 # Copy public methods of the parser.
600 for attr in dir(parser):
601 if not attr.startswith('_'):
602 setattr(self, attr, getattr(parser, attr))
603 self.parser = parser
604 self.add_argument = self.parser.add_argument
605
606 # Argument will be added to all parsers and subparsers.
607 common_args.verbose.add_to(parser)
608
609
610 class CommandParser(ArgParser):
611 '''
612 Main parser which parses command strings and uses those to direct to
613 a subparser.
614 '''
615 def __init__(self):
616 parser = argparse.ArgumentParser()
617 super(CommandParser, self).__init__(parser)
618 self.subparser = self.add_subparsers(dest='command')
619
620
621 class RunParser(ArgParser):
622 '''
623 Parser for the \'run\' command.
624 '''
625 def __init__(self, subparser):
626 parser = subparser.add_parser(
627 'run',
628 help='''Run Tests.'''
629 )
630
631 super(RunParser, self).__init__(parser)
632
633 common_args.uid.add_to(parser)
634 common_args.skip_build.add_to(parser)
635 common_args.directory.add_to(parser)
636 common_args.build_dir.add_to(parser)
637 common_args.base_dir.add_to(parser)
638 common_args.bin_path.add_to(parser)
639 common_args.threads.add_to(parser)
640 common_args.test_threads.add_to(parser)
641 common_args.isa.add_to(parser)
642 common_args.variant.add_to(parser)
643 common_args.length.add_to(parser)
644 common_args.host.add_to(parser)
645 common_args.include_tags.add_to(parser)
646 common_args.exclude_tags.add_to(parser)
647
648
649 class ListParser(ArgParser):
650 '''
651 Parser for the \'list\' command.
652 '''
653 def __init__(self, subparser):
654 parser = subparser.add_parser(
655 'list',
656 help='''List and query test metadata.'''
657 )
658 super(ListParser, self).__init__(parser)
659
660 Argument(
661 '--suites',
662 action='store_true',
663 default=False,
664 help='List all test suites.'
665 ).add_to(parser)
666 Argument(
667 '--tests',
668 action='store_true',
669 default=False,
670 help='List all test cases.'
671 ).add_to(parser)
672 Argument(
673 '--fixtures',
674 action='store_true',
675 default=False,
676 help='List all fixtures.'
677 ).add_to(parser)
678 Argument(
679 '--all-tags',
680 action='store_true',
681 default=False,
682 help='List all tags.'
683 ).add_to(parser)
684 Argument(
685 '-q',
686 dest='quiet',
687 action='store_true',
688 default=False,
689 help='Quiet output (machine readable).'
690 ).add_to(parser)
691
692 common_args.directory.add_to(parser)
693 common_args.bin_path.add_to(parser)
694 common_args.isa.add_to(parser)
695 common_args.variant.add_to(parser)
696 common_args.length.add_to(parser)
697 common_args.host.add_to(parser)
698 common_args.include_tags.add_to(parser)
699 common_args.exclude_tags.add_to(parser)
700
701
702 class RerunParser(ArgParser):
703 def __init__(self, subparser):
704 parser = subparser.add_parser(
705 'rerun',
706 help='''Rerun failed tests.'''
707 )
708 super(RerunParser, self).__init__(parser)
709
710 common_args.skip_build.add_to(parser)
711 common_args.directory.add_to(parser)
712 common_args.build_dir.add_to(parser)
713 common_args.base_dir.add_to(parser)
714 common_args.bin_path.add_to(parser)
715 common_args.threads.add_to(parser)
716 common_args.test_threads.add_to(parser)
717 common_args.isa.add_to(parser)
718 common_args.variant.add_to(parser)
719 common_args.length.add_to(parser)
720 common_args.host.add_to(parser)
721
722 config = _Config()
723 define_constants(config.constants)
724
725 # Constants are directly exposed and available once this module is created.
726 # All constants MUST be defined before this point.
727 config.constants = FrozenAttrDict(config.constants.__dict__)
728 constants = config.constants
729
730 '''
731 This config object is the singleton config object available throughout the
732 framework.
733 '''
734 def initialize_config():
735 '''
736 Parse the commandline arguments and setup the config varibles.
737 '''
738 global config
739
740 # Setup constants and defaults
741 define_defaults(config._defaults)
742 define_post_processors(config)
743 define_common_args(config)
744
745 # Setup parser and subcommands
746 baseparser = CommandParser()
747 runparser = RunParser(baseparser.subparser)
748 listparser = ListParser(baseparser.subparser)
749 rerunparser = RerunParser(baseparser.subparser)
750
751 # Initialize the config by parsing args and running callbacks.
752 config._init(baseparser)