From 85168c4e96ed8e3cbad0013102fa021561df49cd Mon Sep 17 00:00:00 2001 From: Gabe Black Date: Sat, 18 Nov 2017 20:39:48 -0800 Subject: [PATCH] scons: Switch from "guards" to "tags" on source files. Tags are just arbitrary strings which are attached to source files which mark them as having some property. By default, all source files have the "gem5 lib" tag added to them which marks them as part of the gem5 library, the primary component of the gem5 binary but also a seperable component for use in, for example, system C. The tags can be completely overridden by setting the "tags" parameter on Source, etc., functions, and can be augmented by setting "add_tags" which are tags that will be added, or alternatively additional tags. It's possible to specify both, in which case the tags will be set to the union of tags and add_tags. add_tags is supposed to be a way to add extra tags to the default without actually overriding the default. Both tags and add_tags can be a list/tuple/etc of tags, or a single string which will be converted into a set internally. Other existing tags include: 1. "python" for files that need or are used with python and are excluded when the --without-python option is set 2. "main" for the file(s) which implement the gem5 binary's main function. 3. The name of a unit test to group its files together. 4. Tags which group source files for partial linking. By grouping the "tags" into a single parameter instead of taking all extra parameters as tags, the extra parameters can, in the future, be passed to the underlying scons environment. Also, the tags are either present or not. With guards, they could be present and True, present and False, or not present at all. Change-Id: I6d0404211a393968df66f7eddfe019897b6573a2 Reviewed-on: https://gem5-review.googlesource.com/5822 Reviewed-by: Andreas Sandberg Maintainer: Andreas Sandberg --- src/SConscript | 175 +++++++++++++++++++--------------------- src/python/SConscript | 10 +-- src/sim/SConscript | 8 +- src/unittest/SConscript | 2 +- 4 files changed, 94 insertions(+), 101 deletions(-) diff --git a/src/SConscript b/src/SConscript index 6c3f22019..da8476d6c 100755 --- a/src/SConscript +++ b/src/SConscript @@ -59,56 +59,59 @@ from m5.util import code_formatter, compareVersions ######################################################################## # Code for adding source files of various types # -# When specifying a source file of some type, a set of guards can be -# specified for that file. When get() is used to find the files, if -# get specifies a set of filters, only files that match those filters -# will be accepted (unspecified filters on files are assumed to be -# false). Current filters are: -# main -- specifies the gem5 main() function -# skip_lib -- do not put this file into the gem5 library -# skip_no_python -- do not put this file into a no_python library -# as it embeds compiled Python -# -- unit tests use filters based on the unit test name -# -# A parent can now be specified for a source file and default filter -# values will be retrieved recursively from parents (children override -# parents). -# -def guarded_source_iterator(sources, **guards): - '''Iterate over a set of sources, gated by a set of guards.''' - for src in sources: - for flag,value in guards.iteritems(): - # if the flag is found and has a different value, skip - # this file - if src.all_guards.get(flag, False) != value: - break - else: - yield src +# When specifying a source file of some type, a set of tags can be +# specified for that file. + +class SourceList(list): + def with_tags_that(self, predicate): + '''Return a list of sources with tags that satisfy a predicate.''' + def match(source): + return predicate(source.tags) + return SourceList(filter(match, self)) + + def with_any_tags(self, *tags): + '''Return a list of sources with any of the supplied tags.''' + return self.with_tags_that(lambda stags: len(tags & stags) > 0) + + def with_all_tags(self, *tags): + '''Return a list of sources with all of the supplied tags.''' + return self.with_tags_that(lambda stags: tags <= stags) + + def with_tag(self, tag): + '''Return a list of sources with the supplied tag.''' + return self.with_tags_that(lambda stags: tag in stags) + + def without_tags(self, *tags): + '''Return a list of sources without any of the supplied tags.''' + return self.with_tags_that(lambda stags: len(tags & stags) == 0) + + def without_tag(self, tag): + '''Return a list of sources with the supplied tag.''' + return self.with_tags_that(lambda stags: tag not in stags) class SourceMeta(type): '''Meta class for source files that keeps track of all files of a - particular type and has a get function for finding all functions - of a certain type that match a set of guards''' + particular type.''' def __init__(cls, name, bases, dict): super(SourceMeta, cls).__init__(name, bases, dict) - cls.all = [] - - def get(cls, **guards): - '''Find all files that match the specified guards. If a source - file does not specify a flag, the default is False''' - for s in guarded_source_iterator(cls.all, **guards): - yield s + cls.all = SourceList() class SourceFile(object): '''Base object that encapsulates the notion of a source file. This includes, the source node, target node, various manipulations - of those. A source file also specifies a set of guards which - describing which builds the source file applies to. A parent can - also be specified to get default guards from''' + of those. A source file also specifies a set of tags which + describing arbitrary properties of the source file.''' __metaclass__ = SourceMeta - def __init__(self, source, parent=None, **guards): - self.guards = guards - self.parent = parent + def __init__(self, source, tags=None, add_tags=None): + if tags is None: + tags='gem5 lib' + if isinstance(tags, basestring): + tags = set([tags]) + if isinstance(add_tags, basestring): + add_tags = set([add_tags]) + if add_tags: + tags = tags | add_tags + self.tags = set(tags) tnode = source if not isinstance(source, SCons.Node.FS.File): @@ -142,16 +145,6 @@ class SourceFile(object): return self.basename[:index], self.basename[index+1:] - @property - def all_guards(self): - '''find all guards for this object getting default values - recursively from its parents''' - guards = {} - if self.parent: - guards.update(self.parent.guards) - guards.update(self.guards) - return guards - def __lt__(self, other): return self.filename < other.filename def __le__(self, other): return self.filename <= other.filename def __gt__(self, other): return self.filename > other.filename @@ -168,24 +161,31 @@ class SourceFile(object): class Source(SourceFile): - current_group = None - source_groups = { None : [] } + ungrouped_tag = 'No link group' + source_groups = set() + + _current_group_tag = ungrouped_tag + + @staticmethod + def link_group_tag(group): + return 'link group: %s' % group @classmethod def set_group(cls, group): - if not group in Source.source_groups: - Source.source_groups[group] = [] - Source.current_group = group + new_tag = Source.link_group_tag(group) + Source._current_group_tag = new_tag + Source.source_groups.add(group) - '''Add a c/c++ source file to the build''' - def __init__(self, source, Werror=True, **guards): - '''specify the source file, and any guards''' - super(Source, self).__init__(source, **guards) + def _add_link_group_tag(self): + self.tags.add(Source._current_group_tag) + '''Add a c/c++ source file to the build''' + def __init__(self, source, tags=None, add_tags=None, Werror=True): + '''specify the source file, and any tags''' + super(Source, self).__init__(source, tags, add_tags) + self._add_link_group_tag() self.Werror = Werror - Source.source_groups[Source.current_group].append(self) - class PySource(SourceFile): '''Add a python source file to the named package''' invalid_sym_char = re.compile('[^A-z0-9_]') @@ -193,9 +193,9 @@ class PySource(SourceFile): tnodes = {} symnames = {} - def __init__(self, package, source, **guards): - '''specify the python package, the source file, and any guards''' - super(PySource, self).__init__(source, **guards) + def __init__(self, package, source, tags=None, add_tags=None): + '''specify the python package, the source file, and any tags''' + super(PySource, self).__init__(source, tags, add_tags) modname,ext = self.extname assert ext == 'py' @@ -235,10 +235,10 @@ class SimObject(PySource): fixed = False modnames = [] - def __init__(self, source, **guards): - '''Specify the source file and any guards (automatically in + def __init__(self, source, tags=None, add_tags=None): + '''Specify the source file and any tags (automatically in the m5.objects package)''' - super(SimObject, self).__init__('m5.objects', source, **guards) + super(SimObject, self).__init__('m5.objects', source, tags, add_tags) if self.fixed: raise AttributeError, "Too late to call SimObject now." @@ -247,9 +247,9 @@ class SimObject(PySource): class ProtoBuf(SourceFile): '''Add a Protocol Buffer to build''' - def __init__(self, source, **guards): - '''Specify the source file, and any guards''' - super(ProtoBuf, self).__init__(source, **guards) + def __init__(self, source, tags=None, add_tags=None): + '''Specify the source file, and any tags''' + super(ProtoBuf, self).__init__(source, tags, add_tags) # Get the file name and the extension modname,ext = self.extname @@ -267,14 +267,12 @@ class UnitTest(object): def __init__(self, target, *sources, **kwargs): '''Specify the target name and any sources. Sources that are not SourceFiles are evalued with Source(). All files are - guarded with a guard of the same name as the UnitTest - target.''' + tagged with the name of the UnitTest target.''' - srcs = [] + srcs = SourceList() for src in sources: if not isinstance(src, SourceFile): - src = Source(src, skip_lib=True) - src.guards[target] = True + src = Source(src, tags=str(target)) srcs.append(src) self.sources = srcs @@ -770,7 +768,7 @@ if env['HAVE_PROTOBUF']: Transform("PROTOC"))) # Add the C++ source file - Source(proto.cc_file, **proto.guards) + Source(proto.cc_file, tags=proto.tags) elif ProtoBuf.all: print 'Got protobuf to build, but lacks support!' Exit(1) @@ -936,7 +934,7 @@ EmbeddedPython embedded_${sym}( for source in PySource.all: env.Command(source.cpp, source.tnode, MakeAction(embedPyFile, Transform("EMBED PY"))) - Source(source.cpp, skip_no_python=True) + Source(source.cpp, tags=source.tags, add_tags='python') ######################################################################## # @@ -945,7 +943,7 @@ for source in PySource.all: # # List of constructed environments to pass back to SConstruct -date_source = Source('base/date.cc', skip_lib=True) +date_source = Source('base/date.cc', tags=[]) # Function to create a new build environment as clone of current # environment 'env' with modified object suffix and optional stripped @@ -1028,28 +1026,24 @@ def makeEnv(env, label, objsfx, strip=False, disable_partial=False, **kwargs): return obj - lib_guards = {'main': False, 'skip_lib': False} + lib_sources = Source.all.with_tag('gem5 lib') # Without Python, leave out all Python content from the library # builds. The option doesn't affect gem5 built as a program if GetOption('without_python'): - lib_guards['skip_no_python'] = False + lib_sources = lib_sources.without_tag('python') static_objs = [] shared_objs = [] - for s in guarded_source_iterator(Source.source_groups[None], **lib_guards): + + for s in lib_sources.with_tag(Source.ungrouped_tag): static_objs.append(make_obj(s, True)) shared_objs.append(make_obj(s, False)) partial_objs = [] - for group, all_srcs in Source.source_groups.iteritems(): - # If these are the ungrouped source files, skip them. - if not group: - continue - # Get a list of the source files compatible with the current guards. - srcs = [ s for s in guarded_source_iterator(all_srcs, **lib_guards) ] - # If there aren't any left, skip this group. + for group in Source.source_groups: + srcs = lib_sources.with_tag(Source.link_group_tag(group)) if not srcs: continue @@ -1087,11 +1081,10 @@ def makeEnv(env, label, objsfx, strip=False, disable_partial=False, **kwargs): shared_lib = new_env.SharedLibrary(libname, shared_objs) # Now link a stub with main() and the static library. - main_objs = [ make_obj(s, True) for s in Source.get(main=True) ] + main_objs = [ make_obj(s, True) for s in Source.all.with_tag('main') ] for test in UnitTest.all: - flags = { test.target : True } - test_sources = Source.get(**flags) + test_sources = Source.all.with_tag(str(test.target)) test_objs = [ make_obj(s, static=True) for s in test_sources ] if test.main: test_objs += main_objs diff --git a/src/python/SConscript b/src/python/SConscript index 1e7746930..cfd2afeff 100644 --- a/src/python/SConscript +++ b/src/python/SConscript @@ -64,8 +64,8 @@ PySource('m5.util', 'm5/util/pybind.py') PySource('m5.internal', 'm5/internal/__init__.py') PySource('m5.internal', 'm5/internal/params.py') -Source('pybind11/core.cc', skip_no_python=True) -Source('pybind11/debug.cc', skip_no_python=True) -Source('pybind11/event.cc', skip_no_python=True) -Source('pybind11/pyobject.cc', skip_no_python=True) -Source('pybind11/stats.cc', skip_no_python=True) +Source('pybind11/core.cc', add_tags='python') +Source('pybind11/debug.cc', add_tags='python') +Source('pybind11/event.cc', add_tags='python') +Source('pybind11/pyobject.cc', add_tags='python') +Source('pybind11/stats.cc', add_tags='python') diff --git a/src/sim/SConscript b/src/sim/SConscript index a3d5464e8..996a3324a 100644 --- a/src/sim/SConscript +++ b/src/sim/SConscript @@ -48,12 +48,12 @@ Source('cxx_config.cc') Source('cxx_manager.cc') Source('cxx_config_ini.cc') Source('debug.cc') -Source('py_interact.cc', skip_no_python=True) +Source('py_interact.cc', add_tags='python') Source('eventq.cc') Source('global_event.cc') -Source('init.cc', skip_no_python=True) +Source('init.cc', add_tags='python') Source('init_signals.cc') -Source('main.cc', main=True, skip_lib=True) +Source('main.cc', tags='main') Source('root.cc') Source('serialize.cc') Source('drain.cc') @@ -63,7 +63,7 @@ Source('sub_system.cc') Source('ticked_object.cc') Source('simulate.cc') Source('stat_control.cc') -Source('stat_register.cc', skip_no_python=True) +Source('stat_register.cc', add_tags='python') Source('clock_domain.cc') Source('voltage_domain.cc') Source('se_signal.cc') diff --git a/src/unittest/SConscript b/src/unittest/SConscript index 1f723ed2a..70e3c2f95 100644 --- a/src/unittest/SConscript +++ b/src/unittest/SConscript @@ -45,7 +45,7 @@ UnitTest('refcnttest', 'refcnttest.cc') UnitTest('strnumtest', 'strnumtest.cc') UnitTest('trietest', 'trietest.cc') -stattest_py = PySource('m5', 'stattestmain.py', skip_lib=True) +stattest_py = PySource('m5', 'stattestmain.py', tags='stattest') UnitTest('stattest', 'stattest.cc', stattest_py, main=True) UnitTest('symtest', 'symtest.cc') -- 2.30.2