src/SConscript

   1 # -*- mode:python -*-
   2
   3 # Copyright (c) 2004-2005 The Regents of The University of Michigan
   4 # All rights reserved.
   5 #
   6 # Redistribution and use in source and binary forms, with or without
   7 # modification, are permitted provided that the following conditions are
   8 # met: redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer;
  10 # redistributions in binary form must reproduce the above copyright
  11 # notice, this list of conditions and the following disclaimer in the
  12 # documentation and/or other materials provided with the distribution;
  13 # neither the name of the copyright holders nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # Authors: Nathan Binkert
  30
  31 from __future__ import print_function
  32
  33 import array
  34 import bisect
  35 import functools
  36 import imp
  37 import marshal
  38 import os
  39 import re
  40 import subprocess
  41 import sys
  42 import zlib
  43
  44 from os.path import basename, dirname, exists, isdir, isfile, join as joinpath
  45
  46 import SCons
  47
  48 from gem5_scons import Transform
  49
  50 # This file defines how to build a particular configuration of gem5
  51 # based on variable settings in the 'env' build environment.
  52
  53 Import('*')
  54
  55 # Children need to see the environment
  56 Export('env')
  57
  58 build_env = [(opt, env[opt]) for opt in export_vars]
  59
  60 from m5.util import code_formatter, compareVersions
  61
  62 ########################################################################
  63 # Code for adding source files of various types
  64 #
  65 # When specifying a source file of some type, a set of tags can be
  66 # specified for that file.
  67
  68 class SourceFilter(object):
  69     def __init__(self, predicate):
  70         self.predicate = predicate
  71
  72     def __or__(self, other):
  73         return SourceFilter(lambda tags: self.predicate(tags) or
  74                                          other.predicate(tags))
  75
  76     def __and__(self, other):
  77         return SourceFilter(lambda tags: self.predicate(tags) and
  78                                          other.predicate(tags))
  79
  80 def with_tags_that(predicate):
  81     '''Return a list of sources with tags that satisfy a predicate.'''
  82     return SourceFilter(predicate)
  83
  84 def with_any_tags(*tags):
  85     '''Return a list of sources with any of the supplied tags.'''
  86     return SourceFilter(lambda stags: len(set(tags) & stags) > 0)
  87
  88 def with_all_tags(*tags):
  89     '''Return a list of sources with all of the supplied tags.'''
  90     return SourceFilter(lambda stags: set(tags) <= stags)
  91
  92 def with_tag(tag):
  93     '''Return a list of sources with the supplied tag.'''
  94     return SourceFilter(lambda stags: tag in stags)
  95
  96 def without_tags(*tags):
  97     '''Return a list of sources without any of the supplied tags.'''
  98     return SourceFilter(lambda stags: len(set(tags) & stags) == 0)
  99
 100 def without_tag(tag):
 101     '''Return a list of sources with the supplied tag.'''
 102     return SourceFilter(lambda stags: tag not in stags)
 103
 104 source_filter_factories = {
 105     'with_tags_that': with_tags_that,
 106     'with_any_tags': with_any_tags,
 107     'with_all_tags': with_all_tags,
 108     'with_tag': with_tag,
 109     'without_tags': without_tags,
 110     'without_tag': without_tag,
 111 }
 112
 113 Export(source_filter_factories)
 114
 115 class SourceList(list):
 116     def apply_filter(self, f):
 117         def match(source):
 118             return f.predicate(source.tags)
 119         return SourceList(filter(match, self))
 120
 121     def __getattr__(self, name):
 122         func = source_filter_factories.get(name, None)
 123         if not func:
 124             raise AttributeError
 125
 126         @functools.wraps(func)
 127         def wrapper(*args, **kwargs):
 128             return self.apply_filter(func(*args, **kwargs))
 129         return wrapper
 130
 131 class SourceMeta(type):
 132     '''Meta class for source files that keeps track of all files of a
 133     particular type.'''
 134     def __init__(cls, name, bases, dict):
 135         super(SourceMeta, cls).__init__(name, bases, dict)
 136         cls.all = SourceList()
 137
 138 class SourceFile(object):
 139     '''Base object that encapsulates the notion of a source file.
 140     This includes, the source node, target node, various manipulations
 141     of those.  A source file also specifies a set of tags which
 142     describing arbitrary properties of the source file.'''
 143     __metaclass__ = SourceMeta
 144
 145     static_objs = {}
 146     shared_objs = {}
 147
 148     def __init__(self, source, tags=None, add_tags=None):
 149         if tags is None:
 150             tags='gem5 lib'
 151         if isinstance(tags, basestring):
 152             tags = set([tags])
 153         if not isinstance(tags, set):
 154             tags = set(tags)
 155         self.tags = tags
 156
 157         if add_tags:
 158             if isinstance(add_tags, basestring):
 159                 add_tags = set([add_tags])
 160             if not isinstance(add_tags, set):
 161                 add_tags = set(add_tags)
 162             self.tags |= add_tags
 163
 164         tnode = source
 165         if not isinstance(source, SCons.Node.FS.File):
 166             tnode = File(source)
 167
 168         self.tnode = tnode
 169         self.snode = tnode.srcnode()
 170
 171         for base in type(self).__mro__:
 172             if issubclass(base, SourceFile):
 173                 base.all.append(self)
 174
 175     def static(self, env):
 176         key = (self.tnode, env['OBJSUFFIX'])
 177         if not key in self.static_objs:
 178             self.static_objs[key] = env.StaticObject(self.tnode)
 179         return self.static_objs[key]
 180
 181     def shared(self, env):
 182         key = (self.tnode, env['OBJSUFFIX'])
 183         if not key in self.shared_objs:
 184             self.shared_objs[key] = env.SharedObject(self.tnode)
 185         return self.shared_objs[key]
 186
 187     @property
 188     def filename(self):
 189         return str(self.tnode)
 190
 191     @property
 192     def dirname(self):
 193         return dirname(self.filename)
 194
 195     @property
 196     def basename(self):
 197         return basename(self.filename)
 198
 199     @property
 200     def extname(self):
 201         index = self.basename.rfind('.')
 202         if index <= 0:
 203             # dot files aren't extensions
 204             return self.basename, None
 205
 206         return self.basename[:index], self.basename[index+1:]
 207
 208     def __lt__(self, other): return self.filename < other.filename
 209     def __le__(self, other): return self.filename <= other.filename
 210     def __gt__(self, other): return self.filename > other.filename
 211     def __ge__(self, other): return self.filename >= other.filename
 212     def __eq__(self, other): return self.filename == other.filename
 213     def __ne__(self, other): return self.filename != other.filename
 214
 215 class Source(SourceFile):
 216     ungrouped_tag = 'No link group'
 217     source_groups = set()
 218
 219     _current_group_tag = ungrouped_tag
 220
 221     @staticmethod
 222     def link_group_tag(group):
 223         return 'link group: %s' % group
 224
 225     @classmethod
 226     def set_group(cls, group):
 227         new_tag = Source.link_group_tag(group)
 228         Source._current_group_tag = new_tag
 229         Source.source_groups.add(group)
 230
 231     def _add_link_group_tag(self):
 232         self.tags.add(Source._current_group_tag)
 233
 234     '''Add a c/c++ source file to the build'''
 235     def __init__(self, source, tags=None, add_tags=None):
 236         '''specify the source file, and any tags'''
 237         super(Source, self).__init__(source, tags, add_tags)
 238         self._add_link_group_tag()
 239
 240 class PySource(SourceFile):
 241     '''Add a python source file to the named package'''
 242     invalid_sym_char = re.compile('[^A-z0-9_]')
 243     modules = {}
 244     tnodes = {}
 245     symnames = {}
 246
 247     def __init__(self, package, source, tags=None, add_tags=None):
 248         '''specify the python package, the source file, and any tags'''
 249         super(PySource, self).__init__(source, tags, add_tags)
 250
 251         modname,ext = self.extname
 252         assert ext == 'py'
 253
 254         if package:
 255             path = package.split('.')
 256         else:
 257             path = []
 258
 259         modpath = path[:]
 260         if modname != '__init__':
 261             modpath += [ modname ]
 262         modpath = '.'.join(modpath)
 263
 264         arcpath = path + [ self.basename ]
 265         abspath = self.snode.abspath
 266         if not exists(abspath):
 267             abspath = self.tnode.abspath
 268
 269         self.package = package
 270         self.modname = modname
 271         self.modpath = modpath
 272         self.arcname = joinpath(*arcpath)
 273         self.abspath = abspath
 274         self.compiled = File(self.filename + 'c')
 275         self.cpp = File(self.filename + '.cc')
 276         self.symname = PySource.invalid_sym_char.sub('_', modpath)
 277
 278         PySource.modules[modpath] = self
 279         PySource.tnodes[self.tnode] = self
 280         PySource.symnames[self.symname] = self
 281
 282 class SimObject(PySource):
 283     '''Add a SimObject python file as a python source object and add
 284     it to a list of sim object modules'''
 285
 286     fixed = False
 287     modnames = []
 288
 289     def __init__(self, source, tags=None, add_tags=None):
 290         '''Specify the source file and any tags (automatically in
 291         the m5.objects package)'''
 292         super(SimObject, self).__init__('m5.objects', source, tags, add_tags)
 293         if self.fixed:
 294             raise AttributeError, "Too late to call SimObject now."
 295
 296         bisect.insort_right(SimObject.modnames, self.modname)
 297
 298 class ProtoBuf(SourceFile):
 299     '''Add a Protocol Buffer to build'''
 300
 301     def __init__(self, source, tags=None, add_tags=None):
 302         '''Specify the source file, and any tags'''
 303         super(ProtoBuf, self).__init__(source, tags, add_tags)
 304
 305         # Get the file name and the extension
 306         modname,ext = self.extname
 307         assert ext == 'proto'
 308
 309         # Currently, we stick to generating the C++ headers, so we
 310         # only need to track the source and header.
 311         self.cc_file = File(modname + '.pb.cc')
 312         self.hh_file = File(modname + '.pb.h')
 313
 314
 315 exectuable_classes = []
 316 class ExecutableMeta(type):
 317     '''Meta class for Executables.'''
 318     all = []
 319
 320     def __init__(cls, name, bases, d):
 321         if not d.pop('abstract', False):
 322             ExecutableMeta.all.append(cls)
 323         super(ExecutableMeta, cls).__init__(name, bases, d)
 324
 325         cls.all = []
 326
 327 class Executable(object):
 328     '''Base class for creating an executable from sources.'''
 329     __metaclass__ = ExecutableMeta
 330
 331     abstract = True
 332
 333     def __init__(self, target, *srcs_and_filts):
 334         '''Specify the target name and any sources. Sources that are
 335         not SourceFiles are evalued with Source().'''
 336         super(Executable, self).__init__()
 337         self.all.append(self)
 338         self.target = target
 339
 340         isFilter = lambda arg: isinstance(arg, SourceFilter)
 341         self.filters = filter(isFilter, srcs_and_filts)
 342         sources = filter(lambda a: not isFilter(a), srcs_and_filts)
 343
 344         srcs = SourceList()
 345         for src in sources:
 346             if not isinstance(src, SourceFile):
 347                 src = Source(src, tags=[])
 348             srcs.append(src)
 349
 350         self.sources = srcs
 351         self.dir = Dir('.')
 352
 353     def path(self, env):
 354         return self.dir.File(self.target + '.' + env['EXE_SUFFIX'])
 355
 356     def srcs_to_objs(self, env, sources):
 357         return list([ s.static(env) for s in sources ])
 358
 359     @classmethod
 360     def declare_all(cls, env):
 361         return list([ instance.declare(env) for instance in cls.all ])
 362
 363     def declare(self, env, objs=None):
 364         if objs is None:
 365             objs = self.srcs_to_objs(env, self.sources)
 366
 367         if env['STRIP_EXES']:
 368             stripped = self.path(env)
 369             unstripped = env.File(str(stripped) + '.unstripped')
 370             if sys.platform == 'sunos5':
 371                 cmd = 'cp $SOURCE $TARGET; strip $TARGET'
 372             else:
 373                 cmd = 'strip $SOURCE -o $TARGET'
 374             env.Program(unstripped, objs)
 375             return env.Command(stripped, unstripped,
 376                                MakeAction(cmd, Transform("STRIP")))
 377         else:
 378             return env.Program(self.path(env), objs)
 379
 380 class UnitTest(Executable):
 381     '''Create a UnitTest'''
 382     def __init__(self, target, *srcs_and_filts, **kwargs):
 383         super(UnitTest, self).__init__(target, *srcs_and_filts)
 384
 385         self.main = kwargs.get('main', False)
 386
 387     def declare(self, env):
 388         sources = list(self.sources)
 389         for f in self.filters:
 390             sources = Source.all.apply_filter(f)
 391         objs = self.srcs_to_objs(env, sources) + env['STATIC_OBJS']
 392         if self.main:
 393             objs += env['MAIN_OBJS']
 394         return super(UnitTest, self).declare(env, objs)
 395
 396 class GTest(Executable):
 397     '''Create a unit test based on the google test framework.'''
 398     all = []
 399     def __init__(self, *srcs_and_filts, **kwargs):
 400         super(GTest, self).__init__(*srcs_and_filts)
 401
 402         self.skip_lib = kwargs.pop('skip_lib', False)
 403
 404     @classmethod
 405     def declare_all(cls, env):
 406         env = env.Clone()
 407         env.Append(LIBS=env['GTEST_LIBS'])
 408         env.Append(CPPFLAGS=env['GTEST_CPPFLAGS'])
 409         env['GTEST_LIB_SOURCES'] = Source.all.with_tag('gtest lib')
 410         env['GTEST_OUT_DIR'] = \
 411             Dir(env['BUILDDIR']).Dir('unittests.' + env['EXE_SUFFIX'])
 412         return super(GTest, cls).declare_all(env)
 413
 414     def declare(self, env):
 415         sources = list(self.sources)
 416         if not self.skip_lib:
 417             sources += env['GTEST_LIB_SOURCES']
 418         for f in self.filters:
 419             sources += Source.all.apply_filter(f)
 420         objs = self.srcs_to_objs(env, sources)
 421
 422         binary = super(GTest, self).declare(env, objs)
 423
 424         out_dir = env['GTEST_OUT_DIR']
 425         xml_file = out_dir.Dir(str(self.dir)).File(self.target + '.xml')
 426         AlwaysBuild(env.Command(xml_file, binary,
 427             "${SOURCES[0]} --gtest_output=xml:${TARGETS[0]}"))
 428
 429         return binary
 430
 431 class Gem5(Executable):
 432     '''Create a gem5 executable.'''
 433
 434     def __init__(self, target):
 435         super(Gem5, self).__init__(target)
 436
 437     def declare(self, env):
 438         objs = env['MAIN_OBJS'] + env['STATIC_OBJS']
 439         return super(Gem5, self).declare(env, objs)
 440
 441
 442 # Children should have access
 443 Export('Source')
 444 Export('PySource')
 445 Export('SimObject')
 446 Export('ProtoBuf')
 447 Export('Executable')
 448 Export('UnitTest')
 449 Export('GTest')
 450
 451 ########################################################################
 452 #
 453 # Debug Flags
 454 #
 455 debug_flags = {}
 456 def DebugFlag(name, desc=None):
 457     if name in debug_flags:
 458         raise AttributeError, "Flag %s already specified" % name
 459     debug_flags[name] = (name, (), desc)
 460
 461 def CompoundFlag(name, flags, desc=None):
 462     if name in debug_flags:
 463         raise AttributeError, "Flag %s already specified" % name
 464
 465     compound = tuple(flags)
 466     debug_flags[name] = (name, compound, desc)
 467
 468 Export('DebugFlag')
 469 Export('CompoundFlag')
 470
 471 ########################################################################
 472 #
 473 # Set some compiler variables
 474 #
 475
 476 # Include file paths are rooted in this directory.  SCons will
 477 # automatically expand '.' to refer to both the source directory and
 478 # the corresponding build directory to pick up generated include
 479 # files.
 480 env.Append(CPPPATH=Dir('.'))
 481
 482 for extra_dir in extras_dir_list:
 483     env.Append(CPPPATH=Dir(extra_dir))
 484
 485 # Workaround for bug in SCons version > 0.97d20071212
 486 # Scons bug id: 2006 gem5 Bug id: 308
 487 for root, dirs, files in os.walk(base_dir, topdown=True):
 488     Dir(root[len(base_dir) + 1:])
 489
 490 ########################################################################
 491 #
 492 # Walk the tree and execute all SConscripts in subdirectories
 493 #
 494
 495 here = Dir('.').srcnode().abspath
 496 for root, dirs, files in os.walk(base_dir, topdown=True):
 497     if root == here:
 498         # we don't want to recurse back into this SConscript
 499         continue
 500
 501     if 'SConscript' in files:
 502         build_dir = joinpath(env['BUILDDIR'], root[len(base_dir) + 1:])
 503         Source.set_group(build_dir)
 504         SConscript(joinpath(root, 'SConscript'), variant_dir=build_dir)
 505
 506 for extra_dir in extras_dir_list:
 507     prefix_len = len(dirname(extra_dir)) + 1
 508
 509     # Also add the corresponding build directory to pick up generated
 510     # include files.
 511     env.Append(CPPPATH=Dir(joinpath(env['BUILDDIR'], extra_dir[prefix_len:])))
 512
 513     for root, dirs, files in os.walk(extra_dir, topdown=True):
 514         # if build lives in the extras directory, don't walk down it
 515         if 'build' in dirs:
 516             dirs.remove('build')
 517
 518         if 'SConscript' in files:
 519             build_dir = joinpath(env['BUILDDIR'], root[prefix_len:])
 520             SConscript(joinpath(root, 'SConscript'), variant_dir=build_dir)
 521
 522 for opt in export_vars:
 523     env.ConfigFile(opt)
 524
 525 def makeTheISA(source, target, env):
 526     isas = [ src.get_contents() for src in source ]
 527     target_isa = env['TARGET_ISA']
 528     def define(isa):
 529         return isa.upper() + '_ISA'
 530
 531     def namespace(isa):
 532         return isa[0].upper() + isa[1:].lower() + 'ISA'
 533
 534
 535     code = code_formatter()
 536     code('''\
 537 #ifndef __CONFIG_THE_ISA_HH__
 538 #define __CONFIG_THE_ISA_HH__
 539
 540 ''')
 541
 542     # create defines for the preprocessing and compile-time determination
 543     for i,isa in enumerate(isas):
 544         code('#define $0 $1', define(isa), i + 1)
 545     code()
 546
 547     # create an enum for any run-time determination of the ISA, we
 548     # reuse the same name as the namespaces
 549     code('enum class Arch {')
 550     for i,isa in enumerate(isas):
 551         if i + 1 == len(isas):
 552             code('  $0 = $1', namespace(isa), define(isa))
 553         else:
 554             code('  $0 = $1,', namespace(isa), define(isa))
 555     code('};')
 556
 557     code('''
 558
 559 #define THE_ISA ${{define(target_isa)}}
 560 #define TheISA ${{namespace(target_isa)}}
 561 #define THE_ISA_STR "${{target_isa}}"
 562
 563 #endif // __CONFIG_THE_ISA_HH__''')
 564
 565     code.write(str(target[0]))
 566
 567 env.Command('config/the_isa.hh', map(Value, all_isa_list),
 568             MakeAction(makeTheISA, Transform("CFG ISA", 0)))
 569
 570 def makeTheGPUISA(source, target, env):
 571     isas = [ src.get_contents() for src in source ]
 572     target_gpu_isa = env['TARGET_GPU_ISA']
 573     def define(isa):
 574         return isa.upper() + '_ISA'
 575
 576     def namespace(isa):
 577         return isa[0].upper() + isa[1:].lower() + 'ISA'
 578
 579
 580     code = code_formatter()
 581     code('''\
 582 #ifndef __CONFIG_THE_GPU_ISA_HH__
 583 #define __CONFIG_THE_GPU_ISA_HH__
 584
 585 ''')
 586
 587     # create defines for the preprocessing and compile-time determination
 588     for i,isa in enumerate(isas):
 589         code('#define $0 $1', define(isa), i + 1)
 590     code()
 591
 592     # create an enum for any run-time determination of the ISA, we
 593     # reuse the same name as the namespaces
 594     code('enum class GPUArch {')
 595     for i,isa in enumerate(isas):
 596         if i + 1 == len(isas):
 597             code('  $0 = $1', namespace(isa), define(isa))
 598         else:
 599             code('  $0 = $1,', namespace(isa), define(isa))
 600     code('};')
 601
 602     code('''
 603
 604 #define THE_GPU_ISA ${{define(target_gpu_isa)}}
 605 #define TheGpuISA ${{namespace(target_gpu_isa)}}
 606 #define THE_GPU_ISA_STR "${{target_gpu_isa}}"
 607
 608 #endif // __CONFIG_THE_GPU_ISA_HH__''')
 609
 610     code.write(str(target[0]))
 611
 612 env.Command('config/the_gpu_isa.hh', map(Value, all_gpu_isa_list),
 613             MakeAction(makeTheGPUISA, Transform("CFG ISA", 0)))
 614
 615 ########################################################################
 616 #
 617 # Prevent any SimObjects from being added after this point, they
 618 # should all have been added in the SConscripts above
 619 #
 620 SimObject.fixed = True
 621
 622 class DictImporter(object):
 623     '''This importer takes a dictionary of arbitrary module names that
 624     map to arbitrary filenames.'''
 625     def __init__(self, modules):
 626         self.modules = modules
 627         self.installed = set()
 628
 629     def __del__(self):
 630         self.unload()
 631
 632     def unload(self):
 633         import sys
 634         for module in self.installed:
 635             del sys.modules[module]
 636         self.installed = set()
 637
 638     def find_module(self, fullname, path):
 639         if fullname == 'm5.defines':
 640             return self
 641
 642         if fullname == 'm5.objects':
 643             return self
 644
 645         if fullname.startswith('_m5'):
 646             return None
 647
 648         source = self.modules.get(fullname, None)
 649         if source is not None and fullname.startswith('m5.objects'):
 650             return self
 651
 652         return None
 653
 654     def load_module(self, fullname):
 655         mod = imp.new_module(fullname)
 656         sys.modules[fullname] = mod
 657         self.installed.add(fullname)
 658
 659         mod.__loader__ = self
 660         if fullname == 'm5.objects':
 661             mod.__path__ = fullname.split('.')
 662             return mod
 663
 664         if fullname == 'm5.defines':
 665             mod.__dict__['buildEnv'] = m5.util.SmartDict(build_env)
 666             return mod
 667
 668         source = self.modules[fullname]
 669         if source.modname == '__init__':
 670             mod.__path__ = source.modpath
 671         mod.__file__ = source.abspath
 672
 673         exec file(source.abspath, 'r') in mod.__dict__
 674
 675         return mod
 676
 677 import m5.SimObject
 678 import m5.params
 679 from m5.util import code_formatter
 680
 681 m5.SimObject.clear()
 682 m5.params.clear()
 683
 684 # install the python importer so we can grab stuff from the source
 685 # tree itself.  We can't have SimObjects added after this point or
 686 # else we won't know about them for the rest of the stuff.
 687 importer = DictImporter(PySource.modules)
 688 sys.meta_path[0:0] = [ importer ]
 689
 690 # import all sim objects so we can populate the all_objects list
 691 # make sure that we're working with a list, then let's sort it
 692 for modname in SimObject.modnames:
 693     exec('from m5.objects import %s' % modname)
 694
 695 # we need to unload all of the currently imported modules so that they
 696 # will be re-imported the next time the sconscript is run
 697 importer.unload()
 698 sys.meta_path.remove(importer)
 699
 700 sim_objects = m5.SimObject.allClasses
 701 all_enums = m5.params.allEnums
 702
 703 for name,obj in sorted(sim_objects.iteritems()):
 704     for param in obj._params.local.values():
 705         # load the ptype attribute now because it depends on the
 706         # current version of SimObject.allClasses, but when scons
 707         # actually uses the value, all versions of
 708         # SimObject.allClasses will have been loaded
 709         param.ptype
 710
 711 ########################################################################
 712 #
 713 # calculate extra dependencies
 714 #
 715 module_depends = ["m5", "m5.SimObject", "m5.params"]
 716 depends = [ PySource.modules[dep].snode for dep in module_depends ]
 717 depends.sort(key = lambda x: x.name)
 718
 719 ########################################################################
 720 #
 721 # Commands for the basic automatically generated python files
 722 #
 723
 724 # Generate Python file containing a dict specifying the current
 725 # buildEnv flags.
 726 def makeDefinesPyFile(target, source, env):
 727     build_env = source[0].get_contents()
 728
 729     code = code_formatter()
 730     code("""
 731 import _m5.core
 732 import m5.util
 733
 734 buildEnv = m5.util.SmartDict($build_env)
 735
 736 compileDate = _m5.core.compileDate
 737 _globals = globals()
 738 for key,val in _m5.core.__dict__.iteritems():
 739     if key.startswith('flag_'):
 740         flag = key[5:]
 741         _globals[flag] = val
 742 del _globals
 743 """)
 744     code.write(target[0].abspath)
 745
 746 defines_info = Value(build_env)
 747 # Generate a file with all of the compile options in it
 748 env.Command('python/m5/defines.py', defines_info,
 749             MakeAction(makeDefinesPyFile, Transform("DEFINES", 0)))
 750 PySource('m5', 'python/m5/defines.py')
 751
 752 # Generate python file containing info about the M5 source code
 753 def makeInfoPyFile(target, source, env):
 754     code = code_formatter()
 755     for src in source:
 756         data = ''.join(file(src.srcnode().abspath, 'r').xreadlines())
 757         code('$src = ${{repr(data)}}')
 758     code.write(str(target[0]))
 759
 760 # Generate a file that wraps the basic top level files
 761 env.Command('python/m5/info.py',
 762             [ '#/COPYING', '#/LICENSE', '#/README', ],
 763             MakeAction(makeInfoPyFile, Transform("INFO")))
 764 PySource('m5', 'python/m5/info.py')
 765
 766 ########################################################################
 767 #
 768 # Create all of the SimObject param headers and enum headers
 769 #
 770
 771 def createSimObjectParamStruct(target, source, env):
 772     assert len(target) == 1 and len(source) == 1
 773
 774     name = source[0].get_text_contents()
 775     obj = sim_objects[name]
 776
 777     code = code_formatter()
 778     obj.cxx_param_decl(code)
 779     code.write(target[0].abspath)
 780
 781 def createSimObjectCxxConfig(is_header):
 782     def body(target, source, env):
 783         assert len(target) == 1 and len(source) == 1
 784
 785         name = str(source[0].get_contents())
 786         obj = sim_objects[name]
 787
 788         code = code_formatter()
 789         obj.cxx_config_param_file(code, is_header)
 790         code.write(target[0].abspath)
 791     return body
 792
 793 def createEnumStrings(target, source, env):
 794     assert len(target) == 1 and len(source) == 2
 795
 796     name = source[0].get_text_contents()
 797     use_python = source[1].read()
 798     obj = all_enums[name]
 799
 800     code = code_formatter()
 801     obj.cxx_def(code)
 802     if use_python:
 803         obj.pybind_def(code)
 804     code.write(target[0].abspath)
 805
 806 def createEnumDecls(target, source, env):
 807     assert len(target) == 1 and len(source) == 1
 808
 809     name = source[0].get_text_contents()
 810     obj = all_enums[name]
 811
 812     code = code_formatter()
 813     obj.cxx_decl(code)
 814     code.write(target[0].abspath)
 815
 816 def createSimObjectPyBindWrapper(target, source, env):
 817     name = source[0].get_text_contents()
 818     obj = sim_objects[name]
 819
 820     code = code_formatter()
 821     obj.pybind_decl(code)
 822     code.write(target[0].abspath)
 823
 824 # Generate all of the SimObject param C++ struct header files
 825 params_hh_files = []
 826 for name,simobj in sorted(sim_objects.iteritems()):
 827     py_source = PySource.modules[simobj.__module__]
 828     extra_deps = [ py_source.tnode ]
 829
 830     hh_file = File('params/%s.hh' % name)
 831     params_hh_files.append(hh_file)
 832     env.Command(hh_file, Value(name),
 833                 MakeAction(createSimObjectParamStruct, Transform("SO PARAM")))
 834     env.Depends(hh_file, depends + extra_deps)
 835
 836 # C++ parameter description files
 837 if GetOption('with_cxx_config'):
 838     for name,simobj in sorted(sim_objects.iteritems()):
 839         py_source = PySource.modules[simobj.__module__]
 840         extra_deps = [ py_source.tnode ]
 841
 842         cxx_config_hh_file = File('cxx_config/%s.hh' % name)
 843         cxx_config_cc_file = File('cxx_config/%s.cc' % name)
 844         env.Command(cxx_config_hh_file, Value(name),
 845                     MakeAction(createSimObjectCxxConfig(True),
 846                     Transform("CXXCPRHH")))
 847         env.Command(cxx_config_cc_file, Value(name),
 848                     MakeAction(createSimObjectCxxConfig(False),
 849                     Transform("CXXCPRCC")))
 850         env.Depends(cxx_config_hh_file, depends + extra_deps +
 851                     [File('params/%s.hh' % name), File('sim/cxx_config.hh')])
 852         env.Depends(cxx_config_cc_file, depends + extra_deps +
 853                     [cxx_config_hh_file])
 854         Source(cxx_config_cc_file)
 855
 856     cxx_config_init_cc_file = File('cxx_config/init.cc')
 857
 858     def createCxxConfigInitCC(target, source, env):
 859         assert len(target) == 1 and len(source) == 1
 860
 861         code = code_formatter()
 862
 863         for name,simobj in sorted(sim_objects.iteritems()):
 864             if not hasattr(simobj, 'abstract') or not simobj.abstract:
 865                 code('#include "cxx_config/${name}.hh"')
 866         code()
 867         code('void cxxConfigInit()')
 868         code('{')
 869         code.indent()
 870         for name,simobj in sorted(sim_objects.iteritems()):
 871             not_abstract = not hasattr(simobj, 'abstract') or \
 872                 not simobj.abstract
 873             if not_abstract and 'type' in simobj.__dict__:
 874                 code('cxx_config_directory["${name}"] = '
 875                      '${name}CxxConfigParams::makeDirectoryEntry();')
 876         code.dedent()
 877         code('}')
 878         code.write(target[0].abspath)
 879
 880     py_source = PySource.modules[simobj.__module__]
 881     extra_deps = [ py_source.tnode ]
 882     env.Command(cxx_config_init_cc_file, Value(name),
 883         MakeAction(createCxxConfigInitCC, Transform("CXXCINIT")))
 884     cxx_param_hh_files = ["cxx_config/%s.hh" % simobj
 885         for name,simobj in sorted(sim_objects.iteritems())
 886         if not hasattr(simobj, 'abstract') or not simobj.abstract]
 887     Depends(cxx_config_init_cc_file, cxx_param_hh_files +
 888             [File('sim/cxx_config.hh')])
 889     Source(cxx_config_init_cc_file)
 890
 891 # Generate all enum header files
 892 for name,enum in sorted(all_enums.iteritems()):
 893     py_source = PySource.modules[enum.__module__]
 894     extra_deps = [ py_source.tnode ]
 895
 896     cc_file = File('enums/%s.cc' % name)
 897     env.Command(cc_file, [Value(name), Value(env['USE_PYTHON'])],
 898                 MakeAction(createEnumStrings, Transform("ENUM STR")))
 899     env.Depends(cc_file, depends + extra_deps)
 900     Source(cc_file)
 901
 902     hh_file = File('enums/%s.hh' % name)
 903     env.Command(hh_file, Value(name),
 904                 MakeAction(createEnumDecls, Transform("ENUMDECL")))
 905     env.Depends(hh_file, depends + extra_deps)
 906
 907 # Generate SimObject Python bindings wrapper files
 908 if env['USE_PYTHON']:
 909     for name,simobj in sorted(sim_objects.iteritems()):
 910         py_source = PySource.modules[simobj.__module__]
 911         extra_deps = [ py_source.tnode ]
 912         cc_file = File('python/_m5/param_%s.cc' % name)
 913         env.Command(cc_file, Value(name),
 914                     MakeAction(createSimObjectPyBindWrapper,
 915                                Transform("SO PyBind")))
 916         env.Depends(cc_file, depends + extra_deps)
 917         Source(cc_file)
 918
 919 # Build all protocol buffers if we have got protoc and protobuf available
 920 if env['HAVE_PROTOBUF']:
 921     for proto in ProtoBuf.all:
 922         # Use both the source and header as the target, and the .proto
 923         # file as the source. When executing the protoc compiler, also
 924         # specify the proto_path to avoid having the generated files
 925         # include the path.
 926         env.Command([proto.cc_file, proto.hh_file], proto.tnode,
 927                     MakeAction('$PROTOC --cpp_out ${TARGET.dir} '
 928                                '--proto_path ${SOURCE.dir} $SOURCE',
 929                                Transform("PROTOC")))
 930
 931         # Add the C++ source file
 932         Source(proto.cc_file, tags=proto.tags)
 933 elif ProtoBuf.all:
 934     print('Got protobuf to build, but lacks support!')
 935     Exit(1)
 936
 937 #
 938 # Handle debug flags
 939 #
 940 def makeDebugFlagCC(target, source, env):
 941     assert(len(target) == 1 and len(source) == 1)
 942
 943     code = code_formatter()
 944
 945     # delay definition of CompoundFlags until after all the definition
 946     # of all constituent SimpleFlags
 947     comp_code = code_formatter()
 948
 949     # file header
 950     code('''
 951 /*
 952  * DO NOT EDIT THIS FILE! Automatically generated by SCons.
 953  */
 954
 955 #include "base/debug.hh"
 956
 957 namespace Debug {
 958
 959 ''')
 960
 961     for name, flag in sorted(source[0].read().iteritems()):
 962         n, compound, desc = flag
 963         assert n == name
 964
 965         if not compound:
 966             code('SimpleFlag $name("$name", "$desc");')
 967         else:
 968             comp_code('CompoundFlag $name("$name", "$desc",')
 969             comp_code.indent()
 970             last = len(compound) - 1
 971             for i,flag in enumerate(compound):
 972                 if i != last:
 973                     comp_code('&$flag,')
 974                 else:
 975                     comp_code('&$flag);')
 976             comp_code.dedent()
 977
 978     code.append(comp_code)
 979     code()
 980     code('} // namespace Debug')
 981
 982     code.write(str(target[0]))
 983
 984 def makeDebugFlagHH(target, source, env):
 985     assert(len(target) == 1 and len(source) == 1)
 986
 987     val = eval(source[0].get_contents())
 988     name, compound, desc = val
 989
 990     code = code_formatter()
 991
 992     # file header boilerplate
 993     code('''\
 994 /*
 995  * DO NOT EDIT THIS FILE! Automatically generated by SCons.
 996  */
 997
 998 #ifndef __DEBUG_${name}_HH__
 999 #define __DEBUG_${name}_HH__
1000
1001 namespace Debug {
1002 ''')
1003
1004     if compound:
1005         code('class CompoundFlag;')
1006     code('class SimpleFlag;')
1007
1008     if compound:
1009         code('extern CompoundFlag $name;')
1010         for flag in compound:
1011             code('extern SimpleFlag $flag;')
1012     else:
1013         code('extern SimpleFlag $name;')
1014
1015     code('''
1016 }
1017
1018 #endif // __DEBUG_${name}_HH__
1019 ''')
1020
1021     code.write(str(target[0]))
1022
1023 for name,flag in sorted(debug_flags.iteritems()):
1024     n, compound, desc = flag
1025     assert n == name
1026
1027     hh_file = 'debug/%s.hh' % name
1028     env.Command(hh_file, Value(flag),
1029                 MakeAction(makeDebugFlagHH, Transform("TRACING", 0)))
1030
1031 env.Command('debug/flags.cc', Value(debug_flags),
1032             MakeAction(makeDebugFlagCC, Transform("TRACING", 0)))
1033 Source('debug/flags.cc')
1034
1035 # version tags
1036 tags = \
1037 env.Command('sim/tags.cc', None,
1038             MakeAction('util/cpt_upgrader.py --get-cc-file > $TARGET',
1039                        Transform("VER TAGS")))
1040 env.AlwaysBuild(tags)
1041
1042 # Embed python files.  All .py files that have been indicated by a
1043 # PySource() call in a SConscript need to be embedded into the M5
1044 # library.  To do that, we compile the file to byte code, marshal the
1045 # byte code, compress it, and then generate a c++ file that
1046 # inserts the result into an array.
1047 def embedPyFile(target, source, env):
1048     def c_str(string):
1049         if string is None:
1050             return "0"
1051         return '"%s"' % string
1052
1053     '''Action function to compile a .py into a code object, marshal
1054     it, compress it, and stick it into an asm file so the code appears
1055     as just bytes with a label in the data section'''
1056
1057     src = file(str(source[0]), 'r').read()
1058
1059     pysource = PySource.tnodes[source[0]]
1060     compiled = compile(src, pysource.abspath, 'exec')
1061     marshalled = marshal.dumps(compiled)
1062     compressed = zlib.compress(marshalled)
1063     data = compressed
1064     sym = pysource.symname
1065
1066     code = code_formatter()
1067     code('''\
1068 #include "sim/init.hh"
1069
1070 namespace {
1071
1072 const uint8_t data_${sym}[] = {
1073 ''')
1074     code.indent()
1075     step = 16
1076     for i in xrange(0, len(data), step):
1077         x = array.array('B', data[i:i+step])
1078         code(''.join('%d,' % d for d in x))
1079     code.dedent()
1080
1081     code('''};
1082
1083 EmbeddedPython embedded_${sym}(
1084     ${{c_str(pysource.arcname)}},
1085     ${{c_str(pysource.abspath)}},
1086     ${{c_str(pysource.modpath)}},
1087     data_${sym},
1088     ${{len(data)}},
1089     ${{len(marshalled)}});
1090
1091 } // anonymous namespace
1092 ''')
1093     code.write(str(target[0]))
1094
1095 for source in PySource.all:
1096     env.Command(source.cpp, source.tnode,
1097                 MakeAction(embedPyFile, Transform("EMBED PY")))
1098     Source(source.cpp, tags=source.tags, add_tags='python')
1099
1100 ########################################################################
1101 #
1102 # Define binaries.  Each different build type (debug, opt, etc.) gets
1103 # a slightly different build environment.
1104 #
1105
1106 # List of constructed environments to pass back to SConstruct
1107 date_source = Source('base/date.cc', tags=[])
1108
1109 gem5_binary = Gem5('gem5')
1110
1111 # Function to create a new build environment as clone of current
1112 # environment 'env' with modified object suffix and optional stripped
1113 # binary.  Additional keyword arguments are appended to corresponding
1114 # build environment vars.
1115 def makeEnv(env, label, objsfx, strip=False, disable_partial=False, **kwargs):
1116     # SCons doesn't know to append a library suffix when there is a '.' in the
1117     # name.  Use '_' instead.
1118     libname = 'gem5_' + label
1119     secondary_exename = 'm5.' + label
1120
1121     new_env = env.Clone(OBJSUFFIX=objsfx, SHOBJSUFFIX=objsfx + 's')
1122     new_env.Label = label
1123     new_env.Append(**kwargs)
1124
1125     lib_sources = Source.all.with_tag('gem5 lib')
1126
1127     # Without Python, leave out all Python content from the library
1128     # builds.  The option doesn't affect gem5 built as a program
1129     if GetOption('without_python'):
1130         lib_sources = lib_sources.without_tag('python')
1131
1132     static_objs = []
1133     shared_objs = []
1134
1135     for s in lib_sources.with_tag(Source.ungrouped_tag):
1136         static_objs.append(s.static(new_env))
1137         shared_objs.append(s.shared(new_env))
1138
1139     for group in Source.source_groups:
1140         srcs = lib_sources.with_tag(Source.link_group_tag(group))
1141         if not srcs:
1142             continue
1143
1144         group_static = [ s.static(new_env) for s in srcs ]
1145         group_shared = [ s.shared(new_env) for s in srcs ]
1146
1147         # If partial linking is disabled, add these sources to the build
1148         # directly, and short circuit this loop.
1149         if disable_partial:
1150             static_objs.extend(group_static)
1151             shared_objs.extend(group_shared)
1152             continue
1153
1154         # Set up the static partially linked objects.
1155         file_name = new_env.subst("${OBJPREFIX}lib${OBJSUFFIX}.partial")
1156         target = File(joinpath(group, file_name))
1157         partial = env.PartialStatic(target=target, source=group_static)
1158         static_objs.extend(partial)
1159
1160         # Set up the shared partially linked objects.
1161         file_name = new_env.subst("${SHOBJPREFIX}lib${SHOBJSUFFIX}.partial")
1162         target = File(joinpath(group, file_name))
1163         partial = env.PartialShared(target=target, source=group_shared)
1164         shared_objs.extend(partial)
1165
1166     static_date = date_source.static(new_env)
1167     new_env.Depends(static_date, static_objs)
1168     static_objs.extend(static_date)
1169
1170     shared_date = date_source.shared(new_env)
1171     new_env.Depends(shared_date, shared_objs)
1172     shared_objs.extend(shared_date)
1173
1174     main_objs = [ s.static(new_env) for s in Source.all.with_tag('main') ]
1175
1176     # First make a library of everything but main() so other programs can
1177     # link against m5.
1178     static_lib = new_env.StaticLibrary(libname, static_objs)
1179     shared_lib = new_env.SharedLibrary(libname, shared_objs)
1180
1181     # Keep track of the object files generated so far so Executables can
1182     # include them.
1183     new_env['STATIC_OBJS'] = static_objs
1184     new_env['SHARED_OBJS'] = shared_objs
1185     new_env['MAIN_OBJS'] = main_objs
1186
1187     new_env['STATIC_LIB'] = static_lib
1188     new_env['SHARED_LIB'] = shared_lib
1189
1190     # Record some settings for building Executables.
1191     new_env['EXE_SUFFIX'] = label
1192     new_env['STRIP_EXES'] = strip
1193
1194     for cls in ExecutableMeta.all:
1195         cls.declare_all(new_env)
1196
1197     new_env.M5Binary = File(gem5_binary.path(new_env))
1198
1199     new_env.Command(secondary_exename, new_env.M5Binary,
1200             MakeAction('ln $SOURCE $TARGET', Transform("HARDLINK")))
1201
1202     # Set up regression tests.
1203     SConscript(os.path.join(env.root.abspath, 'tests', 'SConscript'),
1204                variant_dir=Dir('tests').Dir(new_env.Label),
1205                exports={ 'env' : new_env }, duplicate=False)
1206
1207 # Start out with the compiler flags common to all compilers,
1208 # i.e. they all use -g for opt and -g -pg for prof
1209 ccflags = {'debug' : [], 'opt' : ['-g'], 'fast' : [], 'prof' : ['-g', '-pg'],
1210            'perf' : ['-g']}
1211
1212 # Start out with the linker flags common to all linkers, i.e. -pg for
1213 # prof, and -lprofiler for perf. The -lprofile flag is surrounded by
1214 # no-as-needed and as-needed as the binutils linker is too clever and
1215 # simply doesn't link to the library otherwise.
1216 ldflags = {'debug' : [], 'opt' : [], 'fast' : [], 'prof' : ['-pg'],
1217            'perf' : ['-Wl,--no-as-needed', '-lprofiler', '-Wl,--as-needed']}
1218
1219 # For Link Time Optimization, the optimisation flags used to compile
1220 # individual files are decoupled from those used at link time
1221 # (i.e. you can compile with -O3 and perform LTO with -O0), so we need
1222 # to also update the linker flags based on the target.
1223 if env['GCC']:
1224     if sys.platform == 'sunos5':
1225         ccflags['debug'] += ['-gstabs+']
1226     else:
1227         ccflags['debug'] += ['-ggdb3']
1228     ldflags['debug'] += ['-O0']
1229     # opt, fast, prof and perf all share the same cc flags, also add
1230     # the optimization to the ldflags as LTO defers the optimization
1231     # to link time
1232     for target in ['opt', 'fast', 'prof', 'perf']:
1233         ccflags[target] += ['-O3']
1234         ldflags[target] += ['-O3']
1235
1236     ccflags['fast'] += env['LTO_CCFLAGS']
1237     ldflags['fast'] += env['LTO_LDFLAGS']
1238 elif env['CLANG']:
1239     ccflags['debug'] += ['-g', '-O0']
1240     # opt, fast, prof and perf all share the same cc flags
1241     for target in ['opt', 'fast', 'prof', 'perf']:
1242         ccflags[target] += ['-O3']
1243 else:
1244     print('Unknown compiler, please fix compiler options')
1245     Exit(1)
1246
1247
1248 # To speed things up, we only instantiate the build environments we
1249 # need.  We try to identify the needed environment for each target; if
1250 # we can't, we fall back on instantiating all the environments just to
1251 # be safe.
1252 target_types = ['debug', 'opt', 'fast', 'prof', 'perf']
1253 obj2target = {'do': 'debug', 'o': 'opt', 'fo': 'fast', 'po': 'prof',
1254               'gpo' : 'perf'}
1255
1256 def identifyTarget(t):
1257     ext = t.split('.')[-1]
1258     if ext in target_types:
1259         return ext
1260     if obj2target.has_key(ext):
1261         return obj2target[ext]
1262     match = re.search(r'/tests/([^/]+)/', t)
1263     if match and match.group(1) in target_types:
1264         return match.group(1)
1265     return 'all'
1266
1267 needed_envs = [identifyTarget(target) for target in BUILD_TARGETS]
1268 if 'all' in needed_envs:
1269     needed_envs += target_types
1270
1271 disable_partial = False
1272 if env['PLATFORM'] == 'darwin':
1273     # Up until Apple LLVM version 10.0.0 (clang-1000.11.45.5), partial
1274     # linked objects do not expose symbols that are marked with the
1275     # hidden visibility and consequently building gem5 on Mac OS
1276     # fails. As a workaround, we disable partial linking, however, we
1277     # may want to revisit in the future.
1278     disable_partial = True
1279
1280 # Debug binary
1281 if 'debug' in needed_envs:
1282     makeEnv(env, 'debug', '.do',
1283             CCFLAGS = Split(ccflags['debug']),
1284             CPPDEFINES = ['DEBUG', 'TRACING_ON=1'],
1285             LINKFLAGS = Split(ldflags['debug']),
1286             disable_partial=disable_partial)
1287
1288 # Optimized binary
1289 if 'opt' in needed_envs:
1290     makeEnv(env, 'opt', '.o',
1291             CCFLAGS = Split(ccflags['opt']),
1292             CPPDEFINES = ['TRACING_ON=1'],
1293             LINKFLAGS = Split(ldflags['opt']),
1294             disable_partial=disable_partial)
1295
1296 # "Fast" binary
1297 if 'fast' in needed_envs:
1298     disable_partial = disable_partial and \
1299             env.get('BROKEN_INCREMENTAL_LTO', False) and \
1300             GetOption('force_lto')
1301     makeEnv(env, 'fast', '.fo', strip = True,
1302             CCFLAGS = Split(ccflags['fast']),
1303             CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'],
1304             LINKFLAGS = Split(ldflags['fast']),
1305             disable_partial=disable_partial)
1306
1307 # Profiled binary using gprof
1308 if 'prof' in needed_envs:
1309     makeEnv(env, 'prof', '.po',
1310             CCFLAGS = Split(ccflags['prof']),
1311             CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'],
1312             LINKFLAGS = Split(ldflags['prof']),
1313             disable_partial=disable_partial)
1314
1315 # Profiled binary using google-pprof
1316 if 'perf' in needed_envs:
1317     makeEnv(env, 'perf', '.gpo',
1318             CCFLAGS = Split(ccflags['perf']),
1319             CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'],
1320             LINKFLAGS = Split(ldflags['perf']),
1321             disable_partial=disable_partial)