src/SConscript

   1 # -*- mode:python -*-
   2
   3 # Copyright (c) 2004-2005 The Regents of The University of Michigan
   4 # All rights reserved.
   5 #
   6 # Redistribution and use in source and binary forms, with or without
   7 # modification, are permitted provided that the following conditions are
   8 # met: redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer;
  10 # redistributions in binary form must reproduce the above copyright
  11 # notice, this list of conditions and the following disclaimer in the
  12 # documentation and/or other materials provided with the distribution;
  13 # neither the name of the copyright holders nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # Authors: Nathan Binkert
  30
  31 from __future__ import print_function
  32
  33 import array
  34 import bisect
  35 import functools
  36 import imp
  37 import marshal
  38 import os
  39 import re
  40 import subprocess
  41 import sys
  42 import zlib
  43
  44 from os.path import basename, dirname, exists, isdir, isfile, join as joinpath
  45
  46 import SCons
  47
  48 from gem5_scons import Transform
  49
  50 # This file defines how to build a particular configuration of gem5
  51 # based on variable settings in the 'env' build environment.
  52
  53 Import('*')
  54
  55 # Children need to see the environment
  56 Export('env')
  57
  58 build_env = [(opt, env[opt]) for opt in export_vars]
  59
  60 from m5.util import code_formatter, compareVersions
  61
  62 ########################################################################
  63 # Code for adding source files of various types
  64 #
  65 # When specifying a source file of some type, a set of tags can be
  66 # specified for that file.
  67
  68 class SourceFilter(object):
  69     def __init__(self, predicate):
  70         self.predicate = predicate
  71
  72     def __or__(self, other):
  73         return SourceFilter(lambda tags: self.predicate(tags) or
  74                                          other.predicate(tags))
  75
  76     def __and__(self, other):
  77         return SourceFilter(lambda tags: self.predicate(tags) and
  78                                          other.predicate(tags))
  79
  80 def with_tags_that(predicate):
  81     '''Return a list of sources with tags that satisfy a predicate.'''
  82     return SourceFilter(predicate)
  83
  84 def with_any_tags(*tags):
  85     '''Return a list of sources with any of the supplied tags.'''
  86     return SourceFilter(lambda stags: len(set(tags) & stags) > 0)
  87
  88 def with_all_tags(*tags):
  89     '''Return a list of sources with all of the supplied tags.'''
  90     return SourceFilter(lambda stags: set(tags) <= stags)
  91
  92 def with_tag(tag):
  93     '''Return a list of sources with the supplied tag.'''
  94     return SourceFilter(lambda stags: tag in stags)
  95
  96 def without_tags(*tags):
  97     '''Return a list of sources without any of the supplied tags.'''
  98     return SourceFilter(lambda stags: len(set(tags) & stags) == 0)
  99
 100 def without_tag(tag):
 101     '''Return a list of sources with the supplied tag.'''
 102     return SourceFilter(lambda stags: tag not in stags)
 103
 104 source_filter_factories = {
 105     'with_tags_that': with_tags_that,
 106     'with_any_tags': with_any_tags,
 107     'with_all_tags': with_all_tags,
 108     'with_tag': with_tag,
 109     'without_tags': without_tags,
 110     'without_tag': without_tag,
 111 }
 112
 113 Export(source_filter_factories)
 114
 115 class SourceList(list):
 116     def apply_filter(self, f):
 117         def match(source):
 118             return f.predicate(source.tags)
 119         return SourceList(filter(match, self))
 120
 121     def __getattr__(self, name):
 122         func = source_filter_factories.get(name, None)
 123         if not func:
 124             raise AttributeError
 125
 126         @functools.wraps(func)
 127         def wrapper(*args, **kwargs):
 128             return self.apply_filter(func(*args, **kwargs))
 129         return wrapper
 130
 131 class SourceMeta(type):
 132     '''Meta class for source files that keeps track of all files of a
 133     particular type.'''
 134     def __init__(cls, name, bases, dict):
 135         super(SourceMeta, cls).__init__(name, bases, dict)
 136         cls.all = SourceList()
 137
 138 class SourceFile(object):
 139     '''Base object that encapsulates the notion of a source file.
 140     This includes, the source node, target node, various manipulations
 141     of those.  A source file also specifies a set of tags which
 142     describing arbitrary properties of the source file.'''
 143     __metaclass__ = SourceMeta
 144
 145     static_objs = {}
 146     shared_objs = {}
 147
 148     def __init__(self, source, tags=None, add_tags=None):
 149         if tags is None:
 150             tags='gem5 lib'
 151         if isinstance(tags, basestring):
 152             tags = set([tags])
 153         if not isinstance(tags, set):
 154             tags = set(tags)
 155         self.tags = tags
 156
 157         if add_tags:
 158             if isinstance(add_tags, basestring):
 159                 add_tags = set([add_tags])
 160             if not isinstance(add_tags, set):
 161                 add_tags = set(add_tags)
 162             self.tags |= add_tags
 163
 164         tnode = source
 165         if not isinstance(source, SCons.Node.FS.File):
 166             tnode = File(source)
 167
 168         self.tnode = tnode
 169         self.snode = tnode.srcnode()
 170
 171         for base in type(self).__mro__:
 172             if issubclass(base, SourceFile):
 173                 base.all.append(self)
 174
 175     def static(self, env):
 176         key = (self.tnode, env['OBJSUFFIX'])
 177         if not key in self.static_objs:
 178             self.static_objs[key] = env.StaticObject(self.tnode)
 179         return self.static_objs[key]
 180
 181     def shared(self, env):
 182         key = (self.tnode, env['OBJSUFFIX'])
 183         if not key in self.shared_objs:
 184             self.shared_objs[key] = env.SharedObject(self.tnode)
 185         return self.shared_objs[key]
 186
 187     @property
 188     def filename(self):
 189         return str(self.tnode)
 190
 191     @property
 192     def dirname(self):
 193         return dirname(self.filename)
 194
 195     @property
 196     def basename(self):
 197         return basename(self.filename)
 198
 199     @property
 200     def extname(self):
 201         index = self.basename.rfind('.')
 202         if index <= 0:
 203             # dot files aren't extensions
 204             return self.basename, None
 205
 206         return self.basename[:index], self.basename[index+1:]
 207
 208     def __lt__(self, other): return self.filename < other.filename
 209     def __le__(self, other): return self.filename <= other.filename
 210     def __gt__(self, other): return self.filename > other.filename
 211     def __ge__(self, other): return self.filename >= other.filename
 212     def __eq__(self, other): return self.filename == other.filename
 213     def __ne__(self, other): return self.filename != other.filename
 214
 215 class Source(SourceFile):
 216     ungrouped_tag = 'No link group'
 217     source_groups = set()
 218
 219     _current_group_tag = ungrouped_tag
 220
 221     @staticmethod
 222     def link_group_tag(group):
 223         return 'link group: %s' % group
 224
 225     @classmethod
 226     def set_group(cls, group):
 227         new_tag = Source.link_group_tag(group)
 228         Source._current_group_tag = new_tag
 229         Source.source_groups.add(group)
 230
 231     def _add_link_group_tag(self):
 232         self.tags.add(Source._current_group_tag)
 233
 234     '''Add a c/c++ source file to the build'''
 235     def __init__(self, source, tags=None, add_tags=None):
 236         '''specify the source file, and any tags'''
 237         super(Source, self).__init__(source, tags, add_tags)
 238         self._add_link_group_tag()
 239
 240 class PySource(SourceFile):
 241     '''Add a python source file to the named package'''
 242     invalid_sym_char = re.compile('[^A-z0-9_]')
 243     modules = {}
 244     tnodes = {}
 245     symnames = {}
 246
 247     def __init__(self, package, source, tags=None, add_tags=None):
 248         '''specify the python package, the source file, and any tags'''
 249         super(PySource, self).__init__(source, tags, add_tags)
 250
 251         modname,ext = self.extname
 252         assert ext == 'py'
 253
 254         if package:
 255             path = package.split('.')
 256         else:
 257             path = []
 258
 259         modpath = path[:]
 260         if modname != '__init__':
 261             modpath += [ modname ]
 262         modpath = '.'.join(modpath)
 263
 264         arcpath = path + [ self.basename ]
 265         abspath = self.snode.abspath
 266         if not exists(abspath):
 267             abspath = self.tnode.abspath
 268
 269         self.package = package
 270         self.modname = modname
 271         self.modpath = modpath
 272         self.arcname = joinpath(*arcpath)
 273         self.abspath = abspath
 274         self.compiled = File(self.filename + 'c')
 275         self.cpp = File(self.filename + '.cc')
 276         self.symname = PySource.invalid_sym_char.sub('_', modpath)
 277
 278         PySource.modules[modpath] = self
 279         PySource.tnodes[self.tnode] = self
 280         PySource.symnames[self.symname] = self
 281
 282 class SimObject(PySource):
 283     '''Add a SimObject python file as a python source object and add
 284     it to a list of sim object modules'''
 285
 286     fixed = False
 287     modnames = []
 288
 289     def __init__(self, source, tags=None, add_tags=None):
 290         '''Specify the source file and any tags (automatically in
 291         the m5.objects package)'''
 292         super(SimObject, self).__init__('m5.objects', source, tags, add_tags)
 293         if self.fixed:
 294             raise AttributeError, "Too late to call SimObject now."
 295
 296         bisect.insort_right(SimObject.modnames, self.modname)
 297
 298 class ProtoBuf(SourceFile):
 299     '''Add a Protocol Buffer to build'''
 300
 301     def __init__(self, source, tags=None, add_tags=None):
 302         '''Specify the source file, and any tags'''
 303         super(ProtoBuf, self).__init__(source, tags, add_tags)
 304
 305         # Get the file name and the extension
 306         modname,ext = self.extname
 307         assert ext == 'proto'
 308
 309         # Currently, we stick to generating the C++ headers, so we
 310         # only need to track the source and header.
 311         self.cc_file = File(modname + '.pb.cc')
 312         self.hh_file = File(modname + '.pb.h')
 313
 314 class UnitTest(object):
 315     '''Create a UnitTest'''
 316
 317     all = []
 318     def __init__(self, target, *sources, **kwargs):
 319         '''Specify the target name and any sources.  Sources that are
 320         not SourceFiles are evalued with Source().  All files are
 321         tagged with the name of the UnitTest target.'''
 322
 323         srcs = SourceList()
 324         for src in sources:
 325             if not isinstance(src, SourceFile):
 326                 src = Source(src, tags=str(target))
 327             srcs.append(src)
 328
 329         self.sources = srcs
 330         self.target = target
 331         self.main = kwargs.get('main', False)
 332         self.all.append(self)
 333
 334 class GTest(UnitTest):
 335     '''Create a unit test based on the google test framework.'''
 336     all = []
 337     def __init__(self, *args, **kwargs):
 338         isFilter = lambda arg: isinstance(arg, SourceFilter)
 339         self.filters = filter(isFilter, args)
 340         args = filter(lambda a: not isFilter(a), args)
 341         super(GTest, self).__init__(*args, **kwargs)
 342         self.dir = Dir('.')
 343         self.skip_lib = kwargs.pop('skip_lib', False)
 344
 345 # Children should have access
 346 Export('Source')
 347 Export('PySource')
 348 Export('SimObject')
 349 Export('ProtoBuf')
 350 Export('UnitTest')
 351 Export('GTest')
 352
 353 ########################################################################
 354 #
 355 # Debug Flags
 356 #
 357 debug_flags = {}
 358 def DebugFlag(name, desc=None):
 359     if name in debug_flags:
 360         raise AttributeError, "Flag %s already specified" % name
 361     debug_flags[name] = (name, (), desc)
 362
 363 def CompoundFlag(name, flags, desc=None):
 364     if name in debug_flags:
 365         raise AttributeError, "Flag %s already specified" % name
 366
 367     compound = tuple(flags)
 368     debug_flags[name] = (name, compound, desc)
 369
 370 Export('DebugFlag')
 371 Export('CompoundFlag')
 372
 373 ########################################################################
 374 #
 375 # Set some compiler variables
 376 #
 377
 378 # Include file paths are rooted in this directory.  SCons will
 379 # automatically expand '.' to refer to both the source directory and
 380 # the corresponding build directory to pick up generated include
 381 # files.
 382 env.Append(CPPPATH=Dir('.'))
 383
 384 for extra_dir in extras_dir_list:
 385     env.Append(CPPPATH=Dir(extra_dir))
 386
 387 # Workaround for bug in SCons version > 0.97d20071212
 388 # Scons bug id: 2006 gem5 Bug id: 308
 389 for root, dirs, files in os.walk(base_dir, topdown=True):
 390     Dir(root[len(base_dir) + 1:])
 391
 392 ########################################################################
 393 #
 394 # Walk the tree and execute all SConscripts in subdirectories
 395 #
 396
 397 here = Dir('.').srcnode().abspath
 398 for root, dirs, files in os.walk(base_dir, topdown=True):
 399     if root == here:
 400         # we don't want to recurse back into this SConscript
 401         continue
 402
 403     if 'SConscript' in files:
 404         build_dir = joinpath(env['BUILDDIR'], root[len(base_dir) + 1:])
 405         Source.set_group(build_dir)
 406         SConscript(joinpath(root, 'SConscript'), variant_dir=build_dir)
 407
 408 for extra_dir in extras_dir_list:
 409     prefix_len = len(dirname(extra_dir)) + 1
 410
 411     # Also add the corresponding build directory to pick up generated
 412     # include files.
 413     env.Append(CPPPATH=Dir(joinpath(env['BUILDDIR'], extra_dir[prefix_len:])))
 414
 415     for root, dirs, files in os.walk(extra_dir, topdown=True):
 416         # if build lives in the extras directory, don't walk down it
 417         if 'build' in dirs:
 418             dirs.remove('build')
 419
 420         if 'SConscript' in files:
 421             build_dir = joinpath(env['BUILDDIR'], root[prefix_len:])
 422             SConscript(joinpath(root, 'SConscript'), variant_dir=build_dir)
 423
 424 for opt in export_vars:
 425     env.ConfigFile(opt)
 426
 427 def makeTheISA(source, target, env):
 428     isas = [ src.get_contents() for src in source ]
 429     target_isa = env['TARGET_ISA']
 430     def define(isa):
 431         return isa.upper() + '_ISA'
 432
 433     def namespace(isa):
 434         return isa[0].upper() + isa[1:].lower() + 'ISA'
 435
 436
 437     code = code_formatter()
 438     code('''\
 439 #ifndef __CONFIG_THE_ISA_HH__
 440 #define __CONFIG_THE_ISA_HH__
 441
 442 ''')
 443
 444     # create defines for the preprocessing and compile-time determination
 445     for i,isa in enumerate(isas):
 446         code('#define $0 $1', define(isa), i + 1)
 447     code()
 448
 449     # create an enum for any run-time determination of the ISA, we
 450     # reuse the same name as the namespaces
 451     code('enum class Arch {')
 452     for i,isa in enumerate(isas):
 453         if i + 1 == len(isas):
 454             code('  $0 = $1', namespace(isa), define(isa))
 455         else:
 456             code('  $0 = $1,', namespace(isa), define(isa))
 457     code('};')
 458
 459     code('''
 460
 461 #define THE_ISA ${{define(target_isa)}}
 462 #define TheISA ${{namespace(target_isa)}}
 463 #define THE_ISA_STR "${{target_isa}}"
 464
 465 #endif // __CONFIG_THE_ISA_HH__''')
 466
 467     code.write(str(target[0]))
 468
 469 env.Command('config/the_isa.hh', map(Value, all_isa_list),
 470             MakeAction(makeTheISA, Transform("CFG ISA", 0)))
 471
 472 def makeTheGPUISA(source, target, env):
 473     isas = [ src.get_contents() for src in source ]
 474     target_gpu_isa = env['TARGET_GPU_ISA']
 475     def define(isa):
 476         return isa.upper() + '_ISA'
 477
 478     def namespace(isa):
 479         return isa[0].upper() + isa[1:].lower() + 'ISA'
 480
 481
 482     code = code_formatter()
 483     code('''\
 484 #ifndef __CONFIG_THE_GPU_ISA_HH__
 485 #define __CONFIG_THE_GPU_ISA_HH__
 486
 487 ''')
 488
 489     # create defines for the preprocessing and compile-time determination
 490     for i,isa in enumerate(isas):
 491         code('#define $0 $1', define(isa), i + 1)
 492     code()
 493
 494     # create an enum for any run-time determination of the ISA, we
 495     # reuse the same name as the namespaces
 496     code('enum class GPUArch {')
 497     for i,isa in enumerate(isas):
 498         if i + 1 == len(isas):
 499             code('  $0 = $1', namespace(isa), define(isa))
 500         else:
 501             code('  $0 = $1,', namespace(isa), define(isa))
 502     code('};')
 503
 504     code('''
 505
 506 #define THE_GPU_ISA ${{define(target_gpu_isa)}}
 507 #define TheGpuISA ${{namespace(target_gpu_isa)}}
 508 #define THE_GPU_ISA_STR "${{target_gpu_isa}}"
 509
 510 #endif // __CONFIG_THE_GPU_ISA_HH__''')
 511
 512     code.write(str(target[0]))
 513
 514 env.Command('config/the_gpu_isa.hh', map(Value, all_gpu_isa_list),
 515             MakeAction(makeTheGPUISA, Transform("CFG ISA", 0)))
 516
 517 ########################################################################
 518 #
 519 # Prevent any SimObjects from being added after this point, they
 520 # should all have been added in the SConscripts above
 521 #
 522 SimObject.fixed = True
 523
 524 class DictImporter(object):
 525     '''This importer takes a dictionary of arbitrary module names that
 526     map to arbitrary filenames.'''
 527     def __init__(self, modules):
 528         self.modules = modules
 529         self.installed = set()
 530
 531     def __del__(self):
 532         self.unload()
 533
 534     def unload(self):
 535         import sys
 536         for module in self.installed:
 537             del sys.modules[module]
 538         self.installed = set()
 539
 540     def find_module(self, fullname, path):
 541         if fullname == 'm5.defines':
 542             return self
 543
 544         if fullname == 'm5.objects':
 545             return self
 546
 547         if fullname.startswith('_m5'):
 548             return None
 549
 550         source = self.modules.get(fullname, None)
 551         if source is not None and fullname.startswith('m5.objects'):
 552             return self
 553
 554         return None
 555
 556     def load_module(self, fullname):
 557         mod = imp.new_module(fullname)
 558         sys.modules[fullname] = mod
 559         self.installed.add(fullname)
 560
 561         mod.__loader__ = self
 562         if fullname == 'm5.objects':
 563             mod.__path__ = fullname.split('.')
 564             return mod
 565
 566         if fullname == 'm5.defines':
 567             mod.__dict__['buildEnv'] = m5.util.SmartDict(build_env)
 568             return mod
 569
 570         source = self.modules[fullname]
 571         if source.modname == '__init__':
 572             mod.__path__ = source.modpath
 573         mod.__file__ = source.abspath
 574
 575         exec file(source.abspath, 'r') in mod.__dict__
 576
 577         return mod
 578
 579 import m5.SimObject
 580 import m5.params
 581 from m5.util import code_formatter
 582
 583 m5.SimObject.clear()
 584 m5.params.clear()
 585
 586 # install the python importer so we can grab stuff from the source
 587 # tree itself.  We can't have SimObjects added after this point or
 588 # else we won't know about them for the rest of the stuff.
 589 importer = DictImporter(PySource.modules)
 590 sys.meta_path[0:0] = [ importer ]
 591
 592 # import all sim objects so we can populate the all_objects list
 593 # make sure that we're working with a list, then let's sort it
 594 for modname in SimObject.modnames:
 595     exec('from m5.objects import %s' % modname)
 596
 597 # we need to unload all of the currently imported modules so that they
 598 # will be re-imported the next time the sconscript is run
 599 importer.unload()
 600 sys.meta_path.remove(importer)
 601
 602 sim_objects = m5.SimObject.allClasses
 603 all_enums = m5.params.allEnums
 604
 605 for name,obj in sorted(sim_objects.iteritems()):
 606     for param in obj._params.local.values():
 607         # load the ptype attribute now because it depends on the
 608         # current version of SimObject.allClasses, but when scons
 609         # actually uses the value, all versions of
 610         # SimObject.allClasses will have been loaded
 611         param.ptype
 612
 613 ########################################################################
 614 #
 615 # calculate extra dependencies
 616 #
 617 module_depends = ["m5", "m5.SimObject", "m5.params"]
 618 depends = [ PySource.modules[dep].snode for dep in module_depends ]
 619 depends.sort(key = lambda x: x.name)
 620
 621 ########################################################################
 622 #
 623 # Commands for the basic automatically generated python files
 624 #
 625
 626 # Generate Python file containing a dict specifying the current
 627 # buildEnv flags.
 628 def makeDefinesPyFile(target, source, env):
 629     build_env = source[0].get_contents()
 630
 631     code = code_formatter()
 632     code("""
 633 import _m5.core
 634 import m5.util
 635
 636 buildEnv = m5.util.SmartDict($build_env)
 637
 638 compileDate = _m5.core.compileDate
 639 _globals = globals()
 640 for key,val in _m5.core.__dict__.iteritems():
 641     if key.startswith('flag_'):
 642         flag = key[5:]
 643         _globals[flag] = val
 644 del _globals
 645 """)
 646     code.write(target[0].abspath)
 647
 648 defines_info = Value(build_env)
 649 # Generate a file with all of the compile options in it
 650 env.Command('python/m5/defines.py', defines_info,
 651             MakeAction(makeDefinesPyFile, Transform("DEFINES", 0)))
 652 PySource('m5', 'python/m5/defines.py')
 653
 654 # Generate python file containing info about the M5 source code
 655 def makeInfoPyFile(target, source, env):
 656     code = code_formatter()
 657     for src in source:
 658         data = ''.join(file(src.srcnode().abspath, 'r').xreadlines())
 659         code('$src = ${{repr(data)}}')
 660     code.write(str(target[0]))
 661
 662 # Generate a file that wraps the basic top level files
 663 env.Command('python/m5/info.py',
 664             [ '#/COPYING', '#/LICENSE', '#/README', ],
 665             MakeAction(makeInfoPyFile, Transform("INFO")))
 666 PySource('m5', 'python/m5/info.py')
 667
 668 ########################################################################
 669 #
 670 # Create all of the SimObject param headers and enum headers
 671 #
 672
 673 def createSimObjectParamStruct(target, source, env):
 674     assert len(target) == 1 and len(source) == 1
 675
 676     name = source[0].get_text_contents()
 677     obj = sim_objects[name]
 678
 679     code = code_formatter()
 680     obj.cxx_param_decl(code)
 681     code.write(target[0].abspath)
 682
 683 def createSimObjectCxxConfig(is_header):
 684     def body(target, source, env):
 685         assert len(target) == 1 and len(source) == 1
 686
 687         name = str(source[0].get_contents())
 688         obj = sim_objects[name]
 689
 690         code = code_formatter()
 691         obj.cxx_config_param_file(code, is_header)
 692         code.write(target[0].abspath)
 693     return body
 694
 695 def createEnumStrings(target, source, env):
 696     assert len(target) == 1 and len(source) == 2
 697
 698     name = source[0].get_text_contents()
 699     use_python = source[1].read()
 700     obj = all_enums[name]
 701
 702     code = code_formatter()
 703     obj.cxx_def(code)
 704     if use_python:
 705         obj.pybind_def(code)
 706     code.write(target[0].abspath)
 707
 708 def createEnumDecls(target, source, env):
 709     assert len(target) == 1 and len(source) == 1
 710
 711     name = source[0].get_text_contents()
 712     obj = all_enums[name]
 713
 714     code = code_formatter()
 715     obj.cxx_decl(code)
 716     code.write(target[0].abspath)
 717
 718 def createSimObjectPyBindWrapper(target, source, env):
 719     name = source[0].get_text_contents()
 720     obj = sim_objects[name]
 721
 722     code = code_formatter()
 723     obj.pybind_decl(code)
 724     code.write(target[0].abspath)
 725
 726 # Generate all of the SimObject param C++ struct header files
 727 params_hh_files = []
 728 for name,simobj in sorted(sim_objects.iteritems()):
 729     py_source = PySource.modules[simobj.__module__]
 730     extra_deps = [ py_source.tnode ]
 731
 732     hh_file = File('params/%s.hh' % name)
 733     params_hh_files.append(hh_file)
 734     env.Command(hh_file, Value(name),
 735                 MakeAction(createSimObjectParamStruct, Transform("SO PARAM")))
 736     env.Depends(hh_file, depends + extra_deps)
 737
 738 # C++ parameter description files
 739 if GetOption('with_cxx_config'):
 740     for name,simobj in sorted(sim_objects.iteritems()):
 741         py_source = PySource.modules[simobj.__module__]
 742         extra_deps = [ py_source.tnode ]
 743
 744         cxx_config_hh_file = File('cxx_config/%s.hh' % name)
 745         cxx_config_cc_file = File('cxx_config/%s.cc' % name)
 746         env.Command(cxx_config_hh_file, Value(name),
 747                     MakeAction(createSimObjectCxxConfig(True),
 748                     Transform("CXXCPRHH")))
 749         env.Command(cxx_config_cc_file, Value(name),
 750                     MakeAction(createSimObjectCxxConfig(False),
 751                     Transform("CXXCPRCC")))
 752         env.Depends(cxx_config_hh_file, depends + extra_deps +
 753                     [File('params/%s.hh' % name), File('sim/cxx_config.hh')])
 754         env.Depends(cxx_config_cc_file, depends + extra_deps +
 755                     [cxx_config_hh_file])
 756         Source(cxx_config_cc_file)
 757
 758     cxx_config_init_cc_file = File('cxx_config/init.cc')
 759
 760     def createCxxConfigInitCC(target, source, env):
 761         assert len(target) == 1 and len(source) == 1
 762
 763         code = code_formatter()
 764
 765         for name,simobj in sorted(sim_objects.iteritems()):
 766             if not hasattr(simobj, 'abstract') or not simobj.abstract:
 767                 code('#include "cxx_config/${name}.hh"')
 768         code()
 769         code('void cxxConfigInit()')
 770         code('{')
 771         code.indent()
 772         for name,simobj in sorted(sim_objects.iteritems()):
 773             not_abstract = not hasattr(simobj, 'abstract') or \
 774                 not simobj.abstract
 775             if not_abstract and 'type' in simobj.__dict__:
 776                 code('cxx_config_directory["${name}"] = '
 777                      '${name}CxxConfigParams::makeDirectoryEntry();')
 778         code.dedent()
 779         code('}')
 780         code.write(target[0].abspath)
 781
 782     py_source = PySource.modules[simobj.__module__]
 783     extra_deps = [ py_source.tnode ]
 784     env.Command(cxx_config_init_cc_file, Value(name),
 785         MakeAction(createCxxConfigInitCC, Transform("CXXCINIT")))
 786     cxx_param_hh_files = ["cxx_config/%s.hh" % simobj
 787         for name,simobj in sorted(sim_objects.iteritems())
 788         if not hasattr(simobj, 'abstract') or not simobj.abstract]
 789     Depends(cxx_config_init_cc_file, cxx_param_hh_files +
 790             [File('sim/cxx_config.hh')])
 791     Source(cxx_config_init_cc_file)
 792
 793 # Generate all enum header files
 794 for name,enum in sorted(all_enums.iteritems()):
 795     py_source = PySource.modules[enum.__module__]
 796     extra_deps = [ py_source.tnode ]
 797
 798     cc_file = File('enums/%s.cc' % name)
 799     env.Command(cc_file, [Value(name), Value(env['USE_PYTHON'])],
 800                 MakeAction(createEnumStrings, Transform("ENUM STR")))
 801     env.Depends(cc_file, depends + extra_deps)
 802     Source(cc_file)
 803
 804     hh_file = File('enums/%s.hh' % name)
 805     env.Command(hh_file, Value(name),
 806                 MakeAction(createEnumDecls, Transform("ENUMDECL")))
 807     env.Depends(hh_file, depends + extra_deps)
 808
 809 # Generate SimObject Python bindings wrapper files
 810 if env['USE_PYTHON']:
 811     for name,simobj in sorted(sim_objects.iteritems()):
 812         py_source = PySource.modules[simobj.__module__]
 813         extra_deps = [ py_source.tnode ]
 814         cc_file = File('python/_m5/param_%s.cc' % name)
 815         env.Command(cc_file, Value(name),
 816                     MakeAction(createSimObjectPyBindWrapper,
 817                                Transform("SO PyBind")))
 818         env.Depends(cc_file, depends + extra_deps)
 819         Source(cc_file)
 820
 821 # Build all protocol buffers if we have got protoc and protobuf available
 822 if env['HAVE_PROTOBUF']:
 823     for proto in ProtoBuf.all:
 824         # Use both the source and header as the target, and the .proto
 825         # file as the source. When executing the protoc compiler, also
 826         # specify the proto_path to avoid having the generated files
 827         # include the path.
 828         env.Command([proto.cc_file, proto.hh_file], proto.tnode,
 829                     MakeAction('$PROTOC --cpp_out ${TARGET.dir} '
 830                                '--proto_path ${SOURCE.dir} $SOURCE',
 831                                Transform("PROTOC")))
 832
 833         # Add the C++ source file
 834         Source(proto.cc_file, tags=proto.tags)
 835 elif ProtoBuf.all:
 836     print('Got protobuf to build, but lacks support!')
 837     Exit(1)
 838
 839 #
 840 # Handle debug flags
 841 #
 842 def makeDebugFlagCC(target, source, env):
 843     assert(len(target) == 1 and len(source) == 1)
 844
 845     code = code_formatter()
 846
 847     # delay definition of CompoundFlags until after all the definition
 848     # of all constituent SimpleFlags
 849     comp_code = code_formatter()
 850
 851     # file header
 852     code('''
 853 /*
 854  * DO NOT EDIT THIS FILE! Automatically generated by SCons.
 855  */
 856
 857 #include "base/debug.hh"
 858
 859 namespace Debug {
 860
 861 ''')
 862
 863     for name, flag in sorted(source[0].read().iteritems()):
 864         n, compound, desc = flag
 865         assert n == name
 866
 867         if not compound:
 868             code('SimpleFlag $name("$name", "$desc");')
 869         else:
 870             comp_code('CompoundFlag $name("$name", "$desc",')
 871             comp_code.indent()
 872             last = len(compound) - 1
 873             for i,flag in enumerate(compound):
 874                 if i != last:
 875                     comp_code('&$flag,')
 876                 else:
 877                     comp_code('&$flag);')
 878             comp_code.dedent()
 879
 880     code.append(comp_code)
 881     code()
 882     code('} // namespace Debug')
 883
 884     code.write(str(target[0]))
 885
 886 def makeDebugFlagHH(target, source, env):
 887     assert(len(target) == 1 and len(source) == 1)
 888
 889     val = eval(source[0].get_contents())
 890     name, compound, desc = val
 891
 892     code = code_formatter()
 893
 894     # file header boilerplate
 895     code('''\
 896 /*
 897  * DO NOT EDIT THIS FILE! Automatically generated by SCons.
 898  */
 899
 900 #ifndef __DEBUG_${name}_HH__
 901 #define __DEBUG_${name}_HH__
 902
 903 namespace Debug {
 904 ''')
 905
 906     if compound:
 907         code('class CompoundFlag;')
 908     code('class SimpleFlag;')
 909
 910     if compound:
 911         code('extern CompoundFlag $name;')
 912         for flag in compound:
 913             code('extern SimpleFlag $flag;')
 914     else:
 915         code('extern SimpleFlag $name;')
 916
 917     code('''
 918 }
 919
 920 #endif // __DEBUG_${name}_HH__
 921 ''')
 922
 923     code.write(str(target[0]))
 924
 925 for name,flag in sorted(debug_flags.iteritems()):
 926     n, compound, desc = flag
 927     assert n == name
 928
 929     hh_file = 'debug/%s.hh' % name
 930     env.Command(hh_file, Value(flag),
 931                 MakeAction(makeDebugFlagHH, Transform("TRACING", 0)))
 932
 933 env.Command('debug/flags.cc', Value(debug_flags),
 934             MakeAction(makeDebugFlagCC, Transform("TRACING", 0)))
 935 Source('debug/flags.cc')
 936
 937 # version tags
 938 tags = \
 939 env.Command('sim/tags.cc', None,
 940             MakeAction('util/cpt_upgrader.py --get-cc-file > $TARGET',
 941                        Transform("VER TAGS")))
 942 env.AlwaysBuild(tags)
 943
 944 # Embed python files.  All .py files that have been indicated by a
 945 # PySource() call in a SConscript need to be embedded into the M5
 946 # library.  To do that, we compile the file to byte code, marshal the
 947 # byte code, compress it, and then generate a c++ file that
 948 # inserts the result into an array.
 949 def embedPyFile(target, source, env):
 950     def c_str(string):
 951         if string is None:
 952             return "0"
 953         return '"%s"' % string
 954
 955     '''Action function to compile a .py into a code object, marshal
 956     it, compress it, and stick it into an asm file so the code appears
 957     as just bytes with a label in the data section'''
 958
 959     src = file(str(source[0]), 'r').read()
 960
 961     pysource = PySource.tnodes[source[0]]
 962     compiled = compile(src, pysource.abspath, 'exec')
 963     marshalled = marshal.dumps(compiled)
 964     compressed = zlib.compress(marshalled)
 965     data = compressed
 966     sym = pysource.symname
 967
 968     code = code_formatter()
 969     code('''\
 970 #include "sim/init.hh"
 971
 972 namespace {
 973
 974 const uint8_t data_${sym}[] = {
 975 ''')
 976     code.indent()
 977     step = 16
 978     for i in xrange(0, len(data), step):
 979         x = array.array('B', data[i:i+step])
 980         code(''.join('%d,' % d for d in x))
 981     code.dedent()
 982
 983     code('''};
 984
 985 EmbeddedPython embedded_${sym}(
 986     ${{c_str(pysource.arcname)}},
 987     ${{c_str(pysource.abspath)}},
 988     ${{c_str(pysource.modpath)}},
 989     data_${sym},
 990     ${{len(data)}},
 991     ${{len(marshalled)}});
 992
 993 } // anonymous namespace
 994 ''')
 995     code.write(str(target[0]))
 996
 997 for source in PySource.all:
 998     env.Command(source.cpp, source.tnode,
 999                 MakeAction(embedPyFile, Transform("EMBED PY")))
1000     Source(source.cpp, tags=source.tags, add_tags='python')
1001
1002 ########################################################################
1003 #
1004 # Define binaries.  Each different build type (debug, opt, etc.) gets
1005 # a slightly different build environment.
1006 #
1007
1008 # List of constructed environments to pass back to SConstruct
1009 date_source = Source('base/date.cc', tags=[])
1010
1011 # Function to create a new build environment as clone of current
1012 # environment 'env' with modified object suffix and optional stripped
1013 # binary.  Additional keyword arguments are appended to corresponding
1014 # build environment vars.
1015 def makeEnv(env, label, objsfx, strip=False, disable_partial=False, **kwargs):
1016     # SCons doesn't know to append a library suffix when there is a '.' in the
1017     # name.  Use '_' instead.
1018     libname = 'gem5_' + label
1019     exename = 'gem5.' + label
1020     secondary_exename = 'm5.' + label
1021
1022     new_env = env.Clone(OBJSUFFIX=objsfx, SHOBJSUFFIX=objsfx + 's')
1023     new_env.Label = label
1024     new_env.Append(**kwargs)
1025
1026     lib_sources = Source.all.with_tag('gem5 lib')
1027
1028     # Without Python, leave out all Python content from the library
1029     # builds.  The option doesn't affect gem5 built as a program
1030     if GetOption('without_python'):
1031         lib_sources = lib_sources.without_tag('python')
1032
1033     static_objs = []
1034     shared_objs = []
1035
1036     for s in lib_sources.with_tag(Source.ungrouped_tag):
1037         static_objs.append(s.static(new_env))
1038         shared_objs.append(s.shared(new_env))
1039
1040     for group in Source.source_groups:
1041         srcs = lib_sources.with_tag(Source.link_group_tag(group))
1042         if not srcs:
1043             continue
1044
1045         group_static = [ s.static(new_env) for s in srcs ]
1046         group_shared = [ s.shared(new_env) for s in srcs ]
1047
1048         # If partial linking is disabled, add these sources to the build
1049         # directly, and short circuit this loop.
1050         if disable_partial:
1051             static_objs.extend(group_static)
1052             shared_objs.extend(group_shared)
1053             continue
1054
1055         # Set up the static partially linked objects.
1056         file_name = new_env.subst("${OBJPREFIX}lib${OBJSUFFIX}.partial")
1057         target = File(joinpath(group, file_name))
1058         partial = env.PartialStatic(target=target, source=group_static)
1059         static_objs.extend(partial)
1060
1061         # Set up the shared partially linked objects.
1062         file_name = new_env.subst("${SHOBJPREFIX}lib${SHOBJSUFFIX}.partial")
1063         target = File(joinpath(group, file_name))
1064         partial = env.PartialShared(target=target, source=group_shared)
1065         shared_objs.extend(partial)
1066
1067     static_date = date_source.static(new_env)
1068     new_env.Depends(static_date, static_objs)
1069     static_objs.extend(static_date)
1070
1071     shared_date = date_source.shared(new_env)
1072     new_env.Depends(shared_date, shared_objs)
1073     shared_objs.extend(shared_date)
1074
1075     # First make a library of everything but main() so other programs can
1076     # link against m5.
1077     static_lib = new_env.StaticLibrary(libname, static_objs)
1078     shared_lib = new_env.SharedLibrary(libname, shared_objs)
1079
1080     # Now link a stub with main() and the static library.
1081     main_objs = [ s.static(new_env) for s in Source.all.with_tag('main') ]
1082
1083     for test in UnitTest.all:
1084         test_sources = Source.all.with_tag(str(test.target))
1085         test_objs = [ s.static(new_env) for s in test_sources ]
1086         if test.main:
1087             test_objs += main_objs
1088         path = 'unittest/%s.%s' % (test.target, label)
1089         new_env.Program(path, test_objs + static_objs)
1090
1091     gtest_env = new_env.Clone()
1092     gtest_env.Append(LIBS=gtest_env['GTEST_LIBS'])
1093     gtest_env.Append(CPPFLAGS=gtest_env['GTEST_CPPFLAGS'])
1094     gtestlib_sources = Source.all.with_tag('gtest lib')
1095     gtest_out_dir = Dir(new_env['BUILDDIR']).Dir('unittests.%s' % label)
1096     for test in GTest.all:
1097         test_sources = list(test.sources)
1098         if not test.skip_lib:
1099             test_sources += gtestlib_sources
1100         for f in test.filters:
1101             test_sources += Source.all.apply_filter(f)
1102         test_objs = [ s.static(gtest_env) for s in test_sources ]
1103         test_binary = gtest_env.Program(
1104             test.dir.File('%s.%s' % (test.target, label)), test_objs)
1105
1106         AlwaysBuild(gtest_env.Command(
1107             gtest_out_dir.File("%s/%s.xml" % (test.dir, test.target)),
1108             test_binary, "${SOURCES[0]} --gtest_output=xml:${TARGETS[0]}"))
1109
1110     progname = exename
1111     if strip:
1112         progname += '.unstripped'
1113
1114     targets = new_env.Program(progname, main_objs + static_objs)
1115
1116     if strip:
1117         if sys.platform == 'sunos5':
1118             cmd = 'cp $SOURCE $TARGET; strip $TARGET'
1119         else:
1120             cmd = 'strip $SOURCE -o $TARGET'
1121         targets = new_env.Command(exename, progname,
1122                     MakeAction(cmd, Transform("STRIP")))
1123
1124     new_env.Command(secondary_exename, exename,
1125             MakeAction('ln $SOURCE $TARGET', Transform("HARDLINK")))
1126
1127     new_env.M5Binary = targets[0]
1128
1129     # Set up regression tests.
1130     SConscript(os.path.join(env.root.abspath, 'tests', 'SConscript'),
1131                variant_dir=Dir('tests').Dir(new_env.Label),
1132                exports={ 'env' : new_env }, duplicate=False)
1133
1134 # Start out with the compiler flags common to all compilers,
1135 # i.e. they all use -g for opt and -g -pg for prof
1136 ccflags = {'debug' : [], 'opt' : ['-g'], 'fast' : [], 'prof' : ['-g', '-pg'],
1137            'perf' : ['-g']}
1138
1139 # Start out with the linker flags common to all linkers, i.e. -pg for
1140 # prof, and -lprofiler for perf. The -lprofile flag is surrounded by
1141 # no-as-needed and as-needed as the binutils linker is too clever and
1142 # simply doesn't link to the library otherwise.
1143 ldflags = {'debug' : [], 'opt' : [], 'fast' : [], 'prof' : ['-pg'],
1144            'perf' : ['-Wl,--no-as-needed', '-lprofiler', '-Wl,--as-needed']}
1145
1146 # For Link Time Optimization, the optimisation flags used to compile
1147 # individual files are decoupled from those used at link time
1148 # (i.e. you can compile with -O3 and perform LTO with -O0), so we need
1149 # to also update the linker flags based on the target.
1150 if env['GCC']:
1151     if sys.platform == 'sunos5':
1152         ccflags['debug'] += ['-gstabs+']
1153     else:
1154         ccflags['debug'] += ['-ggdb3']
1155     ldflags['debug'] += ['-O0']
1156     # opt, fast, prof and perf all share the same cc flags, also add
1157     # the optimization to the ldflags as LTO defers the optimization
1158     # to link time
1159     for target in ['opt', 'fast', 'prof', 'perf']:
1160         ccflags[target] += ['-O3']
1161         ldflags[target] += ['-O3']
1162
1163     ccflags['fast'] += env['LTO_CCFLAGS']
1164     ldflags['fast'] += env['LTO_LDFLAGS']
1165 elif env['CLANG']:
1166     ccflags['debug'] += ['-g', '-O0']
1167     # opt, fast, prof and perf all share the same cc flags
1168     for target in ['opt', 'fast', 'prof', 'perf']:
1169         ccflags[target] += ['-O3']
1170 else:
1171     print('Unknown compiler, please fix compiler options')
1172     Exit(1)
1173
1174
1175 # To speed things up, we only instantiate the build environments we
1176 # need.  We try to identify the needed environment for each target; if
1177 # we can't, we fall back on instantiating all the environments just to
1178 # be safe.
1179 target_types = ['debug', 'opt', 'fast', 'prof', 'perf']
1180 obj2target = {'do': 'debug', 'o': 'opt', 'fo': 'fast', 'po': 'prof',
1181               'gpo' : 'perf'}
1182
1183 def identifyTarget(t):
1184     ext = t.split('.')[-1]
1185     if ext in target_types:
1186         return ext
1187     if obj2target.has_key(ext):
1188         return obj2target[ext]
1189     match = re.search(r'/tests/([^/]+)/', t)
1190     if match and match.group(1) in target_types:
1191         return match.group(1)
1192     return 'all'
1193
1194 needed_envs = [identifyTarget(target) for target in BUILD_TARGETS]
1195 if 'all' in needed_envs:
1196     needed_envs += target_types
1197
1198 # Debug binary
1199 if 'debug' in needed_envs:
1200     makeEnv(env, 'debug', '.do',
1201             CCFLAGS = Split(ccflags['debug']),
1202             CPPDEFINES = ['DEBUG', 'TRACING_ON=1'],
1203             LINKFLAGS = Split(ldflags['debug']))
1204
1205 # Optimized binary
1206 if 'opt' in needed_envs:
1207     makeEnv(env, 'opt', '.o',
1208             CCFLAGS = Split(ccflags['opt']),
1209             CPPDEFINES = ['TRACING_ON=1'],
1210             LINKFLAGS = Split(ldflags['opt']))
1211
1212 # "Fast" binary
1213 if 'fast' in needed_envs:
1214     disable_partial = \
1215             env.get('BROKEN_INCREMENTAL_LTO', False) and \
1216             GetOption('force_lto')
1217     makeEnv(env, 'fast', '.fo', strip = True,
1218             CCFLAGS = Split(ccflags['fast']),
1219             CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'],
1220             LINKFLAGS = Split(ldflags['fast']),
1221             disable_partial=disable_partial)
1222
1223 # Profiled binary using gprof
1224 if 'prof' in needed_envs:
1225     makeEnv(env, 'prof', '.po',
1226             CCFLAGS = Split(ccflags['prof']),
1227             CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'],
1228             LINKFLAGS = Split(ldflags['prof']))
1229
1230 # Profiled binary using google-pprof
1231 if 'perf' in needed_envs:
1232     makeEnv(env, 'perf', '.gpo',
1233             CCFLAGS = Split(ccflags['perf']),
1234             CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'],
1235             LINKFLAGS = Split(ldflags['perf']))