configs/ruby/GPU_VIPER_Region.py

   1 # Copyright (c) 2015 Advanced Micro Devices, Inc.
   2 # All rights reserved.
   3 #
   4 # For use for simulation and test purposes only
   5 #
   6 # Redistribution and use in source and binary forms, with or without
   7 # modification, are permitted provided that the following conditions are met:
   8 #
   9 # 1. Redistributions of source code must retain the above copyright notice,
  10 # this list of conditions and the following disclaimer.
  11 #
  12 # 2. Redistributions in binary form must reproduce the above copyright notice,
  13 # this list of conditions and the following disclaimer in the documentation
  14 # and/or other materials provided with the distribution.
  15 #
  16 # 3. Neither the name of the copyright holder nor the names of its
  17 # contributors may be used to endorse or promote products derived from this
  18 # software without specific prior written permission.
  19 #
  20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23 # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
  24 # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30 # POSSIBILITY OF SUCH DAMAGE.
  31
  32 import six
  33 import math
  34 import m5
  35 from m5.objects import *
  36 from m5.defines import buildEnv
  37 from m5.util import addToPath
  38 from .Ruby import send_evicts
  39
  40 addToPath('../')
  41
  42 from topologies.Cluster import Cluster
  43
  44 if six.PY3:
  45     long = int
  46
  47 class CntrlBase:
  48     _seqs = 0
  49     @classmethod
  50     def seqCount(cls):
  51         # Use SeqCount not class since we need global count
  52         CntrlBase._seqs += 1
  53         return CntrlBase._seqs - 1
  54
  55     _cntrls = 0
  56     @classmethod
  57     def cntrlCount(cls):
  58         # Use CntlCount not class since we need global count
  59         CntrlBase._cntrls += 1
  60         return CntrlBase._cntrls - 1
  61
  62     _version = 0
  63     @classmethod
  64     def versionCount(cls):
  65         cls._version += 1 # Use count for this particular type
  66         return cls._version - 1
  67
  68 #
  69 # Note: the L1 Cache latency is only used by the sequencer on fast path hits
  70 #
  71 class L1Cache(RubyCache):
  72     resourceStalls = False
  73     dataArrayBanks = 2
  74     tagArrayBanks = 2
  75     dataAccessLatency = 1
  76     tagAccessLatency = 1
  77     def create(self, size, assoc, options):
  78         self.size = MemorySize(size)
  79         self.assoc = assoc
  80         self.replacement_policy = TreePLRURP()
  81
  82 class L2Cache(RubyCache):
  83     resourceStalls = False
  84     assoc = 16
  85     dataArrayBanks = 16
  86     tagArrayBanks = 16
  87     def create(self, size, assoc, options):
  88         self.size = MemorySize(size)
  89         self.assoc = assoc
  90         self.replacement_policy = TreePLRURP()
  91
  92 class CPCntrl(CorePair_Controller, CntrlBase):
  93
  94     def create(self, options, ruby_system, system):
  95         self.version = self.versionCount()
  96
  97         self.L1Icache = L1Cache()
  98         self.L1Icache.create(options.l1i_size, options.l1i_assoc, options)
  99         self.L1D0cache = L1Cache()
 100         self.L1D0cache.create(options.l1d_size, options.l1d_assoc, options)
 101         self.L1D1cache = L1Cache()
 102         self.L1D1cache.create(options.l1d_size, options.l1d_assoc, options)
 103         self.L2cache = L2Cache()
 104         self.L2cache.create(options.l2_size, options.l2_assoc, options)
 105
 106         self.sequencer = RubySequencer()
 107         self.sequencer.version = self.seqCount()
 108         self.sequencer.dcache = self.L1D0cache
 109         self.sequencer.ruby_system = ruby_system
 110         self.sequencer.coreid = 0
 111         self.sequencer.is_cpu_sequencer = True
 112
 113         self.sequencer1 = RubySequencer()
 114         self.sequencer1.version = self.seqCount()
 115         self.sequencer1.dcache = self.L1D1cache
 116         self.sequencer1.ruby_system = ruby_system
 117         self.sequencer1.coreid = 1
 118         self.sequencer1.is_cpu_sequencer = True
 119
 120         self.issue_latency = 1
 121         self.send_evictions = send_evicts(options)
 122
 123         self.ruby_system = ruby_system
 124
 125         if options.recycle_latency:
 126             self.recycle_latency = options.recycle_latency
 127
 128 class TCPCache(RubyCache):
 129     size = "16kB"
 130     assoc = 16
 131     dataArrayBanks = 16
 132     tagArrayBanks = 16
 133     dataAccessLatency = 4
 134     tagAccessLatency = 1
 135     def create(self, options):
 136         self.size = MemorySize(options.tcp_size)
 137         self.dataArrayBanks = 16
 138         self.tagArrayBanks = 16
 139         self.dataAccessLatency = 4
 140         self.tagAccessLatency = 1
 141         self.resourceStalls = options.no_tcc_resource_stalls
 142         self.replacement_policy = TreePLRURP(num_leaves = self.assoc)
 143
 144 class TCPCntrl(TCP_Controller, CntrlBase):
 145
 146     def create(self, options, ruby_system, system):
 147         self.version = self.versionCount()
 148         self.L1cache = TCPCache(dataAccessLatency = options.TCP_latency)
 149         self.L1cache.create(options)
 150         self.issue_latency = 1
 151
 152         self.coalescer = VIPERCoalescer()
 153         self.coalescer.version = self.seqCount()
 154         self.coalescer.icache = self.L1cache
 155         self.coalescer.dcache = self.L1cache
 156         self.coalescer.ruby_system = ruby_system
 157         self.coalescer.support_inst_reqs = False
 158         self.coalescer.is_cpu_sequencer = False
 159         if options.tcp_deadlock_threshold:
 160           self.coalescer.deadlock_threshold = \
 161             options.tcp_deadlock_threshold
 162         self.coalescer.max_coalesces_per_cycle = \
 163             options.max_coalesces_per_cycle
 164
 165         self.sequencer = RubySequencer()
 166         self.sequencer.version = self.seqCount()
 167         self.sequencer.dcache = self.L1cache
 168         self.sequencer.ruby_system = ruby_system
 169         self.sequencer.is_cpu_sequencer = True
 170
 171         self.use_seq_not_coal = False
 172
 173         self.ruby_system = ruby_system
 174         if options.recycle_latency:
 175             self.recycle_latency = options.recycle_latency
 176
 177 class SQCCache(RubyCache):
 178     dataArrayBanks = 8
 179     tagArrayBanks = 8
 180     dataAccessLatency = 1
 181     tagAccessLatency = 1
 182
 183     def create(self, options):
 184         self.size = MemorySize(options.sqc_size)
 185         self.assoc = options.sqc_assoc
 186         self.replacement_policy = TreePLRURP(num_leaves = self.assoc)
 187
 188 class SQCCntrl(SQC_Controller, CntrlBase):
 189
 190     def create(self, options, ruby_system, system):
 191         self.version = self.versionCount()
 192         self.L1cache = SQCCache()
 193         self.L1cache.create(options)
 194         self.L1cache.resourceStalls = False
 195         self.sequencer = RubySequencer()
 196         self.sequencer.version = self.seqCount()
 197         self.sequencer.dcache = self.L1cache
 198         self.sequencer.ruby_system = ruby_system
 199         self.sequencer.support_data_reqs = False
 200         self.sequencer.is_cpu_sequencer = False
 201         if options.sqc_deadlock_threshold:
 202           self.sequencer.deadlock_threshold = \
 203             options.sqc_deadlock_threshold
 204
 205         self.ruby_system = ruby_system
 206         if options.recycle_latency:
 207             self.recycle_latency = options.recycle_latency
 208
 209 class TCC(RubyCache):
 210     size = MemorySize("256kB")
 211     assoc = 16
 212     dataAccessLatency = 8
 213     tagAccessLatency = 2
 214     resourceStalls = False
 215     def create(self, options):
 216         self.assoc = options.tcc_assoc
 217         if hasattr(options, 'bw_scalor') and options.bw_scalor > 0:
 218           s = options.num_compute_units
 219           tcc_size = s * 128
 220           tcc_size = str(tcc_size)+'kB'
 221           self.size = MemorySize(tcc_size)
 222           self.dataArrayBanks = 64
 223           self.tagArrayBanks = 64
 224         else:
 225           self.size = MemorySize(options.tcc_size)
 226           self.dataArrayBanks = 256 / options.num_tccs #number of data banks
 227           self.tagArrayBanks = 256 / options.num_tccs #number of tag banks
 228         self.size.value = self.size.value / options.num_tccs
 229         if ((self.size.value / long(self.assoc)) < 128):
 230             self.size.value = long(128 * self.assoc)
 231         self.start_index_bit = math.log(options.cacheline_size, 2) + \
 232                                math.log(options.num_tccs, 2)
 233         self.replacement_policy = TreePLRURP(num_leaves = self.assoc)
 234
 235 class TCCCntrl(TCC_Controller, CntrlBase):
 236     def create(self, options, ruby_system, system):
 237         self.version = self.versionCount()
 238         self.L2cache = TCC()
 239         self.L2cache.create(options)
 240         self.ruby_system = ruby_system
 241         if options.recycle_latency:
 242             self.recycle_latency = options.recycle_latency
 243
 244 class L3Cache(RubyCache):
 245     dataArrayBanks = 16
 246     tagArrayBanks = 16
 247
 248     def create(self, options, ruby_system, system):
 249         self.size = MemorySize(options.l3_size)
 250         self.size.value /= options.num_dirs
 251         self.assoc = options.l3_assoc
 252         self.dataArrayBanks /= options.num_dirs
 253         self.tagArrayBanks /= options.num_dirs
 254         self.dataArrayBanks /= options.num_dirs
 255         self.tagArrayBanks /= options.num_dirs
 256         self.dataAccessLatency = options.l3_data_latency
 257         self.tagAccessLatency = options.l3_tag_latency
 258         self.resourceStalls = False
 259         self.replacement_policy = TreePLRURP(num_leaves = self.assoc)
 260
 261 class L3Cntrl(L3Cache_Controller, CntrlBase):
 262     def create(self, options, ruby_system, system):
 263         self.version = self.versionCount()
 264         self.L3cache = L3Cache()
 265         self.L3cache.create(options, ruby_system, system)
 266         self.l3_response_latency = \
 267             max(self.L3cache.dataAccessLatency, self.L3cache.tagAccessLatency)
 268         self.ruby_system = ruby_system
 269         if options.recycle_latency:
 270             self.recycle_latency = options.recycle_latency
 271
 272     def connectWireBuffers(self, req_to_dir, resp_to_dir, l3_unblock_to_dir,
 273                            req_to_l3, probe_to_l3, resp_to_l3):
 274         self.reqToDir = req_to_dir
 275         self.respToDir = resp_to_dir
 276         self.l3UnblockToDir = l3_unblock_to_dir
 277         self.reqToL3 = req_to_l3
 278         self.probeToL3 = probe_to_l3
 279         self.respToL3 = resp_to_l3
 280
 281 # Directory controller: Contains directory memory, L3 cache and associated
 282 # state machine which is used to accurately redirect a data request to L3 cache
 283 # or memory. The permissions requests do not come to this directory for region
 284 # based protocols as they are handled exclusively by the region directory.
 285 # However, region directory controller uses this directory controller for
 286 # sending probe requests and receiving probe responses.
 287 class DirCntrl(Directory_Controller, CntrlBase):
 288     def create(self, options, dir_ranges, ruby_system, system):
 289         self.version = self.versionCount()
 290         self.response_latency = 25
 291         self.response_latency_regionDir = 1
 292         self.addr_ranges = dir_ranges
 293         self.directory = RubyDirectoryMemory()
 294         self.L3CacheMemory = L3Cache()
 295         self.L3CacheMemory.create(options, ruby_system, system)
 296         self.l3_hit_latency = \
 297             max(self.L3CacheMemory.dataAccessLatency,
 298             self.L3CacheMemory.tagAccessLatency)
 299
 300         self.ruby_system = ruby_system
 301         if options.recycle_latency:
 302             self.recycle_latency = options.recycle_latency
 303
 304     def connectWireBuffers(self, req_to_dir, resp_to_dir, l3_unblock_to_dir,
 305                            req_to_l3, probe_to_l3, resp_to_l3):
 306         self.reqToDir = req_to_dir
 307         self.respToDir = resp_to_dir
 308         self.l3UnblockToDir = l3_unblock_to_dir
 309         self.reqToL3 = req_to_l3
 310         self.probeToL3 = probe_to_l3
 311         self.respToL3 = resp_to_l3
 312
 313 # Region directory : Stores region permissions
 314 class RegionDir(RubyCache):
 315
 316     def create(self, options, ruby_system, system):
 317         self.block_size = "%dB" % (64 * options.blocks_per_region)
 318         self.size = options.region_dir_entries * \
 319             self.block_size * options.num_compute_units
 320         self.assoc = 8
 321         self.tagArrayBanks = 8
 322         self.tagAccessLatency = options.dir_tag_latency
 323         self.dataAccessLatency = 1
 324         self.resourceStalls = options.no_resource_stalls
 325         self.start_index_bit = 6 + int(math.log(options.blocks_per_region, 2))
 326         self.replacement_policy = TreePLRURP(num_leaves = self.assoc)
 327 # Region directory controller : Contains region directory and associated state
 328 # machine for dealing with region coherence requests.
 329 class RegionCntrl(RegionDir_Controller, CntrlBase):
 330     def create(self, options, ruby_system, system):
 331         self.version = self.versionCount()
 332         self.cacheMemory = RegionDir()
 333         self.cacheMemory.create(options, ruby_system, system)
 334         self.blocksPerRegion = options.blocks_per_region
 335         self.toDirLatency = \
 336             max(self.cacheMemory.dataAccessLatency,
 337             self.cacheMemory.tagAccessLatency)
 338         self.ruby_system = ruby_system
 339         self.always_migrate = options.always_migrate
 340         self.sym_migrate = options.symmetric_migrate
 341         self.asym_migrate = options.asymmetric_migrate
 342         if self.always_migrate:
 343             assert(not self.asym_migrate and not self.sym_migrate)
 344         if self.sym_migrate:
 345             assert(not self.always_migrate and not self.asym_migrate)
 346         if self.asym_migrate:
 347             assert(not self.always_migrate and not self.sym_migrate)
 348         if options.recycle_latency:
 349             self.recycle_latency = options.recycle_latency
 350
 351 # Region Buffer: A region directory cache which avoids some potential
 352 # long latency lookup of region directory for getting region permissions
 353 class RegionBuffer(RubyCache):
 354     assoc = 4
 355     dataArrayBanks = 256
 356     tagArrayBanks = 256
 357     dataAccessLatency = 1
 358     tagAccessLatency = 1
 359     resourceStalls = True
 360
 361 class RBCntrl(RegionBuffer_Controller, CntrlBase):
 362     def create(self, options, ruby_system, system):
 363         self.version = self.versionCount()
 364         self.cacheMemory = RegionBuffer()
 365         self.cacheMemory.resourceStalls = options.no_tcc_resource_stalls
 366         self.cacheMemory.dataArrayBanks = 64
 367         self.cacheMemory.tagArrayBanks = 64
 368         self.blocksPerRegion = options.blocks_per_region
 369         self.cacheMemory.block_size = "%dB" % (64 * self.blocksPerRegion)
 370         self.cacheMemory.start_index_bit = \
 371             6 + int(math.log(self.blocksPerRegion, 2))
 372         self.cacheMemory.size = options.region_buffer_entries * \
 373             self.cacheMemory.block_size * options.num_compute_units
 374         self.toDirLatency = options.gpu_to_dir_latency
 375         self.toRegionDirLatency = options.cpu_to_dir_latency
 376         self.noTCCdir = True
 377         TCC_bits = int(math.log(options.num_tccs, 2))
 378         self.TCC_select_num_bits = TCC_bits
 379         self.ruby_system = ruby_system
 380
 381         if options.recycle_latency:
 382             self.recycle_latency = options.recycle_latency
 383         self.cacheMemory.replacement_policy = \
 384             TreePLRURP(num_leaves = self.cacheMemory.assoc)
 385
 386 def define_options(parser):
 387     parser.add_option("--num-subcaches", type="int", default=4)
 388     parser.add_option("--l3-data-latency", type="int", default=20)
 389     parser.add_option("--l3-tag-latency", type="int", default=15)
 390     parser.add_option("--cpu-to-dir-latency", type="int", default=120)
 391     parser.add_option("--gpu-to-dir-latency", type="int", default=60)
 392     parser.add_option("--no-resource-stalls", action="store_false",
 393                       default=True)
 394     parser.add_option("--no-tcc-resource-stalls", action="store_false",
 395                       default=True)
 396     parser.add_option("--num-tbes", type="int", default=32)
 397     parser.add_option("--l2-latency", type="int", default=50) # load to use
 398     parser.add_option("--num-tccs", type="int", default=1,
 399                       help="number of TCC banks in the GPU")
 400
 401     parser.add_option("--sqc-size", type='string', default='32kB',
 402                       help="SQC cache size")
 403     parser.add_option("--sqc-assoc", type='int', default=8,
 404                       help="SQC cache assoc")
 405     parser.add_option("--sqc-deadlock-threshold", type='int',
 406                       help="Set the SQC deadlock threshold to some value")
 407
 408     parser.add_option("--WB_L1", action="store_true",
 409         default=False, help="L2 Writeback Cache")
 410     parser.add_option("--WB_L2", action="store_true",
 411         default=False, help="L2 Writeback Cache")
 412     parser.add_option("--TCP_latency",
 413         type="int", default=4, help="TCP latency")
 414     parser.add_option("--TCC_latency",
 415         type="int", default=16, help="TCC latency")
 416     parser.add_option("--tcc-size", type='string', default='2MB',
 417                       help="agregate tcc size")
 418     parser.add_option("--tcc-assoc", type='int', default=16,
 419                       help="tcc assoc")
 420     parser.add_option("--tcp-size", type='string', default='16kB',
 421                       help="tcp size")
 422     parser.add_option("--tcp-deadlock-threshold", type='int',
 423                       help="Set the TCP deadlock threshold to some value")
 424     parser.add_option("--max-coalesces-per-cycle", type="int", default=1,
 425                       help="Maximum insts that may coalesce in a cycle");
 426
 427     parser.add_option("--dir-tag-latency", type="int", default=4)
 428     parser.add_option("--dir-tag-banks", type="int", default=4)
 429     parser.add_option("--blocks-per-region", type="int", default=16)
 430     parser.add_option("--dir-entries", type="int", default=8192)
 431
 432     # Region buffer is a cache of region directory. Hence region
 433     # directory is inclusive with respect to region directory.
 434     # However, region directory is non-inclusive with respect to
 435     # the caches in the system
 436     parser.add_option("--region-dir-entries", type="int", default=1024)
 437     parser.add_option("--region-buffer-entries", type="int", default=512)
 438
 439     parser.add_option("--always-migrate",
 440         action="store_true", default=False)
 441     parser.add_option("--symmetric-migrate",
 442         action="store_true", default=False)
 443     parser.add_option("--asymmetric-migrate",
 444         action="store_true", default=False)
 445     parser.add_option("--use-L3-on-WT", action="store_true", default=False)
 446
 447 def create_system(options, full_system, system, dma_devices, bootmem,
 448                   ruby_system):
 449     if buildEnv['PROTOCOL'] != 'GPU_VIPER_Region':
 450         panic("This script requires the GPU_VIPER_Region protocol to be built.")
 451
 452     cpu_sequencers = []
 453
 454     #
 455     # The ruby network creation expects the list of nodes in the system to be
 456     # consistent with the NetDest list.  Therefore the l1 controller nodes
 457     # must be listed before the directory nodes and directory nodes before
 458     # dma nodes, etc.
 459     #
 460     dir_cntrl_nodes = []
 461
 462     # For an odd number of CPUs, still create the right number of controllers
 463     TCC_bits = int(math.log(options.num_tccs, 2))
 464
 465     #
 466     # Must create the individual controllers before the network to ensure the
 467     # controller constructors are called before the network constructor
 468     #
 469
 470     # For an odd number of CPUs, still create the right number of controllers
 471     crossbar_bw = 16 * options.num_compute_units #Assuming a 2GHz clock
 472     cpuCluster = Cluster(extBW = (crossbar_bw), intBW=crossbar_bw)
 473     for i in range((options.num_cpus + 1) // 2):
 474
 475         cp_cntrl = CPCntrl()
 476         cp_cntrl.create(options, ruby_system, system)
 477
 478         rb_cntrl = RBCntrl()
 479         rb_cntrl.create(options, ruby_system, system)
 480         rb_cntrl.number_of_TBEs = 256
 481         rb_cntrl.isOnCPU = True
 482
 483         cp_cntrl.regionBufferNum = rb_cntrl.version
 484
 485         exec("system.cp_cntrl%d = cp_cntrl" % i)
 486         exec("system.rb_cntrl%d = rb_cntrl" % i)
 487         #
 488         # Add controllers and sequencers to the appropriate lists
 489         #
 490         cpu_sequencers.extend([cp_cntrl.sequencer, cp_cntrl.sequencer1])
 491
 492         # Connect the CP controllers and the network
 493         cp_cntrl.requestFromCore = MessageBuffer()
 494         cp_cntrl.requestFromCore.master = ruby_system.network.slave
 495
 496         cp_cntrl.responseFromCore = MessageBuffer()
 497         cp_cntrl.responseFromCore.master = ruby_system.network.slave
 498
 499         cp_cntrl.unblockFromCore = MessageBuffer()
 500         cp_cntrl.unblockFromCore.master = ruby_system.network.slave
 501
 502         cp_cntrl.probeToCore = MessageBuffer()
 503         cp_cntrl.probeToCore.slave = ruby_system.network.master
 504
 505         cp_cntrl.responseToCore = MessageBuffer()
 506         cp_cntrl.responseToCore.slave = ruby_system.network.master
 507
 508         cp_cntrl.mandatoryQueue = MessageBuffer()
 509         cp_cntrl.triggerQueue = MessageBuffer(ordered = True)
 510
 511         # Connect the RB controllers to the ruby network
 512         rb_cntrl.requestFromCore = MessageBuffer(ordered = True)
 513         rb_cntrl.requestFromCore.slave = ruby_system.network.master
 514
 515         rb_cntrl.responseFromCore = MessageBuffer()
 516         rb_cntrl.responseFromCore.slave = ruby_system.network.master
 517
 518         rb_cntrl.requestToNetwork = MessageBuffer()
 519         rb_cntrl.requestToNetwork.master = ruby_system.network.slave
 520
 521         rb_cntrl.notifyFromRegionDir = MessageBuffer()
 522         rb_cntrl.notifyFromRegionDir.slave = ruby_system.network.master
 523
 524         rb_cntrl.probeFromRegionDir = MessageBuffer()
 525         rb_cntrl.probeFromRegionDir.slave = ruby_system.network.master
 526
 527         rb_cntrl.unblockFromDir = MessageBuffer()
 528         rb_cntrl.unblockFromDir.slave = ruby_system.network.master
 529
 530         rb_cntrl.responseToRegDir = MessageBuffer()
 531         rb_cntrl.responseToRegDir.master = ruby_system.network.slave
 532
 533         rb_cntrl.triggerQueue = MessageBuffer(ordered = True)
 534
 535         cpuCluster.add(cp_cntrl)
 536         cpuCluster.add(rb_cntrl)
 537
 538     gpuCluster = Cluster(extBW = (crossbar_bw), intBW = crossbar_bw)
 539     for i in range(options.num_compute_units):
 540
 541         tcp_cntrl = TCPCntrl(TCC_select_num_bits = TCC_bits,
 542                              issue_latency = 1,
 543                              number_of_TBEs = 2560)
 544         # TBEs set to max outstanding requests
 545         tcp_cntrl.create(options, ruby_system, system)
 546         tcp_cntrl.WB = options.WB_L1
 547         tcp_cntrl.disableL1 = False
 548
 549         exec("system.tcp_cntrl%d = tcp_cntrl" % i)
 550         #
 551         # Add controllers and sequencers to the appropriate lists
 552         #
 553         cpu_sequencers.append(tcp_cntrl.coalescer)
 554
 555         # Connect the CP (TCP) controllers to the ruby network
 556         tcp_cntrl.requestFromTCP = MessageBuffer(ordered = True)
 557         tcp_cntrl.requestFromTCP.master = ruby_system.network.slave
 558
 559         tcp_cntrl.responseFromTCP = MessageBuffer(ordered = True)
 560         tcp_cntrl.responseFromTCP.master = ruby_system.network.slave
 561
 562         tcp_cntrl.unblockFromCore = MessageBuffer()
 563         tcp_cntrl.unblockFromCore.master = ruby_system.network.slave
 564
 565         tcp_cntrl.probeToTCP = MessageBuffer(ordered = True)
 566         tcp_cntrl.probeToTCP.slave = ruby_system.network.master
 567
 568         tcp_cntrl.responseToTCP = MessageBuffer(ordered = True)
 569         tcp_cntrl.responseToTCP.slave = ruby_system.network.master
 570
 571         tcp_cntrl.mandatoryQueue = MessageBuffer()
 572
 573         gpuCluster.add(tcp_cntrl)
 574
 575     for i in range(options.num_sqc):
 576
 577         sqc_cntrl = SQCCntrl(TCC_select_num_bits = TCC_bits)
 578         sqc_cntrl.create(options, ruby_system, system)
 579
 580         exec("system.sqc_cntrl%d = sqc_cntrl" % i)
 581         #
 582         # Add controllers and sequencers to the appropriate lists
 583         #
 584         cpu_sequencers.append(sqc_cntrl.sequencer)
 585
 586         # Connect the SQC controller to the ruby network
 587         sqc_cntrl.requestFromSQC = MessageBuffer(ordered = True)
 588         sqc_cntrl.requestFromSQC.master = ruby_system.network.slave
 589
 590         sqc_cntrl.probeToSQC = MessageBuffer(ordered = True)
 591         sqc_cntrl.probeToSQC.slave = ruby_system.network.master
 592
 593         sqc_cntrl.responseToSQC = MessageBuffer(ordered = True)
 594         sqc_cntrl.responseToSQC.slave = ruby_system.network.master
 595
 596         sqc_cntrl.mandatoryQueue = MessageBuffer()
 597
 598         # SQC also in GPU cluster
 599         gpuCluster.add(sqc_cntrl)
 600
 601     numa_bit = 6
 602
 603     for i in range(options.num_tccs):
 604
 605         tcc_cntrl = TCCCntrl()
 606         tcc_cntrl.create(options, ruby_system, system)
 607         tcc_cntrl.l2_request_latency = 1
 608         tcc_cntrl.l2_response_latency = options.TCC_latency
 609         tcc_cntrl.WB = options.WB_L2
 610         tcc_cntrl.number_of_TBEs = 2560 * options.num_compute_units
 611
 612         # Connect the TCC controllers to the ruby network
 613         tcc_cntrl.requestFromTCP = MessageBuffer(ordered = True)
 614         tcc_cntrl.requestFromTCP.slave = ruby_system.network.master
 615
 616         tcc_cntrl.responseToCore = MessageBuffer(ordered = True)
 617         tcc_cntrl.responseToCore.master = ruby_system.network.slave
 618
 619         tcc_cntrl.probeFromNB = MessageBuffer()
 620         tcc_cntrl.probeFromNB.slave = ruby_system.network.master
 621
 622         tcc_cntrl.responseFromNB = MessageBuffer()
 623         tcc_cntrl.responseFromNB.slave = ruby_system.network.master
 624
 625         tcc_cntrl.requestToNB = MessageBuffer(ordered = True)
 626         tcc_cntrl.requestToNB.master = ruby_system.network.slave
 627
 628         tcc_cntrl.responseToNB = MessageBuffer()
 629         tcc_cntrl.responseToNB.master = ruby_system.network.slave
 630
 631         tcc_cntrl.unblockToNB = MessageBuffer()
 632         tcc_cntrl.unblockToNB.master = ruby_system.network.slave
 633
 634         tcc_cntrl.triggerQueue = MessageBuffer(ordered = True)
 635
 636         rb_cntrl = RBCntrl()
 637         rb_cntrl.create(options, ruby_system, system)
 638         rb_cntrl.number_of_TBEs = 2560 * options.num_compute_units
 639         rb_cntrl.isOnCPU = False
 640
 641         # Connect the RB controllers to the ruby network
 642         rb_cntrl.requestFromCore = MessageBuffer(ordered = True)
 643         rb_cntrl.requestFromCore.slave = ruby_system.network.master
 644
 645         rb_cntrl.responseFromCore = MessageBuffer()
 646         rb_cntrl.responseFromCore.slave = ruby_system.network.master
 647
 648         rb_cntrl.requestToNetwork = MessageBuffer()
 649         rb_cntrl.requestToNetwork.master = ruby_system.network.slave
 650
 651         rb_cntrl.notifyFromRegionDir = MessageBuffer()
 652         rb_cntrl.notifyFromRegionDir.slave = ruby_system.network.master
 653
 654         rb_cntrl.probeFromRegionDir = MessageBuffer()
 655         rb_cntrl.probeFromRegionDir.slave = ruby_system.network.master
 656
 657         rb_cntrl.unblockFromDir = MessageBuffer()
 658         rb_cntrl.unblockFromDir.slave = ruby_system.network.master
 659
 660         rb_cntrl.responseToRegDir = MessageBuffer()
 661         rb_cntrl.responseToRegDir.master = ruby_system.network.slave
 662
 663         rb_cntrl.triggerQueue = MessageBuffer(ordered = True)
 664
 665         tcc_cntrl.regionBufferNum = rb_cntrl.version
 666
 667         exec("system.tcc_cntrl%d = tcc_cntrl" % i)
 668         exec("system.tcc_rb_cntrl%d = rb_cntrl" % i)
 669
 670         # TCC cntrls added to the GPU cluster
 671         gpuCluster.add(tcc_cntrl)
 672         gpuCluster.add(rb_cntrl)
 673
 674     # Because of wire buffers, num_l3caches must equal num_dirs
 675     # Region coherence only works with 1 dir
 676     assert(options.num_l3caches == options.num_dirs == 1)
 677
 678     # This is the base crossbar that connects the L3s, Dirs, and cpu/gpu
 679     # Clusters
 680     mainCluster = Cluster(intBW = crossbar_bw)
 681
 682     if options.numa_high_bit:
 683         numa_bit = options.numa_high_bit
 684     else:
 685         # if the numa_bit is not specified, set the directory bits as the
 686         # lowest bits above the block offset bits, and the numa_bit as the
 687         # highest of those directory bits
 688         dir_bits = int(math.log(options.num_dirs, 2))
 689         block_size_bits = int(math.log(options.cacheline_size, 2))
 690         numa_bit = block_size_bits + dir_bits - 1
 691
 692     dir_ranges = []
 693     for r in system.mem_ranges:
 694         addr_range = m5.objects.AddrRange(r.start, size = r.size(),
 695                                           intlvHighBit = numa_bit,
 696                                           intlvBits = dir_bits,
 697                                           intlvMatch = i)
 698         dir_ranges.append(addr_range)
 699
 700     dir_cntrl = DirCntrl()
 701     dir_cntrl.create(options, dir_ranges, ruby_system, system)
 702     dir_cntrl.number_of_TBEs = 2560 * options.num_compute_units
 703     dir_cntrl.useL3OnWT = options.use_L3_on_WT
 704
 705     # Connect the Directory controller to the ruby network
 706     dir_cntrl.requestFromCores = MessageBuffer()
 707     dir_cntrl.requestFromCores.slave = ruby_system.network.master
 708
 709     dir_cntrl.responseFromCores = MessageBuffer()
 710     dir_cntrl.responseFromCores.slave = ruby_system.network.master
 711
 712     dir_cntrl.unblockFromCores = MessageBuffer()
 713     dir_cntrl.unblockFromCores.slave = ruby_system.network.master
 714
 715     dir_cntrl.probeToCore = MessageBuffer()
 716     dir_cntrl.probeToCore.master = ruby_system.network.slave
 717
 718     dir_cntrl.responseToCore = MessageBuffer()
 719     dir_cntrl.responseToCore.master = ruby_system.network.slave
 720
 721     dir_cntrl.reqFromRegBuf = MessageBuffer()
 722     dir_cntrl.reqFromRegBuf.slave = ruby_system.network.master
 723
 724     dir_cntrl.reqToRegDir = MessageBuffer(ordered = True)
 725     dir_cntrl.reqToRegDir.master = ruby_system.network.slave
 726
 727     dir_cntrl.reqFromRegDir = MessageBuffer(ordered = True)
 728     dir_cntrl.reqFromRegDir.slave = ruby_system.network.master
 729
 730     dir_cntrl.unblockToRegDir = MessageBuffer()
 731     dir_cntrl.unblockToRegDir.master = ruby_system.network.slave
 732
 733     dir_cntrl.triggerQueue = MessageBuffer(ordered = True)
 734     dir_cntrl.L3triggerQueue = MessageBuffer(ordered = True)
 735     dir_cntrl.requestToMemory = MessageBuffer()
 736     dir_cntrl.responseFromMemory = MessageBuffer()
 737
 738     exec("system.dir_cntrl%d = dir_cntrl" % i)
 739     dir_cntrl_nodes.append(dir_cntrl)
 740
 741     mainCluster.add(dir_cntrl)
 742
 743     reg_cntrl = RegionCntrl(noTCCdir=True,TCC_select_num_bits = TCC_bits)
 744     reg_cntrl.create(options, ruby_system, system)
 745     reg_cntrl.number_of_TBEs = options.num_tbes
 746     reg_cntrl.cpuRegionBufferNum = system.rb_cntrl0.version
 747     reg_cntrl.gpuRegionBufferNum = system.tcc_rb_cntrl0.version
 748
 749     # Connect the Region Dir controllers to the ruby network
 750     reg_cntrl.requestToDir = MessageBuffer(ordered = True)
 751     reg_cntrl.requestToDir.master = ruby_system.network.slave
 752
 753     reg_cntrl.notifyToRBuffer = MessageBuffer()
 754     reg_cntrl.notifyToRBuffer.master = ruby_system.network.slave
 755
 756     reg_cntrl.probeToRBuffer = MessageBuffer()
 757     reg_cntrl.probeToRBuffer.master = ruby_system.network.slave
 758
 759     reg_cntrl.responseFromRBuffer = MessageBuffer()
 760     reg_cntrl.responseFromRBuffer.slave = ruby_system.network.master
 761
 762     reg_cntrl.requestFromRegBuf = MessageBuffer()
 763     reg_cntrl.requestFromRegBuf.slave = ruby_system.network.master
 764
 765     reg_cntrl.triggerQueue = MessageBuffer(ordered = True)
 766
 767     exec("system.reg_cntrl%d = reg_cntrl" % i)
 768
 769     mainCluster.add(reg_cntrl)
 770
 771     # Assuming no DMA devices
 772     assert(len(dma_devices) == 0)
 773
 774     # Add cpu/gpu clusters to main cluster
 775     mainCluster.add(cpuCluster)
 776     mainCluster.add(gpuCluster)
 777
 778     ruby_system.network.number_of_virtual_networks = 10
 779
 780     return (cpu_sequencers, dir_cntrl_nodes, mainCluster)