configs/ruby/GPU_RfO.py

   1 # Copyright (c) 2011-2015 Advanced Micro Devices, Inc.
   2 # All rights reserved.
   3 #
   4 # For use for simulation and test purposes only
   5 #
   6 # Redistribution and use in source and binary forms, with or without
   7 # modification, are permitted provided that the following conditions are met:
   8 #
   9 # 1. Redistributions of source code must retain the above copyright notice,
  10 # this list of conditions and the following disclaimer.
  11 #
  12 # 2. Redistributions in binary form must reproduce the above copyright notice,
  13 # this list of conditions and the following disclaimer in the documentation
  14 # and/or other materials provided with the distribution.
  15 #
  16 # 3. Neither the name of the copyright holder nor the names of its
  17 # contributors may be used to endorse or promote products derived from this
  18 # software without specific prior written permission.
  19 #
  20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23 # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
  24 # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30 # POSSIBILITY OF SUCH DAMAGE.
  31
  32 import math
  33 import m5
  34 from m5.objects import *
  35 from m5.defines import buildEnv
  36 from m5.util import addToPath
  37 from Ruby import create_topology
  38 from Ruby import send_evicts
  39
  40 addToPath('../')
  41
  42 from topologies.Cluster import Cluster
  43 from topologies.Crossbar import Crossbar
  44
  45 class CntrlBase:
  46     _seqs = 0
  47     @classmethod
  48     def seqCount(cls):
  49         # Use SeqCount not class since we need global count
  50         CntrlBase._seqs += 1
  51         return CntrlBase._seqs - 1
  52
  53     _cntrls = 0
  54     @classmethod
  55     def cntrlCount(cls):
  56         # Use CntlCount not class since we need global count
  57         CntrlBase._cntrls += 1
  58         return CntrlBase._cntrls - 1
  59
  60     _version = 0
  61     @classmethod
  62     def versionCount(cls):
  63         cls._version += 1 # Use count for this particular type
  64         return cls._version - 1
  65
  66 class TccDirCache(RubyCache):
  67     size = "512kB"
  68     assoc = 16
  69     resourceStalls = False
  70     def create(self, options):
  71         self.size = MemorySize(options.tcc_size)
  72         self.size.value += (options.num_compute_units *
  73                             (MemorySize(options.tcp_size).value) *
  74                             options.tcc_dir_factor) / long(options.num_tccs)
  75         self.start_index_bit = math.log(options.cacheline_size, 2) + \
  76                                math.log(options.num_tccs, 2)
  77         self.replacement_policy = TreePLRURP()
  78
  79 class L1DCache(RubyCache):
  80     resourceStalls = False
  81     def create(self, options):
  82         self.size = MemorySize(options.l1d_size)
  83         self.assoc = options.l1d_assoc
  84         self.replacement_policy = TreePLRURP()
  85
  86 class L1ICache(RubyCache):
  87     resourceStalls = False
  88     def create(self, options):
  89         self.size = MemorySize(options.l1i_size)
  90         self.assoc = options.l1i_assoc
  91         self.replacement_policy = TreePLRURP()
  92
  93 class L2Cache(RubyCache):
  94     resourceStalls = False
  95     def create(self, options):
  96         self.size = MemorySize(options.l2_size)
  97         self.assoc = options.l2_assoc
  98         self.replacement_policy = TreePLRURP()
  99
 100
 101 class CPCntrl(CorePair_Controller, CntrlBase):
 102
 103     def create(self, options, ruby_system, system):
 104         self.version = self.versionCount()
 105
 106         self.L1Icache = L1ICache()
 107         self.L1Icache.create(options)
 108         self.L1D0cache = L1DCache()
 109         self.L1D0cache.create(options)
 110         self.L1D1cache = L1DCache()
 111         self.L1D1cache.create(options)
 112         self.L2cache = L2Cache()
 113         self.L2cache.create(options)
 114
 115         self.sequencer = RubySequencer()
 116         self.sequencer.version = self.seqCount()
 117         self.sequencer.icache = self.L1Icache
 118         self.sequencer.dcache = self.L1D0cache
 119         self.sequencer.ruby_system = ruby_system
 120         self.sequencer.coreid = 0
 121         self.sequencer.is_cpu_sequencer = True
 122
 123         self.sequencer1 = RubySequencer()
 124         self.sequencer1.version = self.seqCount()
 125         self.sequencer1.icache = self.L1Icache
 126         self.sequencer1.dcache = self.L1D1cache
 127         self.sequencer1.ruby_system = ruby_system
 128         self.sequencer1.coreid = 1
 129         self.sequencer1.is_cpu_sequencer = True
 130
 131         # Defines icache/dcache hit latency
 132         self.mandatory_queue_latency = 2
 133
 134         self.issue_latency = options.cpu_to_dir_latency
 135         self.send_evictions = send_evicts(options)
 136
 137         self.ruby_system = ruby_system
 138
 139         if options.recycle_latency:
 140             self.recycle_latency = options.recycle_latency
 141
 142 class TCPCache(RubyCache):
 143     assoc = 8
 144     dataArrayBanks = 16
 145     tagArrayBanks = 4
 146     dataAccessLatency = 4
 147     tagAccessLatency = 1
 148     def create(self, options):
 149         self.size = MemorySize(options.tcp_size)
 150         self.replacement_policy = TreePLRURP()
 151
 152 class TCPCntrl(TCP_Controller, CntrlBase):
 153
 154     def create(self, options, ruby_system, system):
 155         self.version = self.versionCount()
 156
 157         self.L1cache = TCPCache(tagAccessLatency = options.TCP_latency)
 158         self.L1cache.resourceStalls = options.no_resource_stalls
 159         self.L1cache.create(options)
 160
 161         self.coalescer = RubyGPUCoalescer()
 162         self.coalescer.version = self.seqCount()
 163         self.coalescer.icache = self.L1cache
 164         self.coalescer.dcache = self.L1cache
 165         self.coalescer.ruby_system = ruby_system
 166         self.coalescer.support_inst_reqs = False
 167         self.coalescer.is_cpu_sequencer = False
 168         self.coalescer.max_outstanding_requests = options.simds_per_cu * \
 169                                                   options.wfs_per_simd * \
 170                                                   options.wf_size
 171
 172         self.sequencer = RubySequencer()
 173         self.sequencer.version = self.seqCount()
 174         self.sequencer.icache = self.L1cache
 175         self.sequencer.dcache = self.L1cache
 176         self.sequencer.ruby_system = ruby_system
 177         self.sequencer.is_cpu_sequencer = True
 178
 179         self.use_seq_not_coal = False
 180
 181         self.ruby_system = ruby_system
 182
 183         if options.recycle_latency:
 184             self.recycle_latency = options.recycle_latency
 185
 186     def createCP(self, options, ruby_system, system):
 187         self.version = self.versionCount()
 188
 189         self.L1cache = TCPCache(tagAccessLatency = options.TCP_latency)
 190         self.L1cache.resourceStalls = options.no_resource_stalls
 191         self.L1cache.create(options)
 192
 193         self.coalescer = RubyGPUCoalescer()
 194         self.coalescer.version = self.seqCount()
 195         self.coalescer.icache = self.L1cache
 196         self.coalescer.dcache = self.L1cache
 197         self.coalescer.ruby_system = ruby_system
 198         self.coalescer.support_inst_reqs = False
 199         self.coalescer.is_cpu_sequencer = False
 200
 201         self.sequencer = RubySequencer()
 202         self.sequencer.version = self.seqCount()
 203         self.sequencer.icache = self.L1cache
 204         self.sequencer.dcache = self.L1cache
 205         self.sequencer.ruby_system = ruby_system
 206         self.sequencer.is_cpu_sequencer = True
 207
 208         self.use_seq_not_coal = True
 209
 210         self.ruby_system = ruby_system
 211
 212         if options.recycle_latency:
 213             self.recycle_latency = options.recycle_latency
 214
 215 class SQCCache(RubyCache):
 216     size = "32kB"
 217     assoc = 8
 218     dataArrayBanks = 16
 219     tagArrayBanks = 4
 220     dataAccessLatency = 4
 221     tagAccessLatency = 1
 222     def create(self, options):
 223         self.replacement_policy = TreePLRURP()
 224
 225 class SQCCntrl(SQC_Controller, CntrlBase):
 226
 227     def create(self, options, ruby_system, system):
 228         self.version = self.versionCount()
 229
 230         self.L1cache = SQCCache()
 231         self.L1cache.create(options)
 232         self.L1cache.resourceStalls = options.no_resource_stalls
 233
 234         self.sequencer = RubySequencer()
 235
 236         self.sequencer.version = self.seqCount()
 237         self.sequencer.icache = self.L1cache
 238         self.sequencer.dcache = self.L1cache
 239         self.sequencer.ruby_system = ruby_system
 240         self.sequencer.support_data_reqs = False
 241         self.sequencer.is_cpu_sequencer = False
 242
 243         self.ruby_system = ruby_system
 244
 245         if options.recycle_latency:
 246             self.recycle_latency = options.recycle_latency
 247
 248     def createCP(self, options, ruby_system, system):
 249         self.version = self.versionCount()
 250
 251         self.L1cache = SQCCache()
 252         self.L1cache.create(options)
 253         self.L1cache.resourceStalls = options.no_resource_stalls
 254
 255         self.sequencer = RubySequencer()
 256
 257         self.sequencer.version = self.seqCount()
 258         self.sequencer.icache = self.L1cache
 259         self.sequencer.dcache = self.L1cache
 260         self.sequencer.ruby_system = ruby_system
 261         self.sequencer.support_data_reqs = False
 262
 263         self.ruby_system = ruby_system
 264
 265         if options.recycle_latency:
 266             self.recycle_latency = options.recycle_latency
 267
 268
 269 class TCC(RubyCache):
 270     assoc = 16
 271     dataAccessLatency = 8
 272     tagAccessLatency = 2
 273     resourceStalls = True
 274     def create(self, options):
 275         self.size = MemorySize(options.tcc_size)
 276         self.size = self.size / options.num_tccs
 277         self.dataArrayBanks = 256 / options.num_tccs #number of data banks
 278         self.tagArrayBanks = 256 / options.num_tccs #number of tag banks
 279         if ((self.size.value / long(self.assoc)) < 128):
 280             self.size.value = long(128 * self.assoc)
 281         self.start_index_bit = math.log(options.cacheline_size, 2) + \
 282                                math.log(options.num_tccs, 2)
 283         self.replacement_policy = TreePLRURP()
 284
 285 class TCCCntrl(TCC_Controller, CntrlBase):
 286     def create(self, options, ruby_system, system):
 287         self.version = self.versionCount()
 288         self.L2cache = TCC()
 289         self.L2cache.create(options)
 290         self.l2_response_latency = options.TCC_latency
 291
 292         self.number_of_TBEs = 2048
 293
 294         self.ruby_system = ruby_system
 295
 296         if options.recycle_latency:
 297             self.recycle_latency = options.recycle_latency
 298
 299     def connectWireBuffers(self, req_to_tccdir, resp_to_tccdir,
 300                            tcc_unblock_to_tccdir, req_to_tcc,
 301                            probe_to_tcc, resp_to_tcc):
 302         self.w_reqToTCCDir = req_to_tccdir
 303         self.w_respToTCCDir = resp_to_tccdir
 304         self.w_TCCUnblockToTCCDir = tcc_unblock_to_tccdir
 305         self.w_reqToTCC = req_to_tcc
 306         self.w_probeToTCC = probe_to_tcc
 307         self.w_respToTCC = resp_to_tcc
 308
 309 class TCCDirCntrl(TCCdir_Controller, CntrlBase):
 310     def create(self, options, ruby_system, system):
 311         self.version = self.versionCount()
 312
 313         self.directory = TccDirCache()
 314         self.directory.create(options)
 315
 316         self.number_of_TBEs = 1024
 317
 318         self.ruby_system = ruby_system
 319
 320         if options.recycle_latency:
 321             self.recycle_latency = options.recycle_latency
 322
 323     def connectWireBuffers(self, req_to_tccdir, resp_to_tccdir,
 324                            tcc_unblock_to_tccdir, req_to_tcc,
 325                            probe_to_tcc, resp_to_tcc):
 326         self.w_reqToTCCDir = req_to_tccdir
 327         self.w_respToTCCDir = resp_to_tccdir
 328         self.w_TCCUnblockToTCCDir = tcc_unblock_to_tccdir
 329         self.w_reqToTCC = req_to_tcc
 330         self.w_probeToTCC = probe_to_tcc
 331         self.w_respToTCC = resp_to_tcc
 332
 333 class L3Cache(RubyCache):
 334     assoc = 8
 335     dataArrayBanks = 256
 336     tagArrayBanks = 256
 337
 338     def create(self, options, ruby_system, system):
 339         self.size = MemorySize(options.l3_size)
 340         self.size.value /= options.num_dirs
 341         self.dataArrayBanks /= options.num_dirs
 342         self.tagArrayBanks /= options.num_dirs
 343         self.dataArrayBanks /= options.num_dirs
 344         self.tagArrayBanks /= options.num_dirs
 345         self.dataAccessLatency = options.l3_data_latency
 346         self.tagAccessLatency = options.l3_tag_latency
 347         self.resourceStalls = options.no_resource_stalls
 348         self.replacement_policy = TreePLRURP()
 349
 350 class L3Cntrl(L3Cache_Controller, CntrlBase):
 351     def create(self, options, ruby_system, system):
 352         self.version = self.versionCount()
 353         self.L3cache = L3Cache()
 354         self.L3cache.create(options, ruby_system, system)
 355
 356         self.l3_response_latency = max(self.L3cache.dataAccessLatency,
 357                                        self.L3cache.tagAccessLatency)
 358         self.ruby_system = ruby_system
 359
 360         if options.recycle_latency:
 361             self.recycle_latency = options.recycle_latency
 362
 363     def connectWireBuffers(self, req_to_dir, resp_to_dir, l3_unblock_to_dir,
 364                            req_to_l3, probe_to_l3, resp_to_l3):
 365         self.reqToDir = req_to_dir
 366         self.respToDir = resp_to_dir
 367         self.l3UnblockToDir = l3_unblock_to_dir
 368         self.reqToL3 = req_to_l3
 369         self.probeToL3 = probe_to_l3
 370         self.respToL3 = resp_to_l3
 371
 372 class DirCntrl(Directory_Controller, CntrlBase):
 373     def create(self, options, dir_ranges, ruby_system, system):
 374         self.version = self.versionCount()
 375
 376         self.response_latency = 30
 377
 378         self.addr_ranges = dir_ranges
 379         self.directory = RubyDirectoryMemory()
 380
 381         self.L3CacheMemory = L3Cache()
 382         self.L3CacheMemory.create(options, ruby_system, system)
 383
 384         self.l3_hit_latency = max(self.L3CacheMemory.dataAccessLatency,
 385                                   self.L3CacheMemory.tagAccessLatency)
 386
 387         self.number_of_TBEs = options.num_tbes
 388
 389         self.ruby_system = ruby_system
 390
 391         if options.recycle_latency:
 392             self.recycle_latency = options.recycle_latency
 393
 394     def connectWireBuffers(self, req_to_dir, resp_to_dir, l3_unblock_to_dir,
 395                            req_to_l3, probe_to_l3, resp_to_l3):
 396         self.reqToDir = req_to_dir
 397         self.respToDir = resp_to_dir
 398         self.l3UnblockToDir = l3_unblock_to_dir
 399         self.reqToL3 = req_to_l3
 400         self.probeToL3 = probe_to_l3
 401         self.respToL3 = resp_to_l3
 402
 403
 404
 405 def define_options(parser):
 406     parser.add_option("--num-subcaches", type="int", default=4)
 407     parser.add_option("--l3-data-latency", type="int", default=20)
 408     parser.add_option("--l3-tag-latency", type="int", default=15)
 409     parser.add_option("--cpu-to-dir-latency", type="int", default=15)
 410     parser.add_option("--gpu-to-dir-latency", type="int", default=160)
 411     parser.add_option("--no-resource-stalls", action="store_false",
 412                       default=True)
 413     parser.add_option("--num-tbes", type="int", default=256)
 414     parser.add_option("--l2-latency", type="int", default=50) # load to use
 415     parser.add_option("--num-tccs", type="int", default=1,
 416                       help="number of TCC directories and banks in the GPU")
 417     parser.add_option("--TCP_latency", type="int", default=4,
 418                       help="TCP latency")
 419     parser.add_option("--TCC_latency", type="int", default=16,
 420                       help="TCC latency")
 421     parser.add_option("--tcc-size", type='string', default='256kB',
 422                       help="agregate tcc size")
 423     parser.add_option("--tcp-size", type='string', default='16kB',
 424                       help="tcp size")
 425     parser.add_option("--tcc-dir-factor", type='int', default=4,
 426                       help="TCCdir size = factor *(TCPs + TCC)")
 427
 428 def create_system(options, full_system, system, dma_devices, bootmem,
 429                   ruby_system):
 430     if buildEnv['PROTOCOL'] != 'GPU_RfO':
 431         panic("This script requires the GPU_RfO protocol to be built.")
 432
 433     cpu_sequencers = []
 434
 435     #
 436     # The ruby network creation expects the list of nodes in the system to be
 437     # consistent with the NetDest list.  Therefore the l1 controller nodes
 438     # must be listed before the directory nodes and directory nodes before
 439     # dma nodes, etc.
 440     #
 441     cp_cntrl_nodes = []
 442     tcp_cntrl_nodes = []
 443     sqc_cntrl_nodes = []
 444     tcc_cntrl_nodes = []
 445     tccdir_cntrl_nodes = []
 446     dir_cntrl_nodes = []
 447     l3_cntrl_nodes = []
 448
 449     #
 450     # Must create the individual controllers before the network to ensure the
 451     # controller constructors are called before the network constructor
 452     #
 453
 454     TCC_bits = int(math.log(options.num_tccs, 2))
 455
 456     # This is the base crossbar that connects the L3s, Dirs, and cpu/gpu
 457     # Clusters
 458     mainCluster = Cluster(extBW = 512, intBW = 512) # 1 TB/s
 459
 460     if options.numa_high_bit:
 461         numa_bit = options.numa_high_bit
 462     else:
 463         # if the numa_bit is not specified, set the directory bits as the
 464         # lowest bits above the block offset bits, and the numa_bit as the
 465         # highest of those directory bits
 466         dir_bits = int(math.log(options.num_dirs, 2))
 467         block_size_bits = int(math.log(options.cacheline_size, 2))
 468         numa_bit = block_size_bits + dir_bits - 1
 469
 470     for i in range(options.num_dirs):
 471         dir_ranges = []
 472         for r in system.mem_ranges:
 473             addr_range = m5.objects.AddrRange(r.start, size = r.size(),
 474                                               intlvHighBit = numa_bit,
 475                                               intlvBits = dir_bits,
 476                                               intlvMatch = i)
 477             dir_ranges.append(addr_range)
 478
 479         dir_cntrl = DirCntrl(TCC_select_num_bits = TCC_bits)
 480         dir_cntrl.create(options, dir_ranges, ruby_system, system)
 481         dir_cntrl.number_of_TBEs = 2560 * options.num_compute_units
 482         #Enough TBEs for all TCP TBEs
 483
 484         # Connect the Directory controller to the ruby network
 485         dir_cntrl.requestFromCores = MessageBuffer(ordered = True)
 486         dir_cntrl.requestFromCores.slave = ruby_system.network.master
 487
 488         dir_cntrl.responseFromCores = MessageBuffer()
 489         dir_cntrl.responseFromCores.slave = ruby_system.network.master
 490
 491         dir_cntrl.unblockFromCores = MessageBuffer()
 492         dir_cntrl.unblockFromCores.slave = ruby_system.network.master
 493
 494         dir_cntrl.probeToCore = MessageBuffer()
 495         dir_cntrl.probeToCore.master = ruby_system.network.slave
 496
 497         dir_cntrl.responseToCore = MessageBuffer()
 498         dir_cntrl.responseToCore.master = ruby_system.network.slave
 499
 500         dir_cntrl.triggerQueue = MessageBuffer(ordered = True)
 501         dir_cntrl.L3triggerQueue = MessageBuffer(ordered = True)
 502         dir_cntrl.responseFromMemory = MessageBuffer()
 503
 504         exec("system.dir_cntrl%d = dir_cntrl" % i)
 505         dir_cntrl_nodes.append(dir_cntrl)
 506
 507         mainCluster.add(dir_cntrl)
 508
 509     # For an odd number of CPUs, still create the right number of controllers
 510     cpuCluster = Cluster(extBW = 512, intBW = 512)  # 1 TB/s
 511     for i in range((options.num_cpus + 1) // 2):
 512
 513         cp_cntrl = CPCntrl()
 514         cp_cntrl.create(options, ruby_system, system)
 515
 516         exec("system.cp_cntrl%d = cp_cntrl" % i)
 517         #
 518         # Add controllers and sequencers to the appropriate lists
 519         #
 520         cpu_sequencers.extend([cp_cntrl.sequencer, cp_cntrl.sequencer1])
 521
 522         # Connect the CP controllers and the network
 523         cp_cntrl.requestFromCore = MessageBuffer()
 524         cp_cntrl.requestFromCore.master = ruby_system.network.slave
 525
 526         cp_cntrl.responseFromCore = MessageBuffer()
 527         cp_cntrl.responseFromCore.master = ruby_system.network.slave
 528
 529         cp_cntrl.unblockFromCore = MessageBuffer()
 530         cp_cntrl.unblockFromCore.master = ruby_system.network.slave
 531
 532         cp_cntrl.probeToCore = MessageBuffer()
 533         cp_cntrl.probeToCore.slave = ruby_system.network.master
 534
 535         cp_cntrl.responseToCore = MessageBuffer()
 536         cp_cntrl.responseToCore.slave = ruby_system.network.master
 537
 538         cp_cntrl.mandatoryQueue = MessageBuffer()
 539         cp_cntrl.triggerQueue = MessageBuffer(ordered = True)
 540
 541         cpuCluster.add(cp_cntrl)
 542
 543     gpuCluster = Cluster(extBW = 512, intBW = 512)  # 1 TB/s
 544
 545     for i in range(options.num_compute_units):
 546
 547         tcp_cntrl = TCPCntrl(TCC_select_num_bits = TCC_bits,
 548                              number_of_TBEs = 2560) # max outstanding requests
 549         tcp_cntrl.create(options, ruby_system, system)
 550
 551         exec("system.tcp_cntrl%d = tcp_cntrl" % i)
 552         #
 553         # Add controllers and sequencers to the appropriate lists
 554         #
 555         cpu_sequencers.append(tcp_cntrl.coalescer)
 556         tcp_cntrl_nodes.append(tcp_cntrl)
 557
 558         # Connect the TCP controller to the ruby network
 559         tcp_cntrl.requestFromTCP = MessageBuffer(ordered = True)
 560         tcp_cntrl.requestFromTCP.master = ruby_system.network.slave
 561
 562         tcp_cntrl.responseFromTCP = MessageBuffer(ordered = True)
 563         tcp_cntrl.responseFromTCP.master = ruby_system.network.slave
 564
 565         tcp_cntrl.unblockFromCore = MessageBuffer(ordered = True)
 566         tcp_cntrl.unblockFromCore.master = ruby_system.network.slave
 567
 568         tcp_cntrl.probeToTCP = MessageBuffer(ordered = True)
 569         tcp_cntrl.probeToTCP.slave = ruby_system.network.master
 570
 571         tcp_cntrl.responseToTCP = MessageBuffer(ordered = True)
 572         tcp_cntrl.responseToTCP.slave = ruby_system.network.master
 573
 574         tcp_cntrl.mandatoryQueue = MessageBuffer()
 575
 576         gpuCluster.add(tcp_cntrl)
 577
 578     for i in range(options.num_sqc):
 579
 580         sqc_cntrl = SQCCntrl(TCC_select_num_bits = TCC_bits)
 581         sqc_cntrl.create(options, ruby_system, system)
 582
 583         exec("system.sqc_cntrl%d = sqc_cntrl" % i)
 584         #
 585         # Add controllers and sequencers to the appropriate lists
 586         #
 587         cpu_sequencers.append(sqc_cntrl.sequencer)
 588
 589         # Connect the SQC controller to the ruby network
 590         sqc_cntrl.requestFromSQC = MessageBuffer(ordered = True)
 591         sqc_cntrl.requestFromSQC.master = ruby_system.network.slave
 592
 593         sqc_cntrl.responseFromSQC = MessageBuffer(ordered = True)
 594         sqc_cntrl.responseFromSQC.master = ruby_system.network.slave
 595
 596         sqc_cntrl.unblockFromCore = MessageBuffer(ordered = True)
 597         sqc_cntrl.unblockFromCore.master = ruby_system.network.slave
 598
 599         sqc_cntrl.probeToSQC = MessageBuffer(ordered = True)
 600         sqc_cntrl.probeToSQC.slave = ruby_system.network.master
 601
 602         sqc_cntrl.responseToSQC = MessageBuffer(ordered = True)
 603         sqc_cntrl.responseToSQC.slave = ruby_system.network.master
 604
 605         sqc_cntrl.mandatoryQueue = MessageBuffer()
 606
 607         # SQC also in GPU cluster
 608         gpuCluster.add(sqc_cntrl)
 609
 610     for i in range(options.num_cp):
 611
 612         tcp_cntrl = TCPCntrl(TCC_select_num_bits = TCC_bits,
 613                              number_of_TBEs = 2560) # max outstanding requests
 614         tcp_cntrl.createCP(options, ruby_system, system)
 615
 616         exec("system.tcp_cntrl%d = tcp_cntrl" % (options.num_compute_units + i))
 617         #
 618         # Add controllers and sequencers to the appropriate lists
 619         #
 620         cpu_sequencers.append(tcp_cntrl.sequencer)
 621         tcp_cntrl_nodes.append(tcp_cntrl)
 622
 623         # Connect the TCP controller to the ruby network
 624         tcp_cntrl.requestFromTCP = MessageBuffer(ordered = True)
 625         tcp_cntrl.requestFromTCP.master = ruby_system.network.slave
 626
 627         tcp_cntrl.responseFromTCP = MessageBuffer(ordered = True)
 628         tcp_cntrl.responseFromTCP.master = ruby_system.network.slave
 629
 630         tcp_cntrl.unblockFromCore = MessageBuffer(ordered = True)
 631         tcp_cntrl.unblockFromCore.master = ruby_system.network.slave
 632
 633         tcp_cntrl.probeToTCP = MessageBuffer(ordered = True)
 634         tcp_cntrl.probeToTCP.slave = ruby_system.network.master
 635
 636         tcp_cntrl.responseToTCP = MessageBuffer(ordered = True)
 637         tcp_cntrl.responseToTCP.slave = ruby_system.network.master
 638
 639         tcp_cntrl.mandatoryQueue = MessageBuffer()
 640
 641         gpuCluster.add(tcp_cntrl)
 642
 643         sqc_cntrl = SQCCntrl(TCC_select_num_bits = TCC_bits)
 644         sqc_cntrl.createCP(options, ruby_system, system)
 645
 646         exec("system.sqc_cntrl%d = sqc_cntrl" % (options.num_compute_units + i))
 647         #
 648         # Add controllers and sequencers to the appropriate lists
 649         #
 650         cpu_sequencers.append(sqc_cntrl.sequencer)
 651
 652         # Connect the SQC controller to the ruby network
 653         sqc_cntrl.requestFromSQC = MessageBuffer(ordered = True)
 654         sqc_cntrl.requestFromSQC.master = ruby_system.network.slave
 655
 656         sqc_cntrl.responseFromSQC = MessageBuffer(ordered = True)
 657         sqc_cntrl.responseFromSQC.master = ruby_system.network.slave
 658
 659         sqc_cntrl.unblockFromCore = MessageBuffer(ordered = True)
 660         sqc_cntrl.unblockFromCore.master = ruby_system.network.slave
 661
 662         sqc_cntrl.probeToSQC = MessageBuffer(ordered = True)
 663         sqc_cntrl.probeToSQC.slave = ruby_system.network.master
 664
 665         sqc_cntrl.responseToSQC = MessageBuffer(ordered = True)
 666         sqc_cntrl.responseToSQC.slave = ruby_system.network.master
 667
 668         sqc_cntrl.mandatoryQueue = MessageBuffer()
 669
 670         # SQC also in GPU cluster
 671         gpuCluster.add(sqc_cntrl)
 672
 673     for i in range(options.num_tccs):
 674
 675         tcc_cntrl = TCCCntrl(TCC_select_num_bits = TCC_bits,
 676                              number_of_TBEs = options.num_compute_units * 2560)
 677         #Enough TBEs for all TCP TBEs
 678         tcc_cntrl.create(options, ruby_system, system)
 679         tcc_cntrl_nodes.append(tcc_cntrl)
 680
 681         tccdir_cntrl = TCCDirCntrl(TCC_select_num_bits = TCC_bits,
 682                               number_of_TBEs = options.num_compute_units * 2560)
 683         #Enough TBEs for all TCP TBEs
 684         tccdir_cntrl.create(options, ruby_system, system)
 685         tccdir_cntrl_nodes.append(tccdir_cntrl)
 686
 687         exec("system.tcc_cntrl%d = tcc_cntrl" % i)
 688         exec("system.tccdir_cntrl%d = tccdir_cntrl" % i)
 689
 690         # connect all of the wire buffers between L3 and dirs up
 691         req_to_tccdir = RubyWireBuffer()
 692         resp_to_tccdir = RubyWireBuffer()
 693         tcc_unblock_to_tccdir = RubyWireBuffer()
 694         req_to_tcc = RubyWireBuffer()
 695         probe_to_tcc = RubyWireBuffer()
 696         resp_to_tcc = RubyWireBuffer()
 697
 698         tcc_cntrl.connectWireBuffers(req_to_tccdir, resp_to_tccdir,
 699                                      tcc_unblock_to_tccdir, req_to_tcc,
 700                                      probe_to_tcc, resp_to_tcc)
 701         tccdir_cntrl.connectWireBuffers(req_to_tccdir, resp_to_tccdir,
 702                                         tcc_unblock_to_tccdir, req_to_tcc,
 703                                         probe_to_tcc, resp_to_tcc)
 704
 705         # Connect the TCC controller to the ruby network
 706         tcc_cntrl.responseFromTCC = MessageBuffer(ordered = True)
 707         tcc_cntrl.responseFromTCC.master = ruby_system.network.slave
 708
 709         tcc_cntrl.responseToTCC = MessageBuffer(ordered = True)
 710         tcc_cntrl.responseToTCC.slave = ruby_system.network.master
 711
 712         # Connect the TCC Dir controller to the ruby network
 713         tccdir_cntrl.requestFromTCP = MessageBuffer(ordered = True)
 714         tccdir_cntrl.requestFromTCP.slave = ruby_system.network.master
 715
 716         tccdir_cntrl.responseFromTCP = MessageBuffer(ordered = True)
 717         tccdir_cntrl.responseFromTCP.slave = ruby_system.network.master
 718
 719         tccdir_cntrl.unblockFromTCP = MessageBuffer(ordered = True)
 720         tccdir_cntrl.unblockFromTCP.slave = ruby_system.network.master
 721
 722         tccdir_cntrl.probeToCore = MessageBuffer(ordered = True)
 723         tccdir_cntrl.probeToCore.master = ruby_system.network.slave
 724
 725         tccdir_cntrl.responseToCore = MessageBuffer(ordered = True)
 726         tccdir_cntrl.responseToCore.master = ruby_system.network.slave
 727
 728         tccdir_cntrl.probeFromNB = MessageBuffer()
 729         tccdir_cntrl.probeFromNB.slave = ruby_system.network.master
 730
 731         tccdir_cntrl.responseFromNB = MessageBuffer()
 732         tccdir_cntrl.responseFromNB.slave = ruby_system.network.master
 733
 734         tccdir_cntrl.requestToNB = MessageBuffer()
 735         tccdir_cntrl.requestToNB.master = ruby_system.network.slave
 736
 737         tccdir_cntrl.responseToNB = MessageBuffer()
 738         tccdir_cntrl.responseToNB.master = ruby_system.network.slave
 739
 740         tccdir_cntrl.unblockToNB = MessageBuffer()
 741         tccdir_cntrl.unblockToNB.master = ruby_system.network.slave
 742
 743         tccdir_cntrl.triggerQueue = MessageBuffer(ordered = True)
 744
 745         # TCC cntrls added to the GPU cluster
 746         gpuCluster.add(tcc_cntrl)
 747         gpuCluster.add(tccdir_cntrl)
 748
 749     # Assuming no DMA devices
 750     assert(len(dma_devices) == 0)
 751
 752     # Add cpu/gpu clusters to main cluster
 753     mainCluster.add(cpuCluster)
 754     mainCluster.add(gpuCluster)
 755
 756     ruby_system.network.number_of_virtual_networks = 10
 757
 758     return (cpu_sequencers, dir_cntrl_nodes, mainCluster)