configs/ruby/GPU_RfO.py

   1 # Copyright (c) 2011-2015 Advanced Micro Devices, Inc.
   2 # All rights reserved.
   3 #
   4 # For use for simulation and test purposes only
   5 #
   6 # Redistribution and use in source and binary forms, with or without
   7 # modification, are permitted provided that the following conditions are met:
   8 #
   9 # 1. Redistributions of source code must retain the above copyright notice,
  10 # this list of conditions and the following disclaimer.
  11 #
  12 # 2. Redistributions in binary form must reproduce the above copyright notice,
  13 # this list of conditions and the following disclaimer in the documentation
  14 # and/or other materials provided with the distribution.
  15 #
  16 # 3. Neither the name of the copyright holder nor the names of its
  17 # contributors may be used to endorse or promote products derived from this
  18 # software without specific prior written permission.
  19 #
  20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23 # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
  24 # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30 # POSSIBILITY OF SUCH DAMAGE.
  31
  32 import math
  33 import m5
  34 from m5.objects import *
  35 from m5.defines import buildEnv
  36 from m5.util import addToPath
  37 from Ruby import create_topology
  38 from Ruby import send_evicts
  39
  40 addToPath('../')
  41
  42 from topologies.Cluster import Cluster
  43 from topologies.Crossbar import Crossbar
  44
  45 class CntrlBase:
  46     _seqs = 0
  47     @classmethod
  48     def seqCount(cls):
  49         # Use SeqCount not class since we need global count
  50         CntrlBase._seqs += 1
  51         return CntrlBase._seqs - 1
  52
  53     _cntrls = 0
  54     @classmethod
  55     def cntrlCount(cls):
  56         # Use CntlCount not class since we need global count
  57         CntrlBase._cntrls += 1
  58         return CntrlBase._cntrls - 1
  59
  60     _version = 0
  61     @classmethod
  62     def versionCount(cls):
  63         cls._version += 1 # Use count for this particular type
  64         return cls._version - 1
  65
  66 class TccDirCache(RubyCache):
  67     size = "512kB"
  68     assoc = 16
  69     resourceStalls = False
  70     def create(self, options):
  71         self.size = MemorySize(options.tcc_size)
  72         self.size.value += (options.num_compute_units *
  73                             (MemorySize(options.tcp_size).value) *
  74                             options.tcc_dir_factor) / long(options.num_tccs)
  75         self.start_index_bit = math.log(options.cacheline_size, 2) + \
  76                                math.log(options.num_tccs, 2)
  77         self.replacement_policy = TreePLRURP()
  78
  79 class L1DCache(RubyCache):
  80     resourceStalls = False
  81     def create(self, options):
  82         self.size = MemorySize(options.l1d_size)
  83         self.assoc = options.l1d_assoc
  84         self.replacement_policy = TreePLRURP()
  85
  86 class L1ICache(RubyCache):
  87     resourceStalls = False
  88     def create(self, options):
  89         self.size = MemorySize(options.l1i_size)
  90         self.assoc = options.l1i_assoc
  91         self.replacement_policy = TreePLRURP()
  92
  93 class L2Cache(RubyCache):
  94     resourceStalls = False
  95     def create(self, options):
  96         self.size = MemorySize(options.l2_size)
  97         self.assoc = options.l2_assoc
  98         self.replacement_policy = TreePLRURP()
  99
 100
 101 class CPCntrl(CorePair_Controller, CntrlBase):
 102
 103     def create(self, options, ruby_system, system):
 104         self.version = self.versionCount()
 105
 106         self.L1Icache = L1ICache()
 107         self.L1Icache.create(options)
 108         self.L1D0cache = L1DCache()
 109         self.L1D0cache.create(options)
 110         self.L1D1cache = L1DCache()
 111         self.L1D1cache.create(options)
 112         self.L2cache = L2Cache()
 113         self.L2cache.create(options)
 114
 115         self.sequencer = RubySequencer()
 116         self.sequencer.version = self.seqCount()
 117         self.sequencer.icache = self.L1Icache
 118         self.sequencer.dcache = self.L1D0cache
 119         self.sequencer.ruby_system = ruby_system
 120         self.sequencer.coreid = 0
 121         self.sequencer.is_cpu_sequencer = True
 122
 123         self.sequencer1 = RubySequencer()
 124         self.sequencer1.version = self.seqCount()
 125         self.sequencer1.icache = self.L1Icache
 126         self.sequencer1.dcache = self.L1D1cache
 127         self.sequencer1.ruby_system = ruby_system
 128         self.sequencer1.coreid = 1
 129         self.sequencer1.is_cpu_sequencer = True
 130
 131         # Defines icache/dcache hit latency
 132         self.mandatory_queue_latency = 2
 133
 134         self.issue_latency = options.cpu_to_dir_latency
 135         self.send_evictions = send_evicts(options)
 136
 137         self.ruby_system = ruby_system
 138
 139         if options.recycle_latency:
 140             self.recycle_latency = options.recycle_latency
 141
 142 class TCPCache(RubyCache):
 143     assoc = 8
 144     dataArrayBanks = 16
 145     tagArrayBanks = 4
 146     dataAccessLatency = 4
 147     tagAccessLatency = 1
 148     def create(self, options):
 149         self.size = MemorySize(options.tcp_size)
 150         self.replacement_policy = TreePLRURP()
 151
 152 class TCPCntrl(TCP_Controller, CntrlBase):
 153
 154     def create(self, options, ruby_system, system):
 155         self.version = self.versionCount()
 156
 157         self.L1cache = TCPCache(tagAccessLatency = options.TCP_latency)
 158         self.L1cache.resourceStalls = options.no_resource_stalls
 159         self.L1cache.create(options)
 160
 161         self.coalescer = RubyGPUCoalescer()
 162         self.coalescer.version = self.seqCount()
 163         self.coalescer.icache = self.L1cache
 164         self.coalescer.dcache = self.L1cache
 165         self.coalescer.ruby_system = ruby_system
 166         self.coalescer.support_inst_reqs = False
 167         self.coalescer.is_cpu_sequencer = False
 168         self.coalescer.max_outstanding_requests = options.simds_per_cu * \
 169                                                   options.wfs_per_simd * \
 170                                                   options.wf_size
 171         if options.tcp_deadlock_threshold:
 172           self.coalescer.deadlock_threshold = \
 173             options.tcp_deadlock_threshold
 174         self.coalescer.max_coalesces_per_cycle = \
 175             options.max_coalesces_per_cycle
 176
 177         self.sequencer = RubySequencer()
 178         self.sequencer.version = self.seqCount()
 179         self.sequencer.icache = self.L1cache
 180         self.sequencer.dcache = self.L1cache
 181         self.sequencer.ruby_system = ruby_system
 182         self.sequencer.is_cpu_sequencer = True
 183
 184         self.use_seq_not_coal = False
 185
 186         self.ruby_system = ruby_system
 187
 188         if options.recycle_latency:
 189             self.recycle_latency = options.recycle_latency
 190
 191     def createCP(self, options, ruby_system, system):
 192         self.version = self.versionCount()
 193
 194         self.L1cache = TCPCache(tagAccessLatency = options.TCP_latency)
 195         self.L1cache.resourceStalls = options.no_resource_stalls
 196         self.L1cache.create(options)
 197
 198         self.coalescer = RubyGPUCoalescer()
 199         self.coalescer.version = self.seqCount()
 200         self.coalescer.icache = self.L1cache
 201         self.coalescer.dcache = self.L1cache
 202         self.coalescer.ruby_system = ruby_system
 203         self.coalescer.support_inst_reqs = False
 204         self.coalescer.is_cpu_sequencer = False
 205
 206         self.sequencer = RubySequencer()
 207         self.sequencer.version = self.seqCount()
 208         self.sequencer.icache = self.L1cache
 209         self.sequencer.dcache = self.L1cache
 210         self.sequencer.ruby_system = ruby_system
 211         self.sequencer.is_cpu_sequencer = True
 212
 213         self.use_seq_not_coal = True
 214
 215         self.ruby_system = ruby_system
 216
 217         if options.recycle_latency:
 218             self.recycle_latency = options.recycle_latency
 219
 220 class SQCCache(RubyCache):
 221     size = "32kB"
 222     assoc = 8
 223     dataArrayBanks = 16
 224     tagArrayBanks = 4
 225     dataAccessLatency = 4
 226     tagAccessLatency = 1
 227     def create(self, options):
 228         self.replacement_policy = TreePLRURP()
 229
 230 class SQCCntrl(SQC_Controller, CntrlBase):
 231
 232     def create(self, options, ruby_system, system):
 233         self.version = self.versionCount()
 234
 235         self.L1cache = SQCCache()
 236         self.L1cache.create(options)
 237         self.L1cache.resourceStalls = options.no_resource_stalls
 238
 239         self.sequencer = RubySequencer()
 240
 241         self.sequencer.version = self.seqCount()
 242         self.sequencer.icache = self.L1cache
 243         self.sequencer.dcache = self.L1cache
 244         self.sequencer.ruby_system = ruby_system
 245         self.sequencer.support_data_reqs = False
 246         self.sequencer.is_cpu_sequencer = False
 247
 248         if options.sqc_deadlock_threshold:
 249           self.sequencer.deadlock_threshold = \
 250             options.sqc_deadlock_threshold
 251
 252         self.ruby_system = ruby_system
 253
 254         if options.recycle_latency:
 255             self.recycle_latency = options.recycle_latency
 256
 257     def createCP(self, options, ruby_system, system):
 258         self.version = self.versionCount()
 259
 260         self.L1cache = SQCCache()
 261         self.L1cache.create(options)
 262         self.L1cache.resourceStalls = options.no_resource_stalls
 263
 264         self.sequencer = RubySequencer()
 265
 266         self.sequencer.version = self.seqCount()
 267         self.sequencer.icache = self.L1cache
 268         self.sequencer.dcache = self.L1cache
 269         self.sequencer.ruby_system = ruby_system
 270         self.sequencer.support_data_reqs = False
 271
 272         self.ruby_system = ruby_system
 273
 274         if options.recycle_latency:
 275             self.recycle_latency = options.recycle_latency
 276
 277
 278 class TCC(RubyCache):
 279     assoc = 16
 280     dataAccessLatency = 8
 281     tagAccessLatency = 2
 282     resourceStalls = True
 283     def create(self, options):
 284         self.size = MemorySize(options.tcc_size)
 285         self.size = self.size / options.num_tccs
 286         self.dataArrayBanks = 256 / options.num_tccs #number of data banks
 287         self.tagArrayBanks = 256 / options.num_tccs #number of tag banks
 288         if ((self.size.value / long(self.assoc)) < 128):
 289             self.size.value = long(128 * self.assoc)
 290         self.start_index_bit = math.log(options.cacheline_size, 2) + \
 291                                math.log(options.num_tccs, 2)
 292         self.replacement_policy = TreePLRURP()
 293
 294 class TCCCntrl(TCC_Controller, CntrlBase):
 295     def create(self, options, ruby_system, system):
 296         self.version = self.versionCount()
 297         self.L2cache = TCC()
 298         self.L2cache.create(options)
 299         self.l2_response_latency = options.TCC_latency
 300
 301         self.number_of_TBEs = 2048
 302
 303         self.ruby_system = ruby_system
 304
 305         if options.recycle_latency:
 306             self.recycle_latency = options.recycle_latency
 307
 308     def connectWireBuffers(self, req_to_tccdir, resp_to_tccdir,
 309                            tcc_unblock_to_tccdir, req_to_tcc,
 310                            probe_to_tcc, resp_to_tcc):
 311         self.w_reqToTCCDir = req_to_tccdir
 312         self.w_respToTCCDir = resp_to_tccdir
 313         self.w_TCCUnblockToTCCDir = tcc_unblock_to_tccdir
 314         self.w_reqToTCC = req_to_tcc
 315         self.w_probeToTCC = probe_to_tcc
 316         self.w_respToTCC = resp_to_tcc
 317
 318 class TCCDirCntrl(TCCdir_Controller, CntrlBase):
 319     def create(self, options, ruby_system, system):
 320         self.version = self.versionCount()
 321
 322         self.directory = TccDirCache()
 323         self.directory.create(options)
 324
 325         self.number_of_TBEs = 1024
 326
 327         self.ruby_system = ruby_system
 328
 329         if options.recycle_latency:
 330             self.recycle_latency = options.recycle_latency
 331
 332     def connectWireBuffers(self, req_to_tccdir, resp_to_tccdir,
 333                            tcc_unblock_to_tccdir, req_to_tcc,
 334                            probe_to_tcc, resp_to_tcc):
 335         self.w_reqToTCCDir = req_to_tccdir
 336         self.w_respToTCCDir = resp_to_tccdir
 337         self.w_TCCUnblockToTCCDir = tcc_unblock_to_tccdir
 338         self.w_reqToTCC = req_to_tcc
 339         self.w_probeToTCC = probe_to_tcc
 340         self.w_respToTCC = resp_to_tcc
 341
 342 class L3Cache(RubyCache):
 343     assoc = 8
 344     dataArrayBanks = 256
 345     tagArrayBanks = 256
 346
 347     def create(self, options, ruby_system, system):
 348         self.size = MemorySize(options.l3_size)
 349         self.size.value /= options.num_dirs
 350         self.dataArrayBanks /= options.num_dirs
 351         self.tagArrayBanks /= options.num_dirs
 352         self.dataArrayBanks /= options.num_dirs
 353         self.tagArrayBanks /= options.num_dirs
 354         self.dataAccessLatency = options.l3_data_latency
 355         self.tagAccessLatency = options.l3_tag_latency
 356         self.resourceStalls = options.no_resource_stalls
 357         self.replacement_policy = TreePLRURP()
 358
 359 class L3Cntrl(L3Cache_Controller, CntrlBase):
 360     def create(self, options, ruby_system, system):
 361         self.version = self.versionCount()
 362         self.L3cache = L3Cache()
 363         self.L3cache.create(options, ruby_system, system)
 364
 365         self.l3_response_latency = max(self.L3cache.dataAccessLatency,
 366                                        self.L3cache.tagAccessLatency)
 367         self.ruby_system = ruby_system
 368
 369         if options.recycle_latency:
 370             self.recycle_latency = options.recycle_latency
 371
 372     def connectWireBuffers(self, req_to_dir, resp_to_dir, l3_unblock_to_dir,
 373                            req_to_l3, probe_to_l3, resp_to_l3):
 374         self.reqToDir = req_to_dir
 375         self.respToDir = resp_to_dir
 376         self.l3UnblockToDir = l3_unblock_to_dir
 377         self.reqToL3 = req_to_l3
 378         self.probeToL3 = probe_to_l3
 379         self.respToL3 = resp_to_l3
 380
 381 class DirCntrl(Directory_Controller, CntrlBase):
 382     def create(self, options, dir_ranges, ruby_system, system):
 383         self.version = self.versionCount()
 384
 385         self.response_latency = 30
 386
 387         self.addr_ranges = dir_ranges
 388         self.directory = RubyDirectoryMemory()
 389
 390         self.L3CacheMemory = L3Cache()
 391         self.L3CacheMemory.create(options, ruby_system, system)
 392
 393         self.l3_hit_latency = max(self.L3CacheMemory.dataAccessLatency,
 394                                   self.L3CacheMemory.tagAccessLatency)
 395
 396         self.number_of_TBEs = options.num_tbes
 397
 398         self.ruby_system = ruby_system
 399
 400         if options.recycle_latency:
 401             self.recycle_latency = options.recycle_latency
 402
 403     def connectWireBuffers(self, req_to_dir, resp_to_dir, l3_unblock_to_dir,
 404                            req_to_l3, probe_to_l3, resp_to_l3):
 405         self.reqToDir = req_to_dir
 406         self.respToDir = resp_to_dir
 407         self.l3UnblockToDir = l3_unblock_to_dir
 408         self.reqToL3 = req_to_l3
 409         self.probeToL3 = probe_to_l3
 410         self.respToL3 = resp_to_l3
 411
 412
 413
 414 def define_options(parser):
 415     parser.add_option("--num-subcaches", type="int", default=4)
 416     parser.add_option("--l3-data-latency", type="int", default=20)
 417     parser.add_option("--l3-tag-latency", type="int", default=15)
 418     parser.add_option("--cpu-to-dir-latency", type="int", default=15)
 419     parser.add_option("--gpu-to-dir-latency", type="int", default=160)
 420     parser.add_option("--no-resource-stalls", action="store_false",
 421                       default=True)
 422     parser.add_option("--num-tbes", type="int", default=256)
 423     parser.add_option("--l2-latency", type="int", default=50) # load to use
 424     parser.add_option("--num-tccs", type="int", default=1,
 425                       help="number of TCC directories and banks in the GPU")
 426     parser.add_option("--TCP_latency", type="int", default=4,
 427                       help="TCP latency")
 428     parser.add_option("--tcp-deadlock-threshold", type='int',
 429                       help="Set the TCP deadlock threshold to some value")
 430     parser.add_option("--TCC_latency", type="int", default=16,
 431                       help="TCC latency")
 432     parser.add_option("--tcc-size", type='string', default='256kB',
 433                       help="agregate tcc size")
 434     parser.add_option("--tcp-size", type='string', default='16kB',
 435                       help="tcp size")
 436     parser.add_option("--tcc-dir-factor", type='int', default=4,
 437                       help="TCCdir size = factor *(TCPs + TCC)")
 438     parser.add_option("--sqc-deadlock-threshold", type='int',
 439                       help="Set the SQC deadlock threshold to some value")
 440     parser.add_option("--max-coalesces-per-cycle", type="int", default=1,
 441                       help="Maximum insts that may coalesce in a cycle");
 442
 443 def create_system(options, full_system, system, dma_devices, bootmem,
 444                   ruby_system):
 445     if buildEnv['PROTOCOL'] != 'GPU_RfO':
 446         panic("This script requires the GPU_RfO protocol to be built.")
 447
 448     cpu_sequencers = []
 449
 450     #
 451     # The ruby network creation expects the list of nodes in the system to be
 452     # consistent with the NetDest list.  Therefore the l1 controller nodes
 453     # must be listed before the directory nodes and directory nodes before
 454     # dma nodes, etc.
 455     #
 456     cp_cntrl_nodes = []
 457     tcp_cntrl_nodes = []
 458     sqc_cntrl_nodes = []
 459     tcc_cntrl_nodes = []
 460     tccdir_cntrl_nodes = []
 461     dir_cntrl_nodes = []
 462     l3_cntrl_nodes = []
 463
 464     #
 465     # Must create the individual controllers before the network to ensure the
 466     # controller constructors are called before the network constructor
 467     #
 468
 469     TCC_bits = int(math.log(options.num_tccs, 2))
 470
 471     # This is the base crossbar that connects the L3s, Dirs, and cpu/gpu
 472     # Clusters
 473     mainCluster = Cluster(extBW = 512, intBW = 512) # 1 TB/s
 474
 475     if options.numa_high_bit:
 476         numa_bit = options.numa_high_bit
 477     else:
 478         # if the numa_bit is not specified, set the directory bits as the
 479         # lowest bits above the block offset bits, and the numa_bit as the
 480         # highest of those directory bits
 481         dir_bits = int(math.log(options.num_dirs, 2))
 482         block_size_bits = int(math.log(options.cacheline_size, 2))
 483         numa_bit = block_size_bits + dir_bits - 1
 484
 485     for i in range(options.num_dirs):
 486         dir_ranges = []
 487         for r in system.mem_ranges:
 488             addr_range = m5.objects.AddrRange(r.start, size = r.size(),
 489                                               intlvHighBit = numa_bit,
 490                                               intlvBits = dir_bits,
 491                                               intlvMatch = i)
 492             dir_ranges.append(addr_range)
 493
 494         dir_cntrl = DirCntrl(TCC_select_num_bits = TCC_bits)
 495         dir_cntrl.create(options, dir_ranges, ruby_system, system)
 496         dir_cntrl.number_of_TBEs = 2560 * options.num_compute_units
 497         #Enough TBEs for all TCP TBEs
 498
 499         # Connect the Directory controller to the ruby network
 500         dir_cntrl.requestFromCores = MessageBuffer(ordered = True)
 501         dir_cntrl.requestFromCores.slave = ruby_system.network.master
 502
 503         dir_cntrl.responseFromCores = MessageBuffer()
 504         dir_cntrl.responseFromCores.slave = ruby_system.network.master
 505
 506         dir_cntrl.unblockFromCores = MessageBuffer()
 507         dir_cntrl.unblockFromCores.slave = ruby_system.network.master
 508
 509         dir_cntrl.probeToCore = MessageBuffer()
 510         dir_cntrl.probeToCore.master = ruby_system.network.slave
 511
 512         dir_cntrl.responseToCore = MessageBuffer()
 513         dir_cntrl.responseToCore.master = ruby_system.network.slave
 514
 515         dir_cntrl.triggerQueue = MessageBuffer(ordered = True)
 516         dir_cntrl.L3triggerQueue = MessageBuffer(ordered = True)
 517         dir_cntrl.requestToMemory = MessageBuffer()
 518         dir_cntrl.responseFromMemory = MessageBuffer()
 519
 520         exec("system.dir_cntrl%d = dir_cntrl" % i)
 521         dir_cntrl_nodes.append(dir_cntrl)
 522
 523         mainCluster.add(dir_cntrl)
 524
 525     # For an odd number of CPUs, still create the right number of controllers
 526     cpuCluster = Cluster(extBW = 512, intBW = 512)  # 1 TB/s
 527     for i in range((options.num_cpus + 1) // 2):
 528
 529         cp_cntrl = CPCntrl()
 530         cp_cntrl.create(options, ruby_system, system)
 531
 532         exec("system.cp_cntrl%d = cp_cntrl" % i)
 533         #
 534         # Add controllers and sequencers to the appropriate lists
 535         #
 536         cpu_sequencers.extend([cp_cntrl.sequencer, cp_cntrl.sequencer1])
 537
 538         # Connect the CP controllers and the network
 539         cp_cntrl.requestFromCore = MessageBuffer()
 540         cp_cntrl.requestFromCore.master = ruby_system.network.slave
 541
 542         cp_cntrl.responseFromCore = MessageBuffer()
 543         cp_cntrl.responseFromCore.master = ruby_system.network.slave
 544
 545         cp_cntrl.unblockFromCore = MessageBuffer()
 546         cp_cntrl.unblockFromCore.master = ruby_system.network.slave
 547
 548         cp_cntrl.probeToCore = MessageBuffer()
 549         cp_cntrl.probeToCore.slave = ruby_system.network.master
 550
 551         cp_cntrl.responseToCore = MessageBuffer()
 552         cp_cntrl.responseToCore.slave = ruby_system.network.master
 553
 554         cp_cntrl.mandatoryQueue = MessageBuffer()
 555         cp_cntrl.triggerQueue = MessageBuffer(ordered = True)
 556
 557         cpuCluster.add(cp_cntrl)
 558
 559     gpuCluster = Cluster(extBW = 512, intBW = 512)  # 1 TB/s
 560
 561     for i in range(options.num_compute_units):
 562
 563         tcp_cntrl = TCPCntrl(TCC_select_num_bits = TCC_bits,
 564                              number_of_TBEs = 2560) # max outstanding requests
 565         tcp_cntrl.create(options, ruby_system, system)
 566
 567         exec("system.tcp_cntrl%d = tcp_cntrl" % i)
 568         #
 569         # Add controllers and sequencers to the appropriate lists
 570         #
 571         cpu_sequencers.append(tcp_cntrl.coalescer)
 572         tcp_cntrl_nodes.append(tcp_cntrl)
 573
 574         # Connect the TCP controller to the ruby network
 575         tcp_cntrl.requestFromTCP = MessageBuffer(ordered = True)
 576         tcp_cntrl.requestFromTCP.master = ruby_system.network.slave
 577
 578         tcp_cntrl.responseFromTCP = MessageBuffer(ordered = True)
 579         tcp_cntrl.responseFromTCP.master = ruby_system.network.slave
 580
 581         tcp_cntrl.unblockFromCore = MessageBuffer(ordered = True)
 582         tcp_cntrl.unblockFromCore.master = ruby_system.network.slave
 583
 584         tcp_cntrl.probeToTCP = MessageBuffer(ordered = True)
 585         tcp_cntrl.probeToTCP.slave = ruby_system.network.master
 586
 587         tcp_cntrl.responseToTCP = MessageBuffer(ordered = True)
 588         tcp_cntrl.responseToTCP.slave = ruby_system.network.master
 589
 590         tcp_cntrl.mandatoryQueue = MessageBuffer()
 591
 592         gpuCluster.add(tcp_cntrl)
 593
 594     for i in range(options.num_sqc):
 595
 596         sqc_cntrl = SQCCntrl(TCC_select_num_bits = TCC_bits)
 597         sqc_cntrl.create(options, ruby_system, system)
 598
 599         exec("system.sqc_cntrl%d = sqc_cntrl" % i)
 600         #
 601         # Add controllers and sequencers to the appropriate lists
 602         #
 603         cpu_sequencers.append(sqc_cntrl.sequencer)
 604
 605         # Connect the SQC controller to the ruby network
 606         sqc_cntrl.requestFromSQC = MessageBuffer(ordered = True)
 607         sqc_cntrl.requestFromSQC.master = ruby_system.network.slave
 608
 609         sqc_cntrl.responseFromSQC = MessageBuffer(ordered = True)
 610         sqc_cntrl.responseFromSQC.master = ruby_system.network.slave
 611
 612         sqc_cntrl.unblockFromCore = MessageBuffer(ordered = True)
 613         sqc_cntrl.unblockFromCore.master = ruby_system.network.slave
 614
 615         sqc_cntrl.probeToSQC = MessageBuffer(ordered = True)
 616         sqc_cntrl.probeToSQC.slave = ruby_system.network.master
 617
 618         sqc_cntrl.responseToSQC = MessageBuffer(ordered = True)
 619         sqc_cntrl.responseToSQC.slave = ruby_system.network.master
 620
 621         sqc_cntrl.mandatoryQueue = MessageBuffer()
 622
 623         # SQC also in GPU cluster
 624         gpuCluster.add(sqc_cntrl)
 625
 626     for i in range(options.num_cp):
 627
 628         tcp_cntrl = TCPCntrl(TCC_select_num_bits = TCC_bits,
 629                              number_of_TBEs = 2560) # max outstanding requests
 630         tcp_cntrl.createCP(options, ruby_system, system)
 631
 632         exec("system.tcp_cntrl%d = tcp_cntrl" % (options.num_compute_units + i))
 633         #
 634         # Add controllers and sequencers to the appropriate lists
 635         #
 636         cpu_sequencers.append(tcp_cntrl.sequencer)
 637         tcp_cntrl_nodes.append(tcp_cntrl)
 638
 639         # Connect the TCP controller to the ruby network
 640         tcp_cntrl.requestFromTCP = MessageBuffer(ordered = True)
 641         tcp_cntrl.requestFromTCP.master = ruby_system.network.slave
 642
 643         tcp_cntrl.responseFromTCP = MessageBuffer(ordered = True)
 644         tcp_cntrl.responseFromTCP.master = ruby_system.network.slave
 645
 646         tcp_cntrl.unblockFromCore = MessageBuffer(ordered = True)
 647         tcp_cntrl.unblockFromCore.master = ruby_system.network.slave
 648
 649         tcp_cntrl.probeToTCP = MessageBuffer(ordered = True)
 650         tcp_cntrl.probeToTCP.slave = ruby_system.network.master
 651
 652         tcp_cntrl.responseToTCP = MessageBuffer(ordered = True)
 653         tcp_cntrl.responseToTCP.slave = ruby_system.network.master
 654
 655         tcp_cntrl.mandatoryQueue = MessageBuffer()
 656
 657         gpuCluster.add(tcp_cntrl)
 658
 659         sqc_cntrl = SQCCntrl(TCC_select_num_bits = TCC_bits)
 660         sqc_cntrl.createCP(options, ruby_system, system)
 661
 662         exec("system.sqc_cntrl%d = sqc_cntrl" % (options.num_compute_units + i))
 663         #
 664         # Add controllers and sequencers to the appropriate lists
 665         #
 666         cpu_sequencers.append(sqc_cntrl.sequencer)
 667
 668         # Connect the SQC controller to the ruby network
 669         sqc_cntrl.requestFromSQC = MessageBuffer(ordered = True)
 670         sqc_cntrl.requestFromSQC.master = ruby_system.network.slave
 671
 672         sqc_cntrl.responseFromSQC = MessageBuffer(ordered = True)
 673         sqc_cntrl.responseFromSQC.master = ruby_system.network.slave
 674
 675         sqc_cntrl.unblockFromCore = MessageBuffer(ordered = True)
 676         sqc_cntrl.unblockFromCore.master = ruby_system.network.slave
 677
 678         sqc_cntrl.probeToSQC = MessageBuffer(ordered = True)
 679         sqc_cntrl.probeToSQC.slave = ruby_system.network.master
 680
 681         sqc_cntrl.responseToSQC = MessageBuffer(ordered = True)
 682         sqc_cntrl.responseToSQC.slave = ruby_system.network.master
 683
 684         sqc_cntrl.mandatoryQueue = MessageBuffer()
 685
 686         # SQC also in GPU cluster
 687         gpuCluster.add(sqc_cntrl)
 688
 689     for i in range(options.num_tccs):
 690
 691         tcc_cntrl = TCCCntrl(TCC_select_num_bits = TCC_bits,
 692                              number_of_TBEs = options.num_compute_units * 2560)
 693         #Enough TBEs for all TCP TBEs
 694         tcc_cntrl.create(options, ruby_system, system)
 695         tcc_cntrl_nodes.append(tcc_cntrl)
 696
 697         tccdir_cntrl = TCCDirCntrl(TCC_select_num_bits = TCC_bits,
 698                               number_of_TBEs = options.num_compute_units * 2560)
 699         #Enough TBEs for all TCP TBEs
 700         tccdir_cntrl.create(options, ruby_system, system)
 701         tccdir_cntrl_nodes.append(tccdir_cntrl)
 702
 703         exec("system.tcc_cntrl%d = tcc_cntrl" % i)
 704         exec("system.tccdir_cntrl%d = tccdir_cntrl" % i)
 705
 706         # connect all of the wire buffers between L3 and dirs up
 707         req_to_tccdir = RubyWireBuffer()
 708         resp_to_tccdir = RubyWireBuffer()
 709         tcc_unblock_to_tccdir = RubyWireBuffer()
 710         req_to_tcc = RubyWireBuffer()
 711         probe_to_tcc = RubyWireBuffer()
 712         resp_to_tcc = RubyWireBuffer()
 713
 714         tcc_cntrl.connectWireBuffers(req_to_tccdir, resp_to_tccdir,
 715                                      tcc_unblock_to_tccdir, req_to_tcc,
 716                                      probe_to_tcc, resp_to_tcc)
 717         tccdir_cntrl.connectWireBuffers(req_to_tccdir, resp_to_tccdir,
 718                                         tcc_unblock_to_tccdir, req_to_tcc,
 719                                         probe_to_tcc, resp_to_tcc)
 720
 721         # Connect the TCC controller to the ruby network
 722         tcc_cntrl.responseFromTCC = MessageBuffer(ordered = True)
 723         tcc_cntrl.responseFromTCC.master = ruby_system.network.slave
 724
 725         tcc_cntrl.responseToTCC = MessageBuffer(ordered = True)
 726         tcc_cntrl.responseToTCC.slave = ruby_system.network.master
 727
 728         # Connect the TCC Dir controller to the ruby network
 729         tccdir_cntrl.requestFromTCP = MessageBuffer(ordered = True)
 730         tccdir_cntrl.requestFromTCP.slave = ruby_system.network.master
 731
 732         tccdir_cntrl.responseFromTCP = MessageBuffer(ordered = True)
 733         tccdir_cntrl.responseFromTCP.slave = ruby_system.network.master
 734
 735         tccdir_cntrl.unblockFromTCP = MessageBuffer(ordered = True)
 736         tccdir_cntrl.unblockFromTCP.slave = ruby_system.network.master
 737
 738         tccdir_cntrl.probeToCore = MessageBuffer(ordered = True)
 739         tccdir_cntrl.probeToCore.master = ruby_system.network.slave
 740
 741         tccdir_cntrl.responseToCore = MessageBuffer(ordered = True)
 742         tccdir_cntrl.responseToCore.master = ruby_system.network.slave
 743
 744         tccdir_cntrl.probeFromNB = MessageBuffer()
 745         tccdir_cntrl.probeFromNB.slave = ruby_system.network.master
 746
 747         tccdir_cntrl.responseFromNB = MessageBuffer()
 748         tccdir_cntrl.responseFromNB.slave = ruby_system.network.master
 749
 750         tccdir_cntrl.requestToNB = MessageBuffer()
 751         tccdir_cntrl.requestToNB.master = ruby_system.network.slave
 752
 753         tccdir_cntrl.responseToNB = MessageBuffer()
 754         tccdir_cntrl.responseToNB.master = ruby_system.network.slave
 755
 756         tccdir_cntrl.unblockToNB = MessageBuffer()
 757         tccdir_cntrl.unblockToNB.master = ruby_system.network.slave
 758
 759         tccdir_cntrl.triggerQueue = MessageBuffer(ordered = True)
 760
 761         # TCC cntrls added to the GPU cluster
 762         gpuCluster.add(tcc_cntrl)
 763         gpuCluster.add(tccdir_cntrl)
 764
 765     # Assuming no DMA devices
 766     assert(len(dma_devices) == 0)
 767
 768     # Add cpu/gpu clusters to main cluster
 769     mainCluster.add(cpuCluster)
 770     mainCluster.add(gpuCluster)
 771
 772     ruby_system.network.number_of_virtual_networks = 10
 773
 774     return (cpu_sequencers, dir_cntrl_nodes, mainCluster)