tests/configs/gpu-ruby.py

   1 #
   2 #  Copyright (c) 2015 Advanced Micro Devices, Inc.
   3 #  All rights reserved.
   4 #
   5 #  For use for simulation and test purposes only
   6 #
   7 #  Redistribution and use in source and binary forms, with or without
   8 #  modification, are permitted provided that the following conditions are met:
   9 #
  10 #  1. Redistributions of source code must retain the above copyright notice,
  11 #  this list of conditions and the following disclaimer.
  12 #
  13 #  2. Redistributions in binary form must reproduce the above copyright notice,
  14 #  this list of conditions and the following disclaimer in the documentation
  15 #  and/or other materials provided with the distribution.
  16 #
  17 #  3. Neither the name of the copyright holder nor the names of its contributors
  18 #  may be used to endorse or promote products derived from this software
  19 #  without specific prior written permission.
  20 #
  21 #  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  22 #  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23 #  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24 #  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
  25 #  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  26 #  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  27 #  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  28 #  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  29 #  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  30 #  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  31 #  POSSIBILITY OF SUCH DAMAGE.
  32 #
  33 #  Author: Brad Beckmann
  34 #
  35
  36 from __future__ import print_function
  37
  38 import m5
  39 from m5.objects import *
  40 from m5.defines import buildEnv
  41 from m5.util import addToPath
  42 import os, optparse, sys, math, glob
  43
  44 m5.util.addToPath('../configs/')
  45
  46 from ruby import Ruby
  47 from common import Options
  48 from common import GPUTLBOptions, GPUTLBConfig
  49
  50 ########################## Script Options ########################
  51 def setOption(parser, opt_str, value = 1):
  52     # check to make sure the option actually exists
  53     if not parser.has_option(opt_str):
  54         raise Exception("cannot find %s in list of possible options" % opt_str)
  55
  56     opt = parser.get_option(opt_str)
  57     # set the value
  58     exec("parser.values.%s = %s" % (opt.dest, value))
  59
  60 def getOption(parser, opt_str):
  61     # check to make sure the option actually exists
  62     if not parser.has_option(opt_str):
  63         raise Exception("cannot find %s in list of possible options" % opt_str)
  64
  65     opt = parser.get_option(opt_str)
  66     # get the value
  67     exec("return_value = parser.values.%s" % opt.dest)
  68     return return_value
  69
  70 def run_test(root):
  71     """gpu test requires a specialized run_test implementation to set up the
  72     mmio space."""
  73
  74     # instantiate configuration
  75     m5.instantiate()
  76
  77     # Now that the system has been constructed, setup the mmio space
  78     root.system.cpu[0].workload[0].map(0x10000000, 0x200000000, 4096)
  79
  80     # simulate until program terminates
  81     exit_event = m5.simulate(maxtick)
  82     print('Exiting @ tick', m5.curTick(), 'because', exit_event.getCause())
  83
  84 parser = optparse.OptionParser()
  85 Options.addCommonOptions(parser)
  86 Options.addSEOptions(parser)
  87
  88 parser.add_option("-k", "--kernel-files",
  89                   help="file(s) containing GPU kernel code (colon separated)")
  90 parser.add_option("-u", "--num-compute-units", type="int", default=2,
  91                   help="number of GPU compute units"),
  92 parser.add_option("--num-cp", type="int", default=0,
  93                   help="Number of GPU Command Processors (CP)")
  94 parser.add_option("--simds-per-cu", type="int", default=4, help="SIMD units" \
  95                   "per CU")
  96 parser.add_option("--cu-per-sqc", type="int", default=4, help="number of CUs" \
  97                   "sharing an SQC (icache, and thus icache TLB)")
  98 parser.add_option("--wf-size", type="int", default=64,
  99                   help="Wavefront size(in workitems)")
 100 parser.add_option("--wfs-per-simd", type="int", default=8, help="Number of " \
 101                   "WF slots per SIMD")
 102 parser.add_option("--sp-bypass-path-length", type="int", default=4, \
 103                   help="Number of stages of bypass path in vector ALU for Single "\
 104                   "Precision ops")
 105 parser.add_option("--dp-bypass-path-length", type="int", default=4, \
 106                   help="Number of stages of bypass path in vector ALU for Double "\
 107                   "Precision ops")
 108 parser.add_option("--issue-period", type="int", default=4, \
 109                   help="Number of cycles per vector instruction issue period")
 110 parser.add_option("--glbmem-wr-bus-width", type="int", default=32, \
 111                   help="VGPR to Coalescer (Global Memory) data bus width in bytes")
 112 parser.add_option("--glbmem-rd-bus-width", type="int", default=32, \
 113                   help="Coalescer to VGPR (Global Memory) data bus width in bytes")
 114 parser.add_option("--shr-mem-pipes-per-cu", type="int", default=1, \
 115                   help="Number of Shared Memory pipelines per CU")
 116 parser.add_option("--glb-mem-pipes-per-cu", type="int", default=1, \
 117                   help="Number of Global Memory pipelines per CU")
 118 parser.add_option("--vreg-file-size", type="int", default=2048,
 119                   help="number of physical vector registers per SIMD")
 120 parser.add_option("--bw-scalor", type="int", default=0,
 121                   help="bandwidth scalor for scalability analysis")
 122 parser.add_option("--CPUClock", type="string", default="2GHz",
 123                   help="CPU clock")
 124 parser.add_option("--GPUClock", type="string", default="1GHz",
 125                   help="GPU clock")
 126 parser.add_option("--cpu-voltage", action="store", type="string",
 127                   default='1.0V',
 128                   help = """CPU  voltage domain""")
 129 parser.add_option("--gpu-voltage", action="store", type="string",
 130                   default='1.0V',
 131                   help = """CPU  voltage domain""")
 132 parser.add_option("--CUExecPolicy", type="string", default="OLDEST-FIRST",
 133                   help="WF exec policy (OLDEST-FIRST, ROUND-ROBIN)")
 134 parser.add_option("--xact-cas-mode", action="store_true",
 135                   help="enable load_compare mode (transactional CAS)")
 136 parser.add_option("--SegFaultDebug",action="store_true",
 137                  help="checks for GPU seg fault before TLB access")
 138 parser.add_option("--LocalMemBarrier",action="store_true",
 139                  help="Barrier does not wait for writethroughs to complete")
 140 parser.add_option("--countPages", action="store_true",
 141                  help="Count Page Accesses and output in per-CU output files")
 142 parser.add_option("--TLB-prefetch", type="int", help = "prefetch depth for"\
 143                   "TLBs")
 144 parser.add_option("--pf-type", type="string", help="type of prefetch: "\
 145                   "PF_CU, PF_WF, PF_PHASE, PF_STRIDE")
 146 parser.add_option("--pf-stride", type="int", help="set prefetch stride")
 147 parser.add_option("--numLdsBanks", type="int", default=32,
 148                   help="number of physical banks per LDS module")
 149 parser.add_option("--ldsBankConflictPenalty", type="int", default=1,
 150                   help="number of cycles per LDS bank conflict")
 151
 152 # Add the ruby specific and protocol specific options
 153 Ruby.define_options(parser)
 154
 155 GPUTLBOptions.tlb_options(parser)
 156
 157 (options, args) = parser.parse_args()
 158
 159 # The GPU cache coherence protocols only work with the backing store
 160 setOption(parser, "--access-backing-store")
 161
 162 # Currently, the sqc (I-Cache of GPU) is shared by
 163 # multiple compute units(CUs). The protocol works just fine
 164 # even if sqc is not shared. Overriding this option here
 165 # so that the user need not explicitly set this (assuming
 166 # sharing sqc is the common usage)
 167 n_cu = options.num_compute_units
 168 num_sqc = int(math.ceil(float(n_cu) / options.cu_per_sqc))
 169 options.num_sqc = num_sqc # pass this to Ruby
 170
 171 ########################## Creating the GPU system ########################
 172 # shader is the GPU
 173 shader = Shader(n_wf = options.wfs_per_simd,
 174                 clk_domain = SrcClockDomain(
 175                     clock = options.GPUClock,
 176                     voltage_domain = VoltageDomain(
 177                         voltage = options.gpu_voltage)),
 178                 timing = True)
 179
 180 # GPU_RfO(Read For Ownership) implements SC/TSO memory model.
 181 # Other GPU protocols implement release consistency at GPU side.
 182 # So, all GPU protocols other than GPU_RfO should make their writes
 183 # visible to the global memory and should read from global memory
 184 # during kernal boundary. The pipeline initiates(or do not initiate)
 185 # the acquire/release operation depending on this impl_kern_boundary_sync
 186 # flag. This flag=true means pipeline initiates a acquire/release operation
 187 # at kernel boundary.
 188 if buildEnv['PROTOCOL'] == 'GPU_RfO':
 189     shader.impl_kern_boundary_sync = False
 190 else:
 191     shader.impl_kern_boundary_sync = True
 192
 193 # Switching off per-lane TLB by default
 194 per_lane = False
 195 if options.TLB_config == "perLane":
 196     per_lane = True
 197
 198 # List of compute units; one GPU can have multiple compute units
 199 compute_units = []
 200 for i in range(n_cu):
 201     compute_units.append(ComputeUnit(cu_id = i, perLaneTLB = per_lane,
 202                                      num_SIMDs = options.simds_per_cu,
 203                                      wfSize = options.wf_size,
 204                                      spbypass_pipe_length = \
 205                                      options.sp_bypass_path_length,
 206                                      dpbypass_pipe_length = \
 207                                      options.dp_bypass_path_length,
 208                                      issue_period = options.issue_period,
 209                                      coalescer_to_vrf_bus_width = \
 210                                      options.glbmem_rd_bus_width,
 211                                      vrf_to_coalescer_bus_width = \
 212                                      options.glbmem_wr_bus_width,
 213                                      num_global_mem_pipes = \
 214                                      options.glb_mem_pipes_per_cu,
 215                                      num_shared_mem_pipes = \
 216                                      options.shr_mem_pipes_per_cu,
 217                                      n_wf = options.wfs_per_simd,
 218                                      execPolicy = options.CUExecPolicy,
 219                                      xactCasMode = options.xact_cas_mode,
 220                                      debugSegFault = options.SegFaultDebug,
 221                                      functionalTLB = True,
 222                                      localMemBarrier = options.LocalMemBarrier,
 223                                      countPages = options.countPages,
 224                                      localDataStore = \
 225                                      LdsState(banks = options.numLdsBanks,
 226                                               bankConflictPenalty = \
 227                                               options.ldsBankConflictPenalty)))
 228     wavefronts = []
 229     vrfs = []
 230     for j in range(options.simds_per_cu):
 231         for k in range(int(shader.n_wf)):
 232             wavefronts.append(Wavefront(simdId = j, wf_slot_id = k))
 233         vrfs.append(VectorRegisterFile(simd_id=j,
 234                               num_regs_per_simd=options.vreg_file_size))
 235     compute_units[-1].wavefronts = wavefronts
 236     compute_units[-1].vector_register_file = vrfs
 237     if options.TLB_prefetch:
 238         compute_units[-1].prefetch_depth = options.TLB_prefetch
 239         compute_units[-1].prefetch_prev_type = options.pf_type
 240
 241     # attach the LDS and the CU to the bus (actually a Bridge)
 242     compute_units[-1].ldsPort = compute_units[-1].ldsBus.slave
 243     compute_units[-1].ldsBus.master = compute_units[-1].localDataStore.cuPort
 244
 245 # Attach compute units to GPU
 246 shader.CUs = compute_units
 247
 248 # this is a uniprocessor only test, thus the shader is the second index in the
 249 # list of "system.cpus"
 250 options.num_cpus = 1
 251 shader_idx = 1
 252 cpu = TimingSimpleCPU(cpu_id=0)
 253
 254 ########################## Creating the GPU dispatcher ########################
 255 # Dispatcher dispatches work from host CPU to GPU
 256 host_cpu = cpu
 257 dispatcher = GpuDispatcher()
 258
 259 # Currently does not test for command processors
 260 cpu_list = [cpu] + [shader] + [dispatcher]
 261
 262 system = System(cpu = cpu_list,
 263                 mem_ranges = [AddrRange(options.mem_size)],
 264                 mem_mode = 'timing')
 265
 266 # Dummy voltage domain for all our clock domains
 267 system.voltage_domain = VoltageDomain(voltage = options.sys_voltage)
 268 system.clk_domain = SrcClockDomain(clock = '1GHz',
 269                                    voltage_domain = system.voltage_domain)
 270
 271 # Create a seperate clock domain for components that should run at
 272 # CPUs frequency
 273 system.cpu[0].clk_domain = SrcClockDomain(clock = '2GHz',
 274                                           voltage_domain = \
 275                                           system.voltage_domain)
 276
 277 # configure the TLB hierarchy
 278 GPUTLBConfig.config_tlb_hierarchy(options, system, shader_idx)
 279
 280 # create Ruby system
 281 system.piobus = IOXBar(width=32, response_latency=0,
 282                        frontend_latency=0, forward_latency=0)
 283 Ruby.create_system(options, None, system)
 284
 285 # Create a separate clock for Ruby
 286 system.ruby.clk_domain = SrcClockDomain(clock = options.ruby_clock,
 287                                         voltage_domain = system.voltage_domain)
 288
 289 # create the interrupt controller
 290 cpu.createInterruptController()
 291
 292 #
 293 # Tie the cpu cache ports to the ruby cpu ports and
 294 # physmem, respectively
 295 #
 296 cpu.connectAllPorts(system.ruby._cpu_ports[0])
 297 system.ruby._cpu_ports[0].mem_master_port = system.piobus.slave
 298
 299 # attach CU ports to Ruby
 300 # Because of the peculiarities of the CP core, you may have 1 CPU but 2
 301 # sequencers and thus 2 _cpu_ports created. Your GPUs shouldn't be
 302 # hooked up until after the CP. To make this script generic, figure out
 303 # the index as below, but note that this assumes there is one sequencer
 304 # per compute unit and one sequencer per SQC for the math to work out
 305 # correctly.
 306 gpu_port_idx = len(system.ruby._cpu_ports) \
 307                - options.num_compute_units - options.num_sqc
 308 gpu_port_idx = gpu_port_idx - options.num_cp * 2
 309
 310 wavefront_size = options.wf_size
 311 for i in range(n_cu):
 312     # The pipeline issues wavefront_size number of uncoalesced requests
 313     # in one GPU issue cycle. Hence wavefront_size mem ports.
 314     for j in range(wavefront_size):
 315         system.cpu[shader_idx].CUs[i].memory_port[j] = \
 316                   system.ruby._cpu_ports[gpu_port_idx].slave[j]
 317     gpu_port_idx += 1
 318
 319 for i in range(n_cu):
 320     if i > 0 and not i % options.cu_per_sqc:
 321         gpu_port_idx += 1
 322     system.cpu[shader_idx].CUs[i].sqc_port = \
 323             system.ruby._cpu_ports[gpu_port_idx].slave
 324 gpu_port_idx = gpu_port_idx + 1
 325
 326 # Current regression tests do not support the command processor
 327 assert(options.num_cp == 0)
 328
 329 # connect dispatcher to the system.piobus
 330 dispatcher.pio = system.piobus.master
 331 dispatcher.dma = system.piobus.slave
 332
 333 ################# Connect the CPU and GPU via GPU Dispatcher ###################
 334 # CPU rings the GPU doorbell to notify a pending task
 335 # using this interface.
 336 # And GPU uses this interface to notify the CPU of task completion
 337 # The communcation happens through emulated driver.
 338
 339 # Note this implicit setting of the cpu_pointer, shader_pointer and tlb array
 340 # parameters must be after the explicit setting of the System cpu list
 341 shader.cpu_pointer = host_cpu
 342 dispatcher.cpu = host_cpu
 343 dispatcher.shader_pointer = shader
 344
 345 # -----------------------
 346 # run simulation
 347 # -----------------------
 348
 349 root = Root(full_system = False, system = system)
 350 m5.ticks.setGlobalFrequency('1THz')
 351 root.system.mem_mode = 'timing'