src/soc/experiment/compldst_multi.py

   1 """LOAD / STORE Computation Unit.
   2
   3 This module covers POWER9-compliant Load and Store operations,
   4 with selection on each between immediate and indexed mode as
   5 options for the calculation of the Effective Address (EA),
   6 and also "update" mode which optionally stores that EA into
   7 an additional register.
   8
   9 ----
  10 Note: it took 15 attempts over several weeks to redraw the diagram
  11 needed to capture this FSM properly.  To understand it fully, please
  12 take the time to review the links, video, and diagram.
  13 ----
  14
  15 Stores are activated when Go_Store is enabled, and use a sync'd "ADD" to
  16 compute the "Effective Address", and, when ready the operand (src3_i)
  17 is stored in the computed address (passed through to the PortInterface)
  18
  19 Loads are activated when Go_Write[0] is enabled.  The EA is computed,
  20 and (as long as there was no exception) the data comes out (at any
  21 time from the PortInterface), and is captured by the LDCompSTUnit.
  22
  23 TODO: dcbz, yes, that's going to be complicated, has to be done
  24  with great care, to detect the case when dcbz is set
  25  and *not* expect to read any data, just the address.
  26  so, wait for RA but not RB.
  27
  28 Both LD and ST may request that the address be computed from summing
  29 operand1 (src[0]) with operand2 (src[1]) *or* by summing operand1 with
  30 the immediate (from the opcode).
  31
  32 Both LD and ST may also request "update" mode (op_is_update) which
  33 activates the use of Go_Write[1] to control storage of the EA into
  34 a *second* operand in the register file.
  35
  36 Thus this module has *TWO* write-requests to the register file and
  37 *THREE* read-requests to the register file (not all at the same time!)
  38 The regfile port usage is:
  39
  40     * LD-imm         1R1W
  41     * LD-imm-update  1R2W
  42     * LD-idx         2R1W
  43     * LD-idx-update  2R2W
  44
  45     * ST-imm         2R
  46     * ST-imm-update  2R1W
  47     * ST-idx         3R
  48     * ST-idx-update  3R1W
  49
  50 It's a multi-level Finite State Machine that (unfortunately) nmigen.FSM
  51 is not suited to (nmigen.FSM is clock-driven, and some aspects of
  52 the nested FSMs below are *combinatorial*).
  53
  54     * One FSM covers Operand collection and communication address-side
  55       with the LD/ST PortInterface.  its role ends when "RD_DONE" is asserted
  56
  57     * A second FSM activates to cover LD.  it activates if op_is_ld is true
  58
  59     * A third FSM activates to cover ST.  it activates if op_is_st is true
  60
  61     * TODO document DCBZ (not complete yet)
  62
  63     * The "overall" (fourth) FSM coordinates the progression and completion
  64       of the three other FSMs, firing "WR_RESET" which switches off "busy"
  65
  66 Full diagram:
  67
  68     https://libre-soc.org/3d_gpu/ld_st_comp_unit.jpg
  69
  70 Links including to walk-through videos:
  71
  72     * https://libre-soc.org/3d_gpu/architecture/6600scoreboard/
  73     * http://libre-soc.org/openpower/isa/fixedload
  74     * http://libre-soc.org/openpower/isa/fixedstore
  75
  76 Related Bugreports:
  77
  78     * https://bugs.libre-soc.org/show_bug.cgi?id=302
  79     * https://bugs.libre-soc.org/show_bug.cgi?id=216
  80
  81 Terminology:
  82
  83     * EA - Effective Address
  84     * LD - Load
  85     * ST - Store
  86 """
  87
  88 from nmigen.compat.sim import run_simulation
  89 from nmigen.cli import verilog, rtlil
  90 from nmigen import Module, Signal, Mux, Cat, Elaboratable, Array, Repl, C
  91 from nmigen.hdl.rec import Record, Layout
  92
  93 from nmutil.latch import SRLatch, latchregister
  94 from nmutil.byterev import byte_reverse
  95 from nmutil.extend import exts
  96
  97 from soc.experiment.compalu_multi import go_record, CompUnitRecord
  98 from soc.experiment.l0_cache import PortInterface
  99 from soc.experiment.pimem import LDSTException
 100 from soc.fu.regspec import RegSpecAPI
 101
 102 from openpower.decoder.power_enums import MicrOp, Function, LDSTMode
 103 from soc.fu.ldst.ldst_input_record import CompLDSTOpSubset
 104 from openpower.decoder.power_decoder2 import Data
 105 from openpower.consts import MSR
 106 from soc.config.test.test_loadstore import TestMemPspec
 107
 108 # for debugging dcbz
 109 from nmutil.util import Display
 110
 111
 112 # TODO: LDSTInputData and LDSTOutputData really should be used
 113 # here, to make things more like the other CompUnits.  currently,
 114 # also, RegSpecAPI is used explicitly here
 115
 116
 117 class LDSTCompUnitRecord(CompUnitRecord):
 118     def __init__(self, rwid, opsubset=CompLDSTOpSubset, name=None):
 119         CompUnitRecord.__init__(self, opsubset, rwid,
 120                                 n_src=3, n_dst=2, name=name)
 121
 122         self.ad = go_record(1, name="cu_ad")  # address go in, req out
 123         self.st = go_record(1, name="cu_st")  # store go in, req out
 124
 125         self.exc_o = LDSTException("exc_o")
 126
 127         self.ld_o = Signal(reset_less=True)  # operation is a LD
 128         self.st_o = Signal(reset_less=True)  # operation is a ST
 129
 130         # hmm... are these necessary?
 131         self.load_mem_o = Signal(reset_less=True)  # activate memory LOAD
 132         self.stwd_mem_o = Signal(reset_less=True)  # activate memory STORE
 133
 134
 135 class LDSTCompUnit(RegSpecAPI, Elaboratable):
 136     """LOAD / STORE Computation Unit
 137
 138     Inputs
 139     ------
 140
 141     * :pi:     a PortInterface to the memory subsystem (read-write capable)
 142     * :rwid:   register width
 143     * :awid:   address width
 144
 145     Data inputs
 146     -----------
 147     * :src_i:  Source Operands (RA/RB/RC) - managed by rd[0-3] go/req
 148
 149     Data (outputs)
 150     --------------
 151     * :o_data:  Dest out (LD)          - managed by wr[0] go/req
 152     * :addr_o:  Address out (LD or ST) - managed by wr[1] go/req
 153     * :exc_o:   Address/Data Exception occurred.  LD/ST must terminate
 154
 155     TODO: make exc_o a data-type rather than a single-bit signal
 156           (see bug #302)
 157
 158     Control Signals (In)
 159     --------------------
 160
 161     * :oper_i:     operation being carried out (POWER9 decode LD/ST subset)
 162     * :issue_i:    LD/ST is being "issued".
 163     * :shadown_i:  Inverted-shadow is being held (stops STORE *and* WRITE)
 164     * :go_rd_i:    read is being actioned (latches in src regs)
 165     * :go_wr_i:    write mode (exactly like ALU CompUnit)
 166     * :go_ad_i:    address is being actioned (triggers actual mem LD)
 167     * :go_st_i:    store is being actioned (triggers actual mem STORE)
 168     * :go_die_i:   resets the unit back to "wait for issue"
 169
 170     Control Signals (Out)
 171     ---------------------
 172
 173     * :busy_o:      function unit is busy
 174     * :rd_rel_o:    request src1/src2
 175     * :adr_rel_o:   request address (from mem)
 176     * :sto_rel_o:   request store (to mem)
 177     * :req_rel_o:   request write (result)
 178     * :load_mem_o:  activate memory LOAD
 179     * :stwd_mem_o:  activate memory STORE
 180
 181     Note: load_mem_o, stwd_mem_o and req_rel_o MUST all be acknowledged
 182     in a single cycle and the CompUnit set back to doing another op.
 183     This means deasserting go_st_i, go_ad_i or go_wr_i as appropriate
 184     depending on whether the operation is a ST or LD.
 185
 186     Note: LDSTCompUnit takes care of LE/BE normalisation:
 187     * LD data is normalised after receipt from the PortInterface
 188     * ST data is normalised *prior* to sending onto the PortInterface
 189     TODO: use one module for the byte-reverse as it's quite expensive in gates
 190     """
 191
 192     def __init__(self, pi=None, rwid=64, awid=64, opsubset=CompLDSTOpSubset,
 193                  debugtest=False, name=None):
 194         super().__init__(rwid)
 195         self.awid = awid
 196         self.pi = pi
 197         self.cu = cu = LDSTCompUnitRecord(rwid, opsubset, name=name)
 198         self.debugtest = debugtest # enable debug output for unit testing
 199
 200         # POWER-compliant LD/ST has index and update: *fixed* number of ports
 201         self.n_src = n_src = 3   # RA, RB, RT/RS
 202         self.n_dst = n_dst = 3  # RA, RT/RS, CR0
 203
 204         # set up array of src and dest signals
 205         for i in range(n_src):
 206             j = i + 1  # name numbering to match src1/src2
 207             name = "src%d_i" % j
 208             setattr(self, name, getattr(cu, name))
 209
 210         dst = []
 211         for i in range(n_dst):
 212             j = i + 1  # name numbering to match dest1/2...
 213             name = "dest%d_o" % j
 214             setattr(self, name, getattr(cu, name))
 215
 216         # convenience names
 217         self.rd = cu.rd
 218         self.wr = cu.wr
 219         self.rdmaskn = cu.rdmaskn
 220         self.wrmask = cu.wrmask
 221         self.ad = cu.ad
 222         self.st = cu.st
 223         self.dest = cu._dest
 224
 225         # HACK: get data width from dest[0].  this is used across the board
 226         # (it really shouldn't be)
 227         self.data_wid = self.dest[0].shape()
 228
 229         self.go_rd_i = self.rd.go_i  # temporary naming
 230         self.go_wr_i = self.wr.go_i  # temporary naming
 231         self.go_ad_i = self.ad.go_i  # temp naming: go address in
 232         self.go_st_i = self.st.go_i  # temp naming: go store in
 233
 234         self.rd_rel_o = self.rd.rel_o  # temporary naming
 235         self.req_rel_o = self.wr.rel_o  # temporary naming
 236         self.adr_rel_o = self.ad.rel_o  # request address (from mem)
 237         self.sto_rel_o = self.st.rel_o  # request store (to mem)
 238
 239         self.issue_i = cu.issue_i
 240         self.shadown_i = cu.shadown_i
 241         self.go_die_i = cu.go_die_i
 242
 243         self.oper_i = cu.oper_i
 244         self.src_i = cu._src_i
 245
 246         self.o_data = Data(self.data_wid, name="o")  # Dest1 out: RT
 247         self.addr_o = Data(self.data_wid, name="ea")  # Addr out: Update => RA
 248         self.cr_o = Data(4, name="cr0")  # CR0 (for stdcx etc)
 249         self.exc_o = cu.exc_o
 250         self.done_o = cu.done_o
 251         self.busy_o = cu.busy_o
 252
 253         self.ld_o = cu.ld_o
 254         self.st_o = cu.st_o
 255
 256         self.load_mem_o = cu.load_mem_o
 257         self.stwd_mem_o = cu.stwd_mem_o
 258
 259     def elaborate(self, platform):
 260         m = Module()
 261
 262         # temp/convenience
 263         comb = m.d.comb
 264         sync = m.d.sync
 265         issue_i = self.issue_i
 266
 267         #####################
 268         # latches for the FSM.
 269         m.submodules.opc_l = opc_l = SRLatch(sync=True, name="opc")
 270         m.submodules.src_l = src_l = SRLatch(False, self.n_src, name="src")
 271         m.submodules.alu_l = alu_l = SRLatch(sync=False, name="alu")
 272         m.submodules.adr_l = adr_l = SRLatch(sync=False, name="adr")
 273         m.submodules.lod_l = lod_l = SRLatch(sync=False, name="lod")
 274         m.submodules.sto_l = sto_l = SRLatch(sync=False, name="sto")
 275         m.submodules.wri_l = wri_l = SRLatch(sync=False, name="wri")
 276         m.submodules.upd_l = upd_l = SRLatch(sync=False, name="upd")
 277         m.submodules.cr0_l = cr0_l = SRLatch(sync=False, name="cr0")
 278         m.submodules.rst_l = rst_l = SRLatch(sync=False, name="rst")
 279         m.submodules.lsd_l = lsd_l = SRLatch(sync=False, name="lsd") # done
 280
 281         ####################
 282         # signals
 283
 284         # opcode decode
 285         op_is_ld = Signal(reset_less=True)
 286         op_is_st = Signal(reset_less=True)
 287         op_is_dcbz = Signal(reset_less=True)
 288         op_is_st_or_dcbz = Signal(reset_less=True)
 289         op_is_atomic = Signal(reset_less=True)
 290
 291         # ALU/LD data output control
 292         alu_valid = Signal(reset_less=True)  # ALU operands are valid
 293         alu_ok = Signal(reset_less=True)    # ALU out ok (1 clock delay valid)
 294         addr_ok = Signal(reset_less=True)   # addr ok (from PortInterface)
 295         ld_ok = Signal(reset_less=True)     # LD out ok from PortInterface
 296         wr_any = Signal(reset_less=True)    # any write (incl. store)
 297         rda_any = Signal(reset_less=True)   # any read for address ops
 298         rd_done = Signal(reset_less=True)   # all *necessary* operands read
 299         wr_reset = Signal(reset_less=True)  # final reset condition
 300         canceln = Signal(reset_less=True)   # cancel (active low)
 301         store_done = Signal(reset_less=True) # store has been actioned
 302
 303         # LD and ALU out
 304         alu_o = Signal(self.data_wid, reset_less=True)
 305         ldd_o = Signal(self.data_wid, reset_less=True)
 306
 307         ##############################
 308         # reset conditions for latches
 309
 310         # temporaries (also convenient when debugging)
 311         reset_o = Signal(reset_less=True)             # reset opcode
 312         reset_w = Signal(reset_less=True)             # reset write
 313         reset_u = Signal(reset_less=True)             # reset update
 314         reset_c = Signal(reset_less=True)             # reset cr0
 315         reset_a = Signal(reset_less=True)             # reset adr latch
 316         reset_i = Signal(reset_less=True)             # issue|die (use a lot)
 317         reset_r = Signal(self.n_src, reset_less=True)  # reset src
 318         reset_s = Signal(reset_less=True)             # reset store
 319
 320         # end execution when a terminating condition is detected:
 321         # - go_die_i: a speculative operation was cancelled
 322         # - exc_o.happened: an exception has occurred
 323         terminate = Signal()
 324         comb += terminate.eq(self.go_die_i | self.exc_o.happened)
 325
 326         comb += reset_i.eq(issue_i | terminate)       # various
 327         comb += reset_o.eq(self.done_o | terminate)      # opcode reset
 328         comb += reset_w.eq(self.wr.go_i[0] | terminate)  # write reg 1
 329         comb += reset_u.eq(self.wr.go_i[1] | terminate)  # update (reg 2)
 330         comb += reset_c.eq(self.wr.go_i[2] | terminate)  # cr0 (reg 3)
 331         comb += reset_s.eq(self.go_st_i | terminate)  # store reset
 332         comb += reset_r.eq(self.rd.go_i | Repl(terminate, self.n_src))
 333         comb += reset_a.eq(self.go_ad_i | terminate)
 334
 335         p_st_go = Signal(reset_less=True)
 336         sync += p_st_go.eq(self.st.go_i)
 337
 338         # decode bits of operand (latched)
 339         oper_r = CompLDSTOpSubset(name="oper_r")  # Dest register
 340         comb += op_is_st.eq(oper_r.insn_type == MicrOp.OP_STORE)   # ST
 341         comb += op_is_ld.eq(oper_r.insn_type == MicrOp.OP_LOAD)    # LD
 342         comb += op_is_dcbz.eq(oper_r.insn_type == MicrOp.OP_DCBZ)  # DCBZ
 343         comb += op_is_atomic.eq(oper_r.reserve) # atomic LR/SC
 344         comb += op_is_st_or_dcbz.eq(op_is_st | op_is_dcbz)
 345         # dcbz is special case of store
 346         #uncomment if needed
 347         #comb += Display("compldst_multi: op_is_dcbz = %i",
 348         #                (oper_r.insn_type == MicrOp.OP_DCBZ))
 349         op_is_update = oper_r.ldst_mode == LDSTMode.update           # UPDATE
 350         op_is_cix = oper_r.ldst_mode == LDSTMode.cix           # cache-inhibit
 351         comb += self.load_mem_o.eq(op_is_ld & self.go_ad_i)
 352         comb += self.stwd_mem_o.eq(op_is_st & self.go_st_i)
 353         comb += self.ld_o.eq(op_is_ld)
 354         comb += self.st_o.eq(op_is_st)
 355
 356         ##########################
 357         # FSM implemented through sequence of latches.  approximately this:
 358         # - opc_l       : opcode
 359         #    - src_l[0] : operands
 360         #    - src_l[1]
 361         #       - alu_l : looks after add of src1/2/imm (EA)
 362         #       - adr_l : waits for add (EA)
 363         #       - upd_l : waits for adr and Regfile (port 2)
 364         #       - cr0_l : waits for Rc=1 and CR0 Regfile (port 3)
 365         #    - src_l[2] : ST
 366         # - lod_l       : waits for adr (EA) and for LD Data
 367         # - wri_l       : waits for LD Data and Regfile (port 1)
 368         # - st_l        : waits for alu and operand2
 369         # - rst_l       : waits for all FSM paths to converge.
 370         # NOTE: use sync to stop combinatorial loops.
 371
 372         # opcode latch - inverted so that busy resets to 0
 373         # note this MUST be sync so as to avoid a combinatorial loop
 374         # between busy_o and issue_i on the reset latch (rst_l)
 375         comb += opc_l.s.eq(issue_i)  # XXX NOTE: INVERTED FROM book!
 376         comb += opc_l.r.eq(reset_o)  # XXX NOTE: INVERTED FROM book!
 377
 378         # src operand latch
 379         sync += src_l.s.eq(Repl(issue_i, self.n_src) & ~self.rdmaskn)
 380         sync += src_l.r.eq(reset_r)
 381         #### sync += Display("reset_r = %i",reset_r)
 382
 383         # alu latch.  use sync-delay between alu_ok and valid to generate pulse
 384         comb += alu_l.s.eq(reset_i)
 385         comb += alu_l.r.eq(alu_ok & ~alu_valid & ~rda_any)
 386
 387         # addr latch
 388         comb += adr_l.s.eq(reset_i)
 389         sync += adr_l.r.eq(reset_a)
 390
 391         # ld latch
 392         comb += lod_l.s.eq(reset_i)
 393         comb += lod_l.r.eq(ld_ok)
 394
 395         # dest operand latch
 396         comb += wri_l.s.eq(issue_i)
 397         sync += wri_l.r.eq(reset_w | Repl(wr_reset |
 398                                           (~self.pi.busy_o & op_is_update),
 399                                           #(self.pi.busy_o & op_is_update),
 400                             #self.done_o | (self.pi.busy_o & op_is_update),
 401                                           self.n_dst))
 402
 403         # CR0 operand latch (CR0 written to reg 3 if Rc=1)
 404         op_is_rc1 = self.oper_i.rc.rc & self.oper_i.rc.ok
 405         comb += cr0_l.s.eq(issue_i & op_is_rc1)
 406         sync += cr0_l.r.eq(reset_c)
 407
 408         # update-mode operand latch (EA written to reg 2)
 409         sync += upd_l.s.eq(reset_i)
 410         sync += upd_l.r.eq(reset_u)
 411
 412         # store latch
 413         comb += sto_l.s.eq(addr_ok & op_is_st_or_dcbz)
 414         sync += sto_l.r.eq(reset_s | p_st_go)
 415
 416         # ld/st done.  needed to stop LD/ST from activating repeatedly
 417         comb += lsd_l.s.eq(issue_i)
 418         sync += lsd_l.r.eq(reset_s | p_st_go | ld_ok)
 419
 420         # reset latch
 421         comb += rst_l.s.eq(addr_ok)  # start when address is ready
 422         comb += rst_l.r.eq(issue_i)
 423
 424         # create a latch/register for the operand
 425         with m.If(self.issue_i):
 426             sync += oper_r.eq(self.oper_i)
 427         with m.If(self.done_o | terminate):
 428             sync += oper_r.eq(0)
 429
 430         # and for LD and store-done
 431         ldd_r = Signal(self.data_wid, reset_less=True)  # Dest register
 432         latchregister(m, ldd_o, ldd_r, ld_ok, name="ldo_r")
 433
 434         # store actioned, communicate through CR0 (for atomic LR/SC)
 435         latchregister(m, self.pi.store_done.data, store_done,
 436                          self.pi.store_done.ok,
 437                          name="std_r")
 438
 439         # and for each input from the incoming src operands
 440         srl = []
 441         for i in range(self.n_src):
 442             name = "src_r%d" % i
 443             src_r = Signal(self.data_wid, name=name, reset_less=True)
 444             with m.If(self.rd.go_i[i]):
 445                 sync += src_r.eq(self.src_i[i])
 446             with m.If(self.issue_i):
 447                 sync += src_r.eq(0)
 448             srl.append(src_r)
 449
 450         # and one for the output from the ADD (for the EA)
 451         addr_r = Signal(self.data_wid, reset_less=True)  # Effective Address
 452         latchregister(m, alu_o, addr_r, alu_l.q, "ea_r")
 453
 454         # select either zero or src1 if opcode says so
 455         op_is_z = oper_r.zero_a
 456         src1_or_z = Signal(self.data_wid, reset_less=True)
 457         m.d.comb += src1_or_z.eq(Mux(op_is_z, 0, srl[0]))
 458
 459         # select either immediate or src2 if opcode says so
 460         op_is_imm = oper_r.imm_data.ok
 461         src2_or_imm = Signal(self.data_wid, reset_less=True)
 462         m.d.comb += src2_or_imm.eq(Mux(op_is_imm, oper_r.imm_data.data, srl[1]))
 463
 464         # now do the ALU addr add: one cycle, and say "ready" (next cycle, too)
 465         comb += alu_o.eq(src1_or_z + src2_or_imm)  # actual EA
 466         m.d.sync += alu_ok.eq(alu_valid & canceln) # keep ack in sync with EA
 467
 468         ############################
 469         # Control Signal calculation
 470
 471         # busy signal
 472         busy_o = self.busy_o
 473         comb += self.busy_o.eq(opc_l.q)  # | self.pi.busy_o)  # busy out
 474
 475         # 1st operand read-request only when zero not active
 476         # 2nd operand only needed when immediate is not active
 477         slg = Cat(op_is_z, op_is_imm) #is this correct ?
 478         bro = Repl(self.busy_o, self.n_src)
 479         comb += self.rd.rel_o.eq(src_l.q & bro & ~slg)
 480
 481         # note when the address-related read "go" signals are active
 482         comb += rda_any.eq(self.rd.go_i[0] | self.rd.go_i[1])
 483
 484         # alu input valid when 1st and 2nd ops done (or imm not active)
 485         comb += alu_valid.eq(busy_o & ~(self.rd.rel_o[0] | self.rd.rel_o[1]) &
 486                              canceln)
 487
 488         # 3rd operand only needed when operation is a store
 489         comb += self.rd.rel_o[2].eq(src_l.q[2] & busy_o & op_is_st)
 490
 491         # all reads done when alu is valid and 3rd operand needed
 492         comb += rd_done.eq(alu_valid & ~self.rd.rel_o[2])
 493
 494         # address release only if addr ready, but Port must be idle
 495         comb += self.adr_rel_o.eq(alu_valid & adr_l.q & busy_o)
 496
 497         # the write/store (etc) all must be cancelled if an exception occurs
 498         # note: cancel is active low, like shadown_i,
 499         #       while exc_o.happpened is active high
 500         comb += canceln.eq(~self.exc_o.happened & self.shadown_i)
 501
 502         # store release when st ready *and* all operands read (and no shadow)
 503         # dcbz is special case of store -- TODO verify shadows
 504         comb += self.st.rel_o.eq(sto_l.q & busy_o & rd_done & op_is_st_or_dcbz &
 505                                canceln)
 506
 507         # request write of LD result.  waits until shadow is dropped.
 508         comb += self.wr.rel_o[0].eq(rd_done & wri_l.q & busy_o & lod_l.qn &
 509                                   op_is_ld & canceln)
 510
 511         # request write of EA result only in update mode
 512         comb += self.wr.rel_o[1].eq(upd_l.q & busy_o & op_is_update &
 513                                   alu_valid & canceln)
 514
 515         # request write of CR0 result only in reserve and Rc=1
 516         comb += self.wr.rel_o[2].eq(cr0_l.q & busy_o & op_is_atomic &
 517                                   alu_valid & canceln)
 518
 519         # provide "done" signal: select req_rel for non-LD/ST, adr_rel for LD/ST
 520         comb += wr_any.eq(self.st.go_i | p_st_go |
 521                           self.wr.go_i.bool())
 522         comb += wr_reset.eq(rst_l.q & busy_o & canceln &
 523                             ~(self.st.rel_o | self.wr.rel_o.bool()) &
 524                             (lod_l.qn | op_is_st_or_dcbz)
 525                             )
 526         comb += self.done_o.eq(wr_reset & (~self.pi.busy_o | op_is_ld))
 527
 528         ######################
 529         # Data/Address outputs
 530
 531         # put the LD-output register directly onto the output bus on a go_write
 532         comb += self.o_data.data.eq(self.dest[0])
 533         comb += self.o_data.ok.eq(self.wr.rel_o[0])
 534         with m.If(self.wr.go_i[0]):
 535             comb += self.dest[0].eq(ldd_r)
 536
 537         # "update" mode, put address out on 2nd go-write
 538         comb += self.addr_o.data.eq(self.dest[1])
 539         comb += self.addr_o.ok.eq(self.wr.rel_o[1])
 540         with m.If(op_is_update & self.wr.go_i[1]):
 541             comb += self.dest[1].eq(addr_r)
 542
 543         # fun-fun-fun, calculate CR0 when Rc=1 requested.
 544         cr0 = self.dest[2]
 545         comb += self.cr_o.data.eq(cr0)
 546         comb += self.cr_o.ok.eq(self.wr.rel_o[2])
 547         with m.If(cr0_l.q):
 548             comb += cr0.eq(Cat(C(0, 1), store_done, C(0, 2)))
 549
 550         # need to look like MultiCompUnit: put wrmask out.
 551         # XXX may need to make this enable only when write active
 552         comb += self.wrmask.eq(bro & Cat(op_is_ld, op_is_update, cr0_l.q))
 553
 554         ###########################
 555         # PortInterface connections
 556         pi = self.pi
 557
 558         # connect to LD/ST PortInterface.
 559         comb += pi.is_ld_i.eq(op_is_ld & busy_o)  # decoded-LD
 560         comb += pi.is_nc.eq(op_is_cix & busy_o)  # cache-inhibited
 561         comb += pi.is_st_i.eq(op_is_st_or_dcbz & busy_o)  # decoded-ST
 562         comb += pi.is_dcbz_i.eq(op_is_dcbz & busy_o)  # decoded-DCBZ
 563         comb += pi.reserve.eq(oper_r.reserve & busy_o)  # atomic LR/SC
 564         comb += pi.data_len.eq(oper_r.data_len)  # data_len
 565         # address: use sync to avoid long latency
 566         sync += pi.addr.data.eq(addr_r)           # EA from adder
 567         with m.If(op_is_dcbz):
 568             sync += Display("LDSTCompUnit.DCBZ: EA from adder %x", addr_r)
 569
 570         sync += pi.addr.ok.eq(alu_ok & lsd_l.q)  # "do address stuff" (once)
 571         comb += self.exc_o.eq(pi.exc_o)  # exception occurred
 572         comb += addr_ok.eq(self.pi.addr_ok_o)  # no exc, address fine
 573         # connect MSR.PR etc. for priv/virt operation
 574         comb += pi.priv_mode.eq(~oper_r.msr[MSR.PR])
 575         comb += pi.virt_mode.eq(oper_r.msr[MSR.DR])
 576         comb += pi.mode_32bit.eq(~oper_r.msr[MSR.SF])
 577         with m.If(self.issue_i): # display this only once
 578             sync += Display("LDSTCompUnit: oper_r.msr %x pr=%x dr=%x sf=%x",
 579                                       oper_r.msr,
 580                                       oper_r.msr[MSR.PR],
 581                                       oper_r.msr[MSR.DR],
 582                                       oper_r.msr[MSR.SF])
 583
 584         # byte-reverse on LD
 585         revnorev = Signal(64, reset_less=True)
 586         with m.If(oper_r.byte_reverse):
 587             # byte-reverse the data based on ld/st width (turn it to LE)
 588             data_len = oper_r.data_len
 589             lddata_r = byte_reverse(m, 'lddata_r', pi.ld.data, data_len)
 590             comb += revnorev.eq(lddata_r)  # put reversed- data out
 591         with m.Else():
 592             comb += revnorev.eq(pi.ld.data)  # put data out, straight (as BE)
 593
 594         # then check sign-extend
 595         with m.If(oper_r.sign_extend):
 596             # okok really should "if data_len == 4" and so on here
 597             with m.If(oper_r.data_len == 2):
 598                 comb += ldd_o.eq(exts(revnorev, 16, 64))  # sign-extend hword
 599             with m.Else():
 600                 comb += ldd_o.eq(exts(revnorev, 32, 64))  # sign-extend dword
 601         with m.Else():
 602             comb += ldd_o.eq(revnorev)
 603
 604         # ld - ld gets latched in via lod_l
 605         comb += ld_ok.eq(pi.ld.ok)  # ld.ok *closes* (freezes) ld data
 606
 607         # byte-reverse on ST
 608         op3 = srl[2] # 3rd operand latch
 609         with m.If(oper_r.byte_reverse):
 610             # byte-reverse the data based on width
 611             data_len = oper_r.data_len
 612             stdata_r = byte_reverse(m, 'stdata_r', op3, data_len)
 613             comb += pi.st.data.eq(stdata_r)
 614         with m.Else():
 615             comb += pi.st.data.eq(op3)
 616
 617         # store - data goes in based on go_st
 618         comb += pi.st.ok.eq(self.st.go_i)  # go store signals st data valid
 619
 620         return m
 621
 622     def get_out(self, i):
 623         """make LDSTCompUnit look like RegSpecALUAPI.  these correspond
 624         to LDSTOutputData o and o1 respectively.
 625         """
 626         if i == 0:
 627             return self.o_data # LDSTOutputData.regspec o
 628         if i == 1:
 629             return self.addr_o # LDSTOutputData.regspec o1
 630         if i == 2:
 631             return self.cr_o # LDSTOutputData.regspec cr_a
 632         # return self.dest[i]
 633
 634     def get_fu_out(self, i):
 635         return self.get_out(i)
 636
 637     def __iter__(self):
 638         yield self.rd.go_i
 639         yield self.go_ad_i
 640         yield self.wr.go_i
 641         yield self.go_st_i
 642         yield self.issue_i
 643         yield self.shadown_i
 644         yield self.go_die_i
 645         yield from self.oper_i.ports()
 646         yield from self.src_i
 647         yield self.busy_o
 648         yield self.rd.rel_o
 649         yield self.adr_rel_o
 650         yield self.sto_rel_o
 651         yield self.wr.rel_o
 652         yield from self.o_data.ports()
 653         yield from self.addr_o.ports()
 654         yield from self.cr_o.ports()
 655         yield self.load_mem_o
 656         yield self.stwd_mem_o
 657
 658     def ports(self):
 659         return list(self)
 660
 661
 662 def wait_for(sig, wait=True, test1st=False):
 663     v = (yield sig)
 664     print("wait for", sig, v, wait, test1st)
 665     if test1st and bool(v) == wait:
 666         return
 667     while True:
 668         yield
 669         v = (yield sig)
 670         #print("...wait for", sig, v)
 671         if bool(v) == wait:
 672             break
 673
 674
 675 def store(dut, src1, src2, src3, imm, imm_ok=True, update=False,
 676           byterev=True):
 677     print("ST", src1, src2, src3, imm, imm_ok, update)
 678     yield dut.oper_i.insn_type.eq(MicrOp.OP_STORE)
 679     yield dut.oper_i.data_len.eq(2)  # half-word
 680     yield dut.oper_i.byte_reverse.eq(byterev)
 681     yield dut.src1_i.eq(src1)
 682     yield dut.src2_i.eq(src2)
 683     yield dut.src3_i.eq(src3)
 684     yield dut.oper_i.imm_data.data.eq(imm)
 685     yield dut.oper_i.imm_data.ok.eq(imm_ok)
 686     #guess: this one was removed -- yield dut.oper_i.update.eq(update)
 687     yield dut.issue_i.eq(1)
 688     yield
 689     yield dut.issue_i.eq(0)
 690
 691     if imm_ok:
 692         active_rel = 0b101
 693     else:
 694         active_rel = 0b111
 695     # wait for all active rel signals to come up
 696     while True:
 697         rel = yield dut.rd.rel_o
 698         if rel == active_rel:
 699             break
 700         yield
 701     yield dut.rd.go_i.eq(active_rel)
 702     yield
 703     yield dut.rd.go_i.eq(0)
 704
 705     yield from wait_for(dut.adr_rel_o, False, test1st=True)
 706     # yield from wait_for(dut.adr_rel_o)
 707     # yield dut.ad.go.eq(1)
 708     # yield
 709     # yield dut.ad.go.eq(0)
 710
 711     if update:
 712         yield from wait_for(dut.wr.rel_o[1])
 713         yield dut.wr.go.eq(0b10)
 714         yield
 715         addr = yield dut.addr_o
 716         print("addr", addr)
 717         yield dut.wr.go.eq(0)
 718     else:
 719         addr = None
 720
 721     yield from wait_for(dut.sto_rel_o)
 722     yield dut.go_st_i.eq(1)
 723     yield
 724     yield dut.go_st_i.eq(0)
 725     yield from wait_for(dut.busy_o, False)
 726     # wait_for(dut.stwd_mem_o)
 727     yield
 728     return addr
 729
 730
 731 def load(dut, src1, src2, imm, imm_ok=True, update=False, zero_a=False,
 732          byterev=True):
 733     print("LD", src1, src2, imm, imm_ok, update)
 734     yield dut.oper_i.insn_type.eq(MicrOp.OP_LOAD)
 735     yield dut.oper_i.data_len.eq(2)  # half-word
 736     yield dut.oper_i.byte_reverse.eq(byterev)
 737     yield dut.src1_i.eq(src1)
 738     yield dut.src2_i.eq(src2)
 739     yield dut.oper_i.zero_a.eq(zero_a)
 740     yield dut.oper_i.imm_data.data.eq(imm)
 741     yield dut.oper_i.imm_data.ok.eq(imm_ok)
 742     yield dut.issue_i.eq(1)
 743     yield
 744     yield dut.issue_i.eq(0)
 745     yield
 746
 747     # set up read-operand flags
 748     rd = 0b00
 749     if not imm_ok:  # no immediate means RB register needs to be read
 750         rd |= 0b10
 751     if not zero_a:  # no zero-a means RA needs to be read
 752         rd |= 0b01
 753
 754     # wait for the operands (RA, RB, or both)
 755     if rd:
 756         yield dut.rd.go_i.eq(rd)
 757         yield from wait_for(dut.rd.rel_o)
 758         yield dut.rd.go_i.eq(0)
 759
 760     yield from wait_for(dut.adr_rel_o, False, test1st=True)
 761     # yield dut.ad.go.eq(1)
 762     # yield
 763     # yield dut.ad.go.eq(0)
 764
 765     if update:
 766         yield from wait_for(dut.wr.rel_o[1])
 767         yield dut.wr.go_i.eq(0b10)
 768         yield
 769         addr = yield dut.addr_o
 770         print("addr", addr)
 771         yield dut.wr.go_i.eq(0)
 772     else:
 773         addr = None
 774
 775     yield from wait_for(dut.wr.rel_o[0], test1st=True)
 776     yield dut.wr.go_i.eq(1)
 777     yield
 778     data = yield dut.o_data.o
 779     data_ok = yield dut.o_data.o_ok
 780     yield dut.wr.go_i.eq(0)
 781     yield from wait_for(dut.busy_o)
 782     yield
 783     # wait_for(dut.stwd_mem_o)
 784     return data, data_ok, addr
 785
 786
 787 def ldst_sim(dut):
 788
 789     ###################
 790     # immediate version
 791
 792     # two STs (different addresses)
 793     yield from store(dut, 4, 0, 3, 2)  # ST reg4 into addr rfile[reg3]+2
 794     yield from store(dut, 2, 0, 9, 2)  # ST reg4 into addr rfile[reg9]+2
 795     yield
 796     # two LDs (deliberately LD from the 1st address then 2nd)
 797     data, addr = yield from load(dut, 4, 0, 2)
 798     assert data == 0x0003, "returned %x" % data
 799     data, addr = yield from load(dut, 2, 0, 2)
 800     assert data == 0x0009, "returned %x" % data
 801     yield
 802
 803     # indexed version
 804     yield from store(dut, 9, 5, 3, 0, imm_ok=False)
 805     data, addr = yield from load(dut, 9, 5, 0, imm_ok=False)
 806     assert data == 0x0003, "returned %x" % data
 807
 808     # update-immediate version
 809     addr = yield from store(dut, 9, 6, 3, 2, update=True)
 810     assert addr == 0x000b, "returned %x" % addr
 811
 812     # update-indexed version
 813     data, addr = yield from load(dut, 9, 5, 0, imm_ok=False, update=True)
 814     assert data == 0x0003, "returned %x" % data
 815     assert addr == 0x000e, "returned %x" % addr
 816
 817     # immediate *and* zero version
 818     data, addr = yield from load(dut, 1, 4, 8, imm_ok=True, zero_a=True)
 819     assert data == 0x0008, "returned %x" % data
 820
 821
 822 class TestLDSTCompUnit(LDSTCompUnit):
 823
 824     def __init__(self, rwid, pspec):
 825         from soc.experiment.l0_cache import TstL0CacheBuffer
 826         self.l0 = l0 = TstL0CacheBuffer(pspec)
 827         pi = l0.l0.dports[0]
 828         LDSTCompUnit.__init__(self, pi, rwid, 4)
 829
 830     def elaborate(self, platform):
 831         m = LDSTCompUnit.elaborate(self, platform)
 832         m.submodules.l0 = self.l0
 833         # link addr-go direct to rel
 834         m.d.comb += self.ad.go_i.eq(self.ad.rel_o)
 835         return m
 836
 837
 838 def test_scoreboard():
 839
 840     units = {}
 841     pspec = TestMemPspec(ldst_ifacetype='bare_wb',
 842                          imem_ifacetype='bare_wb',
 843                          addr_wid=64,
 844                          mask_wid=8,
 845                          reg_wid=64,
 846                          units=units)
 847
 848     dut = TestLDSTCompUnit(16,pspec)
 849     vl = rtlil.convert(dut, ports=dut.ports())
 850     with open("test_ldst_comp.il", "w") as f:
 851         f.write(vl)
 852
 853     run_simulation(dut, ldst_sim(dut), vcd_name='test_ldst_comp.vcd')
 854
 855
 856 class TestLDSTCompUnitRegSpec(LDSTCompUnit):
 857
 858     def __init__(self, pspec):
 859         from soc.experiment.l0_cache import TstL0CacheBuffer
 860         from soc.fu.ldst.pipe_data import LDSTPipeSpec
 861         regspec = LDSTPipeSpec.regspec
 862         self.l0 = l0 = TstL0CacheBuffer(pspec)
 863         pi = l0.l0.dports[0]
 864         LDSTCompUnit.__init__(self, pi, regspec, 4)
 865
 866     def elaborate(self, platform):
 867         m = LDSTCompUnit.elaborate(self, platform)
 868         m.submodules.l0 = self.l0
 869         # link addr-go direct to rel
 870         m.d.comb += self.ad.go_i.eq(self.ad.rel_o)
 871         return m
 872
 873
 874 def test_scoreboard_regspec():
 875
 876     units = {}
 877     pspec = TestMemPspec(ldst_ifacetype='bare_wb',
 878                          imem_ifacetype='bare_wb',
 879                          addr_wid=64,
 880                          mask_wid=8,
 881                          reg_wid=64,
 882                          units=units)
 883
 884     dut = TestLDSTCompUnitRegSpec(pspec)
 885     vl = rtlil.convert(dut, ports=dut.ports())
 886     with open("test_ldst_comp.il", "w") as f:
 887         f.write(vl)
 888
 889     run_simulation(dut, ldst_sim(dut), vcd_name='test_ldst_regspec.vcd')
 890
 891
 892 if __name__ == '__main__':
 893     test_scoreboard_regspec()
 894     test_scoreboard()