src/soc/fu/ldst/loadstore.py

   1 """LoadStore1 FSM.
   2
   3 based on microwatt loadstore1.vhdl, but conforming to PortInterface.
   4 unlike loadstore1.vhdl this does *not* deal with actual Load/Store
   5 ops: that job is handled by LDSTCompUnit, which talks to LoadStore1
   6 by way of PortInterface.  PortInterface is where things need extending,
   7 such as adding dcbz support, etc.
   8
   9 this module basically handles "pure" load / store operations, and
  10 its first job is to ask the D-Cache for the data.  if that fails,
  11 the second task (if virtual memory is enabled) is to ask the MMU
  12 to perform a TLB, then to go *back* to the cache and ask again.
  13
  14 Links:
  15
  16 * https://bugs.libre-soc.org/show_bug.cgi?id=465
  17
  18 """
  19
  20 from nmigen import (Elaboratable, Module, Signal, Shape, unsigned, Cat, Mux,
  21                     Record, Memory,
  22                     Const)
  23 from nmutil.iocontrol import RecordObject
  24 from nmutil.util import rising_edge, Display
  25 from enum import Enum, unique
  26
  27 from soc.experiment.dcache import DCache
  28 from soc.experiment.icache import ICache
  29 from soc.experiment.pimem import PortInterfaceBase
  30 from soc.experiment.mem_types import LoadStore1ToMMUType
  31 from soc.experiment.mem_types import MMUToLoadStore1Type
  32
  33 from soc.minerva.wishbone import make_wb_layout
  34 from soc.bus.sram import SRAM
  35 from nmutil.util import Display
  36
  37
  38 @unique
  39 class State(Enum):
  40     IDLE = 0       # ready for instruction
  41     ACK_WAIT = 1   # waiting for ack from dcache
  42     MMU_LOOKUP = 2 # waiting for MMU to look up translation
  43     TLBIE_WAIT = 3 # waiting for MMU to finish doing a tlbie
  44
  45
  46 # captures the LDSTRequest from the PortInterface, which "blips" most
  47 # of this at us (pipeline-style).
  48 class LDSTRequest(RecordObject):
  49     def __init__(self, name=None):
  50         RecordObject.__init__(self, name=name)
  51
  52         self.load          = Signal()
  53         self.dcbz          = Signal()
  54         self.addr          = Signal(64)
  55         # self.store_data    = Signal(64) # this is already sync (on a delay)
  56         self.byte_sel      = Signal(8)
  57         self.nc            = Signal()              # non-cacheable access
  58         self.virt_mode     = Signal()
  59         self.priv_mode     = Signal()
  60         self.mode_32bit    = Signal() # XXX UNUSED AT PRESENT
  61         self.align_intr    = Signal()
  62
  63
  64 # glue logic for microwatt mmu and dcache
  65 class LoadStore1(PortInterfaceBase):
  66     def __init__(self, pspec):
  67         self.pspec = pspec
  68         self.disable_cache = (hasattr(pspec, "disable_cache") and
  69                               pspec.disable_cache == True)
  70         regwid = pspec.reg_wid
  71         addrwid = pspec.addr_wid
  72
  73         super().__init__(regwid, addrwid)
  74         self.dcache = DCache()
  75         self.icache = ICache(pspec)
  76         # these names are from the perspective of here (LoadStore1)
  77         self.d_out  = self.dcache.d_in     # in to dcache is out for LoadStore
  78         self.d_in = self.dcache.d_out      # out from dcache is in for LoadStore
  79         self.i_out  = self.icache.i_in     # in to icache is out for LoadStore
  80         self.i_in = self.icache.i_out      # out from icache is in for LoadStore
  81         self.m_out  = LoadStore1ToMMUType("m_out") # out *to* MMU
  82         self.m_in = MMUToLoadStore1Type("m_in")   # in *from* MMU
  83         self.req = LDSTRequest(name="ldst_req")
  84
  85         # TODO, convert dcache wb_in/wb_out to "standard" nmigen Wishbone bus
  86         self.dbus = Record(make_wb_layout(pspec))
  87         self.ibus = Record(make_wb_layout(pspec))
  88
  89         # for creating a single clock blip to DCache
  90         self.d_valid = Signal()
  91         self.d_w_valid = Signal()
  92         self.d_validblip = Signal()
  93
  94         # state info for LD/ST
  95         self.done          = Signal()
  96         self.done_delay    = Signal()
  97         # latch most of the input request
  98         self.load          = Signal()
  99         self.tlbie         = Signal()
 100         self.dcbz          = Signal()
 101         self.addr          = Signal(64)
 102         self.maddr          = Signal(64)
 103         self.store_data    = Signal(64)
 104         self.load_data     = Signal(64)
 105         self.load_data_delay = Signal(64)
 106         self.byte_sel      = Signal(8)
 107         #self.xerc         : xer_common_t;
 108         #self.reserve       = Signal()
 109         #self.atomic        = Signal()
 110         #self.atomic_last   = Signal()
 111         #self.rc            = Signal()
 112         self.nc            = Signal()              # non-cacheable access
 113         self.virt_mode     = Signal()
 114         self.priv_mode     = Signal()
 115         self.mode_32bit    = Signal() # XXX UNUSED AT PRESENT
 116         self.state         = Signal(State)
 117         self.instr_fault   = Signal()  # indicator to request i-cache MMU lookup
 118         self.r_instr_fault  = Signal() # accessed in external_busy
 119         self.align_intr    = Signal()
 120         self.busy          = Signal()
 121         self.wait_dcache   = Signal()
 122         self.wait_mmu      = Signal()
 123         #self.intr_vec     : integer range 0 to 16#fff#;
 124         #self.nia           = Signal(64)
 125         #self.srr1          = Signal(16)
 126         # use these to set the dsisr or dar respectively
 127         self.mmu_set_spr    = Signal()
 128         self.mmu_set_dsisr  = Signal()
 129         self.mmu_set_dar    = Signal()
 130         self.sprval_in      = Signal(64)
 131
 132         # ONLY access these read-only, do NOT attempt to change
 133         self.dsisr          = Signal(32)
 134         self.dar            = Signal(64)
 135
 136     # when external_busy set, do not allow PortInterface to proceed
 137     def external_busy(self, m):
 138         return self.instr_fault | self.r_instr_fault
 139
 140     def set_wr_addr(self, m, addr, mask, misalign, msr, is_dcbz):
 141         m.d.comb += self.req.load.eq(0) # store operation
 142         m.d.comb += self.req.byte_sel.eq(mask)
 143         m.d.comb += self.req.addr.eq(addr)
 144         m.d.comb += self.req.priv_mode.eq(~msr.pr) # not-problem  ==> priv
 145         m.d.comb += self.req.virt_mode.eq(msr.dr) # DR ==> virt
 146         m.d.comb += self.req.mode_32bit.eq(~msr.sf) # not-sixty-four ==> 32bit
 147         m.d.comb += self.req.align_intr.eq(misalign)
 148         m.d.comb += self.req.dcbz.eq(is_dcbz)
 149
 150         # m.d.comb += Display("set_wr_addr %i dcbz %i",addr,is_dcbz)
 151
 152         # option to disable the cache entirely for write
 153         if self.disable_cache:
 154             m.d.comb += self.req.nc.eq(1)
 155         return None
 156
 157     def set_rd_addr(self, m, addr, mask, misalign, msr):
 158         m.d.comb += self.d_valid.eq(1)
 159         m.d.comb += self.req.load.eq(1) # load operation
 160         m.d.comb += self.req.byte_sel.eq(mask)
 161         m.d.comb += self.req.align_intr.eq(misalign)
 162         m.d.comb += self.req.addr.eq(addr)
 163         m.d.comb += self.req.priv_mode.eq(~msr.pr) # not-problem  ==> priv
 164         m.d.comb += self.req.virt_mode.eq(msr.dr) # DR ==> virt
 165         m.d.comb += self.req.mode_32bit.eq(~msr.sf) # not-sixty-four ==> 32bit
 166         # BAD HACK! disable cacheing on LD when address is 0xCxxx_xxxx
 167         # this is for peripherals. same thing done in Microwatt loadstore1.vhdl
 168         with m.If(addr[28:] == Const(0xc, 4)):
 169             m.d.comb += self.req.nc.eq(1)
 170         # option to disable the cache entirely for read
 171         if self.disable_cache:
 172             m.d.comb += self.req.nc.eq(1)
 173         return None #FIXME return value
 174
 175     def set_wr_data(self, m, data, wen):
 176         # do the "blip" on write data
 177         m.d.comb += self.d_valid.eq(1)
 178         # put data into comb which is picked up in main elaborate()
 179         m.d.comb += self.d_w_valid.eq(1)
 180         m.d.comb += self.store_data.eq(data)
 181         #m.d.sync += self.d_out.byte_sel.eq(wen) # this might not be needed
 182         st_ok = self.done # TODO indicates write data is valid
 183         return st_ok
 184
 185     def get_rd_data(self, m):
 186         ld_ok = self.done_delay # indicates read data is valid
 187         data = self.load_data_delay   # actual read data
 188         return data, ld_ok
 189
 190     def elaborate(self, platform):
 191         m = super().elaborate(platform)
 192         comb, sync = m.d.comb, m.d.sync
 193
 194         # microwatt takes one more cycle before next operation can be issued
 195         sync += self.done_delay.eq(self.done)
 196         sync += self.load_data_delay.eq(self.load_data)
 197
 198         # create dcache and icache module
 199         m.submodules.dcache = dcache = self.dcache
 200         m.submodules.icache = icache = self.icache
 201
 202         # temp vars
 203         d_out, d_in, dbus = self.d_out, self.d_in, self.dbus
 204         i_out, i_in, ibus = self.i_out, self.i_in, self.ibus
 205         m_out, m_in = self.m_out, self.m_in
 206         exc = self.pi.exc_o
 207         exception = exc.happened
 208         mmureq = Signal()
 209
 210         # copy of address, but gets over-ridden for instr_fault
 211         maddr = Signal(64)
 212         m.d.comb += maddr.eq(self.addr)
 213
 214         # create a blip (single pulse) on valid read/write request
 215         # this can be over-ridden in the FSM to get dcache to re-run
 216         # a request when MMU_LOOKUP completes.
 217         m.d.comb += self.d_validblip.eq(rising_edge(m, self.d_valid))
 218         ldst_r = LDSTRequest("ldst_r")
 219         comb += Display("MMUTEST: LoadStore1 d_in.error=%i",d_in.error)
 220
 221         # fsm skeleton
 222         with m.Switch(self.state):
 223             with m.Case(State.IDLE):
 224                 with m.If((self.d_validblip | self.instr_fault) &
 225                           ~exc.happened):
 226                     comb += self.busy.eq(1)
 227                     sync += self.state.eq(State.ACK_WAIT)
 228                     sync += ldst_r.eq(self.req) # copy of LDSTRequest on "blip"
 229 #                   sync += Display("validblip self.req.virt_mode=%i",
 230 #                   self.req.virt_mode)
 231                     with m.If(self.instr_fault):
 232                         comb += mmureq.eq(1)
 233                         sync += self.r_instr_fault.eq(1)
 234                         comb += maddr.eq(self.maddr)
 235                         sync += self.state.eq(State.MMU_LOOKUP)
 236                 with m.Else():
 237                     sync += ldst_r.eq(0)
 238
 239             # waiting for completion
 240             with m.Case(State.ACK_WAIT):
 241                 comb += Display("MMUTEST: ACK_WAIT")
 242                 comb += self.busy.eq(~exc.happened)
 243
 244                 with m.If(d_in.error):
 245                     # cache error is not necessarily "final", it could
 246                     # be that it was just a TLB miss
 247                     with m.If(d_in.cache_paradox):
 248                         comb += exception.eq(1)
 249                         sync += self.state.eq(State.IDLE)
 250                         sync += ldst_r.eq(0)
 251                         sync += Display("cache error -> update dsisr")
 252                         sync += self.dsisr[63 - 38].eq(~self.load)
 253                         # XXX there is no architected bit for this
 254                         # (probably should be a machine check in fact)
 255                         sync += self.dsisr[63 - 35].eq(d_in.cache_paradox)
 256
 257                     with m.Else():
 258                         # Look up the translation for TLB miss
 259                         # and also for permission error and RC error
 260                         # in case the PTE has been updated.
 261                         comb += mmureq.eq(1)
 262                         sync += self.state.eq(State.MMU_LOOKUP)
 263                 with m.If(d_in.valid):
 264                     m.d.comb += self.done.eq(~mmureq) # done if not doing MMU
 265                     with m.If(self.done):
 266                         sync += Display("ACK_WAIT, done %x", self.addr)
 267                     sync += self.state.eq(State.IDLE)
 268                     sync += ldst_r.eq(0)
 269                     with m.If(self.load):
 270                         m.d.comb += self.load_data.eq(d_in.data)
 271
 272             # waiting here for the MMU TLB lookup to complete.
 273             # either re-try the dcache lookup or throw MMU exception
 274             with m.Case(State.MMU_LOOKUP):
 275                 comb += self.busy.eq(~exception)
 276                 with m.If(m_in.done):
 277                     with m.If(~self.r_instr_fault):
 278                         sync += Display("MMU_LOOKUP, done %x -> %x",
 279                                         self.addr, d_out.addr)
 280                         # retry the request now that the MMU has
 281                         # installed a TLB entry, if not exception raised
 282                         m.d.comb += self.d_out.valid.eq(~exception)
 283                         sync += self.state.eq(State.ACK_WAIT)
 284                         sync += ldst_r.eq(0)
 285                     with m.Else():
 286                         sync += self.state.eq(State.IDLE)
 287                         sync += self.r_instr_fault.eq(0)
 288                         comb += self.done.eq(1)
 289
 290                 with m.If(m_in.err):
 291                     # MMU RADIX exception thrown. XXX
 292                     # TODO: critical that the write here has to
 293                     # notify the MMU FSM of the change to dsisr
 294                     comb += exception.eq(1)
 295                     comb += self.done.eq(1)
 296                     sync += Display("MMU RADIX exception thrown")
 297                     sync += Display("TODO: notify MMU of change to dsisr")
 298                     sync += self.dsisr[63 - 33].eq(m_in.invalid)
 299                     sync += self.dsisr[63 - 36].eq(m_in.perm_error) # noexec
 300                     sync += self.dsisr[63 - 38].eq(~self.load)
 301                     sync += self.dsisr[63 - 44].eq(m_in.badtree)
 302                     sync += self.dsisr[63 - 45].eq(m_in.rc_error)
 303                     sync += self.state.eq(State.IDLE)
 304
 305             with m.Case(State.TLBIE_WAIT):
 306                 pass
 307
 308         # MMU FSM communicating a request to update DSISR or DAR (OP_MTSPR)
 309         with m.If(self.mmu_set_spr):
 310             with m.If(self.mmu_set_dsisr):
 311                 sync += self.dsisr.eq(self.sprval_in)
 312             with m.If(self.mmu_set_dar):
 313                 sync += self.dar.eq(self.sprval_in)
 314
 315         # hmmm, alignment occurs in set_rd_addr/set_wr_addr, note exception
 316         with m.If(self.align_intr):
 317             comb += exc.happened.eq(1)
 318         # check for updating DAR
 319         with m.If(exception):
 320             sync += Display("exception %x", self.addr)
 321             # alignment error: store address in DAR
 322             with m.If(self.align_intr):
 323                 sync += Display("alignment error: addr in DAR %x", self.addr)
 324                 sync += self.dar.eq(self.addr)
 325             with m.Elif(~self.r_instr_fault):
 326                 sync += Display("not instr fault, addr in DAR %x", self.addr)
 327                 sync += self.dar.eq(self.addr)
 328
 329         # when done or exception, return to idle state
 330         with m.If(self.done | exception):
 331             sync += self.state.eq(State.IDLE)
 332             comb += self.busy.eq(0)
 333
 334         # happened, alignment, instr_fault, invalid.
 335         # note that all of these flow through - eventually to the TRAP
 336         # pipeline, via PowerDecoder2.
 337         comb += self.align_intr.eq(self.req.align_intr)
 338         comb += exc.invalid.eq(m_in.invalid)
 339         comb += exc.alignment.eq(self.align_intr)
 340         comb += exc.instr_fault.eq(self.r_instr_fault)
 341         # badtree, perm_error, rc_error, segment_fault
 342         comb += exc.badtree.eq(m_in.badtree)
 343         comb += exc.perm_error.eq(m_in.perm_error)
 344         comb += exc.rc_error.eq(m_in.rc_error)
 345         comb += exc.segment_fault.eq(m_in.segerr)
 346
 347         # TODO, connect dcache wb_in/wb_out to "standard" nmigen Wishbone bus
 348         comb += dbus.adr.eq(dcache.bus.adr)
 349         comb += dbus.dat_w.eq(dcache.bus.dat_w)
 350         comb += dbus.sel.eq(dcache.bus.sel)
 351         comb += dbus.cyc.eq(dcache.bus.cyc)
 352         comb += dbus.stb.eq(dcache.bus.stb)
 353         comb += dbus.we.eq(dcache.bus.we)
 354
 355         comb += dcache.bus.dat_r.eq(dbus.dat_r)
 356         comb += dcache.bus.ack.eq(dbus.ack)
 357         if hasattr(dbus, "stall"):
 358             comb += dcache.bus.stall.eq(dbus.stall)
 359
 360         # update out d data when flag set
 361         with m.If(self.d_w_valid):
 362             m.d.sync += d_out.data.eq(self.store_data)
 363         #with m.Else():
 364         #    m.d.sync += d_out.data.eq(0)
 365         # unit test passes with that change
 366
 367         # this must move into the FSM, conditionally noticing that
 368         # the "blip" comes from self.d_validblip.
 369         # task 1: look up in dcache
 370         # task 2: if dcache fails, look up in MMU.
 371         # do **NOT** confuse the two.
 372         with m.If(self.d_validblip):
 373             m.d.comb += self.d_out.valid.eq(~exc.happened)
 374             m.d.comb += d_out.load.eq(self.req.load)
 375             m.d.comb += d_out.byte_sel.eq(self.req.byte_sel)
 376             m.d.comb += self.addr.eq(self.req.addr)
 377             m.d.comb += d_out.nc.eq(self.req.nc)
 378             m.d.comb += d_out.priv_mode.eq(self.req.priv_mode)
 379             m.d.comb += d_out.virt_mode.eq(self.req.virt_mode)
 380             #m.d.comb += Display("validblip dcbz=%i addr=%x",
 381             #self.req.dcbz,self.req.addr)
 382             m.d.comb += d_out.dcbz.eq(self.req.dcbz)
 383         with m.Else():
 384             m.d.comb += d_out.load.eq(ldst_r.load)
 385             m.d.comb += d_out.byte_sel.eq(ldst_r.byte_sel)
 386             m.d.comb += self.addr.eq(ldst_r.addr)
 387             m.d.comb += d_out.nc.eq(ldst_r.nc)
 388             m.d.comb += d_out.priv_mode.eq(ldst_r.priv_mode)
 389             m.d.comb += d_out.virt_mode.eq(ldst_r.virt_mode)
 390             #m.d.comb += Display("no_validblip dcbz=%i addr=%x",
 391             #ldst_r.dcbz,ldst_r.addr)
 392             m.d.comb += d_out.dcbz.eq(ldst_r.dcbz)
 393
 394         # XXX these should be possible to remove but for some reason
 395         # cannot be... yet. TODO, investigate
 396         m.d.comb += self.load_data.eq(d_in.data)
 397         m.d.comb += d_out.addr.eq(self.addr)
 398
 399         # Update outputs to MMU
 400         m.d.comb += m_out.valid.eq(mmureq)
 401         m.d.comb += m_out.iside.eq(self.instr_fault)
 402         m.d.comb += m_out.load.eq(ldst_r.load)
 403         m.d.comb += m_out.priv.eq(self.priv_mode)
 404         # m_out.priv <= r.priv_mode; TODO
 405         m.d.comb += m_out.tlbie.eq(self.tlbie)
 406         # m_out.mtspr <= mmu_mtspr; # TODO
 407         # m_out.sprn <= sprn; # TODO
 408         m.d.comb += m_out.addr.eq(maddr)
 409         # m_out.slbia <= l_in.insn(7); # TODO: no idea what this is
 410         # m_out.rs <= l_in.data; # nope, probably not needed, TODO investigate
 411
 412         return m
 413
 414     def ports(self):
 415         yield from super().ports()
 416         # TODO: memory ports
 417
 418
 419 class TestSRAMLoadStore1(LoadStore1):
 420     def __init__(self, pspec):
 421         super().__init__(pspec)
 422         pspec = self.pspec
 423         # small 32-entry Memory
 424         if (hasattr(pspec, "dmem_test_depth") and
 425                 isinstance(pspec.dmem_test_depth, int)):
 426             depth = pspec.dmem_test_depth
 427         else:
 428             depth = 32
 429         print("TestSRAMBareLoadStoreUnit depth", depth)
 430
 431         self.mem = Memory(width=pspec.reg_wid, depth=depth)
 432
 433     def elaborate(self, platform):
 434         m = super().elaborate(platform)
 435         comb = m.d.comb
 436         m.submodules.sram = sram = SRAM(memory=self.mem, granularity=8,
 437                                         features={'cti', 'bte', 'err'})
 438         dbus = self.dbus
 439
 440         # directly connect the wishbone bus of LoadStoreUnitInterface to SRAM
 441         # note: SRAM is a target (slave), dbus is initiator (master)
 442         fanouts = ['dat_w', 'sel', 'cyc', 'stb', 'we', 'cti', 'bte']
 443         fanins = ['dat_r', 'ack', 'err']
 444         for fanout in fanouts:
 445             print("fanout", fanout, getattr(sram.bus, fanout).shape(),
 446                   getattr(dbus, fanout).shape())
 447             comb += getattr(sram.bus, fanout).eq(getattr(dbus, fanout))
 448             comb += getattr(sram.bus, fanout).eq(getattr(dbus, fanout))
 449         for fanin in fanins:
 450             comb += getattr(dbus, fanin).eq(getattr(sram.bus, fanin))
 451         # connect address
 452         comb += sram.bus.adr.eq(dbus.adr)
 453
 454         return m
 455