src/soc/fu/ldst/loadstore.py

   1 """LoadStore1 FSM.
   2
   3 based on microwatt loadstore1.vhdl, but conforming to PortInterface.
   4 unlike loadstore1.vhdl this does *not* deal with actual Load/Store
   5 ops: that job is handled by LDSTCompUnit, which talks to LoadStore1
   6 by way of PortInterface.  PortInterface is where things need extending,
   7 such as adding dcbz support, etc.
   8
   9 this module basically handles "pure" load / store operations, and
  10 its first job is to ask the D-Cache for the data.  if that fails,
  11 the second task (if virtual memory is enabled) is to ask the MMU
  12 to perform a TLB, then to go *back* to the cache and ask again.
  13
  14 Links:
  15
  16 * https://bugs.libre-soc.org/show_bug.cgi?id=465
  17
  18 """
  19
  20 from nmigen import (Elaboratable, Module, Signal, Shape, unsigned, Cat, Mux,
  21                     Record, Memory,
  22                     Const)
  23 from nmutil.iocontrol import RecordObject
  24 from nmutil.util import rising_edge, Display
  25 from enum import Enum, unique
  26
  27 from soc.experiment.dcache import DCache
  28 from soc.experiment.icache import ICache
  29 from soc.experiment.pimem import PortInterfaceBase
  30 from soc.experiment.mem_types import LoadStore1ToMMUType
  31 from soc.experiment.mem_types import MMUToLoadStore1Type
  32
  33 from soc.minerva.wishbone import make_wb_layout
  34 from soc.bus.sram import SRAM
  35 from nmutil.util import Display
  36
  37
  38 @unique
  39 class State(Enum):
  40     IDLE = 0       # ready for instruction
  41     ACK_WAIT = 1   # waiting for ack from dcache
  42     MMU_LOOKUP = 2 # waiting for MMU to look up translation
  43     TLBIE_WAIT = 3 # waiting for MMU to finish doing a tlbie
  44
  45
  46 # captures the LDSTRequest from the PortInterface, which "blips" most
  47 # of this at us (pipeline-style).
  48 class LDSTRequest(RecordObject):
  49     def __init__(self, name=None):
  50         RecordObject.__init__(self, name=name)
  51
  52         self.load          = Signal()
  53         self.dcbz          = Signal()
  54         self.addr          = Signal(64)
  55         # self.store_data    = Signal(64) # this is already sync (on a delay)
  56         self.byte_sel      = Signal(8)
  57         self.nc            = Signal()              # non-cacheable access
  58         self.virt_mode     = Signal()
  59         self.priv_mode     = Signal()
  60         self.mode_32bit    = Signal() # XXX UNUSED AT PRESENT
  61         self.align_intr    = Signal()
  62
  63
  64 # glue logic for microwatt mmu and dcache
  65 class LoadStore1(PortInterfaceBase):
  66     def __init__(self, pspec):
  67         self.pspec = pspec
  68         self.disable_cache = (hasattr(pspec, "disable_cache") and
  69                               pspec.disable_cache == True)
  70         regwid = pspec.reg_wid
  71         addrwid = pspec.addr_wid
  72
  73         super().__init__(regwid, addrwid)
  74         self.dcache = DCache()
  75         self.icache = ICache(pspec)
  76         # these names are from the perspective of here (LoadStore1)
  77         self.d_out  = self.dcache.d_in     # in to dcache is out for LoadStore
  78         self.d_in = self.dcache.d_out      # out from dcache is in for LoadStore
  79         self.i_out  = self.icache.i_in     # in to icache is out for LoadStore
  80         self.i_in = self.icache.i_out      # out from icache is in for LoadStore
  81         self.m_out  = LoadStore1ToMMUType("m_out") # out *to* MMU
  82         self.m_in = MMUToLoadStore1Type("m_in")   # in *from* MMU
  83         self.req = LDSTRequest(name="ldst_req")
  84
  85         # TODO, convert dcache wb_in/wb_out to "standard" nmigen Wishbone bus
  86         self.dbus = Record(make_wb_layout(pspec))
  87         self.ibus = Record(make_wb_layout(pspec))
  88
  89         # for creating a single clock blip to DCache
  90         self.d_valid = Signal()
  91         self.d_w_valid = Signal()
  92         self.d_validblip = Signal()
  93
  94         # state info for LD/ST
  95         self.done          = Signal()
  96         self.done_delay    = Signal()
  97         # latch most of the input request
  98         self.load          = Signal()
  99         self.tlbie         = Signal()
 100         self.dcbz          = Signal()
 101         self.addr          = Signal(64)
 102         self.maddr          = Signal(64)
 103         self.store_data    = Signal(64)
 104         self.load_data     = Signal(64)
 105         self.load_data_delay = Signal(64)
 106         self.byte_sel      = Signal(8)
 107         #self.xerc         : xer_common_t;
 108         #self.reserve       = Signal()
 109         #self.atomic        = Signal()
 110         #self.atomic_last   = Signal()
 111         #self.rc            = Signal()
 112         self.nc            = Signal()              # non-cacheable access
 113         self.virt_mode     = Signal()
 114         self.priv_mode     = Signal()
 115         self.mode_32bit    = Signal() # XXX UNUSED AT PRESENT
 116         self.state         = Signal(State)
 117         self.instr_fault   = Signal()  # indicator to request i-cache MMU lookup
 118         self.r_instr_fault  = Signal() # accessed in external_busy
 119         self.align_intr    = Signal()
 120         self.busy          = Signal()
 121         self.wait_dcache   = Signal()
 122         self.wait_mmu      = Signal()
 123         #self.intr_vec     : integer range 0 to 16#fff#;
 124         #self.nia           = Signal(64)
 125         #self.srr1          = Signal(16)
 126         # use these to set the dsisr or dar respectively
 127         self.mmu_set_spr    = Signal()
 128         self.mmu_set_dsisr  = Signal()
 129         self.mmu_set_dar    = Signal()
 130         self.sprval_in      = Signal(64)
 131
 132         # ONLY access these read-only, do NOT attempt to change
 133         self.dsisr          = Signal(32)
 134         self.dar            = Signal(64)
 135
 136     # when external_busy set, do not allow PortInterface to proceed
 137     def external_busy(self, m):
 138         return self.instr_fault | self.r_instr_fault
 139
 140     def set_wr_addr(self, m, addr, mask, misalign, msr, is_dcbz):
 141         m.d.comb += self.req.load.eq(0) # store operation
 142         m.d.comb += self.req.byte_sel.eq(mask)
 143         m.d.comb += self.req.addr.eq(addr)
 144         m.d.comb += self.req.priv_mode.eq(~msr.pr) # not-problem  ==> priv
 145         m.d.comb += self.req.virt_mode.eq(msr.dr) # DR ==> virt
 146         m.d.comb += self.req.mode_32bit.eq(~msr.sf) # not-sixty-four ==> 32bit
 147         m.d.comb += self.req.align_intr.eq(misalign)
 148         m.d.comb += self.req.dcbz.eq(is_dcbz)
 149
 150         # m.d.comb += Display("set_wr_addr %i dcbz %i",addr,is_dcbz)
 151
 152         # option to disable the cache entirely for write
 153         if self.disable_cache:
 154             m.d.comb += self.req.nc.eq(1)
 155         return None
 156
 157     def set_rd_addr(self, m, addr, mask, misalign, msr):
 158         m.d.comb += self.d_valid.eq(1)
 159         m.d.comb += self.req.load.eq(1) # load operation
 160         m.d.comb += self.req.byte_sel.eq(mask)
 161         m.d.comb += self.req.align_intr.eq(misalign)
 162         m.d.comb += self.req.addr.eq(addr)
 163         m.d.comb += self.req.priv_mode.eq(~msr.pr) # not-problem  ==> priv
 164         m.d.comb += self.req.virt_mode.eq(msr.dr) # DR ==> virt
 165         m.d.comb += self.req.mode_32bit.eq(~msr.sf) # not-sixty-four ==> 32bit
 166         # BAD HACK! disable cacheing on LD when address is 0xCxxx_xxxx
 167         # this is for peripherals. same thing done in Microwatt loadstore1.vhdl
 168         with m.If(addr[28:] == Const(0xc, 4)):
 169             m.d.comb += self.req.nc.eq(1)
 170         # option to disable the cache entirely for read
 171         if self.disable_cache:
 172             m.d.comb += self.req.nc.eq(1)
 173         return None #FIXME return value
 174
 175     def set_wr_data(self, m, data, wen):
 176         # do the "blip" on write data
 177         m.d.comb += self.d_valid.eq(1)
 178         # put data into comb which is picked up in main elaborate()
 179         m.d.comb += self.d_w_valid.eq(1)
 180         m.d.comb += self.store_data.eq(data)
 181         #m.d.sync += self.d_out.byte_sel.eq(wen) # this might not be needed
 182         st_ok = self.done # TODO indicates write data is valid
 183         return st_ok
 184
 185     def get_rd_data(self, m):
 186         ld_ok = self.done_delay # indicates read data is valid
 187         data = self.load_data_delay   # actual read data
 188         return data, ld_ok
 189
 190     def elaborate(self, platform):
 191         m = super().elaborate(platform)
 192         comb, sync = m.d.comb, m.d.sync
 193
 194         # microwatt takes one more cycle before next operation can be issued
 195         sync += self.done_delay.eq(self.done)
 196         sync += self.load_data_delay.eq(self.load_data)
 197
 198         # create dcache and icache module
 199         m.submodules.dcache = dcache = self.dcache
 200         m.submodules.icache = icache = self.icache
 201
 202         # temp vars
 203         d_out, d_in, dbus = self.d_out, self.d_in, self.dbus
 204         i_out, i_in, ibus = self.i_out, self.i_in, self.ibus
 205         m_out, m_in = self.m_out, self.m_in
 206         exc = self.pi.exc_o
 207         exception = exc.happened
 208         mmureq = Signal()
 209
 210         # copy of address, but gets over-ridden for instr_fault
 211         maddr = Signal(64)
 212         m.d.comb += maddr.eq(self.addr)
 213
 214         # create a blip (single pulse) on valid read/write request
 215         # this can be over-ridden in the FSM to get dcache to re-run
 216         # a request when MMU_LOOKUP completes.
 217         m.d.comb += self.d_validblip.eq(rising_edge(m, self.d_valid))
 218         ldst_r = LDSTRequest("ldst_r")
 219         comb += Display("MMUTEST: LoadStore1 d_in.error=%i",d_in.error)
 220
 221         # fsm skeleton
 222         with m.Switch(self.state):
 223             with m.Case(State.IDLE):
 224                 with m.If((self.d_validblip | self.instr_fault) &
 225                           ~exc.happened):
 226                     comb += self.busy.eq(1)
 227                     sync += self.state.eq(State.ACK_WAIT)
 228                     sync += ldst_r.eq(self.req) # copy of LDSTRequest on "blip"
 229 #                   sync += Display("validblip self.req.virt_mode=%i",
 230 #                   self.req.virt_mode)
 231                     with m.If(self.instr_fault):
 232                         comb += mmureq.eq(1)
 233                         sync += self.r_instr_fault.eq(1)
 234                         comb += maddr.eq(self.maddr)
 235                         sync += self.state.eq(State.MMU_LOOKUP)
 236                     with m.Else():
 237                         sync += self.r_instr_fault.eq(0)
 238                 with m.Else():
 239                     sync += ldst_r.eq(0)
 240
 241             # waiting for completion
 242             with m.Case(State.ACK_WAIT):
 243                 comb += Display("MMUTEST: ACK_WAIT")
 244                 comb += self.busy.eq(~exc.happened)
 245
 246                 with m.If(d_in.error):
 247                     # cache error is not necessarily "final", it could
 248                     # be that it was just a TLB miss
 249                     with m.If(d_in.cache_paradox):
 250                         comb += exception.eq(1)
 251                         sync += self.state.eq(State.IDLE)
 252                         sync += ldst_r.eq(0)
 253                         sync += Display("cache error -> update dsisr")
 254                         sync += self.dsisr[63 - 38].eq(~self.load)
 255                         # XXX there is no architected bit for this
 256                         # (probably should be a machine check in fact)
 257                         sync += self.dsisr[63 - 35].eq(d_in.cache_paradox)
 258
 259                     with m.Else():
 260                         # Look up the translation for TLB miss
 261                         # and also for permission error and RC error
 262                         # in case the PTE has been updated.
 263                         comb += mmureq.eq(1)
 264                         sync += self.state.eq(State.MMU_LOOKUP)
 265                 with m.If(d_in.valid):
 266                     m.d.comb += self.done.eq(~mmureq) # done if not doing MMU
 267                     with m.If(self.done):
 268                         sync += Display("ACK_WAIT, done %x", self.addr)
 269                     sync += self.state.eq(State.IDLE)
 270                     sync += ldst_r.eq(0)
 271                     with m.If(self.load):
 272                         m.d.comb += self.load_data.eq(d_in.data)
 273
 274             # waiting here for the MMU TLB lookup to complete.
 275             # either re-try the dcache lookup or throw MMU exception
 276             with m.Case(State.MMU_LOOKUP):
 277                 comb += self.busy.eq(~exception)
 278                 with m.If(m_in.done):
 279                     with m.If(~self.r_instr_fault):
 280                         sync += Display("MMU_LOOKUP, done %x -> %x",
 281                                         self.addr, d_out.addr)
 282                         # retry the request now that the MMU has
 283                         # installed a TLB entry, if not exception raised
 284                         m.d.comb += self.d_out.valid.eq(~exception)
 285                         sync += self.state.eq(State.ACK_WAIT)
 286                         sync += ldst_r.eq(0)
 287                     with m.Else():
 288                         sync += self.state.eq(State.IDLE)
 289                         sync += self.r_instr_fault.eq(0)
 290                         comb += self.done.eq(1)
 291
 292                 with m.If(m_in.err):
 293                     # MMU RADIX exception thrown. XXX
 294                     # TODO: critical that the write here has to
 295                     # notify the MMU FSM of the change to dsisr
 296                     comb += exception.eq(1)
 297                     comb += self.done.eq(1)
 298                     sync += Display("MMU RADIX exception thrown")
 299                     sync += Display("TODO: notify MMU of change to dsisr")
 300                     sync += self.dsisr[63 - 33].eq(m_in.invalid)
 301                     sync += self.dsisr[63 - 36].eq(m_in.perm_error) # noexec
 302                     sync += self.dsisr[63 - 38].eq(~self.load)
 303                     sync += self.dsisr[63 - 44].eq(m_in.badtree)
 304                     sync += self.dsisr[63 - 45].eq(m_in.rc_error)
 305                     sync += self.state.eq(State.IDLE)
 306
 307             with m.Case(State.TLBIE_WAIT):
 308                 pass
 309
 310         # MMU FSM communicating a request to update DSISR or DAR (OP_MTSPR)
 311         with m.If(self.mmu_set_spr):
 312             with m.If(self.mmu_set_dsisr):
 313                 sync += self.dsisr.eq(self.sprval_in)
 314             with m.If(self.mmu_set_dar):
 315                 sync += self.dar.eq(self.sprval_in)
 316
 317         # hmmm, alignment occurs in set_rd_addr/set_wr_addr, note exception
 318         with m.If(self.align_intr):
 319             comb += exc.happened.eq(1)
 320         # check for updating DAR
 321         with m.If(exception):
 322             sync += Display("exception %x", self.addr)
 323             # alignment error: store address in DAR
 324             with m.If(self.align_intr):
 325                 sync += Display("alignment error: addr in DAR %x", self.addr)
 326                 sync += self.dar.eq(self.addr)
 327             with m.Elif(~self.r_instr_fault):
 328                 sync += Display("not instr fault, addr in DAR %x", self.addr)
 329                 sync += self.dar.eq(self.addr)
 330
 331         # when done or exception, return to idle state
 332         with m.If(self.done | exception):
 333             sync += self.state.eq(State.IDLE)
 334             comb += self.busy.eq(0)
 335
 336         # happened, alignment, instr_fault, invalid.
 337         # note that all of these flow through - eventually to the TRAP
 338         # pipeline, via PowerDecoder2.
 339         comb += self.align_intr.eq(self.req.align_intr)
 340         comb += exc.invalid.eq(m_in.invalid)
 341         comb += exc.alignment.eq(self.align_intr)
 342         comb += exc.instr_fault.eq(self.r_instr_fault)
 343         # badtree, perm_error, rc_error, segment_fault
 344         comb += exc.badtree.eq(m_in.badtree)
 345         comb += exc.perm_error.eq(m_in.perm_error)
 346         comb += exc.rc_error.eq(m_in.rc_error)
 347         comb += exc.segment_fault.eq(m_in.segerr)
 348
 349         # TODO, connect dcache wb_in/wb_out to "standard" nmigen Wishbone bus
 350         comb += dbus.adr.eq(dcache.bus.adr)
 351         comb += dbus.dat_w.eq(dcache.bus.dat_w)
 352         comb += dbus.sel.eq(dcache.bus.sel)
 353         comb += dbus.cyc.eq(dcache.bus.cyc)
 354         comb += dbus.stb.eq(dcache.bus.stb)
 355         comb += dbus.we.eq(dcache.bus.we)
 356
 357         comb += dcache.bus.dat_r.eq(dbus.dat_r)
 358         comb += dcache.bus.ack.eq(dbus.ack)
 359         if hasattr(dbus, "stall"):
 360             comb += dcache.bus.stall.eq(dbus.stall)
 361
 362         # update out d data when flag set
 363         with m.If(self.d_w_valid):
 364             m.d.sync += d_out.data.eq(self.store_data)
 365         #with m.Else():
 366         #    m.d.sync += d_out.data.eq(0)
 367         # unit test passes with that change
 368
 369         # this must move into the FSM, conditionally noticing that
 370         # the "blip" comes from self.d_validblip.
 371         # task 1: look up in dcache
 372         # task 2: if dcache fails, look up in MMU.
 373         # do **NOT** confuse the two.
 374         with m.If(self.d_validblip):
 375             m.d.comb += self.d_out.valid.eq(~exc.happened)
 376             m.d.comb += d_out.load.eq(self.req.load)
 377             m.d.comb += d_out.byte_sel.eq(self.req.byte_sel)
 378             m.d.comb += self.addr.eq(self.req.addr)
 379             m.d.comb += d_out.nc.eq(self.req.nc)
 380             m.d.comb += d_out.priv_mode.eq(self.req.priv_mode)
 381             m.d.comb += d_out.virt_mode.eq(self.req.virt_mode)
 382             #m.d.comb += Display("validblip dcbz=%i addr=%x",
 383             #self.req.dcbz,self.req.addr)
 384             m.d.comb += d_out.dcbz.eq(self.req.dcbz)
 385         with m.Else():
 386             m.d.comb += d_out.load.eq(ldst_r.load)
 387             m.d.comb += d_out.byte_sel.eq(ldst_r.byte_sel)
 388             m.d.comb += self.addr.eq(ldst_r.addr)
 389             m.d.comb += d_out.nc.eq(ldst_r.nc)
 390             m.d.comb += d_out.priv_mode.eq(ldst_r.priv_mode)
 391             m.d.comb += d_out.virt_mode.eq(ldst_r.virt_mode)
 392             #m.d.comb += Display("no_validblip dcbz=%i addr=%x",
 393             #ldst_r.dcbz,ldst_r.addr)
 394             m.d.comb += d_out.dcbz.eq(ldst_r.dcbz)
 395
 396         # XXX these should be possible to remove but for some reason
 397         # cannot be... yet. TODO, investigate
 398         m.d.comb += self.load_data.eq(d_in.data)
 399         m.d.comb += d_out.addr.eq(self.addr)
 400
 401         # Update outputs to MMU
 402         m.d.comb += m_out.valid.eq(mmureq)
 403         m.d.comb += m_out.iside.eq(self.instr_fault)
 404         m.d.comb += m_out.load.eq(ldst_r.load)
 405         m.d.comb += m_out.priv.eq(self.priv_mode)
 406         # m_out.priv <= r.priv_mode; TODO
 407         m.d.comb += m_out.tlbie.eq(self.tlbie)
 408         # m_out.mtspr <= mmu_mtspr; # TODO
 409         # m_out.sprn <= sprn; # TODO
 410         m.d.comb += m_out.addr.eq(maddr)
 411         # m_out.slbia <= l_in.insn(7); # TODO: no idea what this is
 412         # m_out.rs <= l_in.data; # nope, probably not needed, TODO investigate
 413
 414         return m
 415
 416     def ports(self):
 417         yield from super().ports()
 418         # TODO: memory ports
 419
 420
 421 class TestSRAMLoadStore1(LoadStore1):
 422     def __init__(self, pspec):
 423         super().__init__(pspec)
 424         pspec = self.pspec
 425         # small 32-entry Memory
 426         if (hasattr(pspec, "dmem_test_depth") and
 427                 isinstance(pspec.dmem_test_depth, int)):
 428             depth = pspec.dmem_test_depth
 429         else:
 430             depth = 32
 431         print("TestSRAMBareLoadStoreUnit depth", depth)
 432
 433         self.mem = Memory(width=pspec.reg_wid, depth=depth)
 434
 435     def elaborate(self, platform):
 436         m = super().elaborate(platform)
 437         comb = m.d.comb
 438         m.submodules.sram = sram = SRAM(memory=self.mem, granularity=8,
 439                                         features={'cti', 'bte', 'err'})
 440         dbus = self.dbus
 441
 442         # directly connect the wishbone bus of LoadStoreUnitInterface to SRAM
 443         # note: SRAM is a target (slave), dbus is initiator (master)
 444         fanouts = ['dat_w', 'sel', 'cyc', 'stb', 'we', 'cti', 'bte']
 445         fanins = ['dat_r', 'ack', 'err']
 446         for fanout in fanouts:
 447             print("fanout", fanout, getattr(sram.bus, fanout).shape(),
 448                   getattr(dbus, fanout).shape())
 449             comb += getattr(sram.bus, fanout).eq(getattr(dbus, fanout))
 450             comb += getattr(sram.bus, fanout).eq(getattr(dbus, fanout))
 451         for fanin in fanins:
 452             comb += getattr(dbus, fanin).eq(getattr(sram.bus, fanin))
 453         # connect address
 454         comb += sram.bus.adr.eq(dbus.adr)
 455
 456         return m
 457