add a second LD request to dcache which is merged with first,
[soc.git] / src / soc / fu / ldst / loadstore.py
1 """LoadStore1 FSM.
2
3 based on microwatt loadstore1.vhdl, but conforming to PortInterface.
4 unlike loadstore1.vhdl this does *not* deal with actual Load/Store
5 ops: that job is handled by LDSTCompUnit, which talks to LoadStore1
6 by way of PortInterface. PortInterface is where things need extending,
7 such as adding dcbz support, etc.
8
9 this module basically handles "pure" load / store operations, and
10 its first job is to ask the D-Cache for the data. if that fails,
11 the second task (if virtual memory is enabled) is to ask the MMU
12 to perform a TLB, then to go *back* to the cache and ask again.
13
14 Links:
15
16 * https://bugs.libre-soc.org/show_bug.cgi?id=465
17
18 """
19
20 from nmigen import (Elaboratable, Module, Signal, Shape, unsigned, Cat, Mux,
21 Record, Memory,
22 Const)
23 from nmutil.iocontrol import RecordObject
24 from nmutil.util import rising_edge, Display
25 from enum import Enum, unique
26
27 from soc.experiment.dcache import DCache
28 from soc.experiment.icache import ICache
29 from soc.experiment.pimem import PortInterfaceBase
30 from soc.experiment.mem_types import LoadStore1ToMMUType
31 from soc.experiment.mem_types import MMUToLoadStore1Type
32
33 from soc.minerva.wishbone import make_wb_layout
34 from soc.bus.sram import SRAM
35 from nmutil.util import Display
36
37
38 @unique
39 class State(Enum):
40 IDLE = 0 # ready for instruction
41 ACK_WAIT = 1 # waiting for ack from dcache
42 MMU_LOOKUP = 2 # waiting for MMU to look up translation
43 #SECOND_REQ = 3 # second request for unaligned transfer
44
45 @unique
46 class Misalign(Enum):
47 ONEWORD = 0 # only one word needed, all good
48 NEED2WORDS = 1 # need to send/receive two words
49 WAITFIRST = 2 # waiting for the first word
50 WAITSECOND = 3 # waiting for the second word
51
52
53 # captures the LDSTRequest from the PortInterface, which "blips" most
54 # of this at us (pipeline-style).
55 class LDSTRequest(RecordObject):
56 def __init__(self, name=None):
57 RecordObject.__init__(self, name=name)
58
59 self.load = Signal()
60 self.dcbz = Signal()
61 self.raddr = Signal(64)
62 # self.store_data = Signal(64) # this is already sync (on a delay)
63 self.byte_sel = Signal(16)
64 self.nc = Signal() # non-cacheable access
65 self.virt_mode = Signal()
66 self.priv_mode = Signal()
67 self.mode_32bit = Signal() # XXX UNUSED AT PRESENT
68 self.alignstate = Signal(Misalign) # progress of alignment request
69 self.align_intr = Signal()
70
71
72 # glue logic for microwatt mmu and dcache
73 class LoadStore1(PortInterfaceBase):
74 def __init__(self, pspec):
75 self.pspec = pspec
76 self.disable_cache = (hasattr(pspec, "disable_cache") and
77 pspec.disable_cache == True)
78 regwid = pspec.reg_wid
79 addrwid = pspec.addr_wid
80
81 super().__init__(regwid, addrwid)
82 self.dcache = DCache(pspec)
83 self.icache = ICache(pspec)
84 # these names are from the perspective of here (LoadStore1)
85 self.d_out = self.dcache.d_in # in to dcache is out for LoadStore
86 self.d_in = self.dcache.d_out # out from dcache is in for LoadStore
87 self.i_out = self.icache.i_in # in to icache is out for LoadStore
88 self.i_in = self.icache.i_out # out from icache is in for LoadStore
89 self.m_out = LoadStore1ToMMUType("m_out") # out *to* MMU
90 self.m_in = MMUToLoadStore1Type("m_in") # in *from* MMU
91 self.req = LDSTRequest(name="ldst_req")
92
93 # TODO, convert dcache wb_in/wb_out to "standard" nmigen Wishbone bus
94 self.dbus = Record(make_wb_layout(pspec))
95 self.ibus = Record(make_wb_layout(pspec))
96
97 # for creating a single clock blip to DCache
98 self.d_valid = Signal()
99 self.d_w_valid = Signal()
100 self.d_validblip = Signal()
101
102 # state info for LD/ST
103 self.done = Signal()
104 self.done_delay = Signal()
105 # latch most of the input request
106 self.load = Signal()
107 self.tlbie = Signal()
108 self.dcbz = Signal()
109 self.raddr = Signal(64)
110 self.maddr = Signal(64)
111 self.store_data = Signal(128) # 128-bit to cope with
112 self.load_data = Signal(128) # misalignment
113 self.load_data_delay = Signal(128) # perform 2 LD/STs
114 self.byte_sel = Signal(16) # also for misaligned, 16-bit
115 self.alignstate = Signal(Misalign) # progress of alignment request
116 #self.xerc : xer_common_t;
117 #self.reserve = Signal()
118 #self.atomic = Signal()
119 #self.atomic_last = Signal()
120 #self.rc = Signal()
121 self.nc = Signal() # non-cacheable access
122 self.virt_mode = Signal()
123 self.priv_mode = Signal()
124 self.mode_32bit = Signal() # XXX UNUSED AT PRESENT
125 self.state = Signal(State)
126 self.instr_fault = Signal() # indicator to request i-cache MMU lookup
127 self.r_instr_fault = Signal() # accessed in external_busy
128 self.align_intr = Signal()
129 self.busy = Signal()
130 self.wait_dcache = Signal()
131 self.wait_mmu = Signal()
132 #self.intr_vec : integer range 0 to 16#fff#;
133 #self.nia = Signal(64)
134 #self.srr1 = Signal(16)
135 # use these to set the dsisr or dar respectively
136 self.mmu_set_spr = Signal()
137 self.mmu_set_dsisr = Signal()
138 self.mmu_set_dar = Signal()
139 self.sprval_in = Signal(64)
140
141 # ONLY access these read-only, do NOT attempt to change
142 self.dsisr = Signal(32)
143 self.dar = Signal(64)
144
145 # when external_busy set, do not allow PortInterface to proceed
146 def external_busy(self, m):
147 return self.instr_fault | self.r_instr_fault
148
149 def set_wr_addr(self, m, addr, mask, misalign, msr, is_dcbz):
150 m.d.comb += self.req.load.eq(0) # store operation
151 m.d.comb += self.req.byte_sel.eq(mask)
152 m.d.comb += self.req.raddr.eq(addr)
153 m.d.comb += self.req.priv_mode.eq(~msr.pr) # not-problem ==> priv
154 m.d.comb += self.req.virt_mode.eq(msr.dr) # DR ==> virt
155 m.d.comb += self.req.mode_32bit.eq(~msr.sf) # not-sixty-four ==> 32bit
156 m.d.comb += self.req.dcbz.eq(is_dcbz)
157 with m.If(misalign):
158 m.d.comb += self.req.alignstate.eq(Misalign.NEED2WORDS)
159
160 # m.d.comb += Display("set_wr_addr %i dcbz %i",addr,is_dcbz)
161
162 # option to disable the cache entirely for write
163 if self.disable_cache:
164 m.d.comb += self.req.nc.eq(1)
165
166 # dcbz cannot do no-cache
167 with m.If(is_dcbz & self.req.nc):
168 m.d.comb += self.req.align_intr.eq(1)
169
170 return None
171
172 def set_rd_addr(self, m, addr, mask, misalign, msr):
173 m.d.comb += self.d_valid.eq(1)
174 m.d.comb += self.req.load.eq(1) # load operation
175 m.d.comb += self.req.byte_sel.eq(mask)
176 m.d.comb += self.req.raddr.eq(addr)
177 m.d.comb += self.req.priv_mode.eq(~msr.pr) # not-problem ==> priv
178 m.d.comb += self.req.virt_mode.eq(msr.dr) # DR ==> virt
179 m.d.comb += self.req.mode_32bit.eq(~msr.sf) # not-sixty-four ==> 32bit
180 # BAD HACK! disable cacheing on LD when address is 0xCxxx_xxxx
181 # this is for peripherals. same thing done in Microwatt loadstore1.vhdl
182 with m.If(addr[28:] == Const(0xc, 4)):
183 m.d.comb += self.req.nc.eq(1)
184 # option to disable the cache entirely for read
185 if self.disable_cache:
186 m.d.comb += self.req.nc.eq(1)
187 with m.If(misalign):
188 m.d.comb += self.req.alignstate.eq(Misalign.NEED2WORDS)
189 return None #FIXME return value
190
191 def set_wr_data(self, m, data, wen):
192 # do the "blip" on write data
193 m.d.comb += self.d_valid.eq(1)
194 # put data into comb which is picked up in main elaborate()
195 m.d.comb += self.d_w_valid.eq(1)
196 m.d.comb += self.store_data.eq(data)
197 #m.d.sync += self.d_out.byte_sel.eq(wen) # this might not be needed
198 st_ok = self.done # TODO indicates write data is valid
199 return st_ok
200
201 def get_rd_data(self, m):
202 ld_ok = self.done_delay # indicates read data is valid
203 data = self.load_data_delay # actual read data
204 return data, ld_ok
205
206 def elaborate(self, platform):
207 m = super().elaborate(platform)
208 comb, sync = m.d.comb, m.d.sync
209
210 # microwatt takes one more cycle before next operation can be issued
211 sync += self.done_delay.eq(self.done)
212 #sync += self.load_data_delay[0:64].eq(self.load_data[0:64])
213
214 # create dcache and icache module
215 m.submodules.dcache = dcache = self.dcache
216 m.submodules.icache = icache = self.icache
217
218 # temp vars
219 d_out, d_in, dbus = self.d_out, self.d_in, self.dbus
220 i_out, i_in, ibus = self.i_out, self.i_in, self.ibus
221 m_out, m_in = self.m_out, self.m_in
222 exc = self.pi.exc_o
223 exception = exc.happened
224 mmureq = Signal()
225
226 # copy of address, but gets over-ridden for instr_fault
227 maddr = Signal(64)
228 m.d.comb += maddr.eq(self.raddr)
229
230 # create a blip (single pulse) on valid read/write request
231 # this can be over-ridden in the FSM to get dcache to re-run
232 # a request when MMU_LOOKUP completes.
233 m.d.comb += self.d_validblip.eq(rising_edge(m, self.d_valid))
234 ldst_r = LDSTRequest("ldst_r")
235 sync += Display("MMUTEST: LoadStore1 d_in.error=%i",d_in.error)
236
237 # fsm skeleton
238 with m.Switch(self.state):
239 with m.Case(State.IDLE):
240 with m.If((self.d_validblip | self.instr_fault) &
241 ~exc.happened):
242 comb += self.busy.eq(1)
243 sync += self.state.eq(State.ACK_WAIT)
244 sync += ldst_r.eq(self.req) # copy of LDSTRequest on "blip"
245 # sync += Display("validblip self.req.virt_mode=%i",
246 # self.req.virt_mode)
247 with m.If(self.instr_fault):
248 comb += mmureq.eq(1)
249 sync += self.r_instr_fault.eq(1)
250 comb += maddr.eq(self.maddr)
251 sync += self.state.eq(State.MMU_LOOKUP)
252 with m.Else():
253 sync += self.r_instr_fault.eq(0)
254 # if the LD/ST requires two dwords, move to waiting
255 # for first word
256 with m.If(self.req.alignstate == Misalign.NEED2WORDS):
257 sync += ldst_r.alignstate.eq(Misalign.WAITFIRST)
258 with m.Else():
259 sync += ldst_r.eq(0)
260
261 # waiting for completion
262 with m.Case(State.ACK_WAIT):
263 sync += Display("MMUTEST: ACK_WAIT")
264 comb += self.busy.eq(~exc.happened)
265
266 with m.If(d_in.error):
267 # cache error is not necessarily "final", it could
268 # be that it was just a TLB miss
269 with m.If(d_in.cache_paradox):
270 comb += exception.eq(1)
271 sync += self.state.eq(State.IDLE)
272 sync += ldst_r.eq(0)
273 sync += Display("cache error -> update dsisr")
274 sync += self.dsisr[63 - 38].eq(~ldst_r.load)
275 # XXX there is no architected bit for this
276 # (probably should be a machine check in fact)
277 sync += self.dsisr[63 - 35].eq(d_in.cache_paradox)
278 sync += self.r_instr_fault.eq(0)
279
280 with m.Else():
281 # Look up the translation for TLB miss
282 # and also for permission error and RC error
283 # in case the PTE has been updated.
284 comb += mmureq.eq(1)
285 sync += self.state.eq(State.MMU_LOOKUP)
286 with m.If(d_in.valid):
287 with m.If(self.done):
288 sync += Display("ACK_WAIT, done %x", self.raddr)
289 with m.If(ldst_r.alignstate == Misalign.ONEWORD):
290 # done if there is only one dcache operation
291 sync += self.state.eq(State.IDLE)
292 sync += ldst_r.eq(0)
293 with m.If(ldst_r.load):
294 m.d.comb += self.load_data.eq(d_in.data)
295 sync += self.load_data_delay[0:64].eq(d_in.data)
296 m.d.comb += self.done.eq(~mmureq) # done if not MMU
297 with m.Elif(ldst_r.alignstate == Misalign.WAITFIRST):
298 # first LD done: load data, initiate 2nd request.
299 # leave in ACK_WAIT state
300 with m.If(ldst_r.load):
301 m.d.comb += self.load_data[0:63].eq(d_in.data)
302 sync += self.load_data_delay[0:64].eq(d_in.data)
303 # mmm kinda cheating, make a 2nd blip
304 m.d.comb += self.d_validblip.eq(1)
305 comb += self.req.eq(ldst_r) # from copy of request
306 comb += self.req.raddr.eq(ldst_r.raddr + 8)
307 comb += self.req.byte_sel.eq(ldst_r.byte_sel[8:])
308 comb += self.req.alignstate.eq(Misalign.WAITSECOND)
309 sync += ldst_r.alignstate.eq(Misalign.WAITSECOND)
310 sync += Display(" second req %x", self.req.raddr)
311 with m.Elif(ldst_r.alignstate == Misalign.WAITSECOND):
312 sync += Display(" done second %x", d_in.data)
313 # done second load
314 sync += self.state.eq(State.IDLE)
315 sync += ldst_r.eq(0)
316 with m.If(ldst_r.load):
317 m.d.comb += self.load_data[64:128].eq(d_in.data)
318 sync += self.load_data_delay[64:128].eq(d_in.data)
319 m.d.comb += self.done.eq(~mmureq) # done if not MMU
320
321 # waiting here for the MMU TLB lookup to complete.
322 # either re-try the dcache lookup or throw MMU exception
323 with m.Case(State.MMU_LOOKUP):
324 comb += self.busy.eq(~exception)
325 with m.If(m_in.done):
326 with m.If(~self.r_instr_fault):
327 sync += Display("MMU_LOOKUP, done %x -> %x",
328 self.raddr, d_out.addr)
329 # retry the request now that the MMU has
330 # installed a TLB entry, if not exception raised
331 m.d.comb += self.d_out.valid.eq(~exception)
332 sync += self.state.eq(State.ACK_WAIT)
333 sync += ldst_r.eq(0)
334 with m.Else():
335 sync += self.state.eq(State.IDLE)
336 sync += self.r_instr_fault.eq(0)
337 comb += self.done.eq(1)
338
339 with m.If(m_in.err):
340 # MMU RADIX exception thrown. XXX
341 # TODO: critical that the write here has to
342 # notify the MMU FSM of the change to dsisr
343 comb += exception.eq(1)
344 comb += self.done.eq(1)
345 sync += Display("MMU RADIX exception thrown")
346 sync += self.dsisr[63 - 33].eq(m_in.invalid)
347 sync += self.dsisr[63 - 36].eq(m_in.perm_error) # noexec
348 sync += self.dsisr[63 - 38].eq(~ldst_r.load)
349 sync += self.dsisr[63 - 44].eq(m_in.badtree)
350 sync += self.dsisr[63 - 45].eq(m_in.rc_error)
351 sync += self.state.eq(State.IDLE)
352 # exception thrown, clear out instruction fault state
353 sync += self.r_instr_fault.eq(0)
354
355 # MMU FSM communicating a request to update DSISR or DAR (OP_MTSPR)
356 with m.If(self.mmu_set_spr):
357 with m.If(self.mmu_set_dsisr):
358 sync += self.dsisr.eq(self.sprval_in)
359 with m.If(self.mmu_set_dar):
360 sync += self.dar.eq(self.sprval_in)
361
362 # hmmm, alignment occurs in set_rd_addr/set_wr_addr, note exception
363 with m.If(self.align_intr):
364 comb += exc.happened.eq(1)
365 # check for updating DAR
366 with m.If(exception):
367 sync += Display("exception %x", self.raddr)
368 # alignment error: store address in DAR
369 with m.If(self.align_intr):
370 sync += Display("alignment error: addr in DAR %x", self.raddr)
371 sync += self.dar.eq(self.raddr)
372 with m.Elif(~self.r_instr_fault):
373 sync += Display("not instr fault, addr in DAR %x", self.raddr)
374 sync += self.dar.eq(self.raddr)
375
376 # when done or exception, return to idle state
377 with m.If(self.done | exception):
378 sync += self.state.eq(State.IDLE)
379 comb += self.busy.eq(0)
380
381 # happened, alignment, instr_fault, invalid.
382 # note that all of these flow through - eventually to the TRAP
383 # pipeline, via PowerDecoder2.
384 comb += self.align_intr.eq(self.req.align_intr)
385 comb += exc.invalid.eq(m_in.invalid)
386 comb += exc.alignment.eq(self.align_intr)
387 comb += exc.instr_fault.eq(self.r_instr_fault)
388 # badtree, perm_error, rc_error, segment_fault
389 comb += exc.badtree.eq(m_in.badtree)
390 comb += exc.perm_error.eq(m_in.perm_error)
391 comb += exc.rc_error.eq(m_in.rc_error)
392 comb += exc.segment_fault.eq(m_in.segerr)
393
394 # TODO, connect dcache wb_in/wb_out to "standard" nmigen Wishbone bus
395 comb += dbus.adr.eq(dcache.bus.adr)
396 comb += dbus.dat_w.eq(dcache.bus.dat_w)
397 comb += dbus.sel.eq(dcache.bus.sel)
398 comb += dbus.cyc.eq(dcache.bus.cyc)
399 comb += dbus.stb.eq(dcache.bus.stb)
400 comb += dbus.we.eq(dcache.bus.we)
401
402 comb += dcache.bus.dat_r.eq(dbus.dat_r)
403 comb += dcache.bus.ack.eq(dbus.ack)
404 if hasattr(dbus, "stall"):
405 comb += dcache.bus.stall.eq(dbus.stall)
406
407 # update out d data when flag set
408 with m.If(self.d_w_valid):
409 with m.If(ldst_r.alignstate == Misalign.WAITSECOND):
410 m.d.sync += d_out.data.eq(self.store_data[64:128])
411 with m.Else():
412 m.d.sync += d_out.data.eq(self.store_data[0:64])
413 #with m.Else():
414 # m.d.sync += d_out.data.eq(0)
415 # unit test passes with that change
416
417 # this must move into the FSM, conditionally noticing that
418 # the "blip" comes from self.d_validblip.
419 # task 1: look up in dcache
420 # task 2: if dcache fails, look up in MMU.
421 # do **NOT** confuse the two.
422 with m.If(self.d_validblip):
423 m.d.comb += self.d_out.valid.eq(~exc.happened)
424 m.d.comb += d_out.load.eq(self.req.load)
425 m.d.comb += d_out.byte_sel.eq(self.req.byte_sel)
426 m.d.comb += self.raddr.eq(self.req.raddr)
427 m.d.comb += d_out.nc.eq(self.req.nc)
428 m.d.comb += d_out.priv_mode.eq(self.req.priv_mode)
429 m.d.comb += d_out.virt_mode.eq(self.req.virt_mode)
430 #m.d.comb += Display("validblip dcbz=%i addr=%x",
431 #self.req.dcbz,self.req.addr)
432 m.d.comb += d_out.dcbz.eq(self.req.dcbz)
433 with m.Else():
434 m.d.comb += d_out.load.eq(ldst_r.load)
435 m.d.comb += d_out.byte_sel.eq(ldst_r.byte_sel)
436 m.d.comb += self.raddr.eq(ldst_r.raddr)
437 m.d.comb += d_out.nc.eq(ldst_r.nc)
438 m.d.comb += d_out.priv_mode.eq(ldst_r.priv_mode)
439 m.d.comb += d_out.virt_mode.eq(ldst_r.virt_mode)
440 #m.d.comb += Display("no_validblip dcbz=%i addr=%x",
441 #ldst_r.dcbz,ldst_r.addr)
442 m.d.comb += d_out.dcbz.eq(ldst_r.dcbz)
443
444 # XXX these should be possible to remove but for some reason
445 # cannot be... yet. TODO, investigate
446 #m.d.comb += self.load_data.eq(d_in.data)
447 m.d.comb += d_out.addr.eq(self.raddr)
448
449 # Update outputs to MMU
450 m.d.comb += m_out.valid.eq(mmureq)
451 m.d.comb += m_out.iside.eq(self.instr_fault)
452 m.d.comb += m_out.load.eq(ldst_r.load)
453 m.d.comb += m_out.priv.eq(self.priv_mode)
454 # m_out.priv <= r.priv_mode; TODO
455 m.d.comb += m_out.tlbie.eq(self.tlbie)
456 # m_out.mtspr <= mmu_mtspr; # TODO
457 # m_out.sprn <= sprn; # TODO
458 m.d.comb += m_out.addr.eq(maddr)
459 # m_out.slbia <= l_in.insn(7); # TODO: no idea what this is
460 # m_out.rs <= l_in.data; # nope, probably not needed, TODO investigate
461
462 return m
463
464 def ports(self):
465 yield from super().ports()
466 # TODO: memory ports
467
468
469 class TestSRAMLoadStore1(LoadStore1):
470 def __init__(self, pspec):
471 super().__init__(pspec)
472 pspec = self.pspec
473 # small 32-entry Memory
474 if (hasattr(pspec, "dmem_test_depth") and
475 isinstance(pspec.dmem_test_depth, int)):
476 depth = pspec.dmem_test_depth
477 else:
478 depth = 32
479 print("TestSRAMBareLoadStoreUnit depth", depth)
480
481 self.mem = Memory(width=pspec.reg_wid, depth=depth)
482
483 def elaborate(self, platform):
484 m = super().elaborate(platform)
485 comb = m.d.comb
486 m.submodules.sram = sram = SRAM(memory=self.mem, granularity=8,
487 features={'cti', 'bte', 'err'})
488 dbus = self.dbus
489
490 # directly connect the wishbone bus of LoadStoreUnitInterface to SRAM
491 # note: SRAM is a target (slave), dbus is initiator (master)
492 fanouts = ['dat_w', 'sel', 'cyc', 'stb', 'we', 'cti', 'bte']
493 fanins = ['dat_r', 'ack', 'err']
494 for fanout in fanouts:
495 print("fanout", fanout, getattr(sram.bus, fanout).shape(),
496 getattr(dbus, fanout).shape())
497 comb += getattr(sram.bus, fanout).eq(getattr(dbus, fanout))
498 comb += getattr(sram.bus, fanout).eq(getattr(dbus, fanout))
499 for fanin in fanins:
500 comb += getattr(dbus, fanin).eq(getattr(sram.bus, fanin))
501 # connect address
502 comb += sram.bus.adr.eq(dbus.adr)
503
504 return m
505