test_ldst_pi.py: add dcache regression and random test from test_dcache.py
[soc.git] / src / soc / experiment / dcache.py
index 45f17d065b1942494aee489d3896c94b0fa76f59..072d34a90d108e601d0640930607ca205ab2fe4b 100644 (file)
@@ -2,14 +2,36 @@
 
 based on Anton Blanchard microwatt dcache.vhdl
 
+note that the microwatt dcache wishbone interface expects "stall".
+for simplicity at the moment this is hard-coded to cyc & ~ack.
+see WB4 spec, p84, section 5.2.1
+
+IMPORTANT: for store, the data is sampled the cycle AFTER the "valid"
+is raised.  sigh
+
+Links:
+
+* https://libre-soc.org/3d_gpu/architecture/set_associative_cache.jpg
+* https://bugs.libre-soc.org/show_bug.cgi?id=469
+
 """
 
+import sys
+
+from nmutil.gtkw import write_gtkw
+
+sys.setrecursionlimit(1000000)
+
 from enum import Enum, unique
 
 from nmigen import Module, Signal, Elaboratable, Cat, Repl, Array, Const
+from nmutil.util import Display
+
+from copy import deepcopy
+from random import randint, seed
+
 from nmigen.cli import main
 from nmutil.iocontrol import RecordObject
-from nmutil.util import wrap
 from nmigen.utils import log2_int
 from soc.experiment.mem_types import (LoadStore1ToDCacheType,
                                      DCacheToLoadStore1Type,
@@ -23,16 +45,19 @@ from soc.experiment.wb_types import (WB_ADDR_BITS, WB_DATA_BITS, WB_SEL_BITS,
                                 WBIOMasterOut, WBIOSlaveOut)
 
 from soc.experiment.cache_ram import CacheRam
-from soc.experiment.plru import PLRU
+#from soc.experiment.plru import PLRU
+from nmutil.plru import PLRU
 
 # for test
-from nmigen_soc.wishbone.sram import SRAM
+from soc.bus.sram import SRAM
 from nmigen import Memory
 from nmigen.cli import rtlil
-if True:
-    from nmigen.back.pysim import Simulator, Delay, Settle
-else:
-    from nmigen.sim.cxxsim import Simulator, Delay, Settle
+
+# NOTE: to use cxxsim, export NMIGEN_SIM_MODE=cxxsim from the shell
+# Also, check out the cxxsim nmigen branch, and latest yosys from git
+from nmutil.sim_tmp_alternative import Simulator
+
+from nmutil.util import wrap
 
 
 # TODO: make these parameters of DCache at some point
@@ -61,6 +86,10 @@ ROW_PER_LINE = LINE_SIZE // ROW_SIZE
 # to represent the full dcache
 BRAM_ROWS = NUM_LINES * ROW_PER_LINE
 
+print ("ROW_SIZE", ROW_SIZE)
+print ("ROW_PER_LINE", ROW_PER_LINE)
+print ("BRAM_ROWS", BRAM_ROWS)
+print ("NUM_WAYS", NUM_WAYS)
 
 # Bit fields counts in the address
 
@@ -101,26 +130,39 @@ TAG_WIDTH = TAG_BITS + 7 - ((TAG_BITS + 7) % 8)
 WAY_BITS = log2_int(NUM_WAYS)
 
 # Example of layout for 32 lines of 64 bytes:
-#
-# ..  tag    |index|  line  |
-# ..         |   row   |    |
-# ..         |     |---|    | ROW_LINE_BITS  (3)
-# ..         |     |--- - --| LINE_OFF_BITS (6)
-# ..         |         |- --| ROW_OFF_BITS  (3)
-# ..         |----- ---|    | ROW_BITS      (8)
-# ..         |-----|        | INDEX_BITS    (5)
-# .. --------|              | TAG_BITS      (45)
+layout = """\
+  ..  tag    |index|  line  |
+  ..         |   row   |    |
+  ..         |     |---|    | ROW_LINE_BITS  (3)
+  ..         |     |--- - --| LINE_OFF_BITS (6)
+  ..         |         |- --| ROW_OFF_BITS  (3)
+  ..         |----- ---|    | ROW_BITS      (8)
+  ..         |-----|        | INDEX_BITS    (5)
+  .. --------|              | TAG_BITS      (45)
+"""
+print (layout)
+print ("Dcache TAG %d IDX %d ROW_BITS %d ROFF %d LOFF %d RLB %d" % \
+            (TAG_BITS, INDEX_BITS, ROW_BITS,
+             ROW_OFF_BITS, LINE_OFF_BITS, ROW_LINE_BITS))
+print ("index @: %d-%d" % (LINE_OFF_BITS, SET_SIZE_BITS))
+print ("row @: %d-%d" % (LINE_OFF_BITS, ROW_OFF_BITS))
+print ("tag @: %d-%d width %d" % (SET_SIZE_BITS, REAL_ADDR_BITS, TAG_WIDTH))
 
 TAG_RAM_WIDTH = TAG_WIDTH * NUM_WAYS
 
+print ("TAG_RAM_WIDTH", TAG_RAM_WIDTH)
+
 def CacheTagArray():
-    return Array(Signal(TAG_RAM_WIDTH) for x in range(NUM_LINES))
+    return Array(Signal(TAG_RAM_WIDTH, name="cachetag_%d" % x) \
+                        for x in range(NUM_LINES))
 
 def CacheValidBitsArray():
-    return Array(Signal(INDEX_BITS) for x in range(NUM_LINES))
+    return Array(Signal(NUM_WAYS, name="cachevalid_%d" % x) \
+                        for x in range(NUM_LINES))
 
 def RowPerLineValidArray():
-    return Array(Signal() for x in range(ROW_PER_LINE))
+    return Array(Signal(name="rows_valid%d" % x) \
+                        for x in range(ROW_PER_LINE))
 
 # L1 TLB
 TLB_SET_BITS     = log2_int(TLB_SET_SIZE)
@@ -130,10 +172,13 @@ TLB_TAG_WAY_BITS = TLB_NUM_WAYS * TLB_EA_TAG_BITS
 TLB_PTE_BITS     = 64
 TLB_PTE_WAY_BITS = TLB_NUM_WAYS * TLB_PTE_BITS;
 
+def ispow2(x):
+    return (1<<log2_int(x, False)) == x
+
 assert (LINE_SIZE % ROW_SIZE) == 0, "LINE_SIZE not multiple of ROW_SIZE"
-assert (LINE_SIZE % 2) == 0, "LINE_SIZE not power of 2"
-assert (NUM_LINES % 2) == 0, "NUM_LINES not power of 2"
-assert (ROW_PER_LINE % 2) == 0, "ROW_PER_LINE not power of 2"
+assert ispow2(LINE_SIZE), "LINE_SIZE not power of 2"
+assert ispow2(NUM_LINES), "NUM_LINES not power of 2"
+assert ispow2(ROW_PER_LINE), "ROW_PER_LINE not power of 2"
 assert ROW_BITS == (INDEX_BITS + ROW_LINE_BITS), "geometry bits don't add up"
 assert (LINE_OFF_BITS == ROW_OFF_BITS + ROW_LINE_BITS), \
         "geometry bits don't add up"
@@ -146,31 +191,39 @@ assert SET_SIZE_BITS <= TLB_LG_PGSZ, "Set indexed by virtual address"
 
 
 def TLBValidBitsArray():
-    return Array(Signal(TLB_NUM_WAYS) for x in range(TLB_SET_SIZE))
+    return Array(Signal(TLB_NUM_WAYS, name="tlbvalid%d" % x) \
+                for x in range(TLB_SET_SIZE))
 
 def TLBTagEAArray():
-    return Array(Signal(TLB_EA_TAG_BITS) for x in range (TLB_NUM_WAYS))
+    return Array(Signal(TLB_EA_TAG_BITS, name="tlbtagea%d" % x) \
+                for x in range (TLB_NUM_WAYS))
 
 def TLBTagsArray():
-    return Array(Signal(TLB_TAG_WAY_BITS) for x in range (TLB_SET_SIZE))
+    return Array(Signal(TLB_TAG_WAY_BITS, name="tlbtags%d" % x) \
+                for x in range (TLB_SET_SIZE))
 
 def TLBPtesArray():
-    return Array(Signal(TLB_PTE_WAY_BITS) for x in range(TLB_SET_SIZE))
+    return Array(Signal(TLB_PTE_WAY_BITS, name="tlbptes%d" % x) \
+                for x in range(TLB_SET_SIZE))
 
 def HitWaySet():
-    return Array(Signal(WAY_BITS) for x in range(TLB_NUM_WAYS))
+    return Array(Signal(WAY_BITS, name="hitway_%d" % x) \
+                        for x in range(TLB_NUM_WAYS))
 
 # Cache RAM interface
 def CacheRamOut():
-    return Array(Signal(WB_DATA_BITS) for x in range(NUM_WAYS))
+    return Array(Signal(WB_DATA_BITS, name="cache_out%d" % x) \
+                 for x in range(NUM_WAYS))
 
 # PLRU output interface
 def PLRUOut():
-    return Array(Signal(WAY_BITS) for x in range(NUM_LINES))
+    return Array(Signal(WAY_BITS, name="plru_out%d" % x) \
+                for x in range(NUM_LINES))
 
 # TLB PLRU output interface
 def TLBPLRUOut():
-    return Array(Signal(TLB_WAY_BITS) for x in range(TLB_SET_SIZE))
+    return Array(Signal(TLB_WAY_BITS, name="tlbplru_out%d" % x) \
+                for x in range(TLB_SET_SIZE))
 
 # Helper functions to decode incoming requests
 #
@@ -184,7 +237,7 @@ def get_row(addr):
 
 # Return the index of a row within a line
 def get_row_of_line(row):
-    return row[:ROW_LINE_BITS]
+    return row[:ROW_BITS][:ROW_LINE_BITS]
 
 # Returns whether this is the last row of a line
 def is_last_row_addr(addr, last):
@@ -228,8 +281,8 @@ def write_tlb_pte(way, ptes, newpte):
 
 # Record for storing permission, attribute, etc. bits from a PTE
 class PermAttr(RecordObject):
-    def __init__(self):
-        super().__init__()
+    def __init__(self, name=None):
+        super().__init__(name=name)
         self.reference = Signal()
         self.changed   = Signal()
         self.nocache   = Signal()
@@ -240,12 +293,6 @@ class PermAttr(RecordObject):
 
 def extract_perm_attr(pte):
     pa = PermAttr()
-    pa.reference = pte[8]
-    pa.changed   = pte[7]
-    pa.nocache   = pte[5]
-    pa.priv      = pte[3]
-    pa.rd_perm   = pte[2]
-    pa.wr_perm   = pte[1]
     return pa;
 
 
@@ -292,10 +339,11 @@ class RegStage0(RecordObject):
     def __init__(self, name=None):
         super().__init__(name=name)
         self.req     = LoadStore1ToDCacheType(name="lsmem")
-        self.tlbie   = Signal()
-        self.doall   = Signal()
-        self.tlbld   = Signal()
+        self.tlbie   = Signal() # indicates a tlbie request (from MMU)
+        self.doall   = Signal() # with tlbie, indicates flush whole TLB
+        self.tlbld   = Signal() # indicates a TLB load request (from MMU)
         self.mmu_req = Signal() # indicates source of request
+        self.d_valid = Signal() # indicates req.data is valid now
 
 
 class MemAccessRequest(RecordObject):
@@ -349,12 +397,12 @@ class RegStage1(RecordObject):
         self.write_bram       = Signal()
         self.write_tag        = Signal()
         self.slow_valid       = Signal()
-        self.wb               = WBMasterOut()
+        self.wb               = WBMasterOut("wb")
         self.reload_tag       = Signal(TAG_BITS)
         self.store_way        = Signal(WAY_BITS)
         self.store_row        = Signal(ROW_BITS)
         self.store_index      = Signal(INDEX_BITS)
-        self.end_row_ix       = Signal(log2_int(ROW_LINE_BITS, False))
+        self.end_row_ix       = Signal(ROW_LINE_BITS)
         self.rows_valid       = RowPerLineValidArray()
         self.acks_pending     = Signal(3)
         self.inc_acks         = Signal()
@@ -396,11 +444,11 @@ class DTLBUpdate(Elaboratable):
         self.eatag           = Signal(TLB_EA_TAG_BITS)
         self.pte_data        = Signal(TLB_PTE_BITS)
 
-        self.dv = Signal(TLB_PTE_WAY_BITS)
+        self.dv = Signal(TLB_NUM_WAYS) # tlb_way_valids_t
 
-        self.tb_out = Signal(TLB_TAG_WAY_BITS)
-        self.pb_out = Signal(TLB_NUM_WAYS)
-        self.db_out = Signal(TLB_PTE_WAY_BITS)
+        self.tb_out = Signal(TLB_TAG_WAY_BITS) # tlb_way_tags_t
+        self.db_out = Signal(TLB_NUM_WAYS)     # tlb_way_valids_t
+        self.pb_out = Signal(TLB_PTE_WAY_BITS) # tlb_way_ptes_t
 
     def elaborate(self, platform):
         m = Module()
@@ -411,13 +459,13 @@ class DTLBUpdate(Elaboratable):
         pteset   = Signal(TLB_PTE_WAY_BITS)
 
         tb_out, pb_out, db_out = self.tb_out, self.pb_out, self.db_out
+        comb += db_out.eq(self.dv)
 
         with m.If(self.tlbie & self.doall):
             pass # clear all back in parent
         with m.Elif(self.tlbie):
             with m.If(self.tlb_hit):
-                comb += db_out.eq(self.dv)
-                comb += db_out.bit_select(self.tlb_hit_way, 1).eq(1)
+                comb += db_out.bit_select(self.tlb_hit_way, 1).eq(0)
                 comb += self.v_updated.eq(1)
 
         with m.Elif(self.tlbwe):
@@ -437,16 +485,6 @@ class DTLBUpdate(Elaboratable):
 
         return m
 
-    def dcache_request(self, m, r0, ra, req_index, req_row, req_tag,
-                       r0_valid, r1, cache_valid_bits, replace_way,
-                       use_forward1_next, use_forward2_next,
-                       req_hit_way, plru_victim, rc_ok, perm_attr,
-                       valid_ra, perm_ok, access_ok, req_op, req_go,
-                       tlb_pte_way,
-                       tlb_hit, tlb_hit_way, tlb_valid_way, cache_tag_set,
-                       cancel_store, req_same_tag, r0_stall, early_req_row):
-        """Cache request parsing and hit detection
-        """
 
 class DCachePendingHit(Elaboratable):
 
@@ -493,7 +531,8 @@ class DCachePendingHit(Elaboratable):
         req_index = self.req_index
         reload_tag = self.reload_tag
 
-        rel_matches = Array(Signal() for i in range(TLB_NUM_WAYS))
+        rel_matches = Array(Signal(name="rel_matches_%d" % i) \
+                                    for i in range(TLB_NUM_WAYS))
         hit_way_set = HitWaySet()
 
         # Test if pending request is a hit on any way
@@ -503,7 +542,7 @@ class DCachePendingHit(Elaboratable):
         # the TLB, and then decide later which match to use.
 
         with m.If(virt_mode):
-            for j in range(TLB_NUM_WAYS):
+            for j in range(TLB_NUM_WAYS): # tlb_num_way_t
                 s_tag       = Signal(TAG_BITS, name="s_tag%d" % j)
                 s_hit       = Signal()
                 s_pte       = Signal(TLB_PTE_BITS)
@@ -513,8 +552,8 @@ class DCachePendingHit(Elaboratable):
                                     s_pte[TLB_LG_PGSZ:REAL_ADDR_BITS]))
                 comb += s_tag.eq(get_tag(s_ra))
 
-                for i in range(NUM_WAYS):
-                    is_tag_hit = Signal()
+                for i in range(NUM_WAYS): # way_t
+                    is_tag_hit = Signal(name="is_tag_hit_%d_%d" % (j, i))
                     comb += is_tag_hit.eq(go & cache_valid_idx[i] &
                                   (read_tag(i, cache_tag_set) == s_tag)
                                   & tlb_valid_way[j])
@@ -531,10 +570,10 @@ class DCachePendingHit(Elaboratable):
         with m.Else():
             s_tag       = Signal(TAG_BITS)
             comb += s_tag.eq(get_tag(req_addr))
-            for i in range(NUM_WAYS):
-                is_tag_hit = Signal()
+            for i in range(NUM_WAYS): # way_t
+                is_tag_hit = Signal(name="is_tag_hit_%d" % i)
                 comb += is_tag_hit.eq(go & cache_valid_idx[i] &
-                          read_tag(i, cache_tag_set) == s_tag)
+                          (read_tag(i, cache_tag_set) == s_tag))
                 with m.If(is_tag_hit):
                     comb += hit_way.eq(i)
                     comb += is_hit.eq(1)
@@ -546,6 +585,7 @@ class DCachePendingHit(Elaboratable):
 
 class DCache(Elaboratable):
     """Set associative dcache write-through
+
     TODO (in no specific order):
     * See list in icache.vhdl
     * Complete load misses on the cycle when WB data comes instead of
@@ -561,8 +601,8 @@ class DCache(Elaboratable):
 
         self.stall_out = Signal()
 
-        self.wb_out    = WBMasterOut()
-        self.wb_in     = WBSlaveOut()
+        self.wb_out    = WBMasterOut("wb_out")
+        self.wb_in     = WBSlaveOut("wb_in")
 
         self.log_out   = Signal(20)
 
@@ -576,34 +616,49 @@ class DCache(Elaboratable):
         r = RegStage0("stage0")
 
         # TODO, this goes in unit tests and formal proofs
-        with m.If(~(d_in.valid & m_in.valid)):
-            #sync += Display("request collision loadstore vs MMU")
-            pass
+        with m.If(d_in.valid & m_in.valid):
+            sync += Display("request collision loadstore vs MMU")
 
         with m.If(m_in.valid):
-            sync += r.req.valid.eq(1)
-            sync += r.req.load.eq(~(m_in.tlbie | m_in.tlbld))
-            sync += r.req.dcbz.eq(0)
-            sync += r.req.nc.eq(0)
-            sync += r.req.reserve.eq(0)
-            sync += r.req.virt_mode.eq(1)
-            sync += r.req.priv_mode.eq(1)
-            sync += r.req.addr.eq(m_in.addr)
-            sync += r.req.data.eq(m_in.pte)
-            sync += r.req.byte_sel.eq(~0) # Const -1 sets all to 0b111....
-            sync += r.tlbie.eq(m_in.tlbie)
-            sync += r.doall.eq(m_in.doall)
-            sync += r.tlbld.eq(m_in.tlbld)
-            sync += r.mmu_req.eq(1)
+            comb += r.req.valid.eq(1)
+            comb += r.req.load.eq(~(m_in.tlbie | m_in.tlbld))# no invalidate
+            comb += r.req.dcbz.eq(0)
+            comb += r.req.nc.eq(0)
+            comb += r.req.reserve.eq(0)
+            comb += r.req.virt_mode.eq(0)
+            comb += r.req.priv_mode.eq(1)
+            comb += r.req.addr.eq(m_in.addr)
+            comb += r.req.data.eq(m_in.pte)
+            comb += r.req.byte_sel.eq(~0) # Const -1 sets all to 0b111....
+            comb += r.tlbie.eq(m_in.tlbie)
+            comb += r.doall.eq(m_in.doall)
+            comb += r.tlbld.eq(m_in.tlbld)
+            comb += r.mmu_req.eq(1)
+            m.d.sync += Display("    DCACHE req mmu addr %x pte %x ld %d",
+                                 m_in.addr, m_in.pte, r.req.load)
+
         with m.Else():
-            sync += r.req.eq(d_in)
-            sync += r.tlbie.eq(0)
-            sync += r.doall.eq(0)
-            sync += r.tlbld.eq(0)
-            sync += r.mmu_req.eq(0)
-            with m.If(~(r1.full & r0_full)):
-                sync += r0.eq(r)
-                sync += r0_full.eq(r.req.valid)
+            comb += r.req.eq(d_in)
+            comb += r.req.data.eq(0)
+            comb += r.tlbie.eq(0)
+            comb += r.doall.eq(0)
+            comb += r.tlbld.eq(0)
+            comb += r.mmu_req.eq(0)
+        with m.If((~r1.full & ~d_in.hold) | ~r0_full):
+            sync += r0.eq(r)
+            sync += r0_full.eq(r.req.valid)
+            # Sample data the cycle after a request comes in from loadstore1.
+            # If another request has come in already then the data will get
+            # put directly into req.data below.
+            with m.If(r0.req.valid & ~r.req.valid & ~r0.d_valid &
+                     ~r0.mmu_req):
+                sync += r0.req.data.eq(d_in.data)
+                sync += r0.d_valid.eq(1)
+        with m.If(d_in.valid):
+            m.d.sync += Display("    DCACHE req cache "
+                                "virt %d addr %x data %x ld %d",
+                                 r.req.virt_mode, r.req.addr,
+                                 r.req.data, r.req.load)
 
     def tlb_read(self, m, r0_stall, tlb_valid_way,
                  tlb_tag_way, tlb_pte_way, dtlb_valid_bits,
@@ -645,13 +700,13 @@ class DCache(Elaboratable):
             return
         for i in range(TLB_SET_SIZE):
             # TLB PLRU interface
-            tlb_plru        = PLRU(WAY_BITS)
+            tlb_plru        = PLRU(TLB_WAY_BITS)
             setattr(m.submodules, "maybe_plru_%d" % i, tlb_plru)
             tlb_plru_acc_en = Signal()
 
             comb += tlb_plru_acc_en.eq(r1.tlb_hit & (r1.tlb_hit_index == i))
             comb += tlb_plru.acc_en.eq(tlb_plru_acc_en)
-            comb += tlb_plru.acc.eq(r1.tlb_hit_way)
+            comb += tlb_plru.acc_i.eq(r1.tlb_hit_way)
             comb += tlb_plru_victim[i].eq(tlb_plru.lru_o)
 
     def tlb_search(self, m, tlb_req_index, r0, r0_valid,
@@ -659,7 +714,6 @@ class DCache(Elaboratable):
                    tlb_pte_way, pte, tlb_hit, valid_ra, perm_attr, ra):
 
         comb = m.d.comb
-        sync = m.d.sync
 
         hitway = Signal(TLB_WAY_BITS)
         hit    = Signal()
@@ -670,9 +724,10 @@ class DCache(Elaboratable):
         comb += eatag.eq(r0.req.addr[TLB_LG_END : 64 ])
 
         for i in range(TLB_NUM_WAYS):
-            is_tag_hit = Signal()
-            comb += is_tag_hit.eq(tlb_valid_way[i]
-                                  & read_tlb_tag(i, tlb_tag_way) == eatag)
+            is_tag_hit = Signal(name="is_tag_hit%d" % i)
+            tlb_tag = Signal(TLB_EA_TAG_BITS, name="tlb_tag%d" % i)
+            comb += tlb_tag.eq(read_tlb_tag(i, tlb_tag_way))
+            comb += is_tag_hit.eq(tlb_valid_way[i] & (tlb_tag == eatag))
             with m.If(is_tag_hit):
                 comb += hitway.eq(i)
                 comb += hit.eq(1)
@@ -682,29 +737,44 @@ class DCache(Elaboratable):
 
         with m.If(tlb_hit):
             comb += pte.eq(read_tlb_pte(hitway, tlb_pte_way))
-        with m.Else():
-            comb += pte.eq(0)
         comb += valid_ra.eq(tlb_hit | ~r0.req.virt_mode)
+
         with m.If(r0.req.virt_mode):
             comb += ra.eq(Cat(Const(0, ROW_OFF_BITS),
                               r0.req.addr[ROW_OFF_BITS:TLB_LG_PGSZ],
                               pte[TLB_LG_PGSZ:REAL_ADDR_BITS]))
-            comb += perm_attr.eq(extract_perm_attr(pte))
+            comb += perm_attr.reference.eq(pte[8])
+            comb += perm_attr.changed.eq(pte[7])
+            comb += perm_attr.nocache.eq(pte[5])
+            comb += perm_attr.priv.eq(pte[3])
+            comb += perm_attr.rd_perm.eq(pte[2])
+            comb += perm_attr.wr_perm.eq(pte[1])
         with m.Else():
             comb += ra.eq(Cat(Const(0, ROW_OFF_BITS),
                               r0.req.addr[ROW_OFF_BITS:REAL_ADDR_BITS]))
-
             comb += perm_attr.reference.eq(1)
             comb += perm_attr.changed.eq(1)
-            comb += perm_attr.priv.eq(1)
             comb += perm_attr.nocache.eq(0)
+            comb += perm_attr.priv.eq(1)
             comb += perm_attr.rd_perm.eq(1)
             comb += perm_attr.wr_perm.eq(1)
 
+        with m.If(valid_ra):
+            m.d.sync += Display("DCACHE virt mode %d hit %d ra %x pte %x",
+                                r0.req.virt_mode, tlb_hit, ra, pte)
+            m.d.sync += Display("       perm ref=%d", perm_attr.reference)
+            m.d.sync += Display("       perm chg=%d", perm_attr.changed)
+            m.d.sync += Display("       perm noc=%d", perm_attr.nocache)
+            m.d.sync += Display("       perm prv=%d", perm_attr.priv)
+            m.d.sync += Display("       perm rdp=%d", perm_attr.rd_perm)
+            m.d.sync += Display("       perm wrp=%d", perm_attr.wr_perm)
+
     def tlb_update(self, m, r0_valid, r0, dtlb_valid_bits, tlb_req_index,
                     tlb_hit_way, tlb_hit, tlb_plru_victim, tlb_tag_way,
                     dtlb_tags, tlb_pte_way, dtlb_ptes):
 
+        dtlb_valids = TLBValidBitsArray()
+
         comb = m.d.comb
         sync = m.d.sync
 
@@ -760,7 +830,7 @@ class DCache(Elaboratable):
 
             comb += plru_acc_en.eq(r1.cache_hit & (r1.hit_index == i))
             comb += plru.acc_en.eq(plru_acc_en)
-            comb += plru.acc.eq(r1.hit_way)
+            comb += plru.acc_i.eq(r1.hit_way)
             comb += plru_victim[i].eq(plru.lru_o)
 
     def cache_tag_read(self, m, r0_stall, req_index, cache_tag_set, cache_tags):
@@ -781,7 +851,7 @@ class DCache(Elaboratable):
         sync += cache_tag_set.eq(cache_tags[index])
 
     def dcache_request(self, m, r0, ra, req_index, req_row, req_tag,
-                       r0_valid, r1, cache_valid_bits, replace_way,
+                       r0_valid, r1, cache_valids, replace_way,
                        use_forward1_next, use_forward2_next,
                        req_hit_way, plru_victim, rc_ok, perm_attr,
                        valid_ra, perm_ok, access_ok, req_op, req_go,
@@ -792,7 +862,6 @@ class DCache(Elaboratable):
         """
 
         comb = m.d.comb
-        sync = m.d.sync
         m_in, d_in = self.m_in, self.d_in
 
         is_hit      = Signal()
@@ -801,16 +870,21 @@ class DCache(Elaboratable):
         opsel       = Signal(3)
         go          = Signal()
         nc          = Signal()
-        hit_set     = Array(Signal() for i in range(TLB_NUM_WAYS))
-        cache_valid_idx = Signal(INDEX_BITS)
+        hit_set     = Array(Signal(name="hit_set_%d" % i) \
+                                  for i in range(TLB_NUM_WAYS))
+        cache_valid_idx = Signal(NUM_WAYS)
 
         # Extract line, row and tag from request
         comb += req_index.eq(get_index(r0.req.addr))
         comb += req_row.eq(get_row(r0.req.addr))
         comb += req_tag.eq(get_tag(ra))
 
+        if False: # display on comb is a bit... busy.
+            comb += Display("dcache_req addr:%x ra: %x idx: %x tag: %x row: %x",
+                    r0.req.addr, ra, req_index, req_tag, req_row)
+
         comb += go.eq(r0_valid & ~(r0.tlbie | r0.tlbld) & ~r1.ls_error)
-        comb += cache_valid_idx.eq(cache_valid_bits[req_index])
+        comb += cache_valid_idx.eq(cache_valids[req_index])
 
         m.submodules.dcache_pend = dc = DCachePendingHit(tlb_pte_way,
                                 tlb_valid_way, tlb_hit_way,
@@ -833,12 +907,13 @@ class DCache(Elaboratable):
             # For a store, consider this a hit even if the row isn't
             # valid since it will be by the time we perform the store.
             # For a load, check the appropriate row valid bit.
-            valid = r1.rows_valid[req_row % ROW_PER_LINE]
-            comb += is_hit.eq(~r0.req.load | valid)
+            rrow = Signal(ROW_LINE_BITS)
+            comb += rrow.eq(req_row)
+            valid = r1.rows_valid[rrow]
+            comb += is_hit.eq((~r0.req.load) | valid)
             comb += hit_way.eq(replace_way)
 
         # Whether to use forwarded data for a load or not
-        comb += use_forward1_next.eq(0)
         with m.If((get_row(r1.req.real_addr) == req_row) &
                   (r1.req.hit_way == hit_way)):
             # Only need to consider r1.write_bram here, since if we
@@ -850,7 +925,6 @@ class DCache(Elaboratable):
             # cycles after the refill starts before we see the updated
             # cache tag. In that case we don't use the bypass.)
             comb += use_forward1_next.eq(r1.write_bram)
-        comb += use_forward2_next.eq(0)
         with m.If((r1.forward_row1 == req_row) & (r1.forward_way1 == hit_way)):
             comb += use_forward2_next.eq(r1.forward_valid1)
 
@@ -866,12 +940,10 @@ class DCache(Elaboratable):
         # work out whether we have permission for this access
         # NB we don't yet implement AMR, thus no KUAP
         comb += rc_ok.eq(perm_attr.reference
-                         & (r0.req.load | perm_attr.changed)
-                )
-        comb += perm_ok.eq((r0.req.priv_mode | ~perm_attr.priv)
-                           & perm_attr.wr_perm
-                           | (r0.req.load & perm_attr.rd_perm)
-                          )
+                         & (r0.req.load | perm_attr.changed))
+        comb += perm_ok.eq((r0.req.priv_mode | (~perm_attr.priv)) &
+                           (perm_attr.wr_perm |
+                              (r0.req.load & perm_attr.rd_perm)))
         comb += access_ok.eq(valid_ra & perm_ok & rc_ok)
         # Combine the request and cache hit status to decide what
         # operation needs to be done
@@ -879,30 +951,25 @@ class DCache(Elaboratable):
         comb += op.eq(Op.OP_NONE)
         with m.If(go):
             with m.If(~access_ok):
+                m.d.sync += Display("DCACHE access fail valid_ra=%d p=%d rc=%d",
+                                 valid_ra, perm_ok, rc_ok)
                 comb += op.eq(Op.OP_BAD)
             with m.Elif(cancel_store):
+                m.d.sync += Display("DCACHE cancel store")
                 comb += op.eq(Op.OP_STCX_FAIL)
             with m.Else():
+                m.d.sync += Display("DCACHE valid_ra=%d nc=%d ld=%d",
+                                 valid_ra, nc, r0.req.load)
                 comb += opsel.eq(Cat(is_hit, nc, r0.req.load))
                 with m.Switch(opsel):
-                    with m.Case(0b101):
-                        comb += op.eq(Op.OP_LOAD_HIT)
-                    with m.Case(0b100):
-                        comb += op.eq(Op.OP_LOAD_MISS)
-                    with m.Case(0b110):
-                        comb += op.eq(Op.OP_LOAD_NC)
-                    with m.Case(0b001):
-                        comb += op.eq(Op.OP_STORE_HIT)
-                    with m.Case(0b000):
-                        comb += op.eq(Op.OP_STORE_MISS)
-                    with m.Case(0b010):
-                        comb += op.eq(Op.OP_STORE_MISS)
-                    with m.Case(0b011):
-                        comb += op.eq(Op.OP_BAD)
-                    with m.Case(0b111):
-                        comb += op.eq(Op.OP_BAD)
-                    with m.Default():
-                        comb += op.eq(Op.OP_NONE)
+                    with m.Case(0b101): comb += op.eq(Op.OP_LOAD_HIT)
+                    with m.Case(0b100): comb += op.eq(Op.OP_LOAD_MISS)
+                    with m.Case(0b110): comb += op.eq(Op.OP_LOAD_NC)
+                    with m.Case(0b001): comb += op.eq(Op.OP_STORE_HIT)
+                    with m.Case(0b000): comb += op.eq(Op.OP_STORE_MISS)
+                    with m.Case(0b010): comb += op.eq(Op.OP_STORE_MISS)
+                    with m.Case(0b011): comb += op.eq(Op.OP_BAD)
+                    with m.Case(0b111): comb += op.eq(Op.OP_BAD)
         comb += req_op.eq(op)
         comb += req_go.eq(go)
 
@@ -923,17 +990,16 @@ class DCache(Elaboratable):
         """Handle load-with-reservation and store-conditional instructions
         """
         comb = m.d.comb
-        sync = m.d.sync
 
         with m.If(r0_valid & r0.req.reserve):
-
             # XXX generate alignment interrupt if address
             # is not aligned XXX or if r0.req.nc = '1'
             with m.If(r0.req.load):
-                comb += set_rsrv.eq(1) # load with reservation
+                comb += set_rsrv.eq(r0.req.atomic_last) # load with reservation
             with m.Else():
-                comb += clear_rsrv.eq(1) # store conditional
-                with m.If(~reservation.valid | r0.req.addr[LINE_OFF_BITS:64]):
+                comb += clear_rsrv.eq(r0.req.atomic_last) # store conditional
+                with m.If((~reservation.valid) |
+                         (r0.req.addr[LINE_OFF_BITS:64] != reservation.addr)):
                     comb += cancel_store.eq(1)
 
     def reservation_reg(self, m, r0_valid, access_ok, set_rsrv, clear_rsrv,
@@ -949,7 +1015,7 @@ class DCache(Elaboratable):
                 sync += reservation.valid.eq(1)
                 sync += reservation.addr.eq(r0.req.addr[LINE_OFF_BITS:64])
 
-    def writeback_control(self, m, r1, cache_out):
+    def writeback_control(self, m, r1, cache_out_row):
         """Return data for loads & completion control logic
         """
         comb = m.d.comb
@@ -968,7 +1034,7 @@ class DCache(Elaboratable):
         with m.Else():
             comb += data_fwd.eq(r1.forward_data2)
 
-        comb += data_out.eq(cache_out[r1.hit_way])
+        comb += data_out.eq(cache_out_row)
 
         for i in range(8):
             with m.If(r1.forward_sel[i]):
@@ -1009,35 +1075,32 @@ class DCache(Elaboratable):
             # Request came from loadstore1...
             # Load hit case is the standard path
             with m.If(r1.hit_load_valid):
-                #Display(f"completing load hit data={data_out}")
-                pass
+                sync += Display("completing load hit data=%x", data_out)
 
             # error cases complete without stalling
             with m.If(r1.ls_error):
-                # Display("completing ld/st with error")
-                pass
+                sync += Display("completing ld/st with error")
 
             # Slow ops (load miss, NC, stores)
             with m.If(r1.slow_valid):
-                #Display(f"completing store or load miss data={data_out}")
-                pass
+                sync += Display("completing store or load miss adr=%x data=%x",
+                                r1.req.real_addr, data_out)
 
         with m.Else():
             # Request came from MMU
             with m.If(r1.hit_load_valid):
-                # Display(f"completing load hit to MMU, data={m_out.data}")
-                pass
+                sync += Display("completing load hit to MMU, data=%x",
+                                m_out.data)
             # error cases complete without stalling
             with m.If(r1.mmu_error):
-                #Display("combpleting MMU ld with error")
-                pass
+                sync += Display("combpleting MMU ld with error")
 
             # Slow ops (i.e. load miss)
             with m.If(r1.slow_valid):
-                #Display("completing MMU load miss, data={m_out.data}")
-                pass
+                sync += Display("completing MMU load miss, adr=%x data=%x",
+                                r1.req.real_addr, m_out.data)
 
-    def rams(self, m, r1, early_req_row, cache_out, replace_way):
+    def rams(self, m, r1, early_req_row, cache_out_row, replace_way):
         """rams
         Generate a cache RAM for each way. This handles the normal
         reads, writes from reloads and the special store-hit update
@@ -1052,16 +1115,16 @@ class DCache(Elaboratable):
         wb_in = self.wb_in
 
         for i in range(NUM_WAYS):
-            do_read  = Signal()
-            rd_addr  = Signal(ROW_BITS)
-            do_write = Signal()
-            wr_addr  = Signal(ROW_BITS)
-            wr_data  = Signal(WB_DATA_BITS)
+            do_read  = Signal(name="do_rd%d" % i)
+            rd_addr  = Signal(ROW_BITS, name="rd_addr_%d" % i)
+            do_write = Signal(name="do_wr%d" % i)
+            wr_addr  = Signal(ROW_BITS, name="wr_addr_%d" % i)
+            wr_data  = Signal(WB_DATA_BITS, name="din_%d" % i)
             wr_sel   = Signal(ROW_SIZE)
             wr_sel_m = Signal(ROW_SIZE)
-            _d_out   = Signal(WB_DATA_BITS)
+            _d_out   = Signal(WB_DATA_BITS, name="dout_%d" % i) # cache_row_t
 
-            way = CacheRam(ROW_BITS, WB_DATA_BITS, True)
+            way = CacheRam(ROW_BITS, WB_DATA_BITS, ADD_BUF=True, ram_num=i)
             setattr(m.submodules, "cacheram_%d" % i, way)
 
             comb += way.rd_en.eq(do_read)
@@ -1074,7 +1137,8 @@ class DCache(Elaboratable):
             # Cache hit reads
             comb += do_read.eq(1)
             comb += rd_addr.eq(early_req_row)
-            comb += cache_out[i].eq(_d_out)
+            with m.If(r1.hit_way == i):
+                comb += cache_out_row.eq(_d_out)
 
             # Write mux:
             #
@@ -1101,30 +1165,29 @@ class DCache(Elaboratable):
                 comb += wr_addr.eq(r1.store_row)
                 comb += wr_sel.eq(~0) # all 1s
 
-            with m.If((r1.state == State.RELOAD_WAIT_ACK)
-                      & wb_in.ack & (replace_way == i)):
-                comb += do_write.eq(1)
+                with m.If((r1.state == State.RELOAD_WAIT_ACK)
+                          & wb_in.ack & (replace_way == i)):
+                    comb += do_write.eq(1)
 
-                # Mask write selects with do_write since BRAM
-                # doesn't have a global write-enable
-                with m.If(do_write):
-                    comb += wr_sel_m.eq(wr_sel)
+            # Mask write selects with do_write since BRAM
+            # doesn't have a global write-enable
+            with m.If(do_write):
+                comb += wr_sel_m.eq(wr_sel)
 
     # Cache hit synchronous machine for the easy case.
     # This handles load hits.
     # It also handles error cases (TLB miss, cache paradox)
     def dcache_fast_hit(self, m, req_op, r0_valid, r0, r1,
-                        req_hit_way, req_index, access_ok,
+                        req_hit_way, req_index, req_tag, access_ok,
                         tlb_hit, tlb_hit_way, tlb_req_index):
 
         comb = m.d.comb
         sync = m.d.sync
 
         with m.If(req_op != Op.OP_NONE):
-            #Display(f"op:{req_op} addr:{r0.req.addr} nc: {r0.req.nc}" \
-            #      f"idx:{req_index} tag:{req_tag} way: {req_hit_way}"
-            #     )
-            pass
+            sync += Display("op:%d addr:%x nc: %d idx: %x tag: %x way: %x",
+                    req_op, r0.req.addr, r0.req.nc,
+                    req_index, req_tag, req_hit_way)
 
         with m.If(r0_valid):
             sync += r1.mmu_req.eq(r0.mmu_req)
@@ -1151,13 +1214,13 @@ class DCache(Elaboratable):
             sync += r1.mmu_error.eq(r0.mmu_req)
             sync += r1.cache_paradox.eq(access_ok)
 
-            with m.Else():
-                sync += r1.ls_error.eq(0)
-                sync += r1.mmu_error.eq(0)
-                sync += r1.cache_paradox.eq(0)
+        with m.Else():
+            sync += r1.ls_error.eq(0)
+            sync += r1.mmu_error.eq(0)
+            sync += r1.cache_paradox.eq(0)
 
         with m.If(req_op == Op.OP_STCX_FAIL):
-            r1.stcx_fail.eq(1)
+            sync += r1.stcx_fail.eq(1)
         with m.Else():
             sync += r1.stcx_fail.eq(0)
 
@@ -1175,18 +1238,23 @@ class DCache(Elaboratable):
     # All wishbone requests generation is done here.
     # This machine operates at stage 1.
     def dcache_slow(self, m, r1, use_forward1_next, use_forward2_next,
-                    cache_valid_bits, r0, replace_way,
+                    cache_valids, r0, replace_way,
                     req_hit_way, req_same_tag,
-                    r0_valid, req_op, cache_tag, req_go, ra):
+                    r0_valid, req_op, cache_tags, req_go, ra):
 
         comb = m.d.comb
         sync = m.d.sync
         wb_in = self.wb_in
+        d_in = self.d_in
 
-        req         = MemAccessRequest()
-        acks        = Signal(3)
-        adjust_acks = Signal(3)
-        stbs_done = Signal()
+        req         = MemAccessRequest("mreq_ds")
+
+        req_row = Signal(ROW_BITS)
+        req_idx = Signal(INDEX_BITS)
+        req_tag = Signal(TAG_BITS)
+        comb += req_idx.eq(get_index(req.real_addr))
+        comb += req_row.eq(get_row(req.real_addr))
+        comb += req_tag.eq(get_tag(req.real_addr))
 
         sync += r1.use_forward1.eq(use_forward1_next)
         sync += r1.forward_sel.eq(0)
@@ -1223,8 +1291,7 @@ class DCache(Elaboratable):
         # complete tlbies and TLB loads in the third cycle
         sync += r1.mmu_done.eq(r0_valid & (r0.tlbie | r0.tlbld))
 
-        with m.If((req_op == Op.OP_LOAD_HIT)
-                  | (req_op == Op.OP_STCX_FAIL)):
+        with m.If((req_op == Op.OP_LOAD_HIT) | (req_op == Op.OP_STCX_FAIL)):
             with m.If(~r0.mmu_req):
                 sync += r1.ls_valid.eq(1)
             with m.Else():
@@ -1235,9 +1302,14 @@ class DCache(Elaboratable):
             for i in range(NUM_WAYS):
                 with m.If(i == replace_way):
                     ct = Signal(TAG_RAM_WIDTH)
-                    comb += ct.eq(cache_tag[r1.store_index])
+                    comb += ct.eq(cache_tags[r1.store_index])
+                    """
+TODO: check this
+cache_tags(r1.store_index)((i + 1) * TAG_WIDTH - 1 downto i * TAG_WIDTH) <=
+                    (TAG_WIDTH - 1 downto TAG_BITS => '0') & r1.reload_tag;
+                    """
                     comb += ct.word_select(i, TAG_WIDTH).eq(r1.reload_tag)
-                    sync += cache_tag[r1.store_index].eq(ct)
+                    sync += cache_tags[r1.store_index].eq(ct)
             sync += r1.store_way.eq(replace_way)
             sync += r1.write_tag.eq(0)
 
@@ -1252,10 +1324,13 @@ class DCache(Elaboratable):
             comb += req.dcbz.eq(r0.req.dcbz)
             comb += req.real_addr.eq(ra)
 
-            with m.If(~r0.req.dcbz):
+            with m.If(r0.req.dcbz):
+                # force data to 0 for dcbz
+                comb += req.data.eq(0)
+            with m.Elif(r0.d_valid):
                 comb += req.data.eq(r0.req.data)
             with m.Else():
-                comb += req.data.eq(0)
+                comb += req.data.eq(d_in.data)
 
             # Select all bytes for dcbz
             # and for cacheable loads
@@ -1280,23 +1355,17 @@ class DCache(Elaboratable):
         with m.Switch(r1.state):
 
             with m.Case(State.IDLE):
-# XXX check 'left downto.  probably means len(r1.wb.adr)
-#                     r1.wb.adr <= req.real_addr(
-#                                   r1.wb.adr'left downto 0
-#                                  );
-                sync += r1.wb.adr.eq(req.real_addr)
+                sync += r1.wb.adr.eq(req.real_addr[ROW_LINE_BITS:])
                 sync += r1.wb.sel.eq(req.byte_sel)
                 sync += r1.wb.dat.eq(req.data)
                 sync += r1.dcbz.eq(req.dcbz)
 
                 # Keep track of our index and way
                 # for subsequent stores.
-                sync += r1.store_index.eq(get_index(req.real_addr))
-                sync += r1.store_row.eq(get_row(req.real_addr))
-                sync += r1.end_row_ix.eq(
-                         get_row_of_line(get_row(req.real_addr))
-                        )
-                sync += r1.reload_tag.eq(get_tag(req.real_addr))
+                sync += r1.store_index.eq(req_idx)
+                sync += r1.store_row.eq(req_row)
+                sync += r1.end_row_ix.eq(get_row_of_line(req_row)-1)
+                sync += r1.reload_tag.eq(req_tag)
                 sync += r1.req.same_tag.eq(1)
 
                 with m.If(req.op == Op.OP_STORE_HIT):
@@ -1307,17 +1376,18 @@ class DCache(Elaboratable):
                 for i in range(ROW_PER_LINE):
                     sync += r1.rows_valid[i].eq(0)
 
+                with m.If(req_op != Op.OP_NONE):
+                    sync += Display("cache op %d", req.op)
+
                 with m.Switch(req.op):
                     with m.Case(Op.OP_LOAD_HIT):
                         # stay in IDLE state
                         pass
 
                     with m.Case(Op.OP_LOAD_MISS):
-                        #Display(f"cache miss real addr:" \
-                        #      f"{req_real_addr}" \
-                        #      f" idx:{get_index(req_real_addr)}" \
-                        #      f" tag:{get_tag(req.real_addr)}")
-                        pass
+                        sync += Display("cache miss real addr: %x " \
+                                "idx: %x tag: %x",
+                                req.real_addr, req_row, req_tag)
 
                         # Start the wishbone cycle
                         sync += r1.wb.we.eq(0)
@@ -1349,6 +1419,8 @@ class DCache(Elaboratable):
                             with m.If(req.op == Op.OP_STORE_HIT):
                                 sync += r1.write_bram.eq(1)
                         with m.Else():
+                            # dcbz is handled much like a load miss except
+                            # that we are writing to memory instead of reading
                             sync += r1.state.eq(State.RELOAD_WAIT_ACK)
 
                             with m.If(req.op == Op.OP_STORE_MISS):
@@ -1369,39 +1441,42 @@ class DCache(Elaboratable):
                         pass
 
             with m.Case(State.RELOAD_WAIT_ACK):
+                ld_stbs_done = Signal()
                 # Requests are all sent if stb is 0
-                comb += stbs_done.eq(~r1.wb.stb)
-
-                with m.If(~wb_in.stall & ~stbs_done):
-                    # That was the last word?
-                    # We are done sending.
-                    # Clear stb and set stbs_done
-                    # so we can handle an eventual
-                    # last ack on the same cycle.
-                    with m.If(is_last_row_addr(
-                              r1.wb.adr, r1.end_row_ix)):
+                comb += ld_stbs_done.eq(~r1.wb.stb)
+
+                # If we are still sending requests, was one accepted?
+                with m.If((~wb_in.stall) & r1.wb.stb):
+                    # That was the last word?  We are done sending.
+                    # Clear stb and set ld_stbs_done so we can handle an
+                    # eventual last ack on the same cycle.
+                    # sigh - reconstruct wb adr with 3 extra 0s at front
+                    wb_adr = Cat(Const(0, ROW_OFF_BITS), r1.wb.adr)
+                    with m.If(is_last_row_addr(wb_adr, r1.end_row_ix)):
                         sync += r1.wb.stb.eq(0)
-                        comb += stbs_done.eq(0)
+                        comb += ld_stbs_done.eq(1)
 
                     # Calculate the next row address in the current cache line
-                    rarange = r1.wb.adr[ROW_OFF_BITS : LINE_OFF_BITS]
-                    sync += rarange.eq(rarange + 1)
+                    row = Signal(LINE_OFF_BITS-ROW_OFF_BITS)
+                    comb += row.eq(r1.wb.adr)
+                    sync += r1.wb.adr[:LINE_OFF_BITS-ROW_OFF_BITS].eq(row+1)
 
                 # Incoming acks processing
                 sync += r1.forward_valid1.eq(wb_in.ack)
                 with m.If(wb_in.ack):
-                    # XXX needs an Array bit-accessor here
-                    sync += r1.rows_valid[r1.store_row % ROW_PER_LINE].eq(1)
+                    srow = Signal(ROW_LINE_BITS)
+                    comb += srow.eq(r1.store_row)
+                    sync += r1.rows_valid[srow].eq(1)
 
                     # If this is the data we were looking for,
                     # we can complete the request next cycle.
                     # Compare the whole address in case the
                     # request in r1.req is not the one that
                     # started this refill.
-                    with m.If(r1.full & r1.req.same_tag &
+                    with m.If(req.valid & r1.req.same_tag &
                               ((r1.dcbz & r1.req.dcbz) |
                                (~r1.dcbz & (r1.req.op == Op.OP_LOAD_MISS))) &
-                                (r1.store_row == get_row(r1.req.real_addr))):
+                                (r1.store_row == get_row(req.real_addr))):
                         sync += r1.full.eq(0)
                         sync += r1.slow_valid.eq(1)
                         with m.If(~r1.mmu_req):
@@ -1412,22 +1487,31 @@ class DCache(Elaboratable):
                         sync += r1.use_forward1.eq(1)
 
                     # Check for completion
-                    with m.If(stbs_done & is_last_row(r1.store_row,
+                    with m.If(ld_stbs_done & is_last_row(r1.store_row,
                                                       r1.end_row_ix)):
                         # Complete wishbone cycle
                         sync += r1.wb.cyc.eq(0)
 
                         # Cache line is now valid
                         cv = Signal(INDEX_BITS)
-                        sync += cv.eq(cache_valid_bits[r1.store_index])
-                        sync += cv.bit_select(r1.store_way, 1).eq(1)
+                        comb += cv.eq(cache_valids[r1.store_index])
+                        comb += cv.bit_select(r1.store_way, 1).eq(1)
+                        sync += cache_valids[r1.store_index].eq(cv)
+
                         sync += r1.state.eq(State.IDLE)
+                        sync += Display("cache valid set %x "
+                                        "idx %d way %d",
+                                         cv, r1.store_index, r1.store_way)
 
                     # Increment store row counter
                     sync += r1.store_row.eq(next_row(r1.store_row))
 
             with m.Case(State.STORE_WAIT_ACK):
-                comb += stbs_done.eq(~r1.wb.stb)
+                st_stbs_done = Signal()
+                acks        = Signal(3)
+                adjust_acks = Signal(3)
+
+                comb += st_stbs_done.eq(~r1.wb.stb)
                 comb += acks.eq(r1.acks_pending)
 
                 with m.If(r1.inc_acks != r1.dec_acks):
@@ -1445,16 +1529,16 @@ class DCache(Elaboratable):
                     # See if there is another store waiting
                     # to be done which is in the same real page.
                     with m.If(req.valid):
-                        ra = req.real_addr[0:SET_SIZE_BITS]
-                        sync += r1.wb.adr[0:SET_SIZE_BITS].eq(ra)
+                        _ra = req.real_addr[ROW_LINE_BITS:SET_SIZE_BITS]
+                        sync += r1.wb.adr[0:SET_SIZE_BITS].eq(_ra)
                         sync += r1.wb.dat.eq(req.data)
                         sync += r1.wb.sel.eq(req.byte_sel)
 
-                    with m.Elif((adjust_acks < 7) & req.same_tag &
+                    with m.If((adjust_acks < 7) & req.same_tag &
                                 ((req.op == Op.OP_STORE_MISS)
                                  | (req.op == Op.OP_STORE_HIT))):
                         sync += r1.wb.stb.eq(1)
-                        comb += stbs_done.eq(0)
+                        comb += st_stbs_done.eq(0)
 
                         with m.If(req.op == Op.OP_STORE_HIT):
                             sync += r1.write_bram.eq(1)
@@ -1463,15 +1547,15 @@ class DCache(Elaboratable):
 
                         # Store requests never come from the MMU
                         sync += r1.ls_valid.eq(1)
-                        comb += stbs_done.eq(0)
+                        comb += st_stbs_done.eq(0)
                         sync += r1.inc_acks.eq(1)
                     with m.Else():
                         sync += r1.wb.stb.eq(0)
-                        comb += stbs_done.eq(1)
+                        comb += st_stbs_done.eq(1)
 
                 # Got ack ? See if complete.
                 with m.If(wb_in.ack):
-                    with m.If(stbs_done & (adjust_acks == 1)):
+                    with m.If(st_stbs_done & (adjust_acks == 1)):
                         sync += r1.state.eq(State.IDLE)
                         sync += r1.wb.cyc.eq(0)
                         sync += r1.wb.stb.eq(0)
@@ -1506,17 +1590,18 @@ class DCache(Elaboratable):
         sync += log_out.eq(Cat(r1.state[:3], valid_ra, tlb_hit_way[:3],
                                stall_out, req_op[:3], d_out.valid, d_out.error,
                                r1.wb.cyc, r1.wb.stb, wb_in.ack, wb_in.stall,
-                               r1.wb.adr[3:6]))
+                               r1.real_adr[3:6]))
 
     def elaborate(self, platform):
 
         m = Module()
         comb = m.d.comb
+        d_in = self.d_in
 
         # Storage. Hopefully "cache_rows" is a BRAM, the rest is LUTs
         cache_tags       = CacheTagArray()
         cache_tag_set    = Signal(TAG_RAM_WIDTH)
-        cache_valid_bits = CacheValidBitsArray()
+        cache_valids = CacheValidBitsArray()
 
         # TODO attribute ram_style : string;
         # TODO attribute ram_style of cache_tags : signal is "distributed";
@@ -1561,7 +1646,7 @@ class DCache(Elaboratable):
         use_forward1_next = Signal()
         use_forward2_next = Signal()
 
-        cache_out         = CacheRamOut()
+        cache_out_row     = Signal(WB_DATA_BITS)
 
         plru_victim       = PLRUOut()
         replace_way       = Signal(WAY_BITS)
@@ -1579,7 +1664,7 @@ class DCache(Elaboratable):
         pte           = Signal(TLB_PTE_BITS)
         ra            = Signal(REAL_ADDR_BITS)
         valid_ra      = Signal()
-        perm_attr     = PermAttr()
+        perm_attr     = PermAttr("dc_perms")
         rc_ok         = Signal()
         perm_ok       = Signal()
         access_ok     = Signal()
@@ -1591,13 +1676,18 @@ class DCache(Elaboratable):
         comb += self.m_out.stall.eq(0)
 
         # Hold off the request in r0 when r1 has an uncompleted request
-        comb += r0_stall.eq(r0_full & r1.full)
-        comb += r0_valid.eq(r0_full & ~r1.full)
+        comb += r0_stall.eq(r0_full & (r1.full | d_in.hold))
+        comb += r0_valid.eq(r0_full & ~r1.full & ~d_in.hold)
         comb += self.stall_out.eq(r0_stall)
 
         # Wire up wishbone request latch out of stage 1
         comb += self.wb_out.eq(r1.wb)
 
+        # deal with litex not doing wishbone pipeline mode
+        # XXX in wrong way.  FIFOs are needed in the SRAM test
+        # so that stb/ack match up
+        comb += self.wb_in.stall.eq(self.wb_out.cyc & ~self.wb_in.ack)
+
         # call sub-functions putting everything together, using shared
         # signals established above
         self.stage_0(m, r0, r1, r0_full)
@@ -1614,7 +1704,7 @@ class DCache(Elaboratable):
         self.maybe_tlb_plrus(m, r1, tlb_plru_victim)
         self.cache_tag_read(m, r0_stall, req_index, cache_tag_set, cache_tags)
         self.dcache_request(m, r0, ra, req_index, req_row, req_tag,
-                           r0_valid, r1, cache_valid_bits, replace_way,
+                           r0_valid, r1, cache_valids, replace_way,
                            use_forward1_next, use_forward2_next,
                            req_hit_way, plru_victim, rc_ok, perm_attr,
                            valid_ra, perm_ok, access_ok, req_op, req_go,
@@ -1625,13 +1715,13 @@ class DCache(Elaboratable):
                            r0_valid, r0, reservation)
         self.reservation_reg(m, r0_valid, access_ok, set_rsrv, clear_rsrv,
                            reservation, r0)
-        self.writeback_control(m, r1, cache_out)
-        self.rams(m, r1, early_req_row, cache_out, replace_way)
+        self.writeback_control(m, r1, cache_out_row)
+        self.rams(m, r1, early_req_row, cache_out_row, replace_way)
         self.dcache_fast_hit(m, req_op, r0_valid, r0, r1,
-                        req_hit_way, req_index, access_ok,
+                        req_hit_way, req_index, req_tag, access_ok,
                         tlb_hit, tlb_hit_way, tlb_req_index)
         self.dcache_slow(m, r1, use_forward1_next, use_forward2_next,
-                    cache_valid_bits, r0, replace_way,
+                    cache_valids, r0, replace_way,
                     req_hit_way, req_same_tag,
                          r0_valid, req_op, cache_tags, req_go, ra)
         #self.dcache_log(m, r1, valid_ra, tlb_hit_way, stall_out)
@@ -1639,104 +1729,8 @@ class DCache(Elaboratable):
         return m
 
 
-def dcache_sim(dut):
-    # clear stuff
-    yield dut.d_in.valid.eq(0)
-    yield dut.d_in.load.eq(0)
-    yield dut.d_in.nc.eq(0)
-    yield dut.d_in.addr.eq(0)
-    yield dut.d_in.data.eq(0)
-    yield dut.m_in.valid.eq(0)
-    yield dut.m_in.addr.eq(0)
-    yield dut.m_in.pte.eq(0)
-    # wait 4 * clk_period
-    yield
-    yield
-    yield
-    yield
-
-    # Cacheable read of address 4
-    yield dut.d_in.load.eq(1)
-    yield dut.d_in.nc.eq(0)
-    yield dut.d_in.addr.eq(0x0000000000000004)
-    yield dut.d_in.valid.eq(1)
-    yield
-    yield dut.d_in.valid.eq(0)
-    yield
-    while not (yield dut.d_out.valid):
-        yield
-    data = yield dut.d_out.data
-    addr = yield dut.d_in.addr
-    assert data == 0x0000000100000000, \
-        f"data @%x=%x expected 0x0000000100000000" % (data, addr)
-
-    # Cacheable read of address 30
-    yield dut.d_in.load.eq(1)
-    yield dut.d_in.nc.eq(0)
-    yield dut.d_in.addr.eq(Const(0x0000000000000030, 64))
-    yield dut.d_in.valid.eq(1)
-    yield
-    yield dut.d_in.valid.eq(0)
-    yield
-    while not (yield dut.d_out.valid):
-        yield
-    data = yield dut.d_out.data
-    addr = yield dut.d_in.addr
-    assert data == 0x0000000D0000000C, \
-        f"data @%x=%x expected 0000000D0000000C" % (data, addr)
-
-    # Non-cacheable read of address 100
-    yield dut.d_in.load.eq(1)
-    yield dut.d_in.nc.eq(1)
-    yield dut.d_in.addr.eq(Const(0x0000000000000100, 64))
-    yield dut.d_in.valid.eq(1)
-    yield
-    yield dut.d_in.valid.eq(0)
-    yield
-    while not (yield dut.d_out.valid):
-        yield
-    data = yield dut.d_out.data
-    addr = yield dut.d_in.addr
-    assert data == 0x0000004100000040, \
-        f"data @%x=%x expected 0000004100000040" % (data, addr)
-
-    yield
-    yield
-    yield
-    yield
-
-
-def test_dcache():
+if __name__ == '__main__':
     dut = DCache()
     vl = rtlil.convert(dut, ports=[])
     with open("test_dcache.il", "w") as f:
         f.write(vl)
-
-    memory = Memory(width=64, depth=16*8, init=range(128))
-    sram = SRAM(memory=memory, granularity=8)
-
-    m = Module()
-    m.submodules.dcache = dut
-    m.submodules.sram = sram
-
-    m.d.comb += sram.bus.cyc.eq(dut.wb_out.cyc)
-    m.d.comb += sram.bus.stb.eq(dut.wb_out.stb)
-    m.d.comb += sram.bus.we.eq(dut.wb_out.we)
-    m.d.comb += sram.bus.sel.eq(dut.wb_out.sel)
-    m.d.comb += sram.bus.adr.eq(dut.wb_out.adr)
-    m.d.comb += sram.bus.dat_w.eq(dut.wb_out.dat)
-
-    m.d.comb += dut.wb_in.ack.eq(sram.bus.ack)
-    m.d.comb += dut.wb_in.dat.eq(sram.bus.dat_r)
-
-    # nmigen Simulation
-    sim = Simulator(m)
-    sim.add_clock(1e-6)
-
-    sim.add_sync_process(wrap(dcache_sim(dut)))
-    with sim.write_vcd('test_dcache.vcd'):
-        sim.run()
-
-if __name__ == '__main__':
-    test_dcache()
-