1 // See LICENSE for license details.
7 mmu_t::mmu_t(simif_t
* sim
, processor_t
* proc
)
8 : sim(sim
), proc(proc
),
9 check_triggers_fetch(false),
10 check_triggers_load(false),
11 check_triggers_store(false),
15 yield_load_reservation();
22 void mmu_t::flush_icache()
24 for (size_t i
= 0; i
< ICACHE_ENTRIES
; i
++)
28 void mmu_t::flush_tlb()
30 memset(tlb_insn_tag
, -1, sizeof(tlb_insn_tag
));
31 memset(tlb_load_tag
, -1, sizeof(tlb_load_tag
));
32 memset(tlb_store_tag
, -1, sizeof(tlb_store_tag
));
37 reg_t
mmu_t::translate(reg_t addr
, access_type type
)
42 reg_t mode
= proc
->state
.prv
;
44 if (!proc
->state
.dcsr
.cause
&& get_field(proc
->state
.mstatus
, MSTATUS_MPRV
))
45 mode
= get_field(proc
->state
.mstatus
, MSTATUS_MPP
);
48 return walk(addr
, type
, mode
) | (addr
& (PGSIZE
-1));
51 tlb_entry_t
mmu_t::fetch_slow_path(reg_t vaddr
)
53 reg_t paddr
= translate(vaddr
, FETCH
);
55 if (auto host_addr
= sim
->addr_to_mem(paddr
)) {
56 return refill_tlb(vaddr
, paddr
, host_addr
, FETCH
);
58 if (!sim
->mmio_load(paddr
, sizeof fetch_temp
, (uint8_t*)&fetch_temp
))
59 throw trap_instruction_access_fault(vaddr
);
60 tlb_entry_t entry
= {(char*)&fetch_temp
- vaddr
, paddr
- vaddr
};
65 reg_t
reg_from_bytes(size_t len
, const uint8_t* bytes
)
72 (((reg_t
) bytes
[1]) << 8);
75 (((reg_t
) bytes
[1]) << 8) |
76 (((reg_t
) bytes
[2]) << 16) |
77 (((reg_t
) bytes
[3]) << 24);
80 (((reg_t
) bytes
[1]) << 8) |
81 (((reg_t
) bytes
[2]) << 16) |
82 (((reg_t
) bytes
[3]) << 24) |
83 (((reg_t
) bytes
[4]) << 32) |
84 (((reg_t
) bytes
[5]) << 40) |
85 (((reg_t
) bytes
[6]) << 48) |
86 (((reg_t
) bytes
[7]) << 56);
91 void mmu_t::load_slow_path(reg_t addr
, reg_t len
, uint8_t* bytes
)
93 reg_t paddr
= translate(addr
, LOAD
);
95 if (auto host_addr
= sim
->addr_to_mem(paddr
)) {
96 memcpy(bytes
, host_addr
, len
);
97 if (tracer
.interested_in_range(paddr
, paddr
+ PGSIZE
, LOAD
))
98 tracer
.trace(paddr
, len
, LOAD
);
100 refill_tlb(addr
, paddr
, host_addr
, LOAD
);
101 } else if (!sim
->mmio_load(paddr
, len
, bytes
)) {
102 throw trap_load_access_fault(addr
);
105 if (!matched_trigger
) {
106 reg_t data
= reg_from_bytes(len
, bytes
);
107 matched_trigger
= trigger_exception(OPERATION_LOAD
, addr
, data
);
109 throw *matched_trigger
;
113 void mmu_t::store_slow_path(reg_t addr
, reg_t len
, const uint8_t* bytes
)
115 reg_t paddr
= translate(addr
, STORE
);
117 if (!matched_trigger
) {
118 reg_t data
= reg_from_bytes(len
, bytes
);
119 matched_trigger
= trigger_exception(OPERATION_STORE
, addr
, data
);
121 throw *matched_trigger
;
124 if (auto host_addr
= sim
->addr_to_mem(paddr
)) {
125 memcpy(host_addr
, bytes
, len
);
126 if (tracer
.interested_in_range(paddr
, paddr
+ PGSIZE
, STORE
))
127 tracer
.trace(paddr
, len
, STORE
);
129 refill_tlb(addr
, paddr
, host_addr
, STORE
);
130 } else if (!sim
->mmio_store(paddr
, len
, bytes
)) {
131 throw trap_store_access_fault(addr
);
135 tlb_entry_t
mmu_t::refill_tlb(reg_t vaddr
, reg_t paddr
, char* host_addr
, access_type type
)
137 reg_t idx
= (vaddr
>> PGSHIFT
) % TLB_ENTRIES
;
138 reg_t expected_tag
= vaddr
>> PGSHIFT
;
140 if ((tlb_load_tag
[idx
] & ~TLB_CHECK_TRIGGERS
) != expected_tag
)
141 tlb_load_tag
[idx
] = -1;
142 if ((tlb_store_tag
[idx
] & ~TLB_CHECK_TRIGGERS
) != expected_tag
)
143 tlb_store_tag
[idx
] = -1;
144 if ((tlb_insn_tag
[idx
] & ~TLB_CHECK_TRIGGERS
) != expected_tag
)
145 tlb_insn_tag
[idx
] = -1;
147 if ((check_triggers_fetch
&& type
== FETCH
) ||
148 (check_triggers_load
&& type
== LOAD
) ||
149 (check_triggers_store
&& type
== STORE
))
150 expected_tag
|= TLB_CHECK_TRIGGERS
;
152 if (type
== FETCH
) tlb_insn_tag
[idx
] = expected_tag
;
153 else if (type
== STORE
) tlb_store_tag
[idx
] = expected_tag
;
154 else tlb_load_tag
[idx
] = expected_tag
;
156 tlb_entry_t entry
= {host_addr
- vaddr
, paddr
- vaddr
};
157 tlb_data
[idx
] = entry
;
161 reg_t
mmu_t::walk(reg_t addr
, access_type type
, reg_t mode
)
163 vm_info vm
= decode_vm_info(proc
->max_xlen
, mode
, proc
->get_state()->satp
);
165 return addr
& ((reg_t(2) << (proc
->xlen
-1))-1); // zero-extend from xlen
167 bool s_mode
= mode
== PRV_S
;
168 bool sum
= get_field(proc
->state
.mstatus
, MSTATUS_SUM
);
169 bool mxr
= get_field(proc
->state
.mstatus
, MSTATUS_MXR
);
171 // verify bits xlen-1:va_bits-1 are all equal
172 int va_bits
= PGSHIFT
+ vm
.levels
* vm
.idxbits
;
173 reg_t mask
= (reg_t(1) << (proc
->xlen
- (va_bits
-1))) - 1;
174 reg_t masked_msbs
= (addr
>> (va_bits
-1)) & mask
;
175 if (masked_msbs
!= 0 && masked_msbs
!= mask
)
178 reg_t base
= vm
.ptbase
;
179 for (int i
= vm
.levels
- 1; i
>= 0; i
--) {
180 int ptshift
= i
* vm
.idxbits
;
181 reg_t idx
= (addr
>> (PGSHIFT
+ ptshift
)) & ((1 << vm
.idxbits
) - 1);
183 // check that physical address of PTE is legal
184 auto ppte
= sim
->addr_to_mem(base
+ idx
* vm
.ptesize
);
188 reg_t pte
= vm
.ptesize
== 4 ? *(uint32_t*)ppte
: *(uint64_t*)ppte
;
189 reg_t ppn
= pte
>> PTE_PPN_SHIFT
;
191 if (PTE_TABLE(pte
)) { // next level of page table
192 base
= ppn
<< PGSHIFT
;
193 } else if ((pte
& PTE_U
) ? s_mode
&& (type
== FETCH
|| !sum
) : !s_mode
) {
195 } else if (!(pte
& PTE_V
) || (!(pte
& PTE_R
) && (pte
& PTE_W
))) {
197 } else if (type
== FETCH
? !(pte
& PTE_X
) :
198 type
== LOAD
? !(pte
& PTE_R
) && !(mxr
&& (pte
& PTE_X
)) :
199 !((pte
& PTE_R
) && (pte
& PTE_W
))) {
201 } else if ((ppn
& ((reg_t(1) << ptshift
) - 1)) != 0) {
204 reg_t ad
= PTE_A
| ((type
== STORE
) * PTE_D
);
205 #ifdef RISCV_ENABLE_DIRTY
206 // set accessed and possibly dirty bits.
207 *(uint32_t*)ppte
|= ad
;
209 // take exception if access or possibly dirty bit is not set.
210 if ((pte
& ad
) != ad
)
213 // for superpage mappings, make a fake leaf PTE for the TLB's benefit.
214 reg_t vpn
= addr
>> PGSHIFT
;
215 reg_t value
= (ppn
| (vpn
& ((reg_t(1) << ptshift
) - 1))) << PGSHIFT
;
222 case FETCH
: throw trap_instruction_page_fault(addr
);
223 case LOAD
: throw trap_load_page_fault(addr
);
224 case STORE
: throw trap_store_page_fault(addr
);
230 case FETCH
: throw trap_instruction_access_fault(addr
);
231 case LOAD
: throw trap_load_access_fault(addr
);
232 case STORE
: throw trap_store_access_fault(addr
);
237 void mmu_t::register_memtracer(memtracer_t
* t
)