1 // See LICENSE for license details.
11 #include "processor.h"
12 #include "memtracer.h"
16 // virtual memory configuration
18 const reg_t PGSIZE
= 1 << PGSHIFT
;
19 const reg_t PGMASK
= ~(PGSIZE
-1);
27 struct icache_entry_t
{
29 struct icache_entry_t
* next
;
38 class trigger_matched_t
41 trigger_matched_t(int index
,
42 trigger_operation_t operation
, reg_t address
, reg_t data
) :
43 index(index
), operation(operation
), address(address
), data(data
) {}
46 trigger_operation_t operation
;
51 // this class implements a processor's port into the virtual memory system.
52 // an MMU and instruction cache are maintained for simulator performance.
56 mmu_t(simif_t
* sim
, processor_t
* proc
);
59 inline reg_t
misaligned_load(reg_t addr
, size_t size
)
61 #ifdef RISCV_ENABLE_MISALIGNED
63 for (size_t i
= 0; i
< size
; i
++)
64 res
+= (reg_t
)load_uint8(addr
+ i
) << (i
* 8);
67 throw trap_load_address_misaligned(addr
);
71 inline void misaligned_store(reg_t addr
, reg_t data
, size_t size
)
73 #ifdef RISCV_ENABLE_MISALIGNED
74 for (size_t i
= 0; i
< size
; i
++)
75 store_uint8(addr
+ i
, data
>> (i
* 8));
77 throw trap_store_address_misaligned(addr
);
81 // template for functions that load an aligned value from memory
82 #define load_func(type) \
83 inline type##_t load_##type(reg_t addr) { \
84 if (unlikely(addr & (sizeof(type##_t)-1))) \
85 return misaligned_load(addr, sizeof(type##_t)); \
86 reg_t vpn = addr >> PGSHIFT; \
87 if (likely(tlb_load_tag[vpn % TLB_ENTRIES] == vpn)) \
88 return *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr); \
89 if (unlikely(tlb_load_tag[vpn % TLB_ENTRIES] == (vpn | TLB_CHECK_TRIGGERS))) { \
90 type##_t data = *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr); \
91 if (!matched_trigger) { \
92 matched_trigger = trigger_exception(OPERATION_LOAD, addr, data); \
93 if (matched_trigger) \
94 throw *matched_trigger; \
99 load_slow_path(addr, sizeof(type##_t), (uint8_t*)&res); \
103 // load value from memory at aligned address; zero extend to register width
109 // load value from memory at aligned address; sign extend to register width
115 // template for functions that store an aligned value to memory
116 #define store_func(type) \
117 void store_##type(reg_t addr, type##_t val) { \
118 if (unlikely(addr & (sizeof(type##_t)-1))) \
119 return misaligned_store(addr, val, sizeof(type##_t)); \
120 reg_t vpn = addr >> PGSHIFT; \
121 if (likely(tlb_store_tag[vpn % TLB_ENTRIES] == vpn)) \
122 *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr) = val; \
123 else if (unlikely(tlb_store_tag[vpn % TLB_ENTRIES] == (vpn | TLB_CHECK_TRIGGERS))) { \
124 if (!matched_trigger) { \
125 matched_trigger = trigger_exception(OPERATION_STORE, addr, val); \
126 if (matched_trigger) \
127 throw *matched_trigger; \
129 *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr) = val; \
132 store_slow_path(addr, sizeof(type##_t), (const uint8_t*)&val); \
135 // template for functions that perform an atomic memory operation
136 #define amo_func(type) \
137 template<typename op> \
138 type##_t amo_##type(reg_t addr, op f) { \
139 if (addr & (sizeof(type##_t)-1)) \
140 throw trap_store_address_misaligned(addr); \
142 auto lhs = load_##type(addr); \
143 store_##type(addr, f(lhs)); \
145 } catch (trap_load_page_fault& t) { \
146 /* AMO faults should be reported as store faults */ \
147 throw trap_store_page_fault(t.get_tval()); \
148 } catch (trap_load_access_fault& t) { \
149 /* AMO faults should be reported as store faults */ \
150 throw trap_store_access_fault(t.get_tval()); \
154 void store_float128(reg_t addr
, float128_t val
)
156 #ifndef RISCV_ENABLE_MISALIGNED
157 if (unlikely(addr
& (sizeof(float128_t
)-1)))
158 throw trap_store_address_misaligned(addr
);
160 store_uint64(addr
, val
.v
[0]);
161 store_uint64(addr
+ 8, val
.v
[1]);
164 float128_t
load_float128(reg_t addr
)
166 #ifndef RISCV_ENABLE_MISALIGNED
167 if (unlikely(addr
& (sizeof(float128_t
)-1)))
168 throw trap_load_address_misaligned(addr
);
170 return (float128_t
){load_uint64(addr
), load_uint64(addr
+ 8)};
173 // store value to memory at aligned address
179 // perform an atomic memory operation at an aligned address
183 static const reg_t ICACHE_ENTRIES
= 1024;
185 inline size_t icache_index(reg_t addr
)
187 return (addr
/ PC_ALIGN
) % ICACHE_ENTRIES
;
190 inline icache_entry_t
* refill_icache(reg_t addr
, icache_entry_t
* entry
)
192 auto tlb_entry
= translate_insn_addr(addr
);
193 insn_bits_t insn
= *(uint16_t*)(tlb_entry
.host_offset
+ addr
);
194 int length
= insn_length(insn
);
196 if (likely(length
== 4)) {
197 insn
|= (insn_bits_t
)*(const int16_t*)translate_insn_addr_to_host(addr
+ 2) << 16;
198 } else if (length
== 2) {
199 insn
= (int16_t)insn
;
200 } else if (length
== 6) {
201 insn
|= (insn_bits_t
)*(const int16_t*)translate_insn_addr_to_host(addr
+ 4) << 32;
202 insn
|= (insn_bits_t
)*(const uint16_t*)translate_insn_addr_to_host(addr
+ 2) << 16;
204 static_assert(sizeof(insn_bits_t
) == 8, "insn_bits_t must be uint64_t");
205 insn
|= (insn_bits_t
)*(const int16_t*)translate_insn_addr_to_host(addr
+ 6) << 48;
206 insn
|= (insn_bits_t
)*(const uint16_t*)translate_insn_addr_to_host(addr
+ 4) << 32;
207 insn
|= (insn_bits_t
)*(const uint16_t*)translate_insn_addr_to_host(addr
+ 2) << 16;
210 insn_fetch_t fetch
= {proc
->decode_insn(insn
), insn
};
212 entry
->next
= &icache
[icache_index(addr
+ length
)];
215 reg_t paddr
= tlb_entry
.target_offset
+ addr
;;
216 if (tracer
.interested_in_range(paddr
, paddr
+ 1, FETCH
)) {
218 tracer
.trace(paddr
, length
, FETCH
);
223 inline icache_entry_t
* access_icache(reg_t addr
)
225 icache_entry_t
* entry
= &icache
[icache_index(addr
)];
226 if (likely(entry
->tag
== addr
))
228 return refill_icache(addr
, entry
);
231 inline insn_fetch_t
load_insn(reg_t addr
)
233 icache_entry_t entry
;
234 return refill_icache(addr
, &entry
)->data
;
240 void register_memtracer(memtracer_t
*);
242 int is_dirty_enabled()
244 #ifdef RISCV_ENABLE_DIRTY
251 int is_misaligned_enabled()
253 #ifdef RISCV_ENABLE_MISALIGNED
263 memtracer_list_t tracer
;
266 // implement an instruction cache for simulator performance
267 icache_entry_t icache
[ICACHE_ENTRIES
];
269 // implement a TLB for simulator performance
270 static const reg_t TLB_ENTRIES
= 256;
271 // If a TLB tag has TLB_CHECK_TRIGGERS set, then the MMU must check for a
272 // trigger match before completing an access.
273 static const reg_t TLB_CHECK_TRIGGERS
= reg_t(1) << 63;
274 tlb_entry_t tlb_data
[TLB_ENTRIES
];
275 reg_t tlb_insn_tag
[TLB_ENTRIES
];
276 reg_t tlb_load_tag
[TLB_ENTRIES
];
277 reg_t tlb_store_tag
[TLB_ENTRIES
];
279 // finish translation on a TLB miss and update the TLB
280 tlb_entry_t
refill_tlb(reg_t vaddr
, reg_t paddr
, char* host_addr
, access_type type
);
281 const char* fill_from_mmio(reg_t vaddr
, reg_t paddr
);
283 // perform a page table walk for a given VA; set referenced/dirty bits
284 reg_t
walk(reg_t addr
, access_type type
, reg_t prv
);
286 // handle uncommon cases: TLB misses, page faults, MMIO
287 tlb_entry_t
fetch_slow_path(reg_t addr
);
288 void load_slow_path(reg_t addr
, reg_t len
, uint8_t* bytes
);
289 void store_slow_path(reg_t addr
, reg_t len
, const uint8_t* bytes
);
290 reg_t
translate(reg_t addr
, access_type type
);
293 inline tlb_entry_t
translate_insn_addr(reg_t addr
) {
294 reg_t vpn
= addr
>> PGSHIFT
;
295 if (likely(tlb_insn_tag
[vpn
% TLB_ENTRIES
] == vpn
))
296 return tlb_data
[vpn
% TLB_ENTRIES
];
297 if (unlikely(tlb_insn_tag
[vpn
% TLB_ENTRIES
] == (vpn
| TLB_CHECK_TRIGGERS
))) {
298 uint16_t* ptr
= (uint16_t*)(tlb_data
[vpn
% TLB_ENTRIES
].host_offset
+ addr
);
299 int match
= proc
->trigger_match(OPERATION_EXECUTE
, addr
, *ptr
);
301 throw trigger_matched_t(match
, OPERATION_EXECUTE
, addr
, *ptr
);
302 return tlb_data
[vpn
% TLB_ENTRIES
];
304 return fetch_slow_path(addr
);
307 inline const uint16_t* translate_insn_addr_to_host(reg_t addr
) {
308 return (uint16_t*)(translate_insn_addr(addr
).host_offset
+ addr
);
311 inline trigger_matched_t
*trigger_exception(trigger_operation_t operation
,
312 reg_t address
, reg_t data
)
317 int match
= proc
->trigger_match(operation
, address
, data
);
320 if (proc
->state
.mcontrol
[match
].timing
== 0) {
321 throw trigger_matched_t(match
, operation
, address
, data
);
323 return new trigger_matched_t(match
, operation
, address
, data
);
326 bool check_triggers_fetch
;
327 bool check_triggers_load
;
328 bool check_triggers_store
;
329 // The exception describing a matched trigger, or NULL.
330 trigger_matched_t
*matched_trigger
;
332 friend class processor_t
;
342 inline vm_info
decode_vm_info(int xlen
, reg_t prv
, reg_t satp
)
346 } else if (prv
<= PRV_S
&& xlen
== 32) {
347 switch (get_field(satp
, SATP32_MODE
)) {
348 case SATP_MODE_OFF
: return {0, 0, 0, 0};
349 case SATP_MODE_SV32
: return {2, 10, 4, (satp
& SATP32_PPN
) << PGSHIFT
};
352 } else if (prv
<= PRV_S
&& xlen
== 64) {
353 switch (get_field(satp
, SATP64_MODE
)) {
354 case SATP_MODE_OFF
: return {0, 0, 0, 0};
355 case SATP_MODE_SV39
: return {3, 9, 8, (satp
& SATP64_PPN
) << PGSHIFT
};
356 case SATP_MODE_SV48
: return {4, 9, 8, (satp
& SATP64_PPN
) << PGSHIFT
};
357 case SATP_MODE_SV57
: return {5, 9, 8, (satp
& SATP64_PPN
) << PGSHIFT
};
358 case SATP_MODE_SV64
: return {6, 9, 8, (satp
& SATP64_PPN
) << PGSHIFT
};