2 * Copyright 2014 Google, Inc.
3 * Copyright (c) 2012-2013,2015,2017-2019 ARM Limited
6 * The license below extends only to copyright in the software and shall
7 * not be construed as granting a license to any other intellectual
8 * property including but not limited to intellectual property relating
9 * to a hardware implementation of the functionality of the software
10 * licensed hereunder. You may use the software subject to the license
11 * terms below provided that you ensure that this notice is replicated
12 * unmodified and in its entirety in all distributions of the software,
13 * modified or unmodified, in source code or in binary form.
15 * Copyright (c) 2002-2005 The Regents of The University of Michigan
16 * All rights reserved.
18 * Redistribution and use in source and binary forms, with or without
19 * modification, are permitted provided that the following conditions are
20 * met: redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer;
22 * redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution;
25 * neither the name of the copyright holders nor the names of its
26 * contributors may be used to endorse or promote products derived from
27 * this software without specific prior written permission.
29 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
30 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
31 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
32 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
33 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
34 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
35 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
36 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
37 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
38 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
39 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
41 * Authors: Steve Reinhardt
44 #include "cpu/simple/atomic.hh"
46 #include "arch/locked_mem.hh"
47 #include "arch/mmapped_ipr.hh"
48 #include "arch/utility.hh"
49 #include "base/output.hh"
50 #include "config/the_isa.hh"
51 #include "cpu/exetrace.hh"
52 #include "cpu/utils.hh"
53 #include "debug/Drain.hh"
54 #include "debug/ExecFaulting.hh"
55 #include "debug/SimpleCPU.hh"
56 #include "mem/packet.hh"
57 #include "mem/packet_access.hh"
58 #include "mem/physical.hh"
59 #include "params/AtomicSimpleCPU.hh"
60 #include "sim/faults.hh"
61 #include "sim/full_system.hh"
62 #include "sim/system.hh"
65 using namespace TheISA
;
68 AtomicSimpleCPU::init()
70 BaseSimpleCPU::init();
72 int cid
= threadContexts
[0]->contextId();
73 ifetch_req
->setContext(cid
);
74 data_read_req
->setContext(cid
);
75 data_write_req
->setContext(cid
);
76 data_amo_req
->setContext(cid
);
79 AtomicSimpleCPU::AtomicSimpleCPU(AtomicSimpleCPUParams
*p
)
81 tickEvent([this]{ tick(); }, "AtomicSimpleCPU tick",
82 false, Event::CPU_Tick_Pri
),
83 width(p
->width
), locked(false),
84 simulate_data_stalls(p
->simulate_data_stalls
),
85 simulate_inst_stalls(p
->simulate_inst_stalls
),
86 icachePort(name() + ".icache_port", this),
87 dcachePort(name() + ".dcache_port", this),
88 dcache_access(false), dcache_latency(0),
92 ifetch_req
= std::make_shared
<Request
>();
93 data_read_req
= std::make_shared
<Request
>();
94 data_write_req
= std::make_shared
<Request
>();
95 data_amo_req
= std::make_shared
<Request
>();
99 AtomicSimpleCPU::~AtomicSimpleCPU()
101 if (tickEvent
.scheduled()) {
102 deschedule(tickEvent
);
107 AtomicSimpleCPU::drain()
109 // Deschedule any power gating event (if any)
110 deschedulePowerGatingEvent();
113 return DrainState::Drained
;
115 if (!isCpuDrained()) {
116 DPRINTF(Drain
, "Requesting drain.\n");
117 return DrainState::Draining
;
119 if (tickEvent
.scheduled())
120 deschedule(tickEvent
);
122 activeThreads
.clear();
123 DPRINTF(Drain
, "Not executing microcode, no need to drain.\n");
124 return DrainState::Drained
;
129 AtomicSimpleCPU::threadSnoop(PacketPtr pkt
, ThreadID sender
)
131 DPRINTF(SimpleCPU
, "received snoop pkt for addr:%#x %s\n", pkt
->getAddr(),
134 for (ThreadID tid
= 0; tid
< numThreads
; tid
++) {
136 if (getCpuAddrMonitor(tid
)->doMonitor(pkt
)) {
140 TheISA::handleLockedSnoop(threadInfo
[tid
]->thread
,
141 pkt
, dcachePort
.cacheBlockMask
);
147 AtomicSimpleCPU::drainResume()
149 assert(!tickEvent
.scheduled());
153 DPRINTF(SimpleCPU
, "Resume\n");
156 assert(!threadContexts
.empty());
158 _status
= BaseSimpleCPU::Idle
;
160 for (ThreadID tid
= 0; tid
< numThreads
; tid
++) {
161 if (threadInfo
[tid
]->thread
->status() == ThreadContext::Active
) {
162 threadInfo
[tid
]->notIdleFraction
= 1;
163 activeThreads
.push_back(tid
);
164 _status
= BaseSimpleCPU::Running
;
166 // Tick if any threads active
167 if (!tickEvent
.scheduled()) {
168 schedule(tickEvent
, nextCycle());
171 threadInfo
[tid
]->notIdleFraction
= 0;
175 // Reschedule any power gating event (if any)
176 schedulePowerGatingEvent();
180 AtomicSimpleCPU::tryCompleteDrain()
182 if (drainState() != DrainState::Draining
)
185 DPRINTF(Drain
, "tryCompleteDrain.\n");
189 DPRINTF(Drain
, "CPU done draining, processing drain event\n");
197 AtomicSimpleCPU::switchOut()
199 BaseSimpleCPU::switchOut();
201 assert(!tickEvent
.scheduled());
202 assert(_status
== BaseSimpleCPU::Running
|| _status
== Idle
);
203 assert(isCpuDrained());
208 AtomicSimpleCPU::takeOverFrom(BaseCPU
*oldCPU
)
210 BaseSimpleCPU::takeOverFrom(oldCPU
);
212 // The tick event should have been descheduled by drain()
213 assert(!tickEvent
.scheduled());
217 AtomicSimpleCPU::verifyMemoryMode() const
219 if (!system
->isAtomicMode()) {
220 fatal("The atomic CPU requires the memory system to be in "
226 AtomicSimpleCPU::activateContext(ThreadID thread_num
)
228 DPRINTF(SimpleCPU
, "ActivateContext %d\n", thread_num
);
230 assert(thread_num
< numThreads
);
232 threadInfo
[thread_num
]->notIdleFraction
= 1;
233 Cycles delta
= ticksToCycles(threadInfo
[thread_num
]->thread
->lastActivate
-
234 threadInfo
[thread_num
]->thread
->lastSuspend
);
237 if (!tickEvent
.scheduled()) {
238 //Make sure ticks are still on multiples of cycles
239 schedule(tickEvent
, clockEdge(Cycles(0)));
241 _status
= BaseSimpleCPU::Running
;
242 if (std::find(activeThreads
.begin(), activeThreads
.end(), thread_num
)
243 == activeThreads
.end()) {
244 activeThreads
.push_back(thread_num
);
247 BaseCPU::activateContext(thread_num
);
252 AtomicSimpleCPU::suspendContext(ThreadID thread_num
)
254 DPRINTF(SimpleCPU
, "SuspendContext %d\n", thread_num
);
256 assert(thread_num
< numThreads
);
257 activeThreads
.remove(thread_num
);
262 assert(_status
== BaseSimpleCPU::Running
);
264 threadInfo
[thread_num
]->notIdleFraction
= 0;
266 if (activeThreads
.empty()) {
269 if (tickEvent
.scheduled()) {
270 deschedule(tickEvent
);
274 BaseCPU::suspendContext(thread_num
);
278 AtomicSimpleCPU::sendPacket(MasterPort
&port
, const PacketPtr
&pkt
)
280 return port
.sendAtomic(pkt
);
284 AtomicSimpleCPU::AtomicCPUDPort::recvAtomicSnoop(PacketPtr pkt
)
286 DPRINTF(SimpleCPU
, "received snoop pkt for addr:%#x %s\n", pkt
->getAddr(),
289 // X86 ISA: Snooping an invalidation for monitor/mwait
290 AtomicSimpleCPU
*cpu
= (AtomicSimpleCPU
*)(&owner
);
292 for (ThreadID tid
= 0; tid
< cpu
->numThreads
; tid
++) {
293 if (cpu
->getCpuAddrMonitor(tid
)->doMonitor(pkt
)) {
298 // if snoop invalidates, release any associated locks
299 // When run without caches, Invalidation packets will not be received
300 // hence we must check if the incoming packets are writes and wakeup
301 // the processor accordingly
302 if (pkt
->isInvalidate() || pkt
->isWrite()) {
303 DPRINTF(SimpleCPU
, "received invalidation for addr:%#x\n",
305 for (auto &t_info
: cpu
->threadInfo
) {
306 TheISA::handleLockedSnoop(t_info
->thread
, pkt
, cacheBlockMask
);
314 AtomicSimpleCPU::AtomicCPUDPort::recvFunctionalSnoop(PacketPtr pkt
)
316 DPRINTF(SimpleCPU
, "received snoop pkt for addr:%#x %s\n", pkt
->getAddr(),
319 // X86 ISA: Snooping an invalidation for monitor/mwait
320 AtomicSimpleCPU
*cpu
= (AtomicSimpleCPU
*)(&owner
);
321 for (ThreadID tid
= 0; tid
< cpu
->numThreads
; tid
++) {
322 if (cpu
->getCpuAddrMonitor(tid
)->doMonitor(pkt
)) {
327 // if snoop invalidates, release any associated locks
328 if (pkt
->isInvalidate()) {
329 DPRINTF(SimpleCPU
, "received invalidation for addr:%#x\n",
331 for (auto &t_info
: cpu
->threadInfo
) {
332 TheISA::handleLockedSnoop(t_info
->thread
, pkt
, cacheBlockMask
);
338 AtomicSimpleCPU::genMemFragmentRequest(const RequestPtr
& req
, Addr frag_addr
,
339 int size
, Request::Flags flags
,
340 const std::vector
<bool>& byte_enable
,
341 int& frag_size
, int& size_left
) const
343 bool predicate
= true;
344 Addr inst_addr
= threadInfo
[curThread
]->thread
->pcState().instAddr();
346 frag_size
= std::min(
347 cacheLineSize() - addrBlockOffset(frag_addr
, cacheLineSize()),
349 size_left
-= frag_size
;
351 if (!byte_enable
.empty()) {
352 // Set up byte-enable mask for the current fragment
353 auto it_start
= byte_enable
.begin() + (size
- (frag_size
+ size_left
));
354 auto it_end
= byte_enable
.begin() + (size
- size_left
);
355 if (isAnyActiveElement(it_start
, it_end
)) {
356 req
->setVirt(0, frag_addr
, frag_size
, flags
, dataMasterId(),
358 req
->setByteEnable(std::vector
<bool>(it_start
, it_end
));
363 req
->setVirt(0, frag_addr
, frag_size
, flags
, dataMasterId(),
365 req
->setByteEnable(std::vector
<bool>());
372 AtomicSimpleCPU::readMem(Addr addr
, uint8_t * data
, unsigned size
,
373 Request::Flags flags
,
374 const std::vector
<bool>& byte_enable
)
376 SimpleExecContext
& t_info
= *threadInfo
[curThread
];
377 SimpleThread
* thread
= t_info
.thread
;
379 // use the CPU's statically allocated read request and packet objects
380 const RequestPtr
&req
= data_read_req
;
383 traceData
->setMem(addr
, size
, flags
);
387 req
->taskId(taskId());
389 Addr frag_addr
= addr
;
391 int size_left
= size
;
393 Fault fault
= NoFault
;
396 predicate
= genMemFragmentRequest(req
, frag_addr
, size
, flags
,
397 byte_enable
, frag_size
, size_left
);
399 // translate to physical address
401 fault
= thread
->dtb
->translateAtomic(req
, thread
->getTC(),
405 // Now do the access.
406 if (predicate
&& fault
== NoFault
&&
407 !req
->getFlags().isSet(Request::NO_ACCESS
)) {
408 Packet
pkt(req
, Packet::makeReadCmd(req
));
409 pkt
.dataStatic(data
);
411 if (req
->isMmappedIpr()) {
412 dcache_latency
+= TheISA::handleIprRead(thread
->getTC(), &pkt
);
414 dcache_latency
+= sendPacket(dcachePort
, &pkt
);
416 dcache_access
= true;
418 assert(!pkt
.isError());
421 TheISA::handleLockedRead(thread
, req
);
425 //If there's a fault, return it
426 if (fault
!= NoFault
) {
427 if (req
->isPrefetch()) {
434 // If we don't need to access further cache lines, stop now.
435 if (size_left
== 0) {
436 if (req
->isLockedRMW() && fault
== NoFault
) {
444 * Set up for accessing the next cache line.
446 frag_addr
+= frag_size
;
448 //Move the pointer we're reading into to the correct location.
454 AtomicSimpleCPU::writeMem(uint8_t *data
, unsigned size
, Addr addr
,
455 Request::Flags flags
, uint64_t *res
,
456 const std::vector
<bool>& byte_enable
)
458 SimpleExecContext
& t_info
= *threadInfo
[curThread
];
459 SimpleThread
* thread
= t_info
.thread
;
460 static uint8_t zero_array
[64] = {};
464 assert(flags
& Request::STORE_NO_DATA
);
465 // This must be a cache block cleaning request
469 // use the CPU's statically allocated write request and packet objects
470 const RequestPtr
&req
= data_write_req
;
473 traceData
->setMem(addr
, size
, flags
);
477 req
->taskId(taskId());
479 Addr frag_addr
= addr
;
481 int size_left
= size
;
482 int curr_frag_id
= 0;
484 Fault fault
= NoFault
;
487 predicate
= genMemFragmentRequest(req
, frag_addr
, size
, flags
,
488 byte_enable
, frag_size
, size_left
);
490 // translate to physical address
492 fault
= thread
->dtb
->translateAtomic(req
, thread
->getTC(),
495 // Now do the access.
496 if (predicate
&& fault
== NoFault
) {
497 bool do_access
= true; // flag to suppress cache access
500 assert(curr_frag_id
== 0);
502 TheISA::handleLockedWrite(thread
, req
,
503 dcachePort
.cacheBlockMask
);
504 } else if (req
->isSwap()) {
505 assert(curr_frag_id
== 0);
506 if (req
->isCondSwap()) {
508 req
->setExtraData(*res
);
512 if (do_access
&& !req
->getFlags().isSet(Request::NO_ACCESS
)) {
513 Packet
pkt(req
, Packet::makeWriteCmd(req
));
514 pkt
.dataStatic(data
);
516 if (req
->isMmappedIpr()) {
518 TheISA::handleIprWrite(thread
->getTC(), &pkt
);
520 dcache_latency
+= sendPacket(dcachePort
, &pkt
);
522 // Notify other threads on this CPU of write
523 threadSnoop(&pkt
, curThread
);
525 dcache_access
= true;
526 assert(!pkt
.isError());
529 assert(res
&& curr_frag_id
== 0);
530 memcpy(res
, pkt
.getConstPtr
<uint8_t>(), size
);
534 if (res
&& !req
->isSwap()) {
535 *res
= req
->getExtraData();
539 //If there's a fault or we don't need to access a second cache line,
541 if (fault
!= NoFault
|| size_left
== 0)
543 if (req
->isLockedRMW() && fault
== NoFault
) {
544 assert(!req
->isMasked());
548 if (fault
!= NoFault
&& req
->isPrefetch()) {
556 * Set up for accessing the next cache line.
558 frag_addr
+= frag_size
;
560 //Move the pointer we're reading into to the correct location.
568 AtomicSimpleCPU::amoMem(Addr addr
, uint8_t* data
, unsigned size
,
569 Request::Flags flags
, AtomicOpFunctorPtr amo_op
)
571 SimpleExecContext
& t_info
= *threadInfo
[curThread
];
572 SimpleThread
* thread
= t_info
.thread
;
574 // use the CPU's statically allocated amo request and packet objects
575 const RequestPtr
&req
= data_amo_req
;
578 traceData
->setMem(addr
, size
, flags
);
580 //The address of the second part of this access if it needs to be split
581 //across a cache line boundary.
582 Addr secondAddr
= roundDown(addr
+ size
- 1, cacheLineSize());
584 // AMO requests that access across a cache line boundary are not
585 // allowed since the cache does not guarantee AMO ops to be executed
586 // atomically in two cache lines
587 // For ISAs such as x86 that requires AMO operations to work on
588 // accesses that cross cache-line boundaries, the cache needs to be
589 // modified to support locking both cache lines to guarantee the
591 if (secondAddr
> addr
) {
592 panic("AMO request should not access across a cache line boundary\n");
597 req
->taskId(taskId());
598 req
->setVirt(0, addr
, size
, flags
, dataMasterId(),
599 thread
->pcState().instAddr(), std::move(amo_op
));
601 // translate to physical address
602 Fault fault
= thread
->dtb
->translateAtomic(req
, thread
->getTC(),
605 // Now do the access.
606 if (fault
== NoFault
&& !req
->getFlags().isSet(Request::NO_ACCESS
)) {
607 // We treat AMO accesses as Write accesses with SwapReq command
608 // data will hold the return data of the AMO access
609 Packet
pkt(req
, Packet::makeWriteCmd(req
));
610 pkt
.dataStatic(data
);
612 if (req
->isMmappedIpr())
613 dcache_latency
+= TheISA::handleIprRead(thread
->getTC(), &pkt
);
615 dcache_latency
+= sendPacket(dcachePort
, &pkt
);
618 dcache_access
= true;
620 assert(!pkt
.isError());
621 assert(!req
->isLLSC());
624 if (fault
!= NoFault
&& req
->isPrefetch()) {
628 //If there's a fault and we're not doing prefetch, return it
633 AtomicSimpleCPU::tick()
635 DPRINTF(SimpleCPU
, "Tick\n");
637 // Change thread if multi-threaded
640 // Set memroy request ids to current thread
641 if (numThreads
> 1) {
642 ContextID cid
= threadContexts
[curThread
]->contextId();
644 ifetch_req
->setContext(cid
);
645 data_read_req
->setContext(cid
);
646 data_write_req
->setContext(cid
);
647 data_amo_req
->setContext(cid
);
650 SimpleExecContext
& t_info
= *threadInfo
[curThread
];
651 SimpleThread
* thread
= t_info
.thread
;
655 for (int i
= 0; i
< width
|| locked
; ++i
) {
657 updateCycleCounters(BaseCPU::CPU_STATE_ON
);
659 if (!curStaticInst
|| !curStaticInst
->isDelayedCommit()) {
660 checkForInterrupts();
664 // We must have just got suspended by a PC event
665 if (_status
== Idle
) {
670 Fault fault
= NoFault
;
672 TheISA::PCState pcState
= thread
->pcState();
674 bool needToFetch
= !isRomMicroPC(pcState
.microPC()) &&
677 ifetch_req
->taskId(taskId());
678 setupFetchRequest(ifetch_req
);
679 fault
= thread
->itb
->translateAtomic(ifetch_req
, thread
->getTC(),
683 if (fault
== NoFault
) {
684 Tick icache_latency
= 0;
685 bool icache_access
= false;
686 dcache_access
= false; // assume no dcache access
689 // This is commented out because the decoder would act like
690 // a tiny cache otherwise. It wouldn't be flushed when needed
691 // like the I cache. It should be flushed, and when that works
692 // this code should be uncommented.
693 //Fetch more instruction memory if necessary
694 //if (decoder.needMoreBytes())
696 icache_access
= true;
697 Packet ifetch_pkt
= Packet(ifetch_req
, MemCmd::ReadReq
);
698 ifetch_pkt
.dataStatic(&inst
);
700 icache_latency
= sendPacket(icachePort
, &ifetch_pkt
);
702 assert(!ifetch_pkt
.isError());
704 // ifetch_req is initialized to read the instruction directly
705 // into the CPU object's inst field.
711 Tick stall_ticks
= 0;
713 fault
= curStaticInst
->execute(&t_info
, traceData
);
715 // keep an instruction count
716 if (fault
== NoFault
) {
718 ppCommit
->notify(std::make_pair(thread
, curStaticInst
));
720 else if (traceData
&& !DTRACE(ExecFaulting
)) {
725 if (fault
!= NoFault
&&
726 dynamic_pointer_cast
<SyscallRetryFault
>(fault
)) {
727 // Retry execution of system calls after a delay.
728 // Prevents immediate re-execution since conditions which
729 // caused the retry are unlikely to change every tick.
730 stall_ticks
+= clockEdge(syscallRetryLatency
) - curTick();
736 // @todo remove me after debugging with legion done
737 if (curStaticInst
&& (!curStaticInst
->isMicroop() ||
738 curStaticInst
->isFirstMicroop()))
741 if (simulate_inst_stalls
&& icache_access
)
742 stall_ticks
+= icache_latency
;
744 if (simulate_data_stalls
&& dcache_access
)
745 stall_ticks
+= dcache_latency
;
748 // the atomic cpu does its accounting in ticks, so
749 // keep counting in ticks but round to the clock
751 latency
+= divCeil(stall_ticks
, clockPeriod()) *
756 if (fault
!= NoFault
|| !t_info
.stayAtPC
)
760 if (tryCompleteDrain())
763 // instruction takes at least one cycle
764 if (latency
< clockPeriod())
765 latency
= clockPeriod();
768 reschedule(tickEvent
, curTick() + latency
, true);
772 AtomicSimpleCPU::regProbePoints()
774 BaseCPU::regProbePoints();
776 ppCommit
= new ProbePointArg
<pair
<SimpleThread
*, const StaticInstPtr
>>
777 (getProbeManager(), "Commit");
781 AtomicSimpleCPU::printAddr(Addr a
)
783 dcachePort
.printAddr(a
);
786 ////////////////////////////////////////////////////////////////////////
788 // AtomicSimpleCPU Simulation Object
791 AtomicSimpleCPUParams::create()
793 return new AtomicSimpleCPU(this);