/*
- * Copyright (c) 2010 ARM Limited
+ * Copyright (c) 2010-2011 ARM Limited
* All rights reserved
*
* The license below extends only to copyright in the software and shall
* Korey Sewell
*/
+#include "arch/generic/debugfaults.hh"
#include "arch/locked_mem.hh"
+#include "base/str.hh"
#include "config/the_isa.hh"
-#include "config/use_checker.hh"
#include "cpu/o3/lsq.hh"
#include "cpu/o3/lsq_unit.hh"
-#include "base/str.hh"
+#include "debug/Activity.hh"
+#include "debug/IEW.hh"
+#include "debug/LSQUnit.hh"
#include "mem/packet.hh"
#include "mem/request.hh"
template<class Impl>
LSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt,
LSQUnit *lsq_ptr)
- : inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
+ : Event(Default_Pri, AutoDelete),
+ inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
{
- this->setFlags(Event::AutoDelete);
}
template<class Impl>
template <class Impl>
LSQUnit<Impl>::LSQUnit()
- : loads(0), stores(0), storesToWB(0), stalled(false),
+ : loads(0), stores(0), storesToWB(0), cacheBlockMask(0), stalled(false),
isStoreBlocked(false), isLoadBlocked(false),
- loadBlockedHandled(false), hasPendingPkt(false)
+ loadBlockedHandled(false), storeInFlight(false), hasPendingPkt(false)
{
}
switchedOut = false;
+ cacheBlockMask = 0;
+
lsq = lsq_ptr;
lsqID = id;
loadQueue.resize(LQEntries);
storeQueue.resize(SQEntries);
+ depCheckShift = params->LSQDepCheckShift;
+ checkLoads = params->LSQCheckLoads;
+
loadHead = loadTail = 0;
storeHead = storeWBIdx = storeTail = 0;
memDepViolator = NULL;
blockedLoadSeqNum = 0;
+ needsTSO = params->needsTSO;
}
template<class Impl>
if (Impl::MaxThreads == 1) {
return iewStage->name() + ".lsq";
} else {
- return iewStage->name() + ".lsq.thread." + to_string(lsqID);
+ return iewStage->name() + ".lsq.thread" + to_string(lsqID);
}
}
LSQUnit<Impl>::setDcachePort(Port *dcache_port)
{
dcachePort = dcache_port;
-
-#if USE_CHECKER
- if (cpu->checker) {
- cpu->checker->setDcachePort(dcachePort);
- }
-#endif
}
template<class Impl>
stalled = false;
isLoadBlocked = false;
loadBlockedHandled = false;
+
+ // Just incase the memory system changed out from under us
+ cacheBlockMask = 0;
}
template<class Impl>
return retval;
}
+template <class Impl>
+void
+LSQUnit<Impl>::checkSnoop(PacketPtr pkt)
+{
+ int load_idx = loadHead;
+
+ if (!cacheBlockMask) {
+ assert(dcachePort);
+ Addr bs = dcachePort->peerBlockSize();
+
+ // Make sure we actually got a size
+ assert(bs != 0);
+
+ cacheBlockMask = ~(bs - 1);
+ }
+
+ // If this is the only load in the LSQ we don't care
+ if (load_idx == loadTail)
+ return;
+ incrLdIdx(load_idx);
+
+ DPRINTF(LSQUnit, "Got snoop for address %#x\n", pkt->getAddr());
+ Addr invalidate_addr = pkt->getAddr() & cacheBlockMask;
+ while (load_idx != loadTail) {
+ DynInstPtr ld_inst = loadQueue[load_idx];
+
+ if (!ld_inst->effAddrValid || ld_inst->uncacheable()) {
+ incrLdIdx(load_idx);
+ continue;
+ }
+
+ Addr load_addr = ld_inst->physEffAddr & cacheBlockMask;
+ DPRINTF(LSQUnit, "-- inst [sn:%lli] load_addr: %#x to pktAddr:%#x\n",
+ ld_inst->seqNum, load_addr, invalidate_addr);
+
+ if (load_addr == invalidate_addr) {
+ if (ld_inst->possibleLoadViolation) {
+ DPRINTF(LSQUnit, "Conflicting load at addr %#x [sn:%lli]\n",
+ ld_inst->physEffAddr, pkt->getAddr(), ld_inst->seqNum);
+
+ // Mark the load for re-execution
+ ld_inst->fault = new ReExec;
+ } else {
+ // If a older load checks this and it's true
+ // then we might have missed the snoop
+ // in which case we need to invalidate to be sure
+ ld_inst->hitExternalSnoop = true;
+ }
+ }
+ incrLdIdx(load_idx);
+ }
+ return;
+}
+
+template <class Impl>
+Fault
+LSQUnit<Impl>::checkViolations(int load_idx, DynInstPtr &inst)
+{
+ Addr inst_eff_addr1 = inst->effAddr >> depCheckShift;
+ Addr inst_eff_addr2 = (inst->effAddr + inst->effSize - 1) >> depCheckShift;
+
+ /** @todo in theory you only need to check an instruction that has executed
+ * however, there isn't a good way in the pipeline at the moment to check
+ * all instructions that will execute before the store writes back. Thus,
+ * like the implementation that came before it, we're overly conservative.
+ */
+ while (load_idx != loadTail) {
+ DynInstPtr ld_inst = loadQueue[load_idx];
+ if (!ld_inst->effAddrValid || ld_inst->uncacheable()) {
+ incrLdIdx(load_idx);
+ continue;
+ }
+
+ Addr ld_eff_addr1 = ld_inst->effAddr >> depCheckShift;
+ Addr ld_eff_addr2 =
+ (ld_inst->effAddr + ld_inst->effSize - 1) >> depCheckShift;
+
+ if (inst_eff_addr2 >= ld_eff_addr1 && inst_eff_addr1 <= ld_eff_addr2) {
+ if (inst->isLoad()) {
+ // If this load is to the same block as an external snoop
+ // invalidate that we've observed then the load needs to be
+ // squashed as it could have newer data
+ if (ld_inst->hitExternalSnoop) {
+ if (!memDepViolator ||
+ ld_inst->seqNum < memDepViolator->seqNum) {
+ DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] "
+ "and [sn:%lli] at address %#x\n",
+ inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
+ memDepViolator = ld_inst;
+
+ ++lsqMemOrderViolation;
+
+ return new GenericISA::M5PanicFault(
+ "Detected fault with inst [sn:%lli] and "
+ "[sn:%lli] at address %#x\n",
+ inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
+ }
+ }
+
+ // Otherwise, mark the load has a possible load violation
+ // and if we see a snoop before it's commited, we need to squash
+ ld_inst->possibleLoadViolation = true;
+ DPRINTF(LSQUnit, "Found possible load violaiton at addr: %#x"
+ " between instructions [sn:%lli] and [sn:%lli]\n",
+ inst_eff_addr1, inst->seqNum, ld_inst->seqNum);
+ } else {
+ // A load/store incorrectly passed this store.
+ // Check if we already have a violator, or if it's newer
+ // squash and refetch.
+ if (memDepViolator && ld_inst->seqNum > memDepViolator->seqNum)
+ break;
+
+ DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] and "
+ "[sn:%lli] at address %#x\n",
+ inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
+ memDepViolator = ld_inst;
+
+ ++lsqMemOrderViolation;
+
+ return new GenericISA::M5PanicFault("Detected fault with "
+ "inst [sn:%lli] and [sn:%lli] at address %#x\n",
+ inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
+ }
+ }
+
+ incrLdIdx(load_idx);
+ }
+ return NoFault;
+}
+
+
+
+
template <class Impl>
Fault
LSQUnit<Impl>::executeLoad(DynInstPtr &inst)
assert(inst->effAddrValid);
int load_idx = inst->lqIdx;
incrLdIdx(load_idx);
- while (load_idx != loadTail) {
- // Really only need to check loads that have actually executed
-
- // @todo: For now this is extra conservative, detecting a
- // violation if the addresses match assuming all accesses
- // are quad word accesses.
-
- // @todo: Fix this, magic number being used here
-
- // @todo: Uncachable load is not executed until it reaches
- // the head of the ROB. Once this if checks only the executed
- // loads(as noted above), this check can be removed
- if (loadQueue[load_idx]->effAddrValid &&
- ((loadQueue[load_idx]->effAddr >> 8)
- == (inst->effAddr >> 8)) &&
- !loadQueue[load_idx]->uncacheable()) {
- // A load incorrectly passed this load. Squash and refetch.
- // For now return a fault to show that it was unsuccessful.
- DynInstPtr violator = loadQueue[load_idx];
- if (!memDepViolator ||
- (violator->seqNum < memDepViolator->seqNum)) {
- memDepViolator = violator;
- } else {
- break;
- }
- ++lsqMemOrderViolation;
-
- return genMachineCheckFault();
- }
-
- incrLdIdx(load_idx);
- }
+ if (checkLoads)
+ return checkViolations(load_idx, inst);
}
return load_fault;
++storesToWB;
}
- assert(store_inst->effAddrValid);
- while (load_idx != loadTail) {
- // Really only need to check loads that have actually executed
- // It's safe to check all loads because effAddr is set to
- // InvalAddr when the dyn inst is created.
-
- // @todo: For now this is extra conservative, detecting a
- // violation if the addresses match assuming all accesses
- // are quad word accesses.
-
- // @todo: Fix this, magic number being used here
-
- // @todo: Uncachable load is not executed until it reaches
- // the head of the ROB. Once this if checks only the executed
- // loads(as noted above), this check can be removed
- if (loadQueue[load_idx]->effAddrValid &&
- ((loadQueue[load_idx]->effAddr >> 8)
- == (store_inst->effAddr >> 8)) &&
- !loadQueue[load_idx]->uncacheable()) {
- // A load incorrectly passed this store. Squash and refetch.
- // For now return a fault to show that it was unsuccessful.
- DynInstPtr violator = loadQueue[load_idx];
- if (!memDepViolator ||
- (violator->seqNum < memDepViolator->seqNum)) {
- memDepViolator = violator;
- } else {
- break;
- }
-
- ++lsqMemOrderViolation;
-
- return genMachineCheckFault();
- }
-
- incrLdIdx(load_idx);
- }
+ return checkViolations(load_idx, store_inst);
- return store_fault;
}
template <class Impl>
storeWBIdx != storeTail &&
storeQueue[storeWBIdx].inst &&
storeQueue[storeWBIdx].canWB &&
+ ((!needsTSO) || (!storeInFlight)) &&
usedPorts < cachePorts) {
if (isStoreBlocked || lsq->cacheBlocked()) {
DynInstPtr inst = storeQueue[storeWBIdx].inst;
Request *req = storeQueue[storeWBIdx].req;
+ RequestPtr sreqLow = storeQueue[storeWBIdx].sreqLow;
+ RequestPtr sreqHigh = storeQueue[storeWBIdx].sreqHigh;
+
storeQueue[storeWBIdx].committed = true;
assert(!inst->memData);
data_pkt->dataStatic(inst->memData);
data_pkt->senderState = state;
} else {
- RequestPtr sreqLow = storeQueue[storeWBIdx].sreqLow;
- RequestPtr sreqHigh = storeQueue[storeWBIdx].sreqHigh;
-
// Create two packets if the store is split in two.
data_pkt = new Packet(sreqLow, command, Packet::Broadcast);
snd_data_pkt = new Packet(sreqHigh, command, Packet::Broadcast);
inst->seqNum);
WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
cpu->schedule(wb, curTick() + 1);
+#if USE_CHECKER
+ // Make sure to set the LLSC data for verification
+ inst->reqToVerify->setExtraData(0);
+ inst->completeAcc(data_pkt);
+#endif
completeStore(storeWBIdx);
incrStIdx(storeWBIdx);
continue;
state->noWB = true;
}
- if (!sendStore(data_pkt)) {
+ bool split =
+ TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit;
+
+ ThreadContext *thread = cpu->tcBase(lsqID);
+
+ if (req->isMmappedIpr()) {
+ assert(!inst->isStoreConditional());
+ TheISA::handleIprWrite(thread, data_pkt);
+ delete data_pkt;
+ if (split) {
+ assert(snd_data_pkt->req->isMmappedIpr());
+ TheISA::handleIprWrite(thread, snd_data_pkt);
+ delete snd_data_pkt;
+ delete sreqLow;
+ delete sreqHigh;
+ }
+ delete state;
+ delete req;
+ completeStore(storeWBIdx);
+ incrStIdx(storeWBIdx);
+ } else if (!sendStore(data_pkt)) {
DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will"
"retry later\n",
inst->seqNum);
// Need to store the second packet, if split.
- if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) {
+ if (split) {
state->pktToSend = true;
state->pendingPacket = snd_data_pkt;
}
} else {
// If split, try to send the second packet too
- if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) {
+ if (split) {
assert(snd_data_pkt);
// Ensure there are enough ports to use.
#endif
}
+ if (needsTSO) {
+ storeInFlight = true;
+ }
+
incrStIdx(storeWBIdx);
}
storeQueue[store_idx].inst->setCompleted();
+ if (needsTSO) {
+ storeInFlight = false;
+ }
+
// Tell the checker we've completed this instruction. Some stores
// may get reported twice to the checker, but the checker can
// handle that case.
DPRINTF(LSQUnit, "Receiving retry: store blocked\n");
assert(retryPkt != NULL);
- if (dcachePort->sendTiming(retryPkt)) {
- LSQSenderState *state =
- dynamic_cast<LSQSenderState *>(retryPkt->senderState);
+ LSQSenderState *state =
+ dynamic_cast<LSQSenderState *>(retryPkt->senderState);
+ if (dcachePort->sendTiming(retryPkt)) {
// Don't finish the store unless this is the last packet.
if (!TheISA::HasUnalignedMemAcc || !state->pktToSend ||
state->pendingPacket == retryPkt) {
projects / gem5.git / blobdiff