#include "debug/Cache.hh"
#include "debug/CachePort.hh"
#include "debug/CacheTags.hh"
+#include "debug/CacheVerbose.hh"
#include "mem/cache/blk.hh"
#include "mem/cache/mshr.hh"
#include "mem/cache/prefetch/base.hh"
tags(p->tags),
prefetcher(p->prefetcher),
doFastWrites(true),
- prefetchOnAccess(p->prefetch_on_access)
+ prefetchOnAccess(p->prefetch_on_access),
+ clusivity(p->clusivity),
+ writebackClean(p->writeback_clean),
+ tempBlockWriteback(nullptr),
+ writebackTempBlockAtomicEvent(this, false,
+ EventBase::Delayed_Writeback_Pri)
{
tempBlock = new CacheBlk();
tempBlock->data = new uint8_t[blkSize];
// can satisfy a following ReadEx anyway since we can rely on the
// Read requester(s) to have buffered the ReadEx snoop and to
// invalidate their blocks after receiving them.
- // assert(!pkt->needsExclusive() || blk->isWritable());
+ // assert(!pkt->needsWritable() || blk->isWritable());
assert(pkt->getOffset(blkSize) + pkt->getSize() <= blkSize);
// Check RMW operations first since both isRead() and
if (pkt->cmd == MemCmd::SwapReq) {
cmpAndSwap(blk, pkt);
} else if (pkt->isWrite()) {
+ // we have the block in a writable state and can go ahead,
+ // note that the line may be also be considered writable in
+ // downstream caches along the path to memory, but always
+ // Exclusive, and never Modified
assert(blk->isWritable());
- // Write or WriteLine at the first cache with block in Exclusive
+ // Write or WriteLine at the first cache with block in writable state
if (blk->checkWrite(pkt)) {
pkt->writeDataToBlock(blk->data, blkSize);
}
- // Always mark the line as dirty even if we are a failed
- // StoreCond so we supply data to any snoops that have
- // appended themselves to this cache before knowing the store
- // will fail.
+ // Always mark the line as dirty (and thus transition to the
+ // Modified state) even if we are a failed StoreCond so we
+ // supply data to any snoops that have appended themselves to
+ // this cache before knowing the store will fail.
blk->status |= BlkDirty;
- DPRINTF(Cache, "%s for %s addr %#llx size %d (write)\n", __func__,
- pkt->cmdString(), pkt->getAddr(), pkt->getSize());
+ DPRINTF(CacheVerbose, "%s for %s addr %#llx size %d (write)\n",
+ __func__, pkt->cmdString(), pkt->getAddr(), pkt->getSize());
} else if (pkt->isRead()) {
if (pkt->isLLSC()) {
blk->trackLoadLocked(pkt);
}
+
+ // all read responses have a data payload
+ assert(pkt->hasRespData());
pkt->setDataFromBlock(blk->data, blkSize);
+
// determine if this read is from a (coherent) cache, or not
// by looking at the command type; we could potentially add a
// packet attribute such as 'FromCache' to make this check a
assert(pkt->getSize() == blkSize);
// special handling for coherent block requests from
// upper-level caches
- if (pkt->needsExclusive()) {
+ if (pkt->needsWritable()) {
// sanity check
assert(pkt->cmd == MemCmd::ReadExReq ||
pkt->cmd == MemCmd::SCUpgradeFailReq);
// if we have a dirty copy, make sure the recipient
- // keeps it marked dirty
+ // keeps it marked dirty (in the modified state)
if (blk->isDirty()) {
- pkt->assertMemInhibit();
+ pkt->setCacheResponding();
}
- // on ReadExReq we give up our copy unconditionally
- if (blk != tempBlock)
- tags->invalidate(blk);
- blk->invalidate();
+ // on ReadExReq we give up our copy unconditionally,
+ // even if this cache is mostly inclusive, we may want
+ // to revisit this
+ invalidateBlock(blk);
} else if (blk->isWritable() && !pending_downgrade &&
- !pkt->sharedAsserted() &&
+ !pkt->hasSharers() &&
pkt->cmd != MemCmd::ReadCleanReq) {
- // we can give the requester an exclusive copy (by not
- // asserting shared line) on a read request if:
- // - we have an exclusive copy at this level (& below)
+ // we can give the requester a writable copy on a read
+ // request if:
+ // - we have a writable copy at this level (& below)
// - we don't have a pending snoop from below
// signaling another read request
// - no other cache above has a copy (otherwise it
- // would have asseretd shared line on request)
- // - we are not satisfying an instruction fetch (this
- // prevents dirty data in the i-cache)
-
+ // would have set hasSharers flag when
+ // snooping the packet)
+ // - the read has explicitly asked for a clean
+ // copy of the line
if (blk->isDirty()) {
// special considerations if we're owner:
if (!deferred_response) {
- // if we are responding immediately and can
- // signal that we're transferring ownership
- // along with exclusivity, do so
- pkt->assertMemInhibit();
- blk->status &= ~BlkDirty;
+ // respond with the line in Modified state
+ // (cacheResponding set, hasSharers not set)
+ pkt->setCacheResponding();
+
+ if (clusivity == Enums::mostly_excl) {
+ // if this cache is mostly exclusive with
+ // respect to the cache above, drop the
+ // block, no need to first unset the dirty
+ // bit
+ invalidateBlock(blk);
+ } else {
+ // if this cache is mostly inclusive, we
+ // keep the block in the Exclusive state,
+ // and pass it upwards as Modified
+ // (writable and dirty), hence we have
+ // multiple caches, all on the same path
+ // towards memory, all considering the
+ // same block writable, but only one
+ // considering it Modified
+
+ // we get away with multiple caches (on
+ // the same path to memory) considering
+ // the block writeable as we always enter
+ // the cache hierarchy through a cache,
+ // and first snoop upwards in all other
+ // branches
+ blk->status &= ~BlkDirty;
+ }
} else {
// if we're responding after our own miss,
// there's a window where the recipient didn't
// know it was getting ownership and may not
// have responded to snoops correctly, so we
- // can't pass off ownership *or* exclusivity
- pkt->assertShared();
+ // have to respond with a shared line
+ pkt->setHasSharers();
}
}
} else {
// otherwise only respond with a shared copy
- pkt->assertShared();
+ pkt->setHasSharers();
}
}
} else {
- // Upgrade or Invalidate, since we have it Exclusively (E or
- // M), we ack then invalidate.
+ // Upgrade or Invalidate
assert(pkt->isUpgrade() || pkt->isInvalidate());
- assert(blk != tempBlock);
- tags->invalidate(blk);
- blk->invalidate();
- DPRINTF(Cache, "%s for %s addr %#llx size %d (invalidation)\n",
+
+ // for invalidations we could be looking at the temp block
+ // (for upgrades we always allocate)
+ invalidateBlock(blk);
+ DPRINTF(CacheVerbose, "%s for %s addr %#llx size %d (invalidation)\n",
__func__, pkt->cmdString(), pkt->getAddr(), pkt->getSize());
}
}
void
-Cache::markInService(MSHR *mshr, bool pending_dirty_resp)
+Cache::markInService(MSHR *mshr, bool pending_modified_resp)
{
- markInServiceInternal(mshr, pending_dirty_resp);
+ markInServiceInternal(mshr, pending_modified_resp);
}
/////////////////////////////////////////////////////
"Should never see a write in a read-only cache %s\n",
name());
- DPRINTF(Cache, "%s for %s addr %#llx size %d\n", __func__,
+ DPRINTF(CacheVerbose, "%s for %s addr %#llx size %d\n", __func__,
pkt->cmdString(), pkt->getAddr(), pkt->getSize());
if (pkt->req->isUncacheable()) {
// flush and invalidate any existing block
CacheBlk *old_blk(tags->findBlock(pkt->getAddr(), pkt->isSecure()));
if (old_blk && old_blk->isValid()) {
- if (old_blk->isDirty())
+ if (old_blk->isDirty() || writebackClean)
writebacks.push_back(writebackBlk(old_blk));
else
writebacks.push_back(cleanEvictBlk(old_blk));
blk ? "hit " + blk->print() : "miss");
- if (pkt->evictingBlock()) {
+ if (pkt->isEviction()) {
// We check for presence of block in above caches before issuing
// Writeback or CleanEvict to write buffer. Therefore the only
// possible cases can be of a CleanEvict packet coming from above
if (writeBuffer.findMatches(pkt->getAddr(), pkt->isSecure(),
outgoing)) {
assert(outgoing.size() == 1);
- PacketPtr wbPkt = outgoing[0]->getTarget()->pkt;
- assert(pkt->cmd == MemCmd::CleanEvict &&
- wbPkt->cmd == MemCmd::Writeback);
- // As the CleanEvict is coming from above, it would have snooped
- // into other peer caches of the same level while traversing the
- // crossbar. If a copy of the block had been found, the CleanEvict
- // would have been deleted in the crossbar. Now that the
- // CleanEvict is here we can be sure none of the other upper level
- // caches connected to this cache have the block, so we can clear
- // the BLOCK_CACHED flag in the Writeback if set and discard the
- // CleanEvict by returning true.
- wbPkt->clearBlockCached();
- return true;
+ MSHR *wb_entry = outgoing[0];
+ assert(wb_entry->getNumTargets() == 1);
+ PacketPtr wbPkt = wb_entry->getTarget()->pkt;
+ assert(wbPkt->isWriteback());
+
+ if (pkt->isCleanEviction()) {
+ // The CleanEvict and WritebackClean snoops into other
+ // peer caches of the same level while traversing the
+ // crossbar. If a copy of the block is found, the
+ // packet is deleted in the crossbar. Hence, none of
+ // the other upper level caches connected to this
+ // cache have the block, so we can clear the
+ // BLOCK_CACHED flag in the Writeback if set and
+ // discard the CleanEvict by returning true.
+ wbPkt->clearBlockCached();
+ return true;
+ } else {
+ assert(pkt->cmd == MemCmd::WritebackDirty);
+ // Dirty writeback from above trumps our clean
+ // writeback... discard here
+ // Note: markInService will remove entry from writeback buffer.
+ markInService(wb_entry, false);
+ delete wbPkt;
+ }
}
}
// Writeback handling is special case. We can write the block into
// the cache without having a writeable copy (or any copy at all).
- if (pkt->cmd == MemCmd::Writeback) {
+ if (pkt->isWriteback()) {
assert(blkSize == pkt->getSize());
+
+ // we could get a clean writeback while we are having
+ // outstanding accesses to a block, do the simple thing for
+ // now and drop the clean writeback so that we do not upset
+ // any ordering/decisions about ownership already taken
+ if (pkt->cmd == MemCmd::WritebackClean &&
+ mshrQueue.findMatch(pkt->getAddr(), pkt->isSecure())) {
+ DPRINTF(Cache, "Clean writeback %#llx to block with MSHR, "
+ "dropping\n", pkt->getAddr());
+ return true;
+ }
+
if (blk == NULL) {
// need to do a replacement
blk = allocateBlock(pkt->getAddr(), pkt->isSecure(), writebacks);
blk->status |= BlkSecure;
}
}
- blk->status |= BlkDirty;
- // if shared is not asserted we got the writeback in modified
- // state, if it is asserted we are in the owned state
- if (!pkt->sharedAsserted()) {
+ // only mark the block dirty if we got a writeback command,
+ // and leave it as is for a clean writeback
+ if (pkt->cmd == MemCmd::WritebackDirty) {
+ blk->status |= BlkDirty;
+ }
+ // if the packet does not have sharers, it is passing
+ // writable, and we got the writeback in Modified or Exclusive
+ // state, if not we are in the Owned or Shared state
+ if (!pkt->hasSharers()) {
blk->status |= BlkWritable;
}
// nothing else to do; writeback doesn't expect response
// go to next level.
return false;
} else if ((blk != NULL) &&
- (pkt->needsExclusive() ? blk->isWritable()
- : blk->isReadable())) {
+ (pkt->needsWritable() ? blk->isWritable() : blk->isReadable())) {
// OK to satisfy access
incHitCount(pkt);
satisfyCpuSideRequest(pkt, blk);
}
// Can't satisfy access normally... either no block (blk == NULL)
- // or have block but need exclusive & only have shared.
+ // or have block but need writable
incMissCount(pkt);
return false;
}
-
-class ForwardResponseRecord : public Packet::SenderState
-{
- public:
-
- ForwardResponseRecord() {}
-};
-
void
Cache::doWritebacks(PacketList& writebacks, Tick forward_time)
{
// this is a non-snoop request packet which does not require a
// response.
delete wbPkt;
+ } else if (wbPkt->cmd == MemCmd::WritebackClean) {
+ // clean writeback, do not send since the block is
+ // still cached above
+ assert(writebackClean);
+ delete wbPkt;
} else {
+ assert(wbPkt->cmd == MemCmd::WritebackDirty);
// Set BLOCK_CACHED flag in Writeback and send below, so that
// the Writeback does not reset the bit corresponding to this
// address in the snoop filter below.
// isCachedAbove returns true we set BLOCK_CACHED flag in Writebacks
// and discard CleanEvicts.
if (isCachedAbove(wbPkt, false)) {
- if (wbPkt->cmd == MemCmd::Writeback) {
+ if (wbPkt->cmd == MemCmd::WritebackDirty) {
// Set BLOCK_CACHED flag in Writeback and send below,
// so that the Writeback does not reset the bit
// corresponding to this address in the snoop filter
pkt->cmdString(), pkt->getAddr(), pkt->getSize());
assert(pkt->isResponse());
-
- // must be cache-to-cache response from upper to lower level
- ForwardResponseRecord *rec =
- dynamic_cast<ForwardResponseRecord *>(pkt->senderState);
assert(!system->bypassCaches());
- if (rec == NULL) {
- // @todo What guarantee do we have that this HardPFResp is
- // actually for this cache, and not a cache closer to the
- // memory?
+ // determine if the response is from a snoop request we created
+ // (in which case it should be in the outstandingSnoop), or if we
+ // merely forwarded someone else's snoop request
+ const bool forwardAsSnoop = outstandingSnoop.find(pkt->req) ==
+ outstandingSnoop.end();
+
+ if (!forwardAsSnoop) {
+ // the packet came from this cache, so sink it here and do not
+ // forward it
assert(pkt->cmd == MemCmd::HardPFResp);
- // Check if it's a prefetch response and handle it. We shouldn't
- // get any other kinds of responses without FRRs.
- DPRINTF(Cache, "Got prefetch response from above for addr %#llx (%s)\n",
- pkt->getAddr(), pkt->isSecure() ? "s" : "ns");
+
+ outstandingSnoop.erase(pkt->req);
+
+ DPRINTF(Cache, "Got prefetch response from above for addr "
+ "%#llx (%s)\n", pkt->getAddr(), pkt->isSecure() ? "s" : "ns");
recvTimingResp(pkt);
return;
}
- pkt->popSenderState();
- delete rec;
// forwardLatency is set here because there is a response from an
// upper level cache.
// To pay the delay that occurs if the packet comes from the bus,
Cache::recvTimingReq(PacketPtr pkt)
{
DPRINTF(CacheTags, "%s tags: %s\n", __func__, tags->print());
-//@todo Add back in MemDebug Calls
-// MemDebug::cacheAccess(pkt);
-
-
- /// @todo temporary hack to deal with memory corruption issue until
- /// 4-phase transactions are complete
- for (int x = 0; x < pendingDelete.size(); x++)
- delete pendingDelete[x];
- pendingDelete.clear();
assert(pkt->isRequest());
promoteWholeLineWrites(pkt);
- if (pkt->memInhibitAsserted()) {
+ if (pkt->cacheResponding()) {
// a cache above us (but not where the packet came from) is
- // responding to the request
- DPRINTF(Cache, "mem inhibited on addr %#llx (%s): not responding\n",
+ // responding to the request, in other words it has the line
+ // in Modified or Owned state
+ DPRINTF(Cache, "Cache above responding to %#llx (%s): "
+ "not responding\n",
pkt->getAddr(), pkt->isSecure() ? "s" : "ns");
- // if the packet needs exclusive, and the cache that has
- // promised to respond (setting the inhibit flag) is not
- // providing exclusive (it is in O vs M state), we know that
- // there may be other shared copies in the system; go out and
- // invalidate them all
- if (pkt->needsExclusive() && !pkt->isSupplyExclusive()) {
+ // if the packet needs the block to be writable, and the cache
+ // that has promised to respond (setting the cache responding
+ // flag) is not providing writable (it is in Owned rather than
+ // the Modified state), we know that there may be other Shared
+ // copies in the system; go out and invalidate them all
+ if (pkt->needsWritable() && !pkt->responderHadWritable()) {
+ // an upstream cache that had the line in Owned state
+ // (dirty, but not writable), is responding and thus
+ // transferring the dirty line from one branch of the
+ // cache hierarchy to another
+
+ // send out an express snoop and invalidate all other
+ // copies (snooping a packet that needs writable is the
+ // same as an invalidation), thus turning the Owned line
+ // into a Modified line, note that we don't invalidate the
+ // block in the current cache or any other cache on the
+ // path to memory
+
// create a downstream express snoop with cleared packet
// flags, there is no need to allocate any data as the
// packet is merely used to co-ordinate state transitions
snoop_pkt->headerDelay = snoop_pkt->payloadDelay = 0;
// make this an instantaneous express snoop, and let the
- // other caches in the system know that the packet is
- // inhibited, because we have found the authorative copy
- // (O) that will supply the right data
+ // other caches in the system know that the another cache
+ // is responding, because we have found the authorative
+ // copy (Modified or Owned) that will supply the right
+ // data
snoop_pkt->setExpressSnoop();
- snoop_pkt->assertMemInhibit();
+ snoop_pkt->setCacheResponding();
// this express snoop travels towards the memory, and at
// every crossbar it is snooped upwards thus reaching
// express snoops always succeed
assert(success);
- // main memory will delete the packet
+ // main memory will delete the snoop packet
}
- /// @todo nominally we should just delete the packet here,
- /// however, until 4-phase stuff we can't because sending
- /// cache is still relying on it.
- pendingDelete.push_back(pkt);
-
- // no need to take any action in this particular cache as the
- // caches along the path to memory are allowed to keep lines
- // in a shared state, and a cache above us already committed
- // to responding
+ // queue for deletion, as opposed to immediate deletion, as
+ // the sending cache is still relying on the packet
+ pendingDelete.reset(pkt);
+
+ // no need to take any action in this particular cache as an
+ // upstream cache has already committed to responding, and
+ // either the packet does not need writable (and we can let
+ // the cache that set the cache responding flag pass on the
+ // line without any need for intervention), or if the packet
+ // needs writable it is provided, or we have already sent out
+ // any express snoops in the section above
return true;
}
// lat, neglecting responseLatency, modelling hit latency
// just as lookupLatency or or the value of lat overriden
// by access(), that calls accessBlock() function.
- cpuSidePort->schedTimingResp(pkt, request_time);
+ cpuSidePort->schedTimingResp(pkt, request_time, true);
} else {
- /// @todo nominally we should just delete the packet here,
- /// however, until 4-phase stuff we can't because sending cache is
- /// still relying on it. If the block is found in access(),
- /// CleanEvict and Writeback messages will be deleted here as
- /// well.
- pendingDelete.push_back(pkt);
+ DPRINTF(Cache, "%s satisfied %s addr %#llx, no response needed\n",
+ __func__, pkt->cmdString(), pkt->getAddr(),
+ pkt->getSize());
+
+ // queue the packet for deletion, as the sending cache is
+ // still relying on it; if the block is found in access(),
+ // CleanEvict and Writeback messages will be deleted
+ // here as well
+ pendingDelete.reset(pkt);
}
} else {
// miss
}
pkt->makeTimingResponse();
- // for debugging, set all the bits in the response data
- // (also keeps valgrind from complaining when debugging settings
- // print out instruction results)
- std::memset(pkt->getPtr<uint8_t>(), 0xFF, pkt->getSize());
+
// request_time is used here, taking into account lat and the delay
// charged if the packet comes from the xbar.
- cpuSidePort->schedTimingResp(pkt, request_time);
+ cpuSidePort->schedTimingResp(pkt, request_time, true);
// If an outstanding request is in progress (we found an
// MSHR) this is set to null
// Coalesce unless it was a software prefetch (see above).
if (pkt) {
- assert(pkt->cmd != MemCmd::Writeback);
- // CleanEvicts corresponding to blocks which have outstanding
- // requests in MSHRs can be deleted here.
+ assert(!pkt->isWriteback());
+ // CleanEvicts corresponding to blocks which have
+ // outstanding requests in MSHRs are simply sunk here
if (pkt->cmd == MemCmd::CleanEvict) {
- pendingDelete.push_back(pkt);
+ pendingDelete.reset(pkt);
} else {
DPRINTF(Cache, "%s coalescing MSHR for %s addr %#llx size %d\n",
__func__, pkt->cmdString(), pkt->getAddr(),
assert(pkt->req->masterId() < system->maxMasters());
mshr_hits[pkt->cmdToIndex()][pkt->req->masterId()]++;
- if (mshr->threadNum != 0/*pkt->req->threadId()*/) {
- mshr->threadNum = -1;
- }
// We use forward_time here because it is the same
// considering new targets. We have multiple
// requests for the same address here. It
// buffer and to schedule an event to the queued
// port and also takes into account the additional
// delay of the xbar.
- mshr->allocateTarget(pkt, forward_time, order++);
+ mshr->allocateTarget(pkt, forward_time, order++,
+ allocOnFill(pkt->cmd));
if (mshr->getNumTargets() == numTarget) {
noTargetMSHR = mshr;
setBlocked(Blocked_NoTargets);
mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++;
}
- if (pkt->evictingBlock() ||
+ if (pkt->isEviction() ||
(pkt->req->isUncacheable() && pkt->isWrite())) {
// We use forward_time here because there is an
// uncached memory write, forwarded to WriteBuffer.
// internally, and have a sufficiently weak memory
// model, this is probably unnecessary, but at some
// point it must have seemed like we needed it...
- assert(pkt->needsExclusive());
+ assert(pkt->needsWritable());
assert(!blk->isWritable());
blk->status &= ~BlkReadable;
}
// See comment in cache.hh.
PacketPtr
Cache::getBusPacket(PacketPtr cpu_pkt, CacheBlk *blk,
- bool needsExclusive) const
+ bool needsWritable) const
{
bool blkValid = blk && blk->isValid();
if (!blkValid &&
(cpu_pkt->isUpgrade() ||
- cpu_pkt->evictingBlock())) {
+ cpu_pkt->isEviction())) {
// Writebacks that weren't allocated in access() and upgrades
// from upper-level caches that missed completely just go
// through.
// which will clobber the owned copy.
const bool useUpgrades = true;
if (blkValid && useUpgrades) {
- // only reason to be here is that blk is shared
- // (read-only) and we need exclusive
- assert(needsExclusive);
+ // only reason to be here is that blk is read only and we need
+ // it to be writable
+ assert(needsWritable);
assert(!blk->isWritable());
cmd = cpu_pkt->isLLSC() ? MemCmd::SCUpgradeReq : MemCmd::UpgradeReq;
} else if (cpu_pkt->cmd == MemCmd::SCUpgradeFailReq ||
cmd = MemCmd::SCUpgradeFailReq;
} else if (cpu_pkt->cmd == MemCmd::WriteLineReq) {
// forward as invalidate to all other caches, this gives us
- // the line in exclusive state, and invalidates all other
+ // the line in Exclusive state, and invalidates all other
// copies
cmd = MemCmd::InvalidateReq;
} else {
// block is invalid
- cmd = needsExclusive ? MemCmd::ReadExReq :
+ cmd = needsWritable ? MemCmd::ReadExReq :
(isReadOnly ? MemCmd::ReadCleanReq : MemCmd::ReadSharedReq);
}
PacketPtr pkt = new Packet(cpu_pkt->req, cmd, blkSize);
- // if there are sharers in the upper levels, pass that info downstream
- if (cpu_pkt->sharedAsserted()) {
+ // if there are upstream caches that have already marked the
+ // packet as having sharers (not passing writable), pass that info
+ // downstream
+ if (cpu_pkt->hasSharers()) {
// note that cpu_pkt may have spent a considerable time in the
// MSHR queue and that the information could possibly be out
// of date, however, there is no harm in conservatively
- // assuming the block is shared
- pkt->assertShared();
- DPRINTF(Cache, "%s passing shared from %s to %s addr %#llx size %d\n",
+ // assuming the block has sharers
+ pkt->setHasSharers();
+ DPRINTF(Cache, "%s passing hasSharers from %s to %s addr %#llx "
+ "size %d\n",
__func__, cpu_pkt->cmdString(), pkt->cmdString(),
pkt->getAddr(), pkt->getSize());
}
{
// We are in atomic mode so we pay just for lookupLatency here.
Cycles lat = lookupLatency;
- // @TODO: make this a parameter
- bool last_level_cache = false;
// Forward the request if the system is in cache bypass mode.
if (system->bypassCaches())
promoteWholeLineWrites(pkt);
- if (pkt->memInhibitAsserted()) {
- // have to invalidate ourselves and any lower caches even if
- // upper cache will be responding
- if (pkt->isInvalidate()) {
- CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
- if (blk && blk->isValid()) {
- tags->invalidate(blk);
- blk->invalidate();
- DPRINTF(Cache, "rcvd mem-inhibited %s on %#llx (%s):"
- " invalidating\n",
- pkt->cmdString(), pkt->getAddr(),
- pkt->isSecure() ? "s" : "ns");
- }
- if (!last_level_cache) {
- DPRINTF(Cache, "forwarding mem-inhibited %s on %#llx (%s)\n",
- pkt->cmdString(), pkt->getAddr(),
- pkt->isSecure() ? "s" : "ns");
- lat += ticksToCycles(memSidePort->sendAtomic(pkt));
- }
- } else {
- DPRINTF(Cache, "rcvd mem-inhibited %s on %#llx: not responding\n",
- pkt->cmdString(), pkt->getAddr());
+ // follow the same flow as in recvTimingReq, and check if a cache
+ // above us is responding
+ if (pkt->cacheResponding()) {
+ DPRINTF(Cache, "Cache above responding to %#llx (%s): "
+ "not responding\n",
+ pkt->getAddr(), pkt->isSecure() ? "s" : "ns");
+
+ // if a cache is responding, and it had the line in Owned
+ // rather than Modified state, we need to invalidate any
+ // copies that are not on the same path to memory
+ if (pkt->needsWritable() && !pkt->responderHadWritable()) {
+ lat += ticksToCycles(memSidePort->sendAtomic(pkt));
}
return lat * clockPeriod();
if (!satisfied) {
// MISS
- PacketPtr bus_pkt = getBusPacket(pkt, blk, pkt->needsExclusive());
+ PacketPtr bus_pkt = getBusPacket(pkt, blk, pkt->needsWritable());
bool is_forward = (bus_pkt == NULL);
// write-line request to the cache that promoted
// the write to a whole line
- blk = handleFill(pkt, blk, writebacks);
+ blk = handleFill(pkt, blk, writebacks,
+ allocOnFill(pkt->cmd));
satisfyCpuSideRequest(pkt, blk);
} else if (bus_pkt->isRead() ||
bus_pkt->cmd == MemCmd::UpgradeResp) {
// we're updating cache state to allow us to
// satisfy the upstream request from the cache
- blk = handleFill(bus_pkt, blk, writebacks);
+ blk = handleFill(bus_pkt, blk, writebacks,
+ allocOnFill(pkt->cmd));
satisfyCpuSideRequest(pkt, blk);
} else {
// we're satisfying the upstream request without
// immediately rather than calling requestMemSideBus() as we do
// there).
- // Handle writebacks (from the response handling) if needed
+ // do any writebacks resulting from the response handling
doWritebacksAtomic(writebacks);
+ // if we used temp block, check to see if its valid and if so
+ // clear it out, but only do so after the call to recvAtomic is
+ // finished so that any downstream observers (such as a snoop
+ // filter), first see the fill, and only then see the eviction
+ if (blk == tempBlock && tempBlock->isValid()) {
+ // the atomic CPU calls recvAtomic for fetch and load/store
+ // sequentuially, and we may already have a tempBlock
+ // writeback from the fetch that we have not yet sent
+ if (tempBlockWriteback) {
+ // if that is the case, write the prevoius one back, and
+ // do not schedule any new event
+ writebackTempBlockAtomic();
+ } else {
+ // the writeback/clean eviction happens after the call to
+ // recvAtomic has finished (but before any successive
+ // calls), so that the response handling from the fill is
+ // allowed to happen first
+ schedule(writebackTempBlockAtomicEvent, curTick());
+ }
+
+ tempBlockWriteback = (blk->isDirty() || writebackClean) ?
+ writebackBlk(blk) : cleanEvictBlk(blk);
+ blk->invalidate();
+ }
+
if (pkt->needsResponse()) {
pkt->makeAtomicResponse();
}
&& pkt->checkFunctional(&cbpw, blk_addr, is_secure, blkSize,
blk->data);
- // data we have is dirty if marked as such or if valid & ownership
- // pending due to outstanding UpgradeReq
+ // data we have is dirty if marked as such or if we have an
+ // in-service MSHR that is pending a modified line
bool have_dirty =
have_data && (blk->isDirty() ||
- (mshr && mshr->inService && mshr->isPendingDirty()));
+ (mshr && mshr->inService && mshr->isPendingModified()));
bool done = have_dirty
|| cpuSidePort->checkFunctional(pkt)
|| writeBuffer.checkFunctional(pkt, blk_addr)
|| memSidePort->checkFunctional(pkt);
- DPRINTF(Cache, "functional %s %#llx (%s) %s%s%s\n",
+ DPRINTF(CacheVerbose, "functional %s %#llx (%s) %s%s%s\n",
pkt->cmdString(), pkt->getAddr(), is_secure ? "s" : "ns",
(blk && blk->isValid()) ? "valid " : "",
have_data ? "data " : "", done ? "done " : "");
// Initial target is used just for stats
MSHR::Target *initial_tgt = mshr->getTarget();
- CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
int stats_cmd_idx = initial_tgt->pkt->cmdToIndex();
Tick miss_latency = curTick() - initial_tgt->recvTime;
PacketList writebacks;
miss_latency;
}
+ // upgrade deferred targets if the response has no sharers, and is
+ // thus passing writable
+ if (!pkt->hasSharers()) {
+ mshr->promoteWritable();
+ }
+
bool is_fill = !mshr->isForward &&
(pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);
+ CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
+
if (is_fill && !is_error) {
DPRINTF(Cache, "Block for addr %#llx being updated in Cache\n",
pkt->getAddr());
- // give mshr a chance to do some dirty work
- mshr->handleFill(pkt, blk);
-
- blk = handleFill(pkt, blk, writebacks);
+ blk = handleFill(pkt, blk, writebacks, mshr->allocOnFill);
assert(blk != NULL);
}
// from above.
if (tgt_pkt->cmd == MemCmd::WriteLineReq) {
assert(!is_error);
-
+ // we got the block in a writable state, so promote
+ // any deferred targets if possible
+ mshr->promoteWritable();
// NB: we use the original packet here and not the response!
- mshr->handleFill(tgt_pkt, blk);
- blk = handleFill(tgt_pkt, blk, writebacks);
+ blk = handleFill(tgt_pkt, blk, writebacks, mshr->allocOnFill);
assert(blk != NULL);
// treat as a fill, and discard the invalidation
}
// Reset the bus additional time as it is now accounted for
tgt_pkt->headerDelay = tgt_pkt->payloadDelay = 0;
- cpuSidePort->schedTimingResp(tgt_pkt, completion_time);
+ cpuSidePort->schedTimingResp(tgt_pkt, completion_time, true);
break;
case MSHR::Target::FromPrefetcher:
// should not invalidate the block, so check if the
// invalidation should be discarded
if (is_invalidate || mshr->hasPostInvalidate()) {
- assert(blk != tempBlock);
- tags->invalidate(blk);
- blk->invalidate();
+ invalidateBlock(blk);
} else if (mshr->hasPostDowngrade()) {
blk->status &= ~BlkWritable;
}
// Writebacks/CleanEvicts to write buffer. It specifies the latency to
// allocate an internal buffer and to schedule an event to the
// queued port.
- if (blk->isDirty()) {
+ if (blk->isDirty() || writebackClean) {
PacketPtr wbPkt = writebackBlk(blk);
allocateWriteBuffer(wbPkt, forward_time);
// Set BLOCK_CACHED flag if cached above.
blk->invalidate();
}
- DPRINTF(Cache, "Leaving %s with %s for addr %#llx\n", __func__,
+ DPRINTF(CacheVerbose, "Leaving %s with %s for addr %#llx\n", __func__,
pkt->cmdString(), pkt->getAddr());
delete pkt;
}
PacketPtr
Cache::writebackBlk(CacheBlk *blk)
{
- chatty_assert(!isReadOnly, "Writeback from read-only cache");
- assert(blk && blk->isValid() && blk->isDirty());
+ chatty_assert(!isReadOnly || writebackClean,
+ "Writeback from read-only cache");
+ assert(blk && blk->isValid() && (blk->isDirty() || writebackClean));
writebacks[Request::wbMasterId]++;
- Request *writebackReq =
- new Request(tags->regenerateBlkAddr(blk->tag, blk->set), blkSize, 0,
- Request::wbMasterId);
+ Request *req = new Request(tags->regenerateBlkAddr(blk->tag, blk->set),
+ blkSize, 0, Request::wbMasterId);
if (blk->isSecure())
- writebackReq->setFlags(Request::SECURE);
+ req->setFlags(Request::SECURE);
- writebackReq->taskId(blk->task_id);
+ req->taskId(blk->task_id);
blk->task_id= ContextSwitchTaskId::Unknown;
blk->tickInserted = curTick();
- PacketPtr writeback = new Packet(writebackReq, MemCmd::Writeback);
+ PacketPtr pkt =
+ new Packet(req, blk->isDirty() ?
+ MemCmd::WritebackDirty : MemCmd::WritebackClean);
+
+ DPRINTF(Cache, "Create Writeback %#llx writable: %d, dirty: %d\n",
+ pkt->getAddr(), blk->isWritable(), blk->isDirty());
+
if (blk->isWritable()) {
// not asserting shared means we pass the block in modified
// state, mark our own block non-writeable
blk->status &= ~BlkWritable;
} else {
- // we are in the owned state, tell the receiver
- writeback->assertShared();
+ // we are in the Owned state, tell the receiver
+ pkt->setHasSharers();
}
- writeback->allocate();
- std::memcpy(writeback->getPtr<uint8_t>(), blk->data, blkSize);
-
+ // make sure the block is not marked dirty
blk->status &= ~BlkDirty;
- return writeback;
+
+ pkt->allocate();
+ std::memcpy(pkt->getPtr<uint8_t>(), blk->data, blkSize);
+
+ return pkt;
}
PacketPtr
Cache::cleanEvictBlk(CacheBlk *blk)
{
+ assert(!writebackClean);
assert(blk && blk->isValid() && !blk->isDirty());
// Creating a zero sized write, a message to the snoop filter
Request *req =
// must be an outstanding upgrade request
// on a block we're about to replace...
assert(!blk->isWritable() || blk->isDirty());
- assert(repl_mshr->needsExclusive());
+ assert(repl_mshr->needsWritable());
// too hard to replace block with transient state
// allocation failed, block not inserted
return NULL;
// Will send up Writeback/CleanEvict snoops via isCachedAbove
// when pushing this writeback list into the write buffer.
- if (blk->isDirty()) {
+ if (blk->isDirty() || writebackClean) {
// Save writeback packet for handling by caller
writebacks.push_back(writebackBlk(blk));
} else {
return blk;
}
+void
+Cache::invalidateBlock(CacheBlk *blk)
+{
+ if (blk != tempBlock)
+ tags->invalidate(blk);
+ blk->invalidate();
+}
// Note that the reason we return a list of writebacks rather than
// inserting them directly in the write buffer is that this function
// mode we don't mess with the write buffer (we just perform the
// writebacks atomically once the original request is complete).
CacheBlk*
-Cache::handleFill(PacketPtr pkt, CacheBlk *blk, PacketList &writebacks)
+Cache::handleFill(PacketPtr pkt, CacheBlk *blk, PacketList &writebacks,
+ bool allocate)
{
assert(pkt->isResponse() || pkt->cmd == MemCmd::WriteLineReq);
Addr addr = pkt->getAddr();
// happens in the subsequent satisfyCpuSideRequest.
assert(pkt->isRead() || pkt->cmd == MemCmd::WriteLineReq);
- // need to do a replacement
- blk = allocateBlock(addr, is_secure, writebacks);
+ // need to do a replacement if allocating, otherwise we stick
+ // with the temporary storage
+ blk = allocate ? allocateBlock(addr, is_secure, writebacks) : NULL;
+
if (blk == NULL) {
- // No replaceable block... just use temporary storage to
- // complete the current request and then get rid of it
+ // No replaceable block or a mostly exclusive
+ // cache... just use temporary storage to complete the
+ // current request and then get rid of it
assert(!tempBlock->isValid());
blk = tempBlock;
tempBlock->set = tags->extractSet(addr);
// marked as writable as part of the fill, and then later marked
// dirty as part of satisfyCpuSideRequest
if (pkt->cmd == MemCmd::WriteLineReq) {
- assert(!pkt->sharedAsserted());
+ assert(!pkt->hasSharers());
// at the moment other caches do not respond to the
// invalidation requests corresponding to a whole-line write
- assert(!pkt->memInhibitAsserted());
- }
-
- if (!pkt->sharedAsserted()) {
- // we could get non-shared responses from memory (rather than
- // a cache) even in a read-only cache, note that we set this
- // bit even for a read-only cache as we use it to represent
- // the exclusive state
+ assert(!pkt->cacheResponding());
+ }
+
+ // here we deal with setting the appropriate state of the line,
+ // and we start by looking at the hasSharers flag, and ignore the
+ // cacheResponding flag (normally signalling dirty data) if the
+ // packet has sharers, thus the line is never allocated as Owned
+ // (dirty but not writable), and always ends up being either
+ // Shared, Exclusive or Modified, see Packet::setCacheResponding
+ // for more details
+ if (!pkt->hasSharers()) {
+ // we could get a writable line from memory (rather than a
+ // cache) even in a read-only cache, note that we set this bit
+ // even for a read-only cache, possibly revisit this decision
blk->status |= BlkWritable;
- // If we got this via cache-to-cache transfer (i.e., from a
- // cache that was an owner) and took away that owner's copy,
- // then we need to write it back. Normally this happens
- // anyway as a side effect of getting a copy to write it, but
- // there are cases (such as failed store conditionals or
- // compare-and-swaps) where we'll demand an exclusive copy but
- // end up not writing it.
- if (pkt->memInhibitAsserted()) {
+ // check if we got this via cache-to-cache transfer (i.e., from a
+ // cache that had the block in Modified or Owned state)
+ if (pkt->cacheResponding()) {
+ // we got the block in Modified state, and invalidated the
+ // owners copy
blk->status |= BlkDirty;
chatty_assert(!isReadOnly, "Should never see dirty snoop response "
pkt = new Packet(req_pkt, false, req_pkt->isRead());
assert(req_pkt->req->isUncacheable() || req_pkt->isInvalidate() ||
- pkt->sharedAsserted());
+ pkt->hasSharers());
pkt->makeTimingResponse();
if (pkt->isRead()) {
pkt->setDataFromBlock(blk_data, blkSize);
if (pkt->cmd == MemCmd::ReadResp && pending_inval) {
// Assume we defer a response to a read from a far-away cache
// A, then later defer a ReadExcl from a cache B on the same
- // bus as us. We'll assert MemInhibit in both cases, but in
- // the latter case MemInhibit will keep the invalidation from
- // reaching cache A. This special response tells cache A that
- // it gets the block to satisfy its read, but must immediately
- // invalidate it.
+ // bus as us. We'll assert cacheResponding in both cases, but
+ // in the latter case cacheResponding will keep the
+ // invalidation from reaching cache A. This special response
+ // tells cache A that it gets the block to satisfy its read,
+ // but must immediately invalidate it.
pkt->cmd = MemCmd::ReadRespWithInvalidate;
}
// Here we consider forward_time, paying for just forward latency and
Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
// Here we reset the timing of the packet.
pkt->headerDelay = pkt->payloadDelay = 0;
- DPRINTF(Cache, "%s created response: %s addr %#llx size %d tick: %lu\n",
+ DPRINTF(CacheVerbose,
+ "%s created response: %s addr %#llx size %d tick: %lu\n",
__func__, pkt->cmdString(), pkt->getAddr(), pkt->getSize(),
forward_time);
memSidePort->schedTimingSnoopResp(pkt, forward_time, true);
Cache::handleSnoop(PacketPtr pkt, CacheBlk *blk, bool is_timing,
bool is_deferred, bool pending_inval)
{
- DPRINTF(Cache, "%s for %s addr %#llx size %d\n", __func__,
+ DPRINTF(CacheVerbose, "%s for %s addr %#llx size %d\n", __func__,
pkt->cmdString(), pkt->getAddr(), pkt->getSize());
// deferred snoops can only happen in timing mode
assert(!(is_deferred && !is_timing));
// responds in atomic mode, so remember a few things about the
// original packet up front
bool invalidate = pkt->isInvalidate();
- bool M5_VAR_USED needs_exclusive = pkt->needsExclusive();
+ bool M5_VAR_USED needs_writable = pkt->needsWritable();
+
+ // at the moment we could get an uncacheable write which does not
+ // have the invalidate flag, and we need a suitable way of dealing
+ // with this case
+ panic_if(invalidate && pkt->req->isUncacheable(),
+ "%s got an invalidating uncacheable snoop request %s to %#llx",
+ name(), pkt->cmdString(), pkt->getAddr());
uint32_t snoop_delay = 0;
// first propagate snoop upward to see if anyone above us wants to
// handle it. save & restore packet src since it will get
// rewritten to be relative to cpu-side bus (if any)
- bool alreadyResponded = pkt->memInhibitAsserted();
+ bool alreadyResponded = pkt->cacheResponding();
if (is_timing) {
// copy the packet so that we can clear any flags before
// forwarding it upwards, we also allocate data (passing
// there is a snoop hit in upper levels
Packet snoopPkt(pkt, true, true);
snoopPkt.setExpressSnoop();
- snoopPkt.pushSenderState(new ForwardResponseRecord());
// the snoop packet does not need to wait any additional
// time
snoopPkt.headerDelay = snoopPkt.payloadDelay = 0;
// cache
snoop_delay += snoopPkt.headerDelay;
- if (snoopPkt.memInhibitAsserted()) {
+ if (snoopPkt.cacheResponding()) {
// cache-to-cache response from some upper cache
assert(!alreadyResponded);
- pkt->assertMemInhibit();
- } else {
- // no cache (or anyone else for that matter) will
- // respond, so delete the ForwardResponseRecord here
- delete snoopPkt.popSenderState();
+ pkt->setCacheResponding();
}
- if (snoopPkt.sharedAsserted()) {
- pkt->assertShared();
+ // upstream cache has the block, or has an outstanding
+ // MSHR, pass the flag on
+ if (snoopPkt.hasSharers()) {
+ pkt->setHasSharers();
}
// If this request is a prefetch or clean evict and an upper level
// signals block present, make sure to propagate the block
}
} else {
cpuSidePort->sendAtomicSnoop(pkt);
- if (!alreadyResponded && pkt->memInhibitAsserted()) {
+ if (!alreadyResponded && pkt->cacheResponding()) {
// cache-to-cache response from some upper cache:
// forward response to original requester
assert(pkt->isResponse());
}
if (!blk || !blk->isValid()) {
- DPRINTF(Cache, "%s snoop miss for %s addr %#llx size %d\n",
+ DPRINTF(CacheVerbose, "%s snoop miss for %s addr %#llx size %d\n",
__func__, pkt->cmdString(), pkt->getAddr(), pkt->getSize());
return snoop_delay;
} else {
- DPRINTF(Cache, "%s snoop hit for %s for addr %#llx size %d, "
- "old state is %s\n", __func__, pkt->cmdString(),
- pkt->getAddr(), pkt->getSize(), blk->print());
+ DPRINTF(Cache, "%s snoop hit for %s addr %#llx size %d, "
+ "old state is %s\n", __func__, pkt->cmdString(),
+ pkt->getAddr(), pkt->getSize(), blk->print());
}
chatty_assert(!(isReadOnly && blk->isDirty()),
// invalidation itself is taken care of below.
bool respond = blk->isDirty() && pkt->needsResponse() &&
pkt->cmd != MemCmd::InvalidateReq;
- bool have_exclusive = blk->isWritable();
+ bool have_writable = blk->isWritable();
// Invalidate any prefetch's from below that would strip write permissions
// MemCmd::HardPFReq is only observed by upstream caches. After missing
return snoop_delay;
}
- if (!pkt->req->isUncacheable() && pkt->isRead() && !invalidate) {
- // reading non-exclusive shared data, note that we retain
- // the block in owned state if it is dirty, with the response
- // taken care of below, and otherwhise simply downgrade to
- // shared
- assert(!needs_exclusive);
- pkt->assertShared();
- blk->status &= ~BlkWritable;
+ if (pkt->isRead() && !invalidate) {
+ // reading without requiring the line in a writable state
+ assert(!needs_writable);
+ pkt->setHasSharers();
+
+ // if the requesting packet is uncacheable, retain the line in
+ // the current state, otherwhise unset the writable flag,
+ // which means we go from Modified to Owned (and will respond
+ // below), remain in Owned (and will respond below), from
+ // Exclusive to Shared, or remain in Shared
+ if (!pkt->req->isUncacheable())
+ blk->status &= ~BlkWritable;
}
if (respond) {
// prevent anyone else from responding, cache as well as
// memory, and also prevent any memory from even seeing the
- // request (with current inhibited semantics), note that this
- // applies both to reads and writes and that for writes it
- // works thanks to the fact that we still have dirty data and
- // will write it back at a later point
- pkt->assertMemInhibit();
- if (have_exclusive) {
+ // request
+ pkt->setCacheResponding();
+ if (have_writable) {
+ // inform the cache hierarchy that this cache had the line
+ // in the Modified state so that we avoid unnecessary
+ // invalidations (see Packet::setResponderHadWritable)
+ pkt->setResponderHadWritable();
+
// in the case of an uncacheable request there is no point
- // in setting the exclusive flag, but since the recipient
- // does not care there is no harm in doing so, in any case
- // it is just a hint
- pkt->setSupplyExclusive();
+ // in setting the responderHadWritable flag, but since the
+ // recipient does not care there is no harm in doing so
+ } else {
+ // if the packet has needsWritable set we invalidate our
+ // copy below and all other copies will be invalidates
+ // through express snoops, and if needsWritable is not set
+ // we already called setHasSharers above
}
+
+ // if we are returning a writable and dirty (Modified) line,
+ // we should be invalidating the line
+ panic_if(!invalidate && !pkt->hasSharers(),
+ "%s is passing a Modified line through %s to %#llx, "
+ "but keeping the block",
+ name(), pkt->cmdString(), pkt->getAddr());
+
if (is_timing) {
doTimingSupplyResponse(pkt, blk->data, is_deferred, pending_inval);
} else {
pkt->makeAtomicResponse();
- pkt->setDataFromBlock(blk->data, blkSize);
+ // packets such as upgrades do not actually have any data
+ // payload
+ if (pkt->hasData())
+ pkt->setDataFromBlock(blk->data, blkSize);
}
}
if (!respond && is_timing && is_deferred) {
- // if it's a deferred timing snoop then we've made a copy of
- // both the request and the packet, and so if we're not using
- // those copies to respond and delete them here
- DPRINTF(Cache, "Deleting pkt %p and request %p for cmd %s addr: %p\n",
- pkt, pkt->req, pkt->cmdString(), pkt->getAddr());
-
- // the packets needs a response (just not from us), so we also
- // need to delete the request and not rely on the packet
- // destructor
+ // if it's a deferred timing snoop to which we are not
+ // responding, then we've made a copy of both the request and
+ // the packet, delete them here
assert(pkt->needsResponse());
delete pkt->req;
delete pkt;
// Do this last in case it deallocates block data or something
// like that
if (invalidate) {
- if (blk != tempBlock)
- tags->invalidate(blk);
- blk->invalidate();
+ invalidateBlock(blk);
}
DPRINTF(Cache, "new state is %s\n", blk->print());
void
Cache::recvTimingSnoopReq(PacketPtr pkt)
{
- DPRINTF(Cache, "%s for %s addr %#llx size %d\n", __func__,
+ DPRINTF(CacheVerbose, "%s for %s addr %#llx size %d\n", __func__,
pkt->cmdString(), pkt->getAddr(), pkt->getSize());
// Snoops shouldn't happen when bypassing caches
// Writebacks/CleanEvicts.
assert(wb_entry->getNumTargets() == 1);
PacketPtr wb_pkt = wb_entry->getTarget()->pkt;
- assert(wb_pkt->evictingBlock());
+ assert(wb_pkt->isEviction());
- if (pkt->evictingBlock()) {
+ if (pkt->isEviction()) {
// if the block is found in the write queue, set the BLOCK_CACHED
// flag for Writeback/CleanEvict snoop. On return the snoop will
// propagate the BLOCK_CACHED flag in Writeback packets and prevent
return;
}
- if (wb_pkt->cmd == MemCmd::Writeback) {
- assert(!pkt->memInhibitAsserted());
- pkt->assertMemInhibit();
- if (!pkt->needsExclusive()) {
- pkt->assertShared();
- // the writeback is no longer passing exclusivity (the
- // receiving cache should consider the block owned
- // rather than modified)
- wb_pkt->assertShared();
- } else {
- // if we're not asserting the shared line, we need to
- // invalidate our copy. we'll do that below as long as
- // the packet's invalidate flag is set...
- assert(pkt->isInvalidate());
+ // conceptually writebacks are no different to other blocks in
+ // this cache, so the behaviour is modelled after handleSnoop,
+ // the difference being that instead of querying the block
+ // state to determine if it is dirty and writable, we use the
+ // command and fields of the writeback packet
+ bool respond = wb_pkt->cmd == MemCmd::WritebackDirty &&
+ pkt->needsResponse() && pkt->cmd != MemCmd::InvalidateReq;
+ bool have_writable = !wb_pkt->hasSharers();
+ bool invalidate = pkt->isInvalidate();
+
+ if (!pkt->req->isUncacheable() && pkt->isRead() && !invalidate) {
+ assert(!pkt->needsWritable());
+ pkt->setHasSharers();
+ wb_pkt->setHasSharers();
+ }
+
+ if (respond) {
+ pkt->setCacheResponding();
+
+ if (have_writable) {
+ pkt->setResponderHadWritable();
}
+
doTimingSupplyResponse(pkt, wb_pkt->getConstPtr<uint8_t>(),
false, false);
- } else {
- assert(wb_pkt->cmd == MemCmd::CleanEvict);
- // The cache technically holds the block until the
- // corresponding CleanEvict message reaches the crossbar
- // below. Therefore when a snoop encounters a CleanEvict
- // message we must set assertShared (just like when it
- // encounters a Writeback) to avoid the snoop filter
- // prematurely clearing the holder bit in the crossbar
- // below
- if (!pkt->needsExclusive())
- pkt->assertShared();
- else
- assert(pkt->isInvalidate());
}
- if (pkt->isInvalidate()) {
+ if (invalidate) {
// Invalidation trumps our writeback... discard here
// Note: markInService will remove entry from writeback buffer.
markInService(wb_entry, false);
// Assert that packet is either Writeback or CleanEvict and not a
// prefetch request because prefetch requests need an MSHR and may
// generate a snoop response.
- assert(pkt->evictingBlock());
+ assert(pkt->isEviction());
snoop_pkt.senderState = NULL;
cpuSidePort->sendTimingSnoopReq(&snoop_pkt);
// Writeback/CleanEvict snoops do not generate a snoop response.
- assert(!(snoop_pkt.memInhibitAsserted()));
+ assert(!(snoop_pkt.cacheResponding()));
return snoop_pkt.isBlockCached();
} else {
cpuSidePort->sendAtomicSnoop(pkt);
// dirty one.
Packet snoop_pkt(tgt_pkt, true, false);
snoop_pkt.setExpressSnoop();
+ // We are sending this packet upwards, but if it hits we will
+ // get a snoop response that we end up treating just like a
+ // normal response, hence it needs the MSHR as its sender
+ // state
snoop_pkt.senderState = mshr;
cpuSidePort->sendTimingSnoopReq(&snoop_pkt);
// the MSHRs and when it was selected to be sent or if the
// prefetch was squashed by an upper cache.
- // It is important to check memInhibitAsserted before
- // prefetchSquashed. If another cache has asserted MEM_INGIBIT, it
- // will be sending a response which will arrive at the MSHR
- // allocated ofr this request. Checking the prefetchSquash first
- // may result in the MSHR being prematurely deallocated.
+ // It is important to check cacheResponding before
+ // prefetchSquashed. If another cache has committed to
+ // responding, it will be sending a dirty response which will
+ // arrive at the MSHR allocated for this request. Checking the
+ // prefetchSquash first may result in the MSHR being
+ // prematurely deallocated.
+ if (snoop_pkt.cacheResponding()) {
+ auto M5_VAR_USED r = outstandingSnoop.insert(snoop_pkt.req);
+ assert(r.second);
+
+ // if we are getting a snoop response with no sharers it
+ // will be allocated as Modified
+ bool pending_modified_resp = !snoop_pkt.hasSharers();
+ markInService(mshr, pending_modified_resp);
- if (snoop_pkt.memInhibitAsserted()) {
- // If we are getting a non-shared response it is dirty
- bool pending_dirty_resp = !snoop_pkt.sharedAsserted();
- markInService(mshr, pending_dirty_resp);
DPRINTF(Cache, "Upward snoop of prefetch for addr"
" %#x (%s) hit\n",
tgt_pkt->getAddr(), tgt_pkt->isSecure()? "s": "ns");
DPRINTF(Cache, "Block present, prefetch squashed by cache. "
"Deallocating mshr target %#x.\n",
mshr->blkAddr);
-
// Deallocate the mshr target
- if (tgt_pkt->cmd != MemCmd::Writeback) {
- if (mshr->queue->forceDeallocateTarget(mshr)) {
- // Clear block if this deallocation resulted freed an
- // mshr when all had previously been utilized
- clearBlocked((BlockedCause)(mshr->queue->index));
- }
- return NULL;
- } else {
- // If this is a Writeback, and the snoops indicate that the blk
- // is cached above, set the BLOCK_CACHED flag in the Writeback
- // packet, so that it does not reset the bits corresponding to
- // this block in the snoop filter below.
- tgt_pkt->setBlockCached();
+ if (mshr->queue->forceDeallocateTarget(mshr)) {
+ // Clear block if this deallocation resulted freed an
+ // mshr when all had previously been utilized
+ clearBlocked((BlockedCause)(mshr->queue->index));
}
+ return NULL;
}
}
assert(tags->findBlock(mshr->blkAddr, mshr->isSecure) == NULL);
pkt = tgt_pkt;
} else {
- pkt = getBusPacket(tgt_pkt, blk, mshr->needsExclusive());
+ pkt = getBusPacket(tgt_pkt, blk, mshr->needsWritable());
mshr->isForward = (pkt == NULL);
}
assert(pkt != NULL);
- pkt->senderState = mshr;
+ // play it safe and append (rather than set) the sender state, as
+ // forwarded packets may already have existing state
+ pkt->pushSenderState(mshr);
return pkt;
}
bool success = false;
- // always let inhibited requests through, even if blocked,
- // ultimately we should check if this is an express snoop, but at
- // the moment that flag is only set in the cache itself
- if (pkt->memInhibitAsserted()) {
+ // always let packets through if an upstream cache has committed
+ // to responding, even if blocked (we should technically look at
+ // the isExpressSnoop flag, but it is set by the cache itself, and
+ // consequently we have to rely on the cacheResponding flag)
+ if (pkt->cacheResponding()) {
// do not change the current retry state
bool M5_VAR_USED bypass_success = cache->recvTimingReq(pkt);
assert(bypass_success);
// it gets retried
} else {
// As part of the call to sendTimingReq the packet is
- // forwarded to all neighbouring caches (and any
- // caches above them) as a snoop. The packet is also
- // sent to any potential cache below as the
- // interconnect is not allowed to buffer the
- // packet. Thus at this point we know if any of the
- // neighbouring, or the downstream cache is
- // responding, and if so, if it is with a dirty line
- // or not.
- bool pending_dirty_resp = !pkt->sharedAsserted() &&
- pkt->memInhibitAsserted();
-
- cache.markInService(mshr, pending_dirty_resp);
+ // forwarded to all neighbouring caches (and any caches
+ // above them) as a snoop. Thus at this point we know if
+ // any of the neighbouring caches are responding, and if
+ // so, we know it is dirty, and we can determine if it is
+ // being passed as Modified, making our MSHR the ordering
+ // point
+ bool pending_modified_resp = !pkt->hasSharers() &&
+ pkt->cacheResponding();
+
+ cache.markInService(mshr, pending_modified_resp);
}
}
projects / gem5.git / blobdiff