/*
- * Copyright (c) 2010-2018 ARM Limited
+ * Copyright (c) 2010-2019 ARM Limited
* All rights reserved.
*
* The license below extends only to copyright in the software and shall
#include "debug/CacheTags.hh"
#include "debug/CacheVerbose.hh"
#include "enums/Clusivity.hh"
-#include "mem/cache/blk.hh"
+#include "mem/cache/cache_blk.hh"
#include "mem/cache/mshr.hh"
#include "mem/cache/tags/base.hh"
#include "mem/cache/write_queue_entry.hh"
// flush and invalidate any existing block
CacheBlk *old_blk(tags->findBlock(pkt->getAddr(), pkt->isSecure()));
if (old_blk && old_blk->isValid()) {
- evictBlock(old_blk, writebacks);
+ BaseCache::evictBlock(old_blk, writebacks);
}
blk = nullptr;
{
// Cache line clearing instructions
if (doFastWrites && (pkt->cmd == MemCmd::WriteReq) &&
- (pkt->getSize() == blkSize) && (pkt->getOffset(blkSize) == 0)) {
+ (pkt->getSize() == blkSize) && (pkt->getOffset(blkSize) == 0) &&
+ !pkt->isMaskedWrite()) {
pkt->cmd = MemCmd::WriteLineReq;
DPRINTF(Cache, "packet promoted from Write to WriteLineReq\n");
}
// should have flushed and have no valid block
assert(!blk || !blk->isValid());
- mshr_uncacheable[pkt->cmdToIndex()][pkt->req->masterId()]++;
+ stats.cmdStats(pkt).mshr_uncacheable[pkt->req->masterId()]++;
if (pkt->isWrite()) {
allocateWriteBuffer(pkt, forward_time);
if (!mshr) {
// copy the request and create a new SoftPFReq packet
- RequestPtr req = new Request(pkt->req->getPaddr(),
- pkt->req->getSize(),
- pkt->req->getFlags(),
- pkt->req->masterId());
+ RequestPtr req = std::make_shared<Request>(pkt->req->getPaddr(),
+ pkt->req->getSize(),
+ pkt->req->getFlags(),
+ pkt->req->masterId());
pf = new Packet(req, pkt->cmd);
pf->allocate();
- assert(pf->getAddr() == pkt->getAddr());
+ assert(pf->matchAddr(pkt));
assert(pf->getSize() == pkt->getSize());
}
// request_time is used here, taking into account lat and the delay
// charged if the packet comes from the xbar.
- cpuSidePort.schedTimingResp(pkt, request_time, true);
+ cpuSidePort.schedTimingResp(pkt, request_time);
// If an outstanding request is in progress (we found an
// MSHR) this is set to null
PacketPtr
Cache::createMissPacket(PacketPtr cpu_pkt, CacheBlk *blk,
- bool needsWritable) const
+ bool needsWritable,
+ bool is_whole_line_write) const
{
// should never see evictions here
assert(!cpu_pkt->isEviction());
// write miss on a shared owned block will generate a ReadExcl,
// which will clobber the owned copy.
const bool useUpgrades = true;
- if (cpu_pkt->cmd == MemCmd::WriteLineReq) {
+ assert(cpu_pkt->cmd != MemCmd::WriteLineReq || is_whole_line_write);
+ if (is_whole_line_write) {
assert(!blkValid || !blk->isWritable());
// forward as invalidate to all other caches, this gives us
// the line in Exclusive state, and invalidates all other
Cycles
-Cache::handleAtomicReqMiss(PacketPtr pkt, CacheBlk *blk,
+Cache::handleAtomicReqMiss(PacketPtr pkt, CacheBlk *&blk,
PacketList &writebacks)
{
// deal with the packets that go through the write path of
// only misses left
- PacketPtr bus_pkt = createMissPacket(pkt, blk, pkt->needsWritable());
+ PacketPtr bus_pkt = createMissPacket(pkt, blk, pkt->needsWritable(),
+ pkt->isWholeLineWrite(blkSize));
bool is_forward = (bus_pkt == nullptr);
if (bus_pkt->isError()) {
pkt->makeAtomicResponse();
pkt->copyError(bus_pkt);
- } else if (pkt->cmd == MemCmd::WriteLineReq) {
+ } else if (pkt->isWholeLineWrite(blkSize)) {
// note the use of pkt, not bus_pkt here.
// write-line request to the cache that promoted
// the write to a whole line
- blk = handleFill(pkt, blk, writebacks,
- allocOnFill(pkt->cmd));
+ const bool allocate = allocOnFill(pkt->cmd) &&
+ (!writeAllocator || writeAllocator->allocate());
+ blk = handleFill(bus_pkt, blk, writebacks, allocate);
assert(blk != NULL);
is_invalidate = false;
satisfyRequest(pkt, blk);
{
promoteWholeLineWrites(pkt);
+ // follow the same flow as in recvTimingReq, and check if a cache
+ // above us is responding
+ if (pkt->cacheResponding()) {
+ assert(!pkt->req->isCacheInvalidate());
+ DPRINTF(Cache, "Cache above responding to %s: not responding\n",
+ pkt->print());
+
+ // if a cache is responding, and it had the line in Owned
+ // rather than Modified state, we need to invalidate any
+ // copies that are not on the same path to memory
+ assert(pkt->needsWritable() && !pkt->responderHadWritable());
+
+ return memSidePort.sendAtomic(pkt);
+ }
+
return BaseCache::recvAtomic(pkt);
}
void
-Cache::serviceMSHRTargets(MSHR *mshr, const PacketPtr pkt, CacheBlk *blk,
- PacketList &writebacks)
+Cache::serviceMSHRTargets(MSHR *mshr, const PacketPtr pkt, CacheBlk *blk)
{
- MSHR::Target *initial_tgt = mshr->getTarget();
+ QueueEntry::Target *initial_tgt = mshr->getTarget();
// First offset for critical word first calculations
const int initial_offset = initial_tgt->pkt->getOffset(blkSize);
const bool is_error = pkt->isError();
- bool is_fill = !mshr->isForward &&
- (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);
// allow invalidation responses originating from write-line
// requests to be discarded
- bool is_invalidate = pkt->isInvalidate();
+ bool is_invalidate = pkt->isInvalidate() &&
+ !mshr->wasWholeLineWrite;
MSHR::TargetList targets = mshr->extractServiceableTargets(pkt);
for (auto &target: targets) {
// Software prefetch handling for cache closest to core
if (tgt_pkt->cmd.isSWPrefetch()) {
+ if (tgt_pkt->needsWritable()) {
+ // All other copies of the block were invalidated and we
+ // have an exclusive copy.
+
+ // The coherence protocol assumes that if we fetched an
+ // exclusive copy of the block, we have the intention to
+ // modify it. Therefore the MSHR for the PrefetchExReq has
+ // been the point of ordering and this cache has commited
+ // to respond to snoops for the block.
+ //
+ // In most cases this is true anyway - a PrefetchExReq
+ // will be followed by a WriteReq. However, if that
+ // doesn't happen, the block is not marked as dirty and
+ // the cache doesn't respond to snoops that has committed
+ // to do so.
+ //
+ // To avoid deadlocks in cases where there is a snoop
+ // between the PrefetchExReq and the expected WriteReq, we
+ // proactively mark the block as Dirty.
+ assert(blk);
+ blk->status |= BlkDirty;
+
+ panic_if(isReadOnly, "Prefetch exclusive requests from "
+ "read-only cache %s\n", name());
+ }
+
// a software prefetch would have already been ack'd
// immediately with dummy data so the core would be able to
// retire it. This request completes right here, so we
// deallocate it.
- delete tgt_pkt->req;
delete tgt_pkt;
break; // skip response
}
// from above.
if (tgt_pkt->cmd == MemCmd::WriteLineReq) {
assert(!is_error);
- // we got the block in a writable state, so promote
- // any deferred targets if possible
- mshr->promoteWritable();
- // NB: we use the original packet here and not the response!
- blk = handleFill(tgt_pkt, blk, writebacks,
- targets.allocOnFill);
assert(blk);
-
- // treat as a fill, and discard the invalidation
- // response
- is_fill = true;
- is_invalidate = false;
+ assert(blk->isWritable());
}
- if (is_fill) {
+ // Here we decide whether we will satisfy the target using
+ // data from the block or from the response. We use the
+ // block data to satisfy the request when the block is
+ // present and valid and in addition the response in not
+ // forwarding data to the cache above (we didn't fill
+ // either); otherwise we use the packet data.
+ if (blk && blk->isValid() &&
+ (!mshr->isForward || !pkt->hasData())) {
satisfyRequest(tgt_pkt, blk, true, mshr->hasPostDowngrade());
// How many bytes past the first request is this one
assert(!tgt_pkt->req->isUncacheable());
assert(tgt_pkt->req->masterId() < system->maxMasters());
- missLatency[tgt_pkt->cmdToIndex()][tgt_pkt->req->masterId()] +=
+ stats.cmdStats(tgt_pkt)
+ .missLatency[tgt_pkt->req->masterId()] +=
completion_time - target.recvTime;
} else if (pkt->cmd == MemCmd::UpgradeFailResp) {
// failed StoreCond upgrade
pkt->payloadDelay;
tgt_pkt->req->setExtraData(0);
} else {
- // We are about to send a response to a cache above
- // that asked for an invalidation; we need to
- // invalidate our copy immediately as the most
- // up-to-date copy of the block will now be in the
- // cache above. It will also prevent this cache from
- // responding (if the block was previously dirty) to
- // snoops as they should snoop the caches above where
- // they will get the response from.
if (is_invalidate && blk && blk->isValid()) {
+ // We are about to send a response to a cache above
+ // that asked for an invalidation; we need to
+ // invalidate our copy immediately as the most
+ // up-to-date copy of the block will now be in the
+ // cache above. It will also prevent this cache from
+ // responding (if the block was previously dirty) to
+ // snoops as they should snoop the caches above where
+ // they will get the response from.
invalidateBlock(blk);
}
// not a cache fill, just forwarding response
// from lower level cahces/memory to the core.
completion_time += clockEdge(responseLatency) +
pkt->payloadDelay;
- if (pkt->isRead() && !is_error) {
- // sanity check
- assert(pkt->getAddr() == tgt_pkt->getAddr());
- assert(pkt->getSize() >= tgt_pkt->getSize());
+ if (!is_error) {
+ if (pkt->isRead()) {
+ // sanity check
+ assert(pkt->matchAddr(tgt_pkt));
+ assert(pkt->getSize() >= tgt_pkt->getSize());
- tgt_pkt->setData(pkt->getConstPtr<uint8_t>());
+ tgt_pkt->setData(pkt->getConstPtr<uint8_t>());
+ } else {
+ // MSHR targets can read data either from the
+ // block or the response pkt. If we can't get data
+ // from the block (i.e., invalid or has old data)
+ // or the response (did not bring in any data)
+ // then make sure that the target didn't expect
+ // any.
+ assert(!tgt_pkt->hasRespData());
+ }
}
+
+ // this response did not allocate here and therefore
+ // it was not consumed, make sure that any flags are
+ // carried over to cache above
+ tgt_pkt->copyResponderFlags(pkt);
}
tgt_pkt->makeTimingResponse();
// if this packet is an error copy that to the new packet
}
// Reset the bus additional time as it is now accounted for
tgt_pkt->headerDelay = tgt_pkt->payloadDelay = 0;
- cpuSidePort.schedTimingResp(tgt_pkt, completion_time, true);
+ cpuSidePort.schedTimingResp(tgt_pkt, completion_time);
break;
case MSHR::Target::FromPrefetcher:
assert(tgt_pkt->cmd == MemCmd::HardPFReq);
if (blk)
blk->status |= BlkHWPrefetched;
- delete tgt_pkt->req;
delete tgt_pkt;
break;
return pkt;
}
-void
-Cache::evictBlock(CacheBlk *blk, PacketList &writebacks)
-{
- PacketPtr pkt = evictBlock(blk);
- if (pkt) {
- writebacks.push_back(pkt);
- }
-}
-
PacketPtr
Cache::cleanEvictBlk(CacheBlk *blk)
{
assert(!writebackClean);
assert(blk && blk->isValid() && !blk->isDirty());
+
// Creating a zero sized write, a message to the snoop filter
- Request *req =
- new Request(regenerateBlkAddr(blk), blkSize, 0,
- Request::wbMasterId);
+ RequestPtr req = std::make_shared<Request>(
+ regenerateBlkAddr(blk), blkSize, 0, Request::wbMasterId);
+
if (blk->isSecure())
req->setFlags(Request::SECURE);
pkt->headerDelay = pkt->payloadDelay = 0;
DPRINTF(CacheVerbose, "%s: created response: %s tick: %lu\n", __func__,
pkt->print(), forward_time);
- memSidePort.schedTimingSnoopResp(pkt, forward_time, true);
+ memSidePort.schedTimingSnoopResp(pkt, forward_time);
}
uint32_t
// first propagate snoop upward to see if anyone above us wants to
// handle it. save & restore packet src since it will get
// rewritten to be relative to cpu-side bus (if any)
- bool alreadyResponded = pkt->cacheResponding();
if (is_timing) {
// copy the packet so that we can clear any flags before
// forwarding it upwards, we also allocate data (passing
// cache
snoop_delay += snoopPkt.headerDelay;
- if (snoopPkt.cacheResponding()) {
- // cache-to-cache response from some upper cache
- assert(!alreadyResponded);
- pkt->setCacheResponding();
- }
- // upstream cache has the block, or has an outstanding
- // MSHR, pass the flag on
- if (snoopPkt.hasSharers()) {
- pkt->setHasSharers();
- }
// If this request is a prefetch or clean evict and an upper level
// signals block present, make sure to propagate the block
// presence to the requester.
if (snoopPkt.satisfied()) {
pkt->setSatisfied();
}
+
+ // Copy over flags from the snoop response to make sure we
+ // inform the final destination
+ pkt->copyResponderFlags(&snoopPkt);
} else {
+ bool already_responded = pkt->cacheResponding();
cpuSidePort.sendAtomicSnoop(pkt);
- if (!alreadyResponded && pkt->cacheResponding()) {
+ if (!already_responded && pkt->cacheResponding()) {
// cache-to-cache response from some upper cache:
// forward response to original requester
assert(pkt->isResponse());
if (pkt->hasData())
pkt->setDataFromBlock(blk->data, blkSize);
}
+
+ // When a block is compressed, it must first be decompressed before
+ // being read, and this increases the snoop delay.
+ if (compressor && pkt->isRead()) {
+ snoop_delay += compressor->getDecompressionLatency(blk);
+ }
}
if (!respond && is_deferred) {
assert(pkt->needsResponse());
-
- // if we copied the deferred packet with the intention to
- // respond, but are not responding, then a cache above us must
- // be, and we can use this as the indication of whether this
- // is a packet where we created a copy of the request or not
- if (!pkt->cacheResponding()) {
- delete pkt->req;
- }
-
delete pkt;
}
return;
}
- // Bypass any existing cache maintenance requests if the request
- // has been satisfied already (i.e., the dirty block has been
- // found).
- if (mshr && pkt->req->isCacheMaintenance() && pkt->satisfied()) {
- return;
- }
-
// Let the MSHR itself track the snoop and decide whether we want
// to go ahead and do the regular cache snoop
if (mshr && mshr->handleSnoop(pkt, order++)) {
}
// given that no response is expected, delete Request and Packet
- delete tgt_pkt->req;
delete tgt_pkt;
return false;
projects / gem5.git / blobdiff