#include "base/logging.hh"
#include "debug/Cache.hh"
#include "debug/CachePort.hh"
+#include "debug/CacheRepl.hh"
#include "debug/CacheVerbose.hh"
#include "mem/cache/mshr.hh"
#include "mem/cache/prefetch/base.hh"
#include "mem/cache/queue_entry.hh"
#include "params/BaseCache.hh"
+#include "params/WriteAllocator.hh"
#include "sim/core.hh"
class BaseMasterPort;
tags(p->tags),
prefetcher(p->prefetcher),
prefetchOnAccess(p->prefetch_on_access),
+ writeAllocator(p->write_allocator),
writebackClean(p->writeback_clean),
tempBlockWriteback(nullptr),
writebackTempBlockAtomicEvent([this]{ writebackTempBlockAtomic(); },
// forward snoops is overridden in init() once we can query
// whether the connected master is actually snooping or not
- tempBlock = new CacheBlk();
- tempBlock->data = new uint8_t[blkSize];
+ tempBlock = new TempCacheBlk(blkSize);
- tags->setCache(this);
+ tags->init(this);
if (prefetcher)
prefetcher->setCache(this);
}
BaseCache::~BaseCache()
{
- delete [] tempBlock->data;
delete tempBlock;
}
sendRetryReq();
}
+Addr
+BaseCache::regenerateBlkAddr(CacheBlk* blk)
+{
+ if (blk != tempBlock) {
+ return tags->regenerateBlkAddr(blk);
+ } else {
+ return tempBlock->getAddr();
+ }
+}
+
void
BaseCache::init()
{
BaseCache::handleTimingReqMiss(PacketPtr pkt, MSHR *mshr, CacheBlk *blk,
Tick forward_time, Tick request_time)
{
+ if (writeAllocator &&
+ pkt && pkt->isWrite() && !pkt->req->isUncacheable()) {
+ writeAllocator->updateMode(pkt->getAddr(), pkt->getSize(),
+ pkt->getBlockAddr(blkSize));
+ }
+
if (mshr) {
/// MSHR hit
/// @note writebacks will be checked in getNextMSHR()
satisfied = access(pkt, blk, lat, writebacks);
// copy writebacks to write buffer here to ensure they logically
- // proceed anything happening below
+ // precede anything happening below
doWritebacks(writebacks, forward_time);
}
// already allocated for this, we need to let the prefetcher
// know about the request
- // Don't notify prefetcher on SWPrefetch or cache maintenance
- // operations
+ // Don't notify prefetcher on SWPrefetch, cache maintenance
+ // operations or for writes that we are coaslescing.
if (prefetcher && pkt &&
!pkt->cmd.isSWPrefetch() &&
- !pkt->req->isCacheMaintenance()) {
+ !pkt->req->isCacheMaintenance() &&
+ !(writeAllocator && writeAllocator->coalesce() &&
+ pkt->isWrite())) {
next_pf_time = prefetcher->notify(pkt);
}
}
PacketList writebacks;
bool is_fill = !mshr->isForward &&
- (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);
+ (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp ||
+ mshr->wasWholeLineWrite);
+
+ // make sure that if the mshr was due to a whole line write then
+ // the response is an invalidation
+ assert(!mshr->wasWholeLineWrite || pkt->isInvalidate());
CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
DPRINTF(Cache, "Block for addr %#llx being updated in Cache\n",
pkt->getAddr());
- blk = handleFill(pkt, blk, writebacks, mshr->allocOnFill());
+ const bool allocate = (writeAllocator && mshr->wasWholeLineWrite) ?
+ writeAllocator->allocate() : mshr->allocOnFill();
+ blk = handleFill(pkt, blk, writebacks, allocate);
assert(blk != nullptr);
}
// The block was marked not readable while there was a pending
// cache maintenance operation, restore its flag.
blk->status |= BlkReadable;
+
+ // This was a cache clean operation (without invalidate)
+ // and we have a copy of the block already. Since there
+ // is no invalidation, we can promote targets that don't
+ // require a writable copy
+ mshr->promoteReadable();
}
if (blk && blk->isWritable() && !pkt->req->isCacheInvalidate()) {
}
// handle writebacks resulting from the access here to ensure they
- // logically proceed anything happening below
+ // logically precede anything happening below
doWritebacksAtomic(writebacks);
assert(writebacks.empty());
// see if we have data at all (owned or otherwise)
bool have_data = blk && blk->isValid()
- && pkt->checkFunctional(&cbpw, blk_addr, is_secure, blkSize,
- blk->data);
+ && pkt->trySatisfyFunctional(&cbpw, blk_addr, is_secure, blkSize,
+ blk->data);
// data we have is dirty if marked as such or if we have an
// in-service MSHR that is pending a modified line
(mshr && mshr->inService && mshr->isPendingModified()));
bool done = have_dirty ||
- cpuSidePort.checkFunctional(pkt) ||
- mshrQueue.checkFunctional(pkt, blk_addr) ||
- writeBuffer.checkFunctional(pkt, blk_addr) ||
- memSidePort.checkFunctional(pkt);
+ cpuSidePort.trySatisfyFunctional(pkt) ||
+ mshrQueue.trySatisfyFunctional(pkt, blk_addr) ||
+ writeBuffer.trySatisfyFunctional(pkt, blk_addr) ||
+ memSidePort.trySatisfyFunctional(pkt);
DPRINTF(CacheVerbose, "%s: %s %s%s%s\n", __func__, pkt->print(),
(blk && blk->isValid()) ? "valid " : "",
return allocateMissBuffer(pkt, curTick(), false);
} else {
// free the request and packet
- delete pkt->req;
delete pkt;
}
}
// Check RMW operations first since both isRead() and
// isWrite() will be true for them
if (pkt->cmd == MemCmd::SwapReq) {
- cmpAndSwap(blk, pkt);
+ if (pkt->isAtomicOp()) {
+ // extract data from cache and save it into the data field in
+ // the packet as a return value from this atomic op
+ int offset = tags->extractBlkOffset(pkt->getAddr());
+ uint8_t *blk_data = blk->data + offset;
+ pkt->setData(blk_data);
+
+ // execute AMO operation
+ (*(pkt->getAtomicOp()))(blk_data);
+
+ // set block status to dirty
+ blk->status |= BlkDirty;
+ } else {
+ cmpAndSwap(blk, pkt);
+ }
} else if (pkt->isWrite()) {
// we have the block in a writable state and can go ahead,
// note that the line may be also be considered writable in
pkt->setCacheResponding();
blk->status &= ~BlkDirty;
}
+ } else if (pkt->isClean()) {
+ blk->status &= ~BlkDirty;
} else {
assert(pkt->isInvalidate());
invalidateBlock(blk);
if (!blk) {
// need to do a replacement
- blk = allocateBlock(pkt->getAddr(), pkt->isSecure(), writebacks);
+ blk = allocateBlock(pkt, writebacks);
if (!blk) {
// no replaceable block available: give up, fwd to next level.
incMissCount(pkt);
return false;
}
- tags->insertBlock(pkt, blk);
blk->status |= (BlkValid | BlkReadable);
}
return false;
} else {
// a writeback that misses needs to allocate a new block
- blk = allocateBlock(pkt->getAddr(), pkt->isSecure(),
- writebacks);
+ blk = allocateBlock(pkt, writebacks);
if (!blk) {
// no replaceable block available: give up, fwd to
// next level.
incMissCount(pkt);
return false;
}
- tags->insertBlock(pkt, blk);
blk->status |= (BlkValid | BlkReadable);
}
BaseCache::handleFill(PacketPtr pkt, CacheBlk *blk, PacketList &writebacks,
bool allocate)
{
- assert(pkt->isResponse() || pkt->cmd == MemCmd::WriteLineReq);
+ assert(pkt->isResponse());
Addr addr = pkt->getAddr();
bool is_secure = pkt->isSecure();
#if TRACING_ON
if (!blk) {
// better have read new data...
- assert(pkt->hasData());
-
- // only read responses and write-line requests have data;
- // note that we don't write the data here for write-line - that
- // happens in the subsequent call to satisfyRequest
- assert(pkt->isRead() || pkt->cmd == MemCmd::WriteLineReq);
+ assert(pkt->hasData() || pkt->cmd == MemCmd::InvalidateResp);
// need to do a replacement if allocating, otherwise we stick
// with the temporary storage
- blk = allocate ? allocateBlock(addr, is_secure, writebacks) : nullptr;
+ blk = allocate ? allocateBlock(pkt, writebacks) : nullptr;
if (!blk) {
// No replaceable block or a mostly exclusive
// current request and then get rid of it
assert(!tempBlock->isValid());
blk = tempBlock;
- tempBlock->set = tags->extractSet(addr);
- tempBlock->tag = tags->extractTag(addr);
+ tempBlock->insert(addr, is_secure);
DPRINTF(Cache, "using temp block for %#llx (%s)\n", addr,
is_secure ? "s" : "ns");
- } else {
- tags->insertBlock(pkt, blk);
}
// we should never be overwriting a valid block
assert(!blk->isValid());
} else {
// existing block... probably an upgrade
- assert(blk->tag == tags->extractTag(addr));
+ assert(regenerateBlkAddr(blk) == addr);
+ assert(blk->isSecure() == is_secure);
// either we're getting new data or the block should already be valid
assert(pkt->hasData() || blk->isValid());
// don't clear block status... if block is already dirty we
// don't want to lose that
}
- if (is_secure)
- blk->status |= BlkSecure;
blk->status |= BlkValid | BlkReadable;
// sanity check for whole-line writes, which should always be
// marked as writable as part of the fill, and then later marked
// dirty as part of satisfyRequest
- if (pkt->cmd == MemCmd::WriteLineReq) {
+ if (pkt->cmd == MemCmd::InvalidateResp) {
assert(!pkt->hasSharers());
}
}
CacheBlk*
-BaseCache::allocateBlock(Addr addr, bool is_secure, PacketList &writebacks)
+BaseCache::allocateBlock(const PacketPtr pkt, PacketList &writebacks)
{
+ // Get address
+ const Addr addr = pkt->getAddr();
+
+ // Get secure bit
+ const bool is_secure = pkt->isSecure();
+
// Find replacement victim
- CacheBlk *blk = tags->findVictim(addr);
+ std::vector<CacheBlk*> evict_blks;
+ CacheBlk *victim = tags->findVictim(addr, is_secure, evict_blks);
// It is valid to return nullptr if there is no victim
- if (!blk)
+ if (!victim)
return nullptr;
- if (blk->isValid()) {
- Addr repl_addr = tags->regenerateBlkAddr(blk);
- MSHR *repl_mshr = mshrQueue.findMatch(repl_addr, blk->isSecure());
- if (repl_mshr) {
- // must be an outstanding upgrade or clean request
- // on a block we're about to replace...
- assert((!blk->isWritable() && repl_mshr->needsWritable()) ||
- repl_mshr->isCleaning());
- // too hard to replace block with transient state
- // allocation failed, block not inserted
- return nullptr;
- } else {
- DPRINTF(Cache, "replacement: replacing %#llx (%s) with %#llx "
- "(%s): %s\n", repl_addr, blk->isSecure() ? "s" : "ns",
- addr, is_secure ? "s" : "ns",
- blk->isDirty() ? "writeback" : "clean");
+ // Print victim block's information
+ DPRINTF(CacheRepl, "Replacement victim: %s\n", victim->print());
+
+ // Check for transient state allocations. If any of the entries listed
+ // for eviction has a transient state, the allocation fails
+ for (const auto& blk : evict_blks) {
+ if (blk->isValid()) {
+ Addr repl_addr = regenerateBlkAddr(blk);
+ MSHR *repl_mshr = mshrQueue.findMatch(repl_addr, blk->isSecure());
+ if (repl_mshr) {
+ // must be an outstanding upgrade or clean request
+ // on a block we're about to replace...
+ assert((!blk->isWritable() && repl_mshr->needsWritable()) ||
+ repl_mshr->isCleaning());
+
+ // too hard to replace block with transient state
+ // allocation failed, block not inserted
+ return nullptr;
+ }
+ }
+ }
+ // The victim will be replaced by a new entry, so increase the replacement
+ // counter if a valid block is being replaced
+ if (victim->isValid()) {
+ DPRINTF(Cache, "replacement: replacing %#llx (%s) with %#llx "
+ "(%s): %s\n", regenerateBlkAddr(victim),
+ victim->isSecure() ? "s" : "ns",
+ addr, is_secure ? "s" : "ns",
+ victim->isDirty() ? "writeback" : "clean");
+
+ replacements++;
+ }
+
+ // Evict valid blocks associated to this victim block
+ for (const auto& blk : evict_blks) {
+ if (blk->isValid()) {
if (blk->wasPrefetched()) {
unusedPrefetches++;
}
+
evictBlock(blk, writebacks);
- replacements++;
}
}
- return blk;
+ // Insert new block at victimized entry
+ tags->insertBlock(addr, is_secure, pkt->req->masterId(),
+ pkt->req->taskId(), victim);
+
+ return victim;
}
void
BaseCache::invalidateBlock(CacheBlk *blk)
{
- if (blk != tempBlock)
+ // If handling a block present in the Tags, let it do its invalidation
+ // process, which will update stats and invalidate the block itself
+ if (blk != tempBlock) {
tags->invalidate(blk);
- blk->invalidate();
+ } else {
+ tempBlock->invalidate();
+ }
+}
+
+void
+BaseCache::evictBlock(CacheBlk *blk, PacketList &writebacks)
+{
+ PacketPtr pkt = evictBlock(blk);
+ if (pkt) {
+ writebacks.push_back(pkt);
+ }
}
PacketPtr
writebacks[Request::wbMasterId]++;
- Request *req = new Request(tags->regenerateBlkAddr(blk), blkSize, 0,
- Request::wbMasterId);
+ RequestPtr req = std::make_shared<Request>(
+ regenerateBlkAddr(blk), blkSize, 0, Request::wbMasterId);
+
if (blk->isSecure())
req->setFlags(Request::SECURE);
PacketPtr
BaseCache::writecleanBlk(CacheBlk *blk, Request::Flags dest, PacketId id)
{
- Request *req = new Request(tags->regenerateBlkAddr(blk), blkSize, 0,
- Request::wbMasterId);
+ RequestPtr req = std::make_shared<Request>(
+ regenerateBlkAddr(blk), blkSize, 0, Request::wbMasterId);
+
if (blk->isSecure()) {
req->setFlags(Request::SECURE);
}
if (blk.isDirty()) {
assert(blk.isValid());
- Request request(tags->regenerateBlkAddr(&blk),
- blkSize, 0, Request::funcMasterId);
- request.taskId(blk.task_id);
+ RequestPtr request = std::make_shared<Request>(
+ regenerateBlkAddr(&blk), blkSize, 0, Request::funcMasterId);
+
+ request->taskId(blk.task_id);
if (blk.isSecure()) {
- request.setFlags(Request::SECURE);
+ request->setFlags(Request::SECURE);
}
- Packet packet(&request, MemCmd::WriteReq);
+ Packet packet(request, MemCmd::WriteReq);
packet.dataStatic(blk.data);
memSidePort.sendFunctional(&packet);
DPRINTF(Cache, "%s: MSHR %s\n", __func__, tgt_pkt->print());
+ // if the cache is in write coalescing mode or (additionally) in
+ // no allocation mode, and we have a write packet with an MSHR
+ // that is not a whole-line write (due to incompatible flags etc),
+ // then reset the write mode
+ if (writeAllocator && writeAllocator->coalesce() && tgt_pkt->isWrite()) {
+ if (!mshr->isWholeLineWrite()) {
+ // if we are currently write coalescing, hold on the
+ // MSHR as many cycles extra as we need to completely
+ // write a cache line
+ if (writeAllocator->delay(mshr->blkAddr)) {
+ Tick delay = blkSize / tgt_pkt->getSize() * clockPeriod();
+ DPRINTF(CacheVerbose, "Delaying pkt %s %llu ticks to allow "
+ "for write coalescing\n", tgt_pkt->print(), delay);
+ mshrQueue.delay(mshr, delay);
+ return false;
+ } else {
+ writeAllocator->reset();
+ }
+ } else {
+ writeAllocator->resetDelay(mshr->blkAddr);
+ }
+ }
+
CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure);
// either a prefetch that is not present upstream, or a normal
// MSHR request, proceed to get the packet to send downstream
- PacketPtr pkt = createMissPacket(tgt_pkt, blk, mshr->needsWritable());
+ PacketPtr pkt = createMissPacket(tgt_pkt, blk, mshr->needsWritable(),
+ mshr->isWholeLineWrite());
mshr->isForward = (pkt == nullptr);
// should writebacks be included here? prior code was inconsistent...
#define SUM_NON_DEMAND(s) \
- (s[MemCmd::SoftPFReq] + s[MemCmd::HardPFReq])
+ (s[MemCmd::SoftPFReq] + s[MemCmd::HardPFReq] + s[MemCmd::SoftPFExReq])
demandHits
.name(name() + ".demand_hits")
_snoopRespQueue(*_cache, *this, _label), cache(_cache)
{
}
+
+void
+WriteAllocator::updateMode(Addr write_addr, unsigned write_size,
+ Addr blk_addr)
+{
+ // check if we are continuing where the last write ended
+ if (nextAddr == write_addr) {
+ delayCtr[blk_addr] = delayThreshold;
+ // stop if we have already saturated
+ if (mode != WriteMode::NO_ALLOCATE) {
+ byteCount += write_size;
+ // switch to streaming mode if we have passed the lower
+ // threshold
+ if (mode == WriteMode::ALLOCATE &&
+ byteCount > coalesceLimit) {
+ mode = WriteMode::COALESCE;
+ DPRINTF(Cache, "Switched to write coalescing\n");
+ } else if (mode == WriteMode::COALESCE &&
+ byteCount > noAllocateLimit) {
+ // and continue and switch to non-allocating mode if we
+ // pass the upper threshold
+ mode = WriteMode::NO_ALLOCATE;
+ DPRINTF(Cache, "Switched to write-no-allocate\n");
+ }
+ }
+ } else {
+ // we did not see a write matching the previous one, start
+ // over again
+ byteCount = write_size;
+ mode = WriteMode::ALLOCATE;
+ resetDelay(blk_addr);
+ }
+ nextAddr = write_addr + write_size;
+}
+
+WriteAllocator*
+WriteAllocatorParams::create()
+{
+ return new WriteAllocator(this);
+}
projects / gem5.git / blobdiff