+ // at this point either this is a writeback or a write-through
+ // write clean operation and the block is already in this
+ // cache, we need to update the data and the block flags
+ assert(blk);
+ // TODO: the coherent cache can assert(!blk->isDirty());
+ if (!pkt->writeThrough()) {
+ blk->status |= BlkDirty;
+ }
+ // nothing else to do; writeback doesn't expect response
+ assert(!pkt->needsResponse());
+ pkt->writeDataToBlock(blk->data, blkSize);
+ DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print());
+
+ incHitCount(pkt);
+
+ // When the packet metadata arrives, the tag lookup will be done while
+ // the payload is arriving. Then the block will be ready to access as
+ // soon as the fill is done
+ blk->setWhenReady(clockEdge(fillLatency) + pkt->headerDelay +
+ std::max(cyclesToTicks(tag_latency), (uint64_t)pkt->payloadDelay));
+
+ // If this a write-through packet it will be sent to cache below
+ return !pkt->writeThrough();
+ } else if (blk && (pkt->needsWritable() ? blk->isWritable() :
+ blk->isReadable())) {
+ // OK to satisfy access
+ incHitCount(pkt);
+
+ // Calculate access latency based on the need to access the data array
+ if (pkt->isRead()) {
+ lat = calculateAccessLatency(blk, pkt->headerDelay, tag_latency);
+
+ // When a block is compressed, it must first be decompressed
+ // before being read. This adds to the access latency.
+ if (compressor) {
+ lat += compressor->getDecompressionLatency(blk);
+ }
+ } else {
+ lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency);
+ }
+
+ satisfyRequest(pkt, blk);
+ maintainClusivity(pkt->fromCache(), blk);
+
+ return true;
+ }
+
+ // Can't satisfy access normally... either no block (blk == nullptr)
+ // or have block but need writable
+
+ incMissCount(pkt);
+
+ lat = calculateAccessLatency(blk, pkt->headerDelay, tag_latency);
+
+ if (!blk && pkt->isLLSC() && pkt->isWrite()) {
+ // complete miss on store conditional... just give up now
+ pkt->req->setExtraData(0);
+ return true;
+ }
+
+ return false;
+}
+
+void
+BaseCache::maintainClusivity(bool from_cache, CacheBlk *blk)
+{
+ if (from_cache && blk && blk->isValid() && !blk->isDirty() &&
+ clusivity == Enums::mostly_excl) {
+ // if we have responded to a cache, and our block is still
+ // valid, but not dirty, and this cache is mostly exclusive
+ // with respect to the cache above, drop the block
+ invalidateBlock(blk);
+ }
+}
+
+CacheBlk*
+BaseCache::handleFill(PacketPtr pkt, CacheBlk *blk, PacketList &writebacks,
+ bool allocate)
+{
+ assert(pkt->isResponse());
+ Addr addr = pkt->getAddr();
+ bool is_secure = pkt->isSecure();
+#if TRACING_ON
+ CacheBlk::State old_state = blk ? blk->status : 0;
+#endif
+
+ // When handling a fill, we should have no writes to this line.
+ assert(addr == pkt->getBlockAddr(blkSize));
+ assert(!writeBuffer.findMatch(addr, is_secure));
+
+ if (!blk) {
+ // better have read new data...
+ assert(pkt->hasData() || pkt->cmd == MemCmd::InvalidateResp);
+
+ // need to do a replacement if allocating, otherwise we stick
+ // with the temporary storage
+ blk = allocate ? allocateBlock(pkt, writebacks) : nullptr;
+
+ if (!blk) {
+ // No replaceable block or a mostly exclusive
+ // cache... just use temporary storage to complete the
+ // current request and then get rid of it
+ blk = tempBlock;
+ tempBlock->insert(addr, is_secure);
+ DPRINTF(Cache, "using temp block for %#llx (%s)\n", addr,
+ is_secure ? "s" : "ns");
+ }
+ } else {
+ // existing block... probably an upgrade
+ // don't clear block status... if block is already dirty we
+ // don't want to lose that
+ }
+
+ // Block is guaranteed to be valid at this point
+ assert(blk->isValid());
+ assert(blk->isSecure() == is_secure);
+ assert(regenerateBlkAddr(blk) == addr);
+
+ blk->status |= BlkReadable;
+
+ // sanity check for whole-line writes, which should always be
+ // marked as writable as part of the fill, and then later marked
+ // dirty as part of satisfyRequest
+ if (pkt->cmd == MemCmd::InvalidateResp) {
+ assert(!pkt->hasSharers());
+ }
+
+ // here we deal with setting the appropriate state of the line,
+ // and we start by looking at the hasSharers flag, and ignore the
+ // cacheResponding flag (normally signalling dirty data) if the
+ // packet has sharers, thus the line is never allocated as Owned
+ // (dirty but not writable), and always ends up being either
+ // Shared, Exclusive or Modified, see Packet::setCacheResponding
+ // for more details
+ if (!pkt->hasSharers()) {
+ // we could get a writable line from memory (rather than a
+ // cache) even in a read-only cache, note that we set this bit
+ // even for a read-only cache, possibly revisit this decision
+ blk->status |= BlkWritable;
+
+ // check if we got this via cache-to-cache transfer (i.e., from a
+ // cache that had the block in Modified or Owned state)
+ if (pkt->cacheResponding()) {
+ // we got the block in Modified state, and invalidated the
+ // owners copy
+ blk->status |= BlkDirty;
+
+ chatty_assert(!isReadOnly, "Should never see dirty snoop response "
+ "in read-only cache %s\n", name());
+
+ }
+ }
+
+ DPRINTF(Cache, "Block addr %#llx (%s) moving from state %x to %s\n",
+ addr, is_secure ? "s" : "ns", old_state, blk->print());
+
+ // if we got new data, copy it in (checking for a read response
+ // and a response that has data is the same in the end)
+ if (pkt->isRead()) {
+ // sanity checks
+ assert(pkt->hasData());
+ assert(pkt->getSize() == blkSize);
+
+ pkt->writeDataToBlock(blk->data, blkSize);
+ }
+ // The block will be ready when the payload arrives and the fill is done
+ blk->setWhenReady(clockEdge(fillLatency) + pkt->headerDelay +
+ pkt->payloadDelay);
+
+ return blk;
+}
+
+CacheBlk*
+BaseCache::allocateBlock(const PacketPtr pkt, PacketList &writebacks)
+{
+ // Get address
+ const Addr addr = pkt->getAddr();
+
+ // Get secure bit
+ const bool is_secure = pkt->isSecure();
+
+ // Block size and compression related access latency. Only relevant if
+ // using a compressor, otherwise there is no extra delay, and the block
+ // is fully sized
+ std::size_t blk_size_bits = blkSize*8;
+ Cycles compression_lat = Cycles(0);
+ Cycles decompression_lat = Cycles(0);
+
+ // If a compressor is being used, it is called to compress data before
+ // insertion. Although in Gem5 the data is stored uncompressed, even if a
+ // compressor is used, the compression/decompression methods are called to
+ // calculate the amount of extra cycles needed to read or write compressed
+ // blocks.
+ if (compressor) {
+ compressor->compress(pkt->getConstPtr<uint64_t>(), compression_lat,
+ decompression_lat, blk_size_bits);
+ }
+
+ // Find replacement victim
+ std::vector<CacheBlk*> evict_blks;
+ CacheBlk *victim = tags->findVictim(addr, is_secure, blk_size_bits,
+ evict_blks);
+
+ // It is valid to return nullptr if there is no victim
+ if (!victim)
+ return nullptr;
+
+ // Print victim block's information
+ DPRINTF(CacheRepl, "Replacement victim: %s\n", victim->print());
+
+ // Check for transient state allocations. If any of the entries listed
+ // for eviction has a transient state, the allocation fails
+ bool replacement = false;
+ for (const auto& blk : evict_blks) {
+ if (blk->isValid()) {
+ replacement = true;
+
+ Addr repl_addr = regenerateBlkAddr(blk);
+ MSHR *repl_mshr = mshrQueue.findMatch(repl_addr, blk->isSecure());
+ if (repl_mshr) {
+ // must be an outstanding upgrade or clean request
+ // on a block we're about to replace...
+ assert((!blk->isWritable() && repl_mshr->needsWritable()) ||
+ repl_mshr->isCleaning());
+
+ // too hard to replace block with transient state
+ // allocation failed, block not inserted
+ return nullptr;
+ }
+ }
+ }
+
+ // The victim will be replaced by a new entry, so increase the replacement
+ // counter if a valid block is being replaced
+ if (replacement) {
+ // Evict valid blocks associated to this victim block
+ for (const auto& blk : evict_blks) {
+ if (blk->isValid()) {
+ DPRINTF(CacheRepl, "Evicting %s (%#llx) to make room for " \
+ "%#llx (%s)\n", blk->print(), regenerateBlkAddr(blk),
+ addr, is_secure);
+
+ if (blk->wasPrefetched()) {
+ stats.unusedPrefetches++;
+ }
+
+ evictBlock(blk, writebacks);
+ }
+ }
+
+ stats.replacements++;
+ }
+
+ // If using a compressor, set compression data. This must be done before
+ // block insertion, as compressed tags use this information.
+ if (compressor) {
+ compressor->setSizeBits(victim, blk_size_bits);
+ compressor->setDecompressionLatency(victim, decompression_lat);
+ }
+
+ // Insert new block at victimized entry
+ tags->insertBlock(pkt, victim);
+
+ return victim;
+}
+
+void
+BaseCache::invalidateBlock(CacheBlk *blk)
+{
+ // If handling a block present in the Tags, let it do its invalidation
+ // process, which will update stats and invalidate the block itself
+ if (blk != tempBlock) {
+ tags->invalidate(blk);
+ } else {
+ tempBlock->invalidate();
+ }
+}
+
+void
+BaseCache::evictBlock(CacheBlk *blk, PacketList &writebacks)
+{
+ PacketPtr pkt = evictBlock(blk);
+ if (pkt) {
+ writebacks.push_back(pkt);
+ }
+}
+
+PacketPtr
+BaseCache::writebackBlk(CacheBlk *blk)
+{
+ chatty_assert(!isReadOnly || writebackClean,
+ "Writeback from read-only cache");
+ assert(blk && blk->isValid() && (blk->isDirty() || writebackClean));
+
+ stats.writebacks[Request::wbMasterId]++;
+
+ RequestPtr req = std::make_shared<Request>(
+ regenerateBlkAddr(blk), blkSize, 0, Request::wbMasterId);
+
+ if (blk->isSecure())
+ req->setFlags(Request::SECURE);
+
+ req->taskId(blk->task_id);
+
+ PacketPtr pkt =
+ new Packet(req, blk->isDirty() ?
+ MemCmd::WritebackDirty : MemCmd::WritebackClean);
+
+ DPRINTF(Cache, "Create Writeback %s writable: %d, dirty: %d\n",
+ pkt->print(), blk->isWritable(), blk->isDirty());
+
+ if (blk->isWritable()) {
+ // not asserting shared means we pass the block in modified
+ // state, mark our own block non-writeable
+ blk->status &= ~BlkWritable;
+ } else {
+ // we are in the Owned state, tell the receiver
+ pkt->setHasSharers();
+ }
+
+ // make sure the block is not marked dirty
+ blk->status &= ~BlkDirty;
+
+ pkt->allocate();
+ pkt->setDataFromBlock(blk->data, blkSize);
+
+ // When a block is compressed, it must first be decompressed before being
+ // sent for writeback.
+ if (compressor) {
+ pkt->payloadDelay = compressor->getDecompressionLatency(blk);
+ }
+
+ return pkt;
+}
+
+PacketPtr
+BaseCache::writecleanBlk(CacheBlk *blk, Request::Flags dest, PacketId id)
+{
+ RequestPtr req = std::make_shared<Request>(
+ regenerateBlkAddr(blk), blkSize, 0, Request::wbMasterId);
+
+ if (blk->isSecure()) {
+ req->setFlags(Request::SECURE);
+ }
+ req->taskId(blk->task_id);
+
+ PacketPtr pkt = new Packet(req, MemCmd::WriteClean, blkSize, id);
+
+ if (dest) {
+ req->setFlags(dest);
+ pkt->setWriteThrough();
+ }
+
+ DPRINTF(Cache, "Create %s writable: %d, dirty: %d\n", pkt->print(),
+ blk->isWritable(), blk->isDirty());
+
+ if (blk->isWritable()) {
+ // not asserting shared means we pass the block in modified
+ // state, mark our own block non-writeable
+ blk->status &= ~BlkWritable;
+ } else {
+ // we are in the Owned state, tell the receiver
+ pkt->setHasSharers();
+ }
+
+ // make sure the block is not marked dirty
+ blk->status &= ~BlkDirty;
+
+ pkt->allocate();
+ pkt->setDataFromBlock(blk->data, blkSize);
+
+ // When a block is compressed, it must first be decompressed before being
+ // sent for writeback.
+ if (compressor) {
+ pkt->payloadDelay = compressor->getDecompressionLatency(blk);
+ }
+
+ return pkt;
+}
+
+
+void
+BaseCache::memWriteback()
+{
+ tags->forEachBlk([this](CacheBlk &blk) { writebackVisitor(blk); });
+}
+
+void
+BaseCache::memInvalidate()
+{
+ tags->forEachBlk([this](CacheBlk &blk) { invalidateVisitor(blk); });
+}
+
+bool
+BaseCache::isDirty() const
+{
+ return tags->anyBlk([](CacheBlk &blk) { return blk.isDirty(); });
+}
+
+bool
+BaseCache::coalesce() const
+{
+ return writeAllocator && writeAllocator->coalesce();
+}
+
+void
+BaseCache::writebackVisitor(CacheBlk &blk)
+{
+ if (blk.isDirty()) {
+ assert(blk.isValid());
+
+ RequestPtr request = std::make_shared<Request>(
+ regenerateBlkAddr(&blk), blkSize, 0, Request::funcMasterId);
+
+ request->taskId(blk.task_id);
+ if (blk.isSecure()) {
+ request->setFlags(Request::SECURE);
+ }
+
+ Packet packet(request, MemCmd::WriteReq);
+ packet.dataStatic(blk.data);
+
+ memSidePort.sendFunctional(&packet);
+
+ blk.status &= ~BlkDirty;
+ }
+}
+
+void
+BaseCache::invalidateVisitor(CacheBlk &blk)
+{
+ if (blk.isDirty())
+ warn_once("Invalidating dirty cache lines. " \
+ "Expect things to break.\n");
+
+ if (blk.isValid()) {
+ assert(!blk.isDirty());
+ invalidateBlock(&blk);
+ }
+}
+
+Tick
+BaseCache::nextQueueReadyTime() const
+{
+ Tick nextReady = std::min(mshrQueue.nextReadyTime(),
+ writeBuffer.nextReadyTime());
+
+ // Don't signal prefetch ready time if no MSHRs available
+ // Will signal once enoguh MSHRs are deallocated
+ if (prefetcher && mshrQueue.canPrefetch()) {
+ nextReady = std::min(nextReady,
+ prefetcher->nextPrefetchReadyTime());
+ }
+
+ return nextReady;
+}
+
+
+bool
+BaseCache::sendMSHRQueuePacket(MSHR* mshr)
+{
+ assert(mshr);
+
+ // use request from 1st target
+ PacketPtr tgt_pkt = mshr->getTarget()->pkt;
+
+ DPRINTF(Cache, "%s: MSHR %s\n", __func__, tgt_pkt->print());
+
+ // if the cache is in write coalescing mode or (additionally) in
+ // no allocation mode, and we have a write packet with an MSHR
+ // that is not a whole-line write (due to incompatible flags etc),
+ // then reset the write mode
+ if (writeAllocator && writeAllocator->coalesce() && tgt_pkt->isWrite()) {
+ if (!mshr->isWholeLineWrite()) {
+ // if we are currently write coalescing, hold on the
+ // MSHR as many cycles extra as we need to completely
+ // write a cache line
+ if (writeAllocator->delay(mshr->blkAddr)) {
+ Tick delay = blkSize / tgt_pkt->getSize() * clockPeriod();
+ DPRINTF(CacheVerbose, "Delaying pkt %s %llu ticks to allow "
+ "for write coalescing\n", tgt_pkt->print(), delay);
+ mshrQueue.delay(mshr, delay);
+ return false;
+ } else {
+ writeAllocator->reset();
+ }
+ } else {
+ writeAllocator->resetDelay(mshr->blkAddr);
+ }
+ }
+
+ CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure);
+
+ // either a prefetch that is not present upstream, or a normal
+ // MSHR request, proceed to get the packet to send downstream
+ PacketPtr pkt = createMissPacket(tgt_pkt, blk, mshr->needsWritable(),
+ mshr->isWholeLineWrite());
+
+ mshr->isForward = (pkt == nullptr);
+
+ if (mshr->isForward) {
+ // not a cache block request, but a response is expected
+ // make copy of current packet to forward, keep current
+ // copy for response handling
+ pkt = new Packet(tgt_pkt, false, true);
+ assert(!pkt->isWrite());
+ }
+
+ // play it safe and append (rather than set) the sender state,
+ // as forwarded packets may already have existing state
+ pkt->pushSenderState(mshr);
+
+ if (pkt->isClean() && blk && blk->isDirty()) {
+ // A cache clean opearation is looking for a dirty block. Mark
+ // the packet so that the destination xbar can determine that
+ // there will be a follow-up write packet as well.
+ pkt->setSatisfied();
+ }
+
+ if (!memSidePort.sendTimingReq(pkt)) {
+ // we are awaiting a retry, but we
+ // delete the packet and will be creating a new packet
+ // when we get the opportunity
+ delete pkt;
+
+ // note that we have now masked any requestBus and
+ // schedSendEvent (we will wait for a retry before
+ // doing anything), and this is so even if we do not
+ // care about this packet and might override it before
+ // it gets retried
+ return true;
+ } else {
+ // As part of the call to sendTimingReq the packet is
+ // forwarded to all neighbouring caches (and any caches
+ // above them) as a snoop. Thus at this point we know if
+ // any of the neighbouring caches are responding, and if
+ // so, we know it is dirty, and we can determine if it is
+ // being passed as Modified, making our MSHR the ordering
+ // point
+ bool pending_modified_resp = !pkt->hasSharers() &&
+ pkt->cacheResponding();
+ markInService(mshr, pending_modified_resp);
+
+ if (pkt->isClean() && blk && blk->isDirty()) {
+ // A cache clean opearation is looking for a dirty
+ // block. If a dirty block is encountered a WriteClean
+ // will update any copies to the path to the memory
+ // until the point of reference.
+ DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n",
+ __func__, pkt->print(), blk->print());
+ PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(),
+ pkt->id);
+ PacketList writebacks;
+ writebacks.push_back(wb_pkt);
+ doWritebacks(writebacks, 0);
+ }
+
+ return false;
+ }
+}
+
+bool
+BaseCache::sendWriteQueuePacket(WriteQueueEntry* wq_entry)
+{
+ assert(wq_entry);
+
+ // always a single target for write queue entries
+ PacketPtr tgt_pkt = wq_entry->getTarget()->pkt;
+
+ DPRINTF(Cache, "%s: write %s\n", __func__, tgt_pkt->print());
+
+ // forward as is, both for evictions and uncacheable writes
+ if (!memSidePort.sendTimingReq(tgt_pkt)) {
+ // note that we have now masked any requestBus and
+ // schedSendEvent (we will wait for a retry before
+ // doing anything), and this is so even if we do not
+ // care about this packet and might override it before
+ // it gets retried
+ return true;
+ } else {
+ markInService(wq_entry);
+ return false;
+ }
+}
+
+void
+BaseCache::serialize(CheckpointOut &cp) const
+{
+ bool dirty(isDirty());
+
+ if (dirty) {
+ warn("*** The cache still contains dirty data. ***\n");
+ warn(" Make sure to drain the system using the correct flags.\n");
+ warn(" This checkpoint will not restore correctly " \
+ "and dirty data in the cache will be lost!\n");
+ }
+
+ // Since we don't checkpoint the data in the cache, any dirty data
+ // will be lost when restoring from a checkpoint of a system that
+ // wasn't drained properly. Flag the checkpoint as invalid if the
+ // cache contains dirty data.
+ bool bad_checkpoint(dirty);
+ SERIALIZE_SCALAR(bad_checkpoint);
+}
+
+void
+BaseCache::unserialize(CheckpointIn &cp)
+{
+ bool bad_checkpoint;
+ UNSERIALIZE_SCALAR(bad_checkpoint);
+ if (bad_checkpoint) {
+ fatal("Restoring from checkpoints with dirty caches is not "
+ "supported in the classic memory system. Please remove any "
+ "caches or drain them properly before taking checkpoints.\n");
+ }
+}
+
+
+BaseCache::CacheCmdStats::CacheCmdStats(BaseCache &c,
+ const std::string &name)
+ : Stats::Group(&c), cache(c),
+
+ hits(
+ this, (name + "_hits").c_str(),
+ ("number of " + name + " hits").c_str()),
+ misses(
+ this, (name + "_misses").c_str(),
+ ("number of " + name + " misses").c_str()),
+ missLatency(
+ this, (name + "_miss_latency").c_str(),
+ ("number of " + name + " miss cycles").c_str()),
+ accesses(
+ this, (name + "_accesses").c_str(),
+ ("number of " + name + " accesses(hits+misses)").c_str()),
+ missRate(
+ this, (name + "_miss_rate").c_str(),
+ ("miss rate for " + name + " accesses").c_str()),
+ avgMissLatency(
+ this, (name + "_avg_miss_latency").c_str(),
+ ("average " + name + " miss latency").c_str()),
+ mshr_hits(
+ this, (name + "_mshr_hits").c_str(),
+ ("number of " + name + " MSHR hits").c_str()),
+ mshr_misses(
+ this, (name + "_mshr_misses").c_str(),
+ ("number of " + name + " MSHR misses").c_str()),
+ mshr_uncacheable(
+ this, (name + "_mshr_uncacheable").c_str(),
+ ("number of " + name + " MSHR uncacheable").c_str()),
+ mshr_miss_latency(
+ this, (name + "_mshr_miss_latency").c_str(),
+ ("number of " + name + " MSHR miss cycles").c_str()),
+ mshr_uncacheable_lat(
+ this, (name + "_mshr_uncacheable_latency").c_str(),
+ ("number of " + name + " MSHR uncacheable cycles").c_str()),
+ mshrMissRate(
+ this, (name + "_mshr_miss_rate").c_str(),
+ ("mshr miss rate for " + name + " accesses").c_str()),
+ avgMshrMissLatency(
+ this, (name + "_avg_mshr_miss_latency").c_str(),
+ ("average " + name + " mshr miss latency").c_str()),
+ avgMshrUncacheableLatency(
+ this, (name + "_avg_mshr_uncacheable_latency").c_str(),
+ ("average " + name + " mshr uncacheable latency").c_str())
+{
+}
+
+void
+BaseCache::CacheCmdStats::regStatsFromParent()
+{
+ using namespace Stats;
+
+ Stats::Group::regStats();
+ System *system = cache.system;
+ const auto max_masters = system->maxMasters();
+
+ hits
+ .init(max_masters)