X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;ds=sidebyside;f=src%2Fmem%2Fcache%2Fmshr.cc;h=5b9302925831ae67751ff64fbbe28d497b9d5f14;hb=338a173e822298bd22741342a7b24352450afdd1;hp=2dd23440a9c5cb48ceb57a392272dc746325ca59;hpb=912b20d02a4ec12b076e675ff150e3f1e89febaa;p=gem5.git diff --git a/src/mem/cache/mshr.cc b/src/mem/cache/mshr.cc index 2dd23440a..5b9302925 100644 --- a/src/mem/cache/mshr.cc +++ b/src/mem/cache/mshr.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2013, 2015-2016 ARM Limited + * Copyright (c) 2012-2013, 2015-2018 ARM Limited * All rights reserved. * * The license below extends only to copyright in the software and shall @@ -40,6 +40,7 @@ * * Authors: Erik Hallnor * Dave Greene + * Nikos Nikoleris */ /** @@ -49,28 +50,27 @@ #include "mem/cache/mshr.hh" -#include #include #include -#include -#include "base/misc.hh" +#include "base/logging.hh" +#include "base/trace.hh" #include "base/types.hh" #include "debug/Cache.hh" -#include "mem/cache/cache.hh" +#include "mem/cache/base.hh" +#include "mem/request.hh" #include "sim/core.hh" -using namespace std; - MSHR::MSHR() : downstreamPending(false), pendingModified(false), postInvalidate(false), postDowngrade(false), - isForward(false) + wasWholeLineWrite(false), isForward(false) { } MSHR::TargetList::TargetList() - : needsWritable(false), hasUpgrade(false), allocOnFill(false) + : needsWritable(false), hasUpgrade(false), allocOnFill(false), + hasFromCache(false) {} @@ -93,6 +93,12 @@ MSHR::TargetList::updateFlags(PacketPtr pkt, Target::Source source, // potentially re-evaluate whether we should allocate on a fill or // not allocOnFill = allocOnFill || alloc_on_fill; + + if (source != Target::FromPrefetcher) { + hasFromCache = hasFromCache || pkt->fromCache(); + + updateWriteFlags(pkt); + } } } @@ -176,31 +182,38 @@ MSHR::TargetList::replaceUpgrades() void -MSHR::TargetList::clearDownstreamPending() +MSHR::TargetList::clearDownstreamPending(MSHR::TargetList::iterator begin, + MSHR::TargetList::iterator end) { - for (auto& t : *this) { - if (t.markedPending) { + for (auto t = begin; t != end; t++) { + if (t->markedPending) { // Iterate over the SenderState stack and see if we find // an MSHR entry. If we find one, clear the // downstreamPending flag by calling // clearDownstreamPending(). This recursively clears the // downstreamPending flag in all caches this packet has // passed through. - MSHR *mshr = t.pkt->findNextSenderState(); + MSHR *mshr = t->pkt->findNextSenderState(); if (mshr != nullptr) { mshr->clearDownstreamPending(); } - t.markedPending = false; + t->markedPending = false; } } } +void +MSHR::TargetList::clearDownstreamPending() +{ + clearDownstreamPending(begin(), end()); +} + bool -MSHR::TargetList::checkFunctional(PacketPtr pkt) +MSHR::TargetList::trySatisfyFunctional(PacketPtr pkt) { for (auto& t : *this) { - if (pkt->checkFunctional(t.pkt)) { + if (pkt->trySatisfyFunctional(t.pkt)) { return true; } } @@ -247,16 +260,19 @@ MSHR::allocate(Addr blk_addr, unsigned blk_size, PacketPtr target, order = _order; assert(target); isForward = false; + wasWholeLineWrite = false; _isUncacheable = target->req->isUncacheable(); inService = false; downstreamPending = false; - assert(targets.isReset()); + + targets.init(blkAddr, blkSize); + deferredTargets.init(blkAddr, blkSize); + // Don't know of a case where we would allocate a new MSHR for a // snoop (mem-side request), so set source according to request here Target::Source source = (target->cmd == MemCmd::HardPFReq) ? Target::FromPrefetcher : Target::FromCPU; targets.add(target, when_ready, _order, source, true, alloc_on_fill); - assert(deferredTargets.isReset()); } @@ -284,6 +300,10 @@ MSHR::markInService(bool pending_modified_resp) // level where it's going to get a response targets.clearDownstreamPending(); } + // if the line is not considered a whole-line write when sent + // downstream, make sure it is also not considered a whole-line + // write when receiving the response, and vice versa + wasWholeLineWrite = isWholeLineWrite(); } @@ -307,27 +327,30 @@ MSHR::allocateTarget(PacketPtr pkt, Tick whenReady, Counter _order, // outstanding miss assert(pkt->cmd != MemCmd::HardPFReq); - // uncacheable accesses always allocate a new MSHR, and cacheable - // accesses ignore any uncacheable MSHRs, thus we should never - // have targets addded if originally allocated uncacheable - assert(!_isUncacheable); - // if there's a request already in service for this MSHR, we will // have to defer the new target until after the response if any of // the following are true: // - there are other targets already deferred // - there's a pending invalidate to be applied after the response // comes back (but before this target is processed) + // - the MSHR's first (and only) non-deferred target is a cache + // maintenance packet + // - the new target is a cache maintenance packet (this is probably + // overly conservative but certainly safe) // - this target requires a writable block and either we're not // getting a writable block back or we have already snooped // another read request that will downgrade our writable block // to non-writable (Shared or Owned) - if (inService && - (!deferredTargets.empty() || hasPostInvalidate() || - (pkt->needsWritable() && - (!isPendingModified() || hasPostDowngrade() || isForward)))) { + PacketPtr tgt_pkt = targets.front().pkt; + if (pkt->req->isCacheMaintenance() || + tgt_pkt->req->isCacheMaintenance() || + !deferredTargets.empty() || + (inService && + (hasPostInvalidate() || + (pkt->needsWritable() && + (!isPendingModified() || hasPostDowngrade() || isForward))))) { // need to put on deferred list - if (hasPostInvalidate()) + if (inService && hasPostInvalidate()) replaceUpgrade(pkt); deferredTargets.add(pkt, whenReady, _order, Target::FromCPU, true, alloc_on_fill); @@ -349,10 +372,10 @@ MSHR::handleSnoop(PacketPtr pkt, Counter _order) // when we snoop packets the needsWritable and isInvalidate flags // should always be the same, however, this assumes that we never // snoop writes as they are currently not marked as invalidations - panic_if(pkt->needsWritable() != pkt->isInvalidate(), + panic_if((pkt->needsWritable() != pkt->isInvalidate()) && + !pkt->req->isCacheMaintenance(), "%s got snoop %s where needsWritable, " - "does not match isInvalidate", name(), pkt->print(), - pkt->getAddr()); + "does not match isInvalidate", name(), pkt->print()); if (!inService || (pkt->isExpressSnoop() && downstreamPending)) { // Request has not been issued yet, or it's been issued @@ -368,7 +391,7 @@ MSHR::handleSnoop(PacketPtr pkt, Counter _order) // That is, even though the upper-level cache got out on its // local bus first, some other invalidating transaction // reached the global bus before the upgrade did. - if (pkt->needsWritable()) { + if (pkt->needsWritable() || pkt->req->isCacheInvalidate()) { targets.replaceUpgrades(); deferredTargets.replaceUpgrades(); } @@ -378,16 +401,18 @@ MSHR::handleSnoop(PacketPtr pkt, Counter _order) // From here on down, the request issued by this MSHR logically // precedes the request we're snooping. - if (pkt->needsWritable()) { + if (pkt->needsWritable() || pkt->req->isCacheInvalidate()) { // snooped request still precedes the re-request we'll have to // issue for deferred targets, if any... deferredTargets.replaceUpgrades(); } - if (hasPostInvalidate()) { - // a prior snoop has already appended an invalidation, so - // logically we don't have the block anymore; no need for - // further snooping. + PacketPtr tgt_pkt = targets.front().pkt; + if (hasPostInvalidate() || tgt_pkt->req->isCacheInvalidate()) { + // a prior snoop has already appended an invalidation or a + // cache invalidation operation is in progress, so logically + // we don't have the block anymore; no need for further + // snooping. return true; } @@ -402,7 +427,8 @@ MSHR::handleSnoop(PacketPtr pkt, Counter _order) // Start by determining if we will eventually respond or not, // matching the conditions checked in Cache::handleSnoop - bool will_respond = isPendingModified() && pkt->needsResponse(); + bool will_respond = isPendingModified() && pkt->needsResponse() && + !pkt->isClean(); // The packet we are snooping may be deleted by the time we // actually process the target, and we consequently need to @@ -415,7 +441,8 @@ MSHR::handleSnoop(PacketPtr pkt, Counter _order) // the packet and the request as part of handling the deferred // snoop. PacketPtr cp_pkt = will_respond ? new Packet(pkt, true, true) : - new Packet(new Request(*pkt->req), pkt->cmd, blkSize); + new Packet(std::make_shared(*pkt->req), pkt->cmd, + blkSize, pkt->id); if (will_respond) { // we are the ordering point, and will consequently @@ -436,10 +463,14 @@ MSHR::handleSnoop(PacketPtr pkt, Counter _order) targets.add(cp_pkt, curTick(), _order, Target::FromSnoop, downstreamPending && targets.needsWritable, false); - if (pkt->needsWritable()) { + if (pkt->needsWritable() || pkt->isInvalidate()) { // This transaction will take away our pending copy postInvalidate = true; } + + if (isPendingModified() && pkt->isClean()) { + pkt->setSatisfied(); + } } if (!pkt->needsWritable() && !pkt->req->isUncacheable()) { @@ -459,6 +490,7 @@ MSHR::TargetList MSHR::extractServiceableTargets(PacketPtr pkt) { TargetList ready_targets; + ready_targets.init(blkAddr, blkSize); // If the downstream MSHR got an invalidation request then we only // service the first of the FromCPU targets and any other // non-FromCPU target. This way the remaining FromCPU targets @@ -466,7 +498,8 @@ MSHR::extractServiceableTargets(PacketPtr pkt) // avoid memory consistency violations. if (pkt->cmd == MemCmd::ReadRespWithInvalidate) { auto it = targets.begin(); - assert(it->source == Target::FromCPU); + assert((it->source == Target::FromCPU) || + (it->source == Target::FromPrefetcher)); ready_targets.push_back(*it); it = targets.erase(it); while (it != targets.end()) { @@ -490,29 +523,85 @@ MSHR::extractServiceableTargets(PacketPtr pkt) bool MSHR::promoteDeferredTargets() { - if (targets.empty()) { - if (deferredTargets.empty()) { - return false; - } + if (targets.empty() && deferredTargets.empty()) { + // nothing to promote + return false; + } - std::swap(targets, deferredTargets); + // the deferred targets can be generally promoted unless they + // contain a cache maintenance request + + // find the first target that is a cache maintenance request + auto it = std::find_if(deferredTargets.begin(), deferredTargets.end(), + [](MSHR::Target &t) { + return t.pkt->req->isCacheMaintenance(); + }); + if (it == deferredTargets.begin()) { + // if the first deferred target is a cache maintenance packet + // then we can promote provided the targets list is empty and + // we can service it on its own + if (targets.empty()) { + targets.splice(targets.end(), deferredTargets, it); + } } else { - // If the targets list is not empty then we have one targets - // from the deferredTargets list to the targets list. A new - // request will then service the targets list. - targets.splice(targets.end(), deferredTargets); - targets.populateFlags(); + // if a cache maintenance operation exists, we promote all the + // deferred targets that precede it, or all deferred targets + // otherwise + targets.splice(targets.end(), deferredTargets, + deferredTargets.begin(), it); } - // clear deferredTargets flags - deferredTargets.resetFlags(); - + deferredTargets.populateFlags(); + targets.populateFlags(); order = targets.front().order; readyTime = std::max(curTick(), targets.front().readyTime); return true; } +void +MSHR::promoteIf(const std::function& pred) +{ + // if any of the deferred targets were upper-level cache + // requests marked downstreamPending, need to clear that + assert(!downstreamPending); // not pending here anymore + + // find the first target does not satisfy the condition + auto last_it = std::find_if_not(deferredTargets.begin(), + deferredTargets.end(), + pred); + + // for the prefix of the deferredTargets [begin(), last_it) clear + // the downstreamPending flag and move them to the target list + deferredTargets.clearDownstreamPending(deferredTargets.begin(), + last_it); + targets.splice(targets.end(), deferredTargets, + deferredTargets.begin(), last_it); + // We need to update the flags for the target lists after the + // modifications + deferredTargets.populateFlags(); +} + +void +MSHR::promoteReadable() +{ + if (!deferredTargets.empty() && !hasPostInvalidate()) { + // We got a non invalidating response, and we have the block + // but we have deferred targets which are waiting and they do + // not need writable. This can happen if the original request + // was for a cache clean operation and we had a copy of the + // block. Since we serviced the cache clean operation and we + // have the block, there's no need to defer the targets, so + // move them up to the regular target list. + + auto pred = [](Target &t) { + assert(t.source == Target::FromCPU); + return !t.pkt->req->isCacheInvalidate() && + !t.pkt->needsWritable(); + }; + promoteIf(pred); + } +} void MSHR::promoteWritable() @@ -528,34 +617,34 @@ MSHR::promoteWritable() // target list. assert(!targets.needsWritable); targets.needsWritable = true; - // if any of the deferred targets were upper-level cache - // requests marked downstreamPending, need to clear that - assert(!downstreamPending); // not pending here anymore - deferredTargets.clearDownstreamPending(); - // this clears out deferredTargets too - targets.splice(targets.end(), deferredTargets); - deferredTargets.resetFlags(); + + auto pred = [](Target &t) { + assert(t.source == Target::FromCPU); + return !t.pkt->req->isCacheInvalidate(); + }; + + promoteIf(pred); } } bool -MSHR::checkFunctional(PacketPtr pkt) +MSHR::trySatisfyFunctional(PacketPtr pkt) { // For printing, we treat the MSHR as a whole as single entity. // For other requests, we iterate over the individual targets // since that's where the actual data lies. if (pkt->isPrint()) { - pkt->checkFunctional(this, blkAddr, isSecure, blkSize, nullptr); + pkt->trySatisfyFunctional(this, blkAddr, isSecure, blkSize, nullptr); return false; } else { - return (targets.checkFunctional(pkt) || - deferredTargets.checkFunctional(pkt)); + return (targets.trySatisfyFunctional(pkt) || + deferredTargets.trySatisfyFunctional(pkt)); } } bool -MSHR::sendPacket(Cache &cache) +MSHR::sendPacket(BaseCache &cache) { return cache.sendMSHRQueuePacket(this); } @@ -563,7 +652,7 @@ MSHR::sendPacket(Cache &cache) void MSHR::print(std::ostream &os, int verbosity, const std::string &prefix) const { - ccprintf(os, "%s[%#llx:%#llx](%s) %s %s %s state: %s %s %s %s %s\n", + ccprintf(os, "%s[%#llx:%#llx](%s) %s %s %s state: %s %s %s %s %s %s\n", prefix, blkAddr, blkAddr + blkSize - 1, isSecure ? "s" : "ns", isForward ? "Forward" : "", @@ -573,7 +662,8 @@ MSHR::print(std::ostream &os, int verbosity, const std::string &prefix) const inService ? "InSvc" : "", downstreamPending ? "DwnPend" : "", postInvalidate ? "PostInv" : "", - postDowngrade ? "PostDowngr" : ""); + postDowngrade ? "PostDowngr" : "", + hasFromCache() ? "HasFromCache" : ""); if (!targets.empty()) { ccprintf(os, "%s Targets:\n", prefix); @@ -588,7 +678,7 @@ MSHR::print(std::ostream &os, int verbosity, const std::string &prefix) const std::string MSHR::print() const { - ostringstream str; + std::ostringstream str; print(str); return str.str(); }