/*
- * Copyright (c) 2012-2013, 2015-2017 ARM Limited
+ * Copyright (c) 2012-2013, 2015-2019 ARM Limited
* All rights reserved.
*
* The license below extends only to copyright in the software and shall
*
* Authors: Erik Hallnor
* Dave Greene
+ * Nikos Nikoleris
*/
/**
#include "mem/cache/mshr.hh"
-#include <algorithm>
#include <cassert>
#include <string>
-#include <vector>
#include "base/logging.hh"
+#include "base/trace.hh"
#include "base/types.hh"
#include "debug/Cache.hh"
-#include "mem/cache/cache.hh"
+#include "mem/cache/base.hh"
+#include "mem/request.hh"
#include "sim/core.hh"
-using namespace std;
-
MSHR::MSHR() : downstreamPending(false),
pendingModified(false),
postInvalidate(false), postDowngrade(false),
- isForward(false)
+ wasWholeLineWrite(false), isForward(false)
{
}
MSHR::TargetList::TargetList()
- : needsWritable(false), hasUpgrade(false), allocOnFill(false)
+ : needsWritable(false), hasUpgrade(false), allocOnFill(false),
+ hasFromCache(false)
{}
// potentially re-evaluate whether we should allocate on a fill or
// not
allocOnFill = allocOnFill || alloc_on_fill;
+
+ if (source != Target::FromPrefetcher) {
+ hasFromCache = hasFromCache || pkt->fromCache();
+
+ updateWriteFlags(pkt);
+ }
}
}
}
}
+void
+MSHR::TargetList::updateWriteFlags(PacketPtr pkt)
+{
+ if (isWholeLineWrite()) {
+ // if we have already seen writes for the full block
+ // stop here, this might be a full line write followed
+ // by other compatible requests (e.g., reads)
+ return;
+ }
+
+ if (canMergeWrites) {
+ if (!pkt->isWrite()) {
+ // We won't allow further merging if this hasn't
+ // been a write
+ canMergeWrites = false;
+ return;
+ }
+
+ // Avoid merging requests with special flags (e.g.,
+ // strictly ordered)
+ const Request::FlagsType no_merge_flags =
+ Request::UNCACHEABLE | Request::STRICT_ORDER |
+ Request::MMAPPED_IPR | Request::PRIVILEGED |
+ Request::LLSC | Request::MEM_SWAP |
+ Request::MEM_SWAP_COND | Request::SECURE;
+ const auto &req_flags = pkt->req->getFlags();
+ bool compat_write = !req_flags.isSet(no_merge_flags);
+
+ // if this is the first write, it might be a whole
+ // line write and even if we can't merge any
+ // subsequent write requests, we still need to service
+ // it as a whole line write (e.g., SECURE whole line
+ // write)
+ bool first_write = empty();
+ if (first_write || compat_write) {
+ auto offset = pkt->getOffset(blkSize);
+ auto begin = writesBitmap.begin() + offset;
+ std::fill(begin, begin + pkt->getSize(), true);
+ }
+
+ // We won't allow further merging if this has been a
+ // special write
+ canMergeWrites &= compat_write;
+ }
+}
+
inline void
MSHR::TargetList::add(PacketPtr pkt, Tick readyTime,
Counter order, Target::Source source, bool markPending,
void
-MSHR::TargetList::clearDownstreamPending()
+MSHR::TargetList::clearDownstreamPending(MSHR::TargetList::iterator begin,
+ MSHR::TargetList::iterator end)
{
- for (auto& t : *this) {
- if (t.markedPending) {
+ for (auto t = begin; t != end; t++) {
+ if (t->markedPending) {
// Iterate over the SenderState stack and see if we find
// an MSHR entry. If we find one, clear the
// downstreamPending flag by calling
// clearDownstreamPending(). This recursively clears the
// downstreamPending flag in all caches this packet has
// passed through.
- MSHR *mshr = t.pkt->findNextSenderState<MSHR>();
+ MSHR *mshr = t->pkt->findNextSenderState<MSHR>();
if (mshr != nullptr) {
mshr->clearDownstreamPending();
}
- t.markedPending = false;
+ t->markedPending = false;
}
}
}
+void
+MSHR::TargetList::clearDownstreamPending()
+{
+ clearDownstreamPending(begin(), end());
+}
+
bool
-MSHR::TargetList::checkFunctional(PacketPtr pkt)
+MSHR::TargetList::trySatisfyFunctional(PacketPtr pkt)
{
for (auto& t : *this) {
- if (pkt->checkFunctional(t.pkt)) {
+ if (pkt->trySatisfyFunctional(t.pkt)) {
return true;
}
}
order = _order;
assert(target);
isForward = false;
+ wasWholeLineWrite = false;
_isUncacheable = target->req->isUncacheable();
inService = false;
downstreamPending = false;
- assert(targets.isReset());
+
+ targets.init(blkAddr, blkSize);
+ deferredTargets.init(blkAddr, blkSize);
+
// Don't know of a case where we would allocate a new MSHR for a
// snoop (mem-side request), so set source according to request here
Target::Source source = (target->cmd == MemCmd::HardPFReq) ?
Target::FromPrefetcher : Target::FromCPU;
targets.add(target, when_ready, _order, source, true, alloc_on_fill);
- assert(deferredTargets.isReset());
+
+ // All targets must refer to the same block
+ assert(target->matchBlockAddr(targets.front().pkt, blkSize));
}
// level where it's going to get a response
targets.clearDownstreamPending();
}
+ // if the line is not considered a whole-line write when sent
+ // downstream, make sure it is also not considered a whole-line
+ // write when receiving the response, and vice versa
+ wasWholeLineWrite = isWholeLineWrite();
}
// outstanding miss
assert(pkt->cmd != MemCmd::HardPFReq);
- // uncacheable accesses always allocate a new MSHR, and cacheable
- // accesses ignore any uncacheable MSHRs, thus we should never
- // have targets addded if originally allocated uncacheable
- assert(!_isUncacheable);
-
// if there's a request already in service for this MSHR, we will
// have to defer the new target until after the response if any of
// the following are true:
return true;
}
+ // Start by determining if we will eventually respond or not,
+ // matching the conditions checked in Cache::handleSnoop
+ const bool will_respond = isPendingModified() && pkt->needsResponse() &&
+ !pkt->isClean();
if (isPendingModified() || pkt->isInvalidate()) {
// We need to save and replay the packet in two cases:
// 1. We're awaiting a writable copy (Modified or Exclusive),
// to forward the snoop up the hierarchy after the current
// transaction completes.
- // Start by determining if we will eventually respond or not,
- // matching the conditions checked in Cache::handleSnoop
- bool will_respond = isPendingModified() && pkt->needsResponse() &&
- !pkt->isClean();
-
// The packet we are snooping may be deleted by the time we
// actually process the target, and we consequently need to
// save a copy here. Clear flags and also allocate new data as
// the packet and the request as part of handling the deferred
// snoop.
PacketPtr cp_pkt = will_respond ? new Packet(pkt, true, true) :
- new Packet(new Request(*pkt->req), pkt->cmd, blkSize, pkt->id);
+ new Packet(std::make_shared<Request>(*pkt->req), pkt->cmd,
+ blkSize, pkt->id);
if (will_respond) {
// we are the ordering point, and will consequently
// in the case of an uncacheable request there is no need
// to set the responderHadWritable flag, but since the
// recipient does not care there is no harm in doing so
+ } else if (isPendingModified() && pkt->isClean()) {
+ // this cache doesn't respond to the clean request, a
+ // destination xbar will respond to this request, but to
+ // do so it needs to know if it should wait for the
+ // WriteCleanReq
+ pkt->setSatisfied();
}
+
targets.add(cp_pkt, curTick(), _order, Target::FromSnoop,
downstreamPending && targets.needsWritable, false);
// This transaction will take away our pending copy
postInvalidate = true;
}
-
- if (pkt->isClean()) {
- pkt->setSatisfied();
- }
}
if (!pkt->needsWritable() && !pkt->req->isUncacheable()) {
pkt->setHasSharers();
}
- return true;
+ return will_respond;
}
MSHR::TargetList
MSHR::extractServiceableTargets(PacketPtr pkt)
{
TargetList ready_targets;
+ ready_targets.init(blkAddr, blkSize);
// If the downstream MSHR got an invalidation request then we only
// service the first of the FromCPU targets and any other
// non-FromCPU target. This way the remaining FromCPU targets
return true;
}
+void
+MSHR::promoteIf(const std::function<bool (Target &)>& pred)
+{
+ // if any of the deferred targets were upper-level cache
+ // requests marked downstreamPending, need to clear that
+ assert(!downstreamPending); // not pending here anymore
+
+ // find the first target does not satisfy the condition
+ auto last_it = std::find_if_not(deferredTargets.begin(),
+ deferredTargets.end(),
+ pred);
+
+ // for the prefix of the deferredTargets [begin(), last_it) clear
+ // the downstreamPending flag and move them to the target list
+ deferredTargets.clearDownstreamPending(deferredTargets.begin(),
+ last_it);
+ targets.splice(targets.end(), deferredTargets,
+ deferredTargets.begin(), last_it);
+ // We need to update the flags for the target lists after the
+ // modifications
+ deferredTargets.populateFlags();
+}
+
+void
+MSHR::promoteReadable()
+{
+ if (!deferredTargets.empty() && !hasPostInvalidate()) {
+ // We got a non invalidating response, and we have the block
+ // but we have deferred targets which are waiting and they do
+ // not need writable. This can happen if the original request
+ // was for a cache clean operation and we had a copy of the
+ // block. Since we serviced the cache clean operation and we
+ // have the block, there's no need to defer the targets, so
+ // move them up to the regular target list.
+
+ auto pred = [](Target &t) {
+ assert(t.source == Target::FromCPU);
+ return !t.pkt->req->isCacheInvalidate() &&
+ !t.pkt->needsWritable();
+ };
+ promoteIf(pred);
+ }
+}
void
MSHR::promoteWritable()
{
+ PacketPtr def_tgt_pkt = deferredTargets.front().pkt;
if (deferredTargets.needsWritable &&
- !(hasPostInvalidate() || hasPostDowngrade())) {
+ !(hasPostInvalidate() || hasPostDowngrade()) &&
+ !def_tgt_pkt->req->isCacheInvalidate()) {
// We got a writable response, but we have deferred targets
// which are waiting to request a writable copy (not because
// of a pending invalidate). This can happen if the original
// target list.
assert(!targets.needsWritable);
targets.needsWritable = true;
- // if any of the deferred targets were upper-level cache
- // requests marked downstreamPending, need to clear that
- assert(!downstreamPending); // not pending here anymore
- deferredTargets.clearDownstreamPending();
- // this clears out deferredTargets too
- targets.splice(targets.end(), deferredTargets);
- deferredTargets.resetFlags();
+
+ auto pred = [](Target &t) {
+ assert(t.source == Target::FromCPU);
+ return !t.pkt->req->isCacheInvalidate();
+ };
+
+ promoteIf(pred);
}
}
bool
-MSHR::checkFunctional(PacketPtr pkt)
+MSHR::trySatisfyFunctional(PacketPtr pkt)
{
// For printing, we treat the MSHR as a whole as single entity.
// For other requests, we iterate over the individual targets
// since that's where the actual data lies.
if (pkt->isPrint()) {
- pkt->checkFunctional(this, blkAddr, isSecure, blkSize, nullptr);
+ pkt->trySatisfyFunctional(this, blkAddr, isSecure, blkSize, nullptr);
return false;
} else {
- return (targets.checkFunctional(pkt) ||
- deferredTargets.checkFunctional(pkt));
+ return (targets.trySatisfyFunctional(pkt) ||
+ deferredTargets.trySatisfyFunctional(pkt));
}
}
bool
-MSHR::sendPacket(Cache &cache)
+MSHR::sendPacket(BaseCache &cache)
{
return cache.sendMSHRQueuePacket(this);
}
void
MSHR::print(std::ostream &os, int verbosity, const std::string &prefix) const
{
- ccprintf(os, "%s[%#llx:%#llx](%s) %s %s %s state: %s %s %s %s %s\n",
+ ccprintf(os, "%s[%#llx:%#llx](%s) %s %s %s state: %s %s %s %s %s %s\n",
prefix, blkAddr, blkAddr + blkSize - 1,
isSecure ? "s" : "ns",
isForward ? "Forward" : "",
inService ? "InSvc" : "",
downstreamPending ? "DwnPend" : "",
postInvalidate ? "PostInv" : "",
- postDowngrade ? "PostDowngr" : "");
+ postDowngrade ? "PostDowngr" : "",
+ hasFromCache() ? "HasFromCache" : "");
if (!targets.empty()) {
ccprintf(os, "%s Targets:\n", prefix);
std::string
MSHR::print() const
{
- ostringstream str;
+ std::ostringstream str;
print(str);
return str.str();
}
+
+bool
+MSHR::matchBlockAddr(const Addr addr, const bool is_secure) const
+{
+ assert(hasTargets());
+ return (blkAddr == addr) && (isSecure == is_secure);
+}
+
+bool
+MSHR::matchBlockAddr(const PacketPtr pkt) const
+{
+ assert(hasTargets());
+ return pkt->matchBlockAddr(blkAddr, isSecure, blkSize);
+}
+
+bool
+MSHR::conflictAddr(const QueueEntry* entry) const
+{
+ assert(hasTargets());
+ return entry->matchBlockAddr(blkAddr, isSecure);
+}
projects / gem5.git / blobdiff