/*
- * Copyright (c) 2013 ARM Limited
+ * Copyright (c) 2013-2017,2019 ARM Limited
* All rights reserved
*
* The license below extends only to copyright in the software and shall
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
- * Authors: Stephan Diestelhorst <stephan.diestelhorst@arm.com>
+ * Authors: Stephan Diestelhorst
*/
/**
* @file
- * Definition of a snoop filter.
+ * Implementation of a snoop filter.
*/
-#include "base/misc.hh"
+#include "mem/snoop_filter.hh"
+
+#include "base/logging.hh"
#include "base/trace.hh"
#include "debug/SnoopFilter.hh"
-#include "mem/snoop_filter.hh"
#include "sim/system.hh"
+const int SnoopFilter::SNOOP_MASK_SIZE;
+
+void
+SnoopFilter::eraseIfNullEntry(SnoopFilterCache::iterator& sf_it)
+{
+ SnoopItem& sf_item = sf_it->second;
+ if ((sf_item.requested | sf_item.holder).none()) {
+ cachedLocations.erase(sf_it);
+ DPRINTF(SnoopFilter, "%s: Removed SF entry.\n",
+ __func__);
+ }
+}
+
std::pair<SnoopFilter::SnoopList, Cycles>
SnoopFilter::lookupRequest(const Packet* cpkt, const SlavePort& slave_port)
{
- DPRINTF(SnoopFilter, "%s: packet src %s addr 0x%x cmd %s\n",
- __func__, slave_port.name(), cpkt->getAddr(), cpkt->cmdString());
-
- Addr line_addr = cpkt->getAddr() & ~(linesize - 1);
+ DPRINTF(SnoopFilter, "%s: src %s packet %s\n", __func__,
+ slave_port.name(), cpkt->print());
+
+ // check if the packet came from a cache
+ bool allocate = !cpkt->req->isUncacheable() && slave_port.isSnooping() &&
+ cpkt->fromCache();
+ Addr line_addr = cpkt->getBlockAddr(linesize);
+ if (cpkt->isSecure()) {
+ line_addr |= LineSecure;
+ }
SnoopMask req_port = portToMask(slave_port);
- auto sf_it = cachedLocations.find(line_addr);
- bool is_hit = (sf_it != cachedLocations.end());
- // Create a new element through operator[] and modify in-place
- SnoopItem& sf_item = is_hit ? sf_it->second : cachedLocations[line_addr];
+ reqLookupResult.it = cachedLocations.find(line_addr);
+ bool is_hit = (reqLookupResult.it != cachedLocations.end());
+
+ // If the snoop filter has no entry, and we should not allocate,
+ // do not create a new snoop filter entry, simply return a NULL
+ // portlist.
+ if (!is_hit && !allocate)
+ return snoopDown(lookupLatency);
+
+ // If no hit in snoop filter create a new element and update iterator
+ if (!is_hit) {
+ reqLookupResult.it =
+ cachedLocations.emplace(line_addr, SnoopItem()).first;
+ }
+ SnoopItem& sf_item = reqLookupResult.it->second;
SnoopMask interested = sf_item.holder | sf_item.requested;
+ // Store unmodified value of snoop filter item in temp storage in
+ // case we need to revert because of a send retry in
+ // updateRequest.
+ reqLookupResult.retryItem = sf_item;
+
totRequests++;
if (is_hit) {
- // Single bit set -> value is a power of two
- if (isPow2(interested))
+ if (interested.count() == 1)
hitSingleRequests++;
else
hitMultiRequests++;
DPRINTF(SnoopFilter, "%s: SF value %x.%x\n",
__func__, sf_item.requested, sf_item.holder);
- if (!cpkt->req->isUncacheable() && cpkt->needsResponse()) {
- if (!cpkt->memInhibitAsserted()) {
+ // If we are not allocating, we are done
+ if (!allocate)
+ return snoopSelected(maskToPortList(interested & ~req_port),
+ lookupLatency);
+
+ if (cpkt->needsResponse()) {
+ if (!cpkt->cacheResponding()) {
// Max one request per address per port
- panic_if(sf_item.requested & req_port, "double request :( "\
- "SF value %x.%x\n", sf_item.requested, sf_item.holder);
+ panic_if((sf_item.requested & req_port).any(),
+ "double request :( SF value %x.%x\n",
+ sf_item.requested, sf_item.holder);
// Mark in-flight requests to distinguish later on
sf_item.requested |= req_port;
+ DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
+ __func__, sf_item.requested, sf_item.holder);
} else {
// NOTE: The memInhibit might have been asserted by a cache closer
// to the CPU, already -> the response will not be seen by this
// filter -> we do not need to keep the in-flight request, but make
// sure that we know that that cluster has a copy
- panic_if(!(sf_item.holder & req_port), "Need to hold the value!");
- DPRINTF(SnoopFilter, "%s: not marking request. SF value %x.%x\n",
+ panic_if((sf_item.holder & req_port).none(),
+ "Need to hold the value!");
+ DPRINTF(SnoopFilter,
+ "%s: not marking request. SF value %x.%x\n",
+ __func__, sf_item.requested, sf_item.holder);
+ }
+ } else { // if (!cpkt->needsResponse())
+ assert(cpkt->isEviction());
+ // make sure that the sender actually had the line
+ panic_if((sf_item.holder & req_port).none(), "requester %x is not a " \
+ "holder :( SF value %x.%x\n", req_port,
+ sf_item.requested, sf_item.holder);
+ // CleanEvicts and Writebacks -> the sender and all caches above
+ // it may not have the line anymore.
+ if (!cpkt->isBlockCached()) {
+ sf_item.holder &= ~req_port;
+ DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
__func__, sf_item.requested, sf_item.holder);
}
- DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
- __func__, sf_item.requested, sf_item.holder);
}
+
return snoopSelected(maskToPortList(interested & ~req_port), lookupLatency);
}
void
-SnoopFilter::updateRequest(const Packet* cpkt, const SlavePort& slave_port,
- bool will_retry)
+SnoopFilter::finishRequest(bool will_retry, Addr addr, bool is_secure)
{
- DPRINTF(SnoopFilter, "%s: packet src %s addr 0x%x cmd %s\n",
- __func__, slave_port.name(), cpkt->getAddr(), cpkt->cmdString());
-
- if (cpkt->req->isUncacheable())
- return;
-
- Addr line_addr = cpkt->getAddr() & ~(linesize - 1);
- SnoopMask req_port = portToMask(slave_port);
- SnoopItem& sf_item = cachedLocations[line_addr];
-
- DPRINTF(SnoopFilter, "%s: old SF value %x.%x retry: %i\n",
- __func__, sf_item.requested, sf_item.holder, will_retry);
-
- if (will_retry) {
- // Unmark a request that will come again.
- sf_item.requested &= ~req_port;
- return;
- }
-
- // will_retry == false
- if (!cpkt->needsResponse()) {
- // Packets that will not evoke a response but still need updates of the
- // snoop filter; WRITEBACKs for now only
- if (cpkt->cmd == MemCmd::Writeback) {
- // make sure that the sender actually had the line
- panic_if(sf_item.requested & req_port, "double request :( "\
- "SF value %x.%x\n", sf_item.requested, sf_item.holder);
- panic_if(!(sf_item.holder & req_port), "requester %x is not a "\
- "holder :( SF value %x.%x\n", req_port,
- sf_item.requested, sf_item.holder);
- // Writebacks -> the sender does not have the line anymore
- sf_item.holder &= ~req_port;
- } else {
- // @todo Add CleanEvicts
- assert(cpkt->cmd == MemCmd::CleanEvict);
+ if (reqLookupResult.it != cachedLocations.end()) {
+ // since we rely on the caller, do a basic check to ensure
+ // that finishRequest is being called following lookupRequest
+ Addr line_addr = (addr & ~(Addr(linesize - 1)));
+ if (is_secure) {
+ line_addr |= LineSecure;
}
- DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
- __func__, sf_item.requested, sf_item.holder);
+ assert(reqLookupResult.it->first == line_addr);
+ if (will_retry) {
+ SnoopItem retry_item = reqLookupResult.retryItem;
+ // Undo any changes made in lookupRequest to the snoop filter
+ // entry if the request will come again. retryItem holds
+ // the previous value of the snoopfilter entry.
+ reqLookupResult.it->second = retry_item;
+
+ DPRINTF(SnoopFilter, "%s: restored SF value %x.%x\n",
+ __func__, retry_item.requested, retry_item.holder);
+ }
+
+ eraseIfNullEntry(reqLookupResult.it);
}
}
std::pair<SnoopFilter::SnoopList, Cycles>
SnoopFilter::lookupSnoop(const Packet* cpkt)
{
- DPRINTF(SnoopFilter, "%s: packet addr 0x%x cmd %s\n",
- __func__, cpkt->getAddr(), cpkt->cmdString());
+ DPRINTF(SnoopFilter, "%s: packet %s\n", __func__, cpkt->print());
assert(cpkt->isRequest());
- // Broadcast / filter upward snoops
- const bool filter_upward = true; // @todo: Make configurable
-
- if (!filter_upward)
- return snoopAll(lookupLatency);
-
- Addr line_addr = cpkt->getAddr() & ~(linesize - 1);
+ Addr line_addr = cpkt->getBlockAddr(linesize);
+ if (cpkt->isSecure()) {
+ line_addr |= LineSecure;
+ }
auto sf_it = cachedLocations.find(line_addr);
bool is_hit = (sf_it != cachedLocations.end());
- // Create a new element through operator[] and modify in-place
- SnoopItem& sf_item = is_hit ? sf_it->second : cachedLocations[line_addr];
- DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n",
- __func__, sf_item.requested, sf_item.holder);
+ panic_if(!is_hit && (cachedLocations.size() >= maxEntryCount),
+ "snoop filter exceeded capacity of %d cache blocks\n",
+ maxEntryCount);
+
+ // If the snoop filter has no entry, simply return a NULL
+ // portlist, there is no point creating an entry only to remove it
+ // later
+ if (!is_hit)
+ return snoopDown(lookupLatency);
+
+ SnoopItem& sf_item = sf_it->second;
SnoopMask interested = (sf_item.holder | sf_item.requested);
totSnoops++;
- if (is_hit) {
- // Single bit set -> value is a power of two
- if (isPow2(interested))
- hitSingleSnoops++;
- else
- hitMultiSnoops++;
- }
+
+ if (interested.count() == 1)
+ hitSingleSnoops++;
+ else
+ hitMultiSnoops++;
+
// ReadEx and Writes require both invalidation and exlusivity, while reads
// require neither. Writebacks on the other hand require exclusivity but
// not the invalidation. Previously Writebacks did not generate upward
- // snoops so this was never an aissue. Now that Writebacks generate snoops
- // we need to special case for Writebacks.
- assert(cpkt->cmd == MemCmd::Writeback ||
- (cpkt->isInvalidate() == cpkt->needsExclusive()));
- if (cpkt->isInvalidate() && !sf_item.requested) {
+ // snoops so this was never an issue. Now that Writebacks generate snoops
+ // we need a special case for Writebacks. Additionally cache maintenance
+ // operations can generate snoops as they clean and/or invalidate all
+ // caches down to the specified point of reference.
+ assert(cpkt->isWriteback() || cpkt->req->isUncacheable() ||
+ (cpkt->isInvalidate() == cpkt->needsWritable()) ||
+ cpkt->req->isCacheMaintenance());
+ if (cpkt->isInvalidate() && sf_item.requested.none()) {
// Early clear of the holder, if no other request is currently going on
// @todo: This should possibly be updated even though we do not filter
// upward snoops
+ DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n",
+ __func__, sf_item.requested, sf_item.holder);
sf_item.holder = 0;
+ DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
+ __func__, sf_item.requested, sf_item.holder);
+ eraseIfNullEntry(sf_it);
}
- DPRINTF(SnoopFilter, "%s: new SF value %x.%x interest: %x \n",
- __func__, sf_item.requested, sf_item.holder, interested);
-
return snoopSelected(maskToPortList(interested), lookupLatency);
}
const SlavePort& rsp_port,
const SlavePort& req_port)
{
- DPRINTF(SnoopFilter, "%s: packet rsp %s req %s addr 0x%x cmd %s\n",
- __func__, rsp_port.name(), req_port.name(), cpkt->getAddr(),
- cpkt->cmdString());
+ DPRINTF(SnoopFilter, "%s: rsp %s req %s packet %s\n",
+ __func__, rsp_port.name(), req_port.name(), cpkt->print());
assert(cpkt->isResponse());
- assert(cpkt->memInhibitAsserted());
+ assert(cpkt->cacheResponding());
- if (cpkt->req->isUncacheable())
+ // if this snoop response is due to an uncacheable request, or is
+ // being turned into a normal response, there is nothing more to
+ // do
+ if (cpkt->req->isUncacheable() || !req_port.isSnooping()) {
return;
+ }
- Addr line_addr = cpkt->getAddr() & ~(linesize - 1);
+ Addr line_addr = cpkt->getBlockAddr(linesize);
+ if (cpkt->isSecure()) {
+ line_addr |= LineSecure;
+ }
SnoopMask rsp_mask = portToMask(rsp_port);
SnoopMask req_mask = portToMask(req_port);
SnoopItem& sf_item = cachedLocations[line_addr];
__func__, sf_item.requested, sf_item.holder);
// The source should have the line
- panic_if(!(sf_item.holder & rsp_mask), "SF value %x.%x does not have "\
- "the line\n", sf_item.requested, sf_item.holder);
+ panic_if((sf_item.holder & rsp_mask).none(),
+ "SF value %x.%x does not have the line\n",
+ sf_item.requested, sf_item.holder);
// The destination should have had a request in
- panic_if(!(sf_item.requested & req_mask), "SF value %x.%x missing "\
+ panic_if((sf_item.requested & req_mask).none(), "SF value %x.%x missing "\
"the original request\n", sf_item.requested, sf_item.holder);
- // Update the residency of the cache line.
- if (cpkt->needsExclusive() || !cpkt->sharedAsserted()) {
- DPRINTF(SnoopFilter, "%s: dropping %x because needs: %i shared: %i "\
- "SF val: %x.%x\n", __func__, rsp_mask,
- cpkt->needsExclusive(), cpkt->sharedAsserted(),
+ // If the snoop response has no sharers the line is passed in
+ // Modified state, and we know that there are no other copies, or
+ // they will all be invalidated imminently
+ if (!cpkt->hasSharers()) {
+ DPRINTF(SnoopFilter,
+ "%s: dropping %x because non-shared snoop "
+ "response SF val: %x.%x\n", __func__, rsp_mask,
sf_item.requested, sf_item.holder);
-
- sf_item.holder &= ~rsp_mask;
- // The snoop filter does not see any ACKs from non-responding sharers
- // that have been invalidated :( So below assert would be nice, but..
- //assert(sf_item.holder == 0);
sf_item.holder = 0;
}
- assert(cpkt->cmd != MemCmd::Writeback);
+ assert(!cpkt->isWriteback());
+ // @todo Deal with invalidating responses
sf_item.holder |= req_mask;
sf_item.requested &= ~req_mask;
+ assert((sf_item.requested | sf_item.holder).any());
DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
__func__, sf_item.requested, sf_item.holder);
}
SnoopFilter::updateSnoopForward(const Packet* cpkt,
const SlavePort& rsp_port, const MasterPort& req_port)
{
- DPRINTF(SnoopFilter, "%s: packet rsp %s req %s addr 0x%x cmd %s\n",
- __func__, rsp_port.name(), req_port.name(), cpkt->getAddr(),
- cpkt->cmdString());
-
- Addr line_addr = cpkt->getAddr() & ~(linesize - 1);
- SnoopItem& sf_item = cachedLocations[line_addr];
- SnoopMask rsp_mask M5_VAR_USED = portToMask(rsp_port);
+ DPRINTF(SnoopFilter, "%s: rsp %s req %s packet %s\n",
+ __func__, rsp_port.name(), req_port.name(), cpkt->print());
assert(cpkt->isResponse());
- assert(cpkt->memInhibitAsserted());
+ assert(cpkt->cacheResponding());
- DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n",
- __func__, sf_item.requested, sf_item.holder);
+ Addr line_addr = cpkt->getBlockAddr(linesize);
+ if (cpkt->isSecure()) {
+ line_addr |= LineSecure;
+ }
+ auto sf_it = cachedLocations.find(line_addr);
+ bool is_hit = sf_it != cachedLocations.end();
+
+ // Nothing to do if it is not a hit
+ if (!is_hit)
+ return;
- // Remote (to this snoop filter) snoops update the filter already when they
- // arrive from below, because we may not see any response.
- if (cpkt->needsExclusive()) {
- // If the request to this snoop response hit an in-flight transaction,
- // the holder was not reset -> no assertion & do that here, now!
- //assert(sf_item.holder == 0);
+ // If the snoop response has no sharers the line is passed in
+ // Modified state, and we know that there are no other copies, or
+ // they will all be invalidated imminently
+ if (!cpkt->hasSharers()) {
+ SnoopItem& sf_item = sf_it->second;
+
+ DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n",
+ __func__, sf_item.requested, sf_item.holder);
sf_item.holder = 0;
+ DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
+ __func__, sf_item.requested, sf_item.holder);
+
+ eraseIfNullEntry(sf_it);
}
- DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
- __func__, sf_item.requested, sf_item.holder);
}
void
SnoopFilter::updateResponse(const Packet* cpkt, const SlavePort& slave_port)
{
- DPRINTF(SnoopFilter, "%s: packet src %s addr 0x%x cmd %s\n",
- __func__, slave_port.name(), cpkt->getAddr(), cpkt->cmdString());
+ DPRINTF(SnoopFilter, "%s: src %s packet %s\n",
+ __func__, slave_port.name(), cpkt->print());
assert(cpkt->isResponse());
- if (cpkt->req->isUncacheable())
+ // we only allocate if the packet actually came from a cache, but
+ // start by checking if the port is snooping
+ if (cpkt->req->isUncacheable() || !slave_port.isSnooping())
+ return;
+
+ // next check if we actually allocated an entry
+ Addr line_addr = cpkt->getBlockAddr(linesize);
+ if (cpkt->isSecure()) {
+ line_addr |= LineSecure;
+ }
+ auto sf_it = cachedLocations.find(line_addr);
+ if (sf_it == cachedLocations.end())
return;
- Addr line_addr = cpkt->getAddr() & ~(linesize - 1);
SnoopMask slave_mask = portToMask(slave_port);
- SnoopItem& sf_item = cachedLocations[line_addr];
+ SnoopItem& sf_item = sf_it->second;
DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n",
__func__, sf_item.requested, sf_item.holder);
// Make sure we have seen the actual request, too
- panic_if(!(sf_item.requested & slave_mask), "SF value %x.%x missing "\
- "request bit\n", sf_item.requested, sf_item.holder);
+ panic_if((sf_item.requested & slave_mask).none(),
+ "SF value %x.%x missing request bit\n",
+ sf_item.requested, sf_item.holder);
- // Update the residency of the cache line.
- if (cpkt->needsExclusive() || !cpkt->sharedAsserted())
- sf_item.holder = 0;
- sf_item.holder |= slave_mask;
sf_item.requested &= ~slave_mask;
+ // Update the residency of the cache line.
+
+ if (cpkt->req->isCacheMaintenance()) {
+ // A cache clean response does not carry any data so it
+ // shouldn't change the holders, unless it is invalidating.
+ if (cpkt->isInvalidate()) {
+ sf_item.holder &= ~slave_mask;
+ }
+ eraseIfNullEntry(sf_it);
+ } else {
+ // Any other response implies that a cache above will have the
+ // block.
+ sf_item.holder |= slave_mask;
+ assert((sf_item.holder | sf_item.requested).any());
+ }
DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
__func__, sf_item.requested, sf_item.holder);
}
void
SnoopFilter::regStats()
{
+ SimObject::regStats();
+
totRequests
.name(name() + ".tot_requests")
.desc("Total number of requests made to the snoop filter.");
projects / gem5.git / blobdiff