/*
- * Copyright (c) 2011-2017 ARM Limited
+ * Copyright (c) 2011-2019 ARM Limited
* All rights reserved
*
* The license below extends only to copyright in the software and shall
* Authors: Ali Saidi
* Andreas Hansson
* William Wang
+ * Nikos Nikoleris
*/
/**
#include "mem/coherent_xbar.hh"
-#include "base/misc.hh"
+#include "base/logging.hh"
#include "base/trace.hh"
#include "debug/AddrRanges.hh"
#include "debug/CoherentXBar.hh"
CoherentXBar::CoherentXBar(const CoherentXBarParams *p)
: BaseXBar(p), system(p->system), snoopFilter(p->snoop_filter),
snoopResponseLatency(p->snoop_response_latency),
- pointOfCoherency(p->point_of_coherency)
+ maxOutstandingSnoopCheck(p->max_outstanding_snoops),
+ maxRoutingTableSizeCheck(p->max_routing_table_size),
+ pointOfCoherency(p->point_of_coherency),
+ pointOfUnification(p->point_of_unification),
+
+ snoops(this, "snoops", "Total snoops (count)"),
+ snoopTraffic(this, "snoopTraffic", "Total snoop traffic (bytes)"),
+ snoopFanout(this, "snoop_fanout", "Request fanout histogram")
{
// create the ports based on the size of the master and slave
// vector ports, and the presence of the default port, the ports
MasterPort* bp = new CoherentXBarMasterPort(portName, *this, i);
masterPorts.push_back(bp);
reqLayers.push_back(new ReqLayer(*bp, *this,
- csprintf(".reqLayer%d", i)));
- snoopLayers.push_back(new SnoopRespLayer(*bp, *this,
- csprintf(".snoopLayer%d", i)));
+ csprintf("reqLayer%d", i)));
+ snoopLayers.push_back(
+ new SnoopRespLayer(*bp, *this, csprintf("snoopLayer%d", i)));
}
// see if we have a default slave device connected and if so add
defaultPortID = masterPorts.size();
std::string portName = name() + ".default";
MasterPort* bp = new CoherentXBarMasterPort(portName, *this,
- defaultPortID);
+ defaultPortID);
masterPorts.push_back(bp);
- reqLayers.push_back(new ReqLayer(*bp, *this, csprintf(".reqLayer%d",
- defaultPortID)));
+ reqLayers.push_back(new ReqLayer(*bp, *this, csprintf("reqLayer%d",
+ defaultPortID)));
snoopLayers.push_back(new SnoopRespLayer(*bp, *this,
- csprintf(".snoopLayer%d",
+ csprintf("snoopLayer%d",
defaultPortID)));
}
QueuedSlavePort* bp = new CoherentXBarSlavePort(portName, *this, i);
slavePorts.push_back(bp);
respLayers.push_back(new RespLayer(*bp, *this,
- csprintf(".respLayer%d", i)));
+ csprintf("respLayer%d", i)));
snoopRespPorts.push_back(new SnoopRespPort(*bp, *this));
}
-
- clearPortCache();
}
CoherentXBar::~CoherentXBar()
for (const auto& p: slavePorts) {
// check if the connected master port is snooping
if (p->isSnooping()) {
- DPRINTF(AddrRanges, "Adding snooping master %s\n",
- p->getMasterPort().name());
+ DPRINTF(AddrRanges, "Adding snooping master %s\n", p->getPeer());
snoopPorts.push_back(p);
}
}
// and the cache responding flag should always be the same
assert(is_express_snoop == cache_responding);
- // determine the destination based on the address
- PortID master_port_id = findPort(pkt->getAddr());
+ // determine the destination based on the destination address range
+ PortID master_port_id = findPort(pkt->getAddrRange());
// test if the crossbar should be considered occupied for the current
// port, and exclude express snoops from the check
// determine how long to be crossbar layer is busy
Tick packetFinishTime = clockEdge(Cycles(1)) + pkt->payloadDelay;
- if (!system->bypassCaches()) {
+ // is this the destination point for this packet? (e.g. true if
+ // this xbar is the PoC for a cache maintenance operation to the
+ // PoC) otherwise the destination is any cache that can satisfy
+ // the request
+ const bool is_destination = isDestination(pkt);
+
+ const bool snoop_caches = !system->bypassCaches() &&
+ pkt->cmd != MemCmd::WriteClean;
+ if (snoop_caches) {
assert(pkt->snoopDelay == 0);
+ if (pkt->isClean() && !is_destination) {
+ // before snooping we need to make sure that the memory
+ // below is not busy and the cache clean request can be
+ // forwarded to it
+ if (!masterPorts[master_port_id]->tryTiming(pkt)) {
+ DPRINTF(CoherentXBar, "%s: src %s packet %s RETRY\n", __func__,
+ src_port->name(), pkt->print());
+
+ // update the layer state and schedule an idle event
+ reqLayers[master_port_id]->failedTiming(src_port,
+ clockEdge(Cycles(1)));
+ return false;
+ }
+ }
+
+
// the packet is a memory-mapped request and should be
// broadcasted to our snoopers but the source
if (snoopFilter) {
} else {
// determine if we are forwarding the packet, or responding to
// it
- if (!pointOfCoherency || pkt->isRead() || pkt->isWrite()) {
+ if (forwardPacket(pkt)) {
// if we are passing on, rather than sinking, a packet to
// which an upstream cache has committed to responding,
// the line was needs writable, and the responding only
pkt->setExpressSnoop();
}
+ // make sure that the write request (e.g., WriteClean)
+ // will stop at the memory below if this crossbar is its
+ // destination
+ if (pkt->isWrite() && is_destination) {
+ pkt->clearWriteThrough();
+ }
+
// since it is a normal request, attempt to send the packet
success = masterPorts[master_port_id]->sendTimingReq(pkt);
} else {
}
}
- if (snoopFilter && !system->bypassCaches()) {
+ if (snoopFilter && snoop_caches) {
// Let the snoop filter know about the success of the send operation
snoopFilter->finishRequest(!success, addr, pkt->isSecure());
}
outstandingSnoop.insert(pkt->req);
// basic sanity check on the outstanding snoops
- panic_if(outstandingSnoop.size() > 512,
- "Outstanding snoop requests exceeded 512\n");
+ panic_if(outstandingSnoop.size() > maxOutstandingSnoopCheck,
+ "%s: Outstanding snoop requests exceeded %d\n",
+ name(), maxOutstandingSnoopCheck);
}
// remember where to route the normal response to
assert(routeTo.find(pkt->req) == routeTo.end());
routeTo[pkt->req] = slave_port_id;
- panic_if(routeTo.size() > 512,
- "Routing table exceeds 512 packets\n");
+ panic_if(routeTo.size() > maxRoutingTableSizeCheck,
+ "%s: Routing table exceeds %d packets\n",
+ name(), maxRoutingTableSizeCheck);
}
// update the layer state and schedule an idle event
// queue the packet for deletion
pendingDelete.reset(pkt);
+ // normally we respond to the packet we just received if we need to
+ PacketPtr rsp_pkt = pkt;
+ PortID rsp_port_id = slave_port_id;
+
+ // If this is the destination of the cache clean operation the
+ // crossbar is responsible for responding. This crossbar will
+ // respond when the cache clean is complete. A cache clean
+ // is complete either:
+ // * direcly, if no cache above had a dirty copy of the block
+ // as indicated by the satisfied flag of the packet, or
+ // * when the crossbar has seen both the cache clean request
+ // (CleanSharedReq, CleanInvalidReq) and the corresponding
+ // write (WriteClean) which updates the block in the memory
+ // below.
+ if (success &&
+ ((pkt->isClean() && pkt->satisfied()) ||
+ pkt->cmd == MemCmd::WriteClean) &&
+ is_destination) {
+ PacketPtr deferred_rsp = pkt->isWrite() ? nullptr : pkt;
+ auto cmo_lookup = outstandingCMO.find(pkt->id);
+ if (cmo_lookup != outstandingCMO.end()) {
+ // the cache clean request has already reached this xbar
+ respond_directly = true;
+ if (pkt->isWrite()) {
+ rsp_pkt = cmo_lookup->second;
+ assert(rsp_pkt);
+
+ // determine the destination
+ const auto route_lookup = routeTo.find(rsp_pkt->req);
+ assert(route_lookup != routeTo.end());
+ rsp_port_id = route_lookup->second;
+ assert(rsp_port_id != InvalidPortID);
+ assert(rsp_port_id < respLayers.size());
+ // remove the request from the routing table
+ routeTo.erase(route_lookup);
+ }
+ outstandingCMO.erase(cmo_lookup);
+ } else {
+ respond_directly = false;
+ outstandingCMO.emplace(pkt->id, deferred_rsp);
+ if (!pkt->isWrite()) {
+ assert(routeTo.find(pkt->req) == routeTo.end());
+ routeTo[pkt->req] = slave_port_id;
+
+ panic_if(routeTo.size() > maxRoutingTableSizeCheck,
+ "%s: Routing table exceeds %d packets\n",
+ name(), maxRoutingTableSizeCheck);
+ }
+ }
+ }
+
+
if (respond_directly) {
- assert(pkt->needsResponse());
+ assert(rsp_pkt->needsResponse());
assert(success);
- pkt->makeResponse();
+ rsp_pkt->makeResponse();
if (snoopFilter && !system->bypassCaches()) {
// let the snoop filter inspect the response and update its state
- snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]);
+ snoopFilter->updateResponse(rsp_pkt, *slavePorts[rsp_port_id]);
}
+ // we send the response after the current packet, even if the
+ // response is not for this packet (e.g. cache clean operation
+ // where both the request and the write packet have to cross
+ // the destination xbar before the response is sent.)
Tick response_time = clockEdge() + pkt->headerDelay;
- pkt->headerDelay = 0;
+ rsp_pkt->headerDelay = 0;
- slavePorts[slave_port_id]->schedTimingResp(pkt, response_time);
+ slavePorts[rsp_port_id]->schedTimingResp(rsp_pkt, response_time);
}
return success;
// device responsible for the address range something is
// wrong, hence there is nothing further to do as the packet
// would be going back to where it came from
- assert(master_port_id == findPort(pkt->getAddr()));
+ assert(findPort(pkt->getAddrRange()) == master_port_id);
}
bool
}
Tick
-CoherentXBar::recvAtomic(PacketPtr pkt, PortID slave_port_id)
+CoherentXBar::recvAtomicBackdoor(PacketPtr pkt, PortID slave_port_id,
+ MemBackdoorPtr *backdoor)
{
DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__,
slavePorts[slave_port_id]->name(), pkt->print());
MemCmd snoop_response_cmd = MemCmd::InvalidCmd;
Tick snoop_response_latency = 0;
- if (!system->bypassCaches()) {
+ // is this the destination point for this packet? (e.g. true if
+ // this xbar is the PoC for a cache maintenance operation to the
+ // PoC) otherwise the destination is any cache that can satisfy
+ // the request
+ const bool is_destination = isDestination(pkt);
+
+ const bool snoop_caches = !system->bypassCaches() &&
+ pkt->cmd != MemCmd::WriteClean;
+ if (snoop_caches) {
// forward to all snoopers but the source
std::pair<MemCmd, Tick> snoop_result;
if (snoopFilter) {
// even if we had a snoop response, we must continue and also
// perform the actual request at the destination
- PortID master_port_id = findPort(pkt->getAddr());
+ PortID master_port_id = findPort(pkt->getAddrRange());
if (sink_packet) {
DPRINTF(CoherentXBar, "%s: Not forwarding %s\n", __func__,
pkt->print());
} else {
- if (!pointOfCoherency || pkt->isRead() || pkt->isWrite()) {
+ if (forwardPacket(pkt)) {
+ // make sure that the write request (e.g., WriteClean)
+ // will stop at the memory below if this crossbar is its
+ // destination
+ if (pkt->isWrite() && is_destination) {
+ pkt->clearWriteThrough();
+ }
+
// forward the request to the appropriate destination
- response_latency = masterPorts[master_port_id]->sendAtomic(pkt);
+ auto master = masterPorts[master_port_id];
+ response_latency = backdoor ?
+ master->sendAtomicBackdoor(pkt, *backdoor) :
+ master->sendAtomic(pkt);
} else {
// if it does not need a response we sink the packet above
assert(pkt->needsResponse());
response_latency = snoop_response_latency;
}
+ // If this is the destination of the cache clean operation the
+ // crossbar is responsible for responding. This crossbar will
+ // respond when the cache clean is complete. An atomic cache clean
+ // is complete when the crossbars receives the cache clean
+ // request (CleanSharedReq, CleanInvalidReq), as either:
+ // * no cache above had a dirty copy of the block as indicated by
+ // the satisfied flag of the packet, or
+ // * the crossbar has already seen the corresponding write
+ // (WriteClean) which updates the block in the memory below.
+ if (pkt->isClean() && isDestination(pkt) && pkt->satisfied()) {
+ auto it = outstandingCMO.find(pkt->id);
+ assert(it != outstandingCMO.end());
+ // we are responding right away
+ outstandingCMO.erase(it);
+ } else if (pkt->cmd == MemCmd::WriteClean && isDestination(pkt)) {
+ // if this is the destination of the operation, the xbar
+ // sends the responce to the cache clean operation only
+ // after having encountered the cache clean request
+ auto M5_VAR_USED ret = outstandingCMO.emplace(pkt->id, nullptr);
+ // in atomic mode we know that the WriteClean packet should
+ // precede the clean request
+ assert(ret.second);
+ }
+
// add the response data
if (pkt->isResponse()) {
pkt_size = pkt->hasData() ? pkt->getSize() : 0;
// if we find a response that has the data, then the
// downstream caches/memories may be out of date, so simply stop
// here
- if (p->checkFunctional(pkt)) {
+ if (p->trySatisfyFunctional(pkt)) {
if (pkt->needsResponse())
pkt->makeResponse();
return;
}
}
- PortID dest_id = findPort(pkt->getAddr());
+ PortID dest_id = findPort(pkt->getAddrRange());
masterPorts[dest_id]->sendFunctional(pkt);
}
}
for (const auto& p : slavePorts) {
- if (p->checkFunctional(pkt)) {
+ if (p->trySatisfyFunctional(pkt)) {
if (pkt->needsResponse())
pkt->makeResponse();
return;
(!pkt->needsWritable() || pkt->responderHadWritable()));
}
+bool
+CoherentXBar::forwardPacket(const PacketPtr pkt)
+{
+ // we are forwarding the packet if:
+ // 1) this is a cache clean request to the PoU/PoC and this
+ // crossbar is above the PoU/PoC
+ // 2) this is a read or a write
+ // 3) this crossbar is above the point of coherency
+ if (pkt->isClean()) {
+ return !isDestination(pkt);
+ }
+ return pkt->isRead() || pkt->isWrite() || !pointOfCoherency;
+}
+
+
void
CoherentXBar::regStats()
{
- // register the stats of the base class and our layers
BaseXBar::regStats();
- for (auto l: reqLayers)
- l->regStats();
- for (auto l: respLayers)
- l->regStats();
- for (auto l: snoopLayers)
- l->regStats();
-
- snoops
- .name(name() + ".snoops")
- .desc("Total snoops (count)")
- ;
-
- snoopTraffic
- .name(name() + ".snoopTraffic")
- .desc("Total snoop traffic (bytes)")
- ;
-
- snoopFanout
- .init(0, snoopPorts.size(), 1)
- .name(name() + ".snoop_fanout")
- .desc("Request fanout histogram")
- ;
+
+ snoopFanout.init(0, snoopPorts.size(), 1);
}
CoherentXBar *
projects / gem5.git / blobdiff