X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fmem%2Fcoherent_xbar.cc;h=713c6421b011c996ffd132665bb0a96d63062995;hb=44e3c95555b380f62c3fa4d878d78f26ad035475;hp=6aec0b335b34985e41c182ca5ae125ac275da3f6;hpb=2f6d69ee08bd92d9fcfb9b78f84b2d7c0ba113d0;p=gem5.git diff --git a/src/mem/coherent_xbar.cc b/src/mem/coherent_xbar.cc index 6aec0b335..713c6421b 100644 --- a/src/mem/coherent_xbar.cc +++ b/src/mem/coherent_xbar.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2017 ARM Limited + * Copyright (c) 2011-2019 ARM Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -40,6 +40,7 @@ * Authors: Ali Saidi * Andreas Hansson * William Wang + * Nikos Nikoleris */ /** @@ -58,8 +59,14 @@ CoherentXBar::CoherentXBar(const CoherentXBarParams *p) : BaseXBar(p), system(p->system), snoopFilter(p->snoop_filter), snoopResponseLatency(p->snoop_response_latency), + maxOutstandingSnoopCheck(p->max_outstanding_snoops), + maxRoutingTableSizeCheck(p->max_routing_table_size), pointOfCoherency(p->point_of_coherency), - pointOfUnification(p->point_of_unification) + pointOfUnification(p->point_of_unification), + + snoops(this, "snoops", "Total snoops (count)"), + snoopTraffic(this, "snoopTraffic", "Total snoop traffic (bytes)"), + snoopFanout(this, "snoop_fanout", "Request fanout histogram") { // create the ports based on the size of the master and slave // vector ports, and the presence of the default port, the ports @@ -69,9 +76,9 @@ CoherentXBar::CoherentXBar(const CoherentXBarParams *p) MasterPort* bp = new CoherentXBarMasterPort(portName, *this, i); masterPorts.push_back(bp); reqLayers.push_back(new ReqLayer(*bp, *this, - csprintf(".reqLayer%d", i))); - snoopLayers.push_back(new SnoopRespLayer(*bp, *this, - csprintf(".snoopLayer%d", i))); + csprintf("reqLayer%d", i))); + snoopLayers.push_back( + new SnoopRespLayer(*bp, *this, csprintf("snoopLayer%d", i))); } // see if we have a default slave device connected and if so add @@ -80,12 +87,12 @@ CoherentXBar::CoherentXBar(const CoherentXBarParams *p) defaultPortID = masterPorts.size(); std::string portName = name() + ".default"; MasterPort* bp = new CoherentXBarMasterPort(portName, *this, - defaultPortID); + defaultPortID); masterPorts.push_back(bp); - reqLayers.push_back(new ReqLayer(*bp, *this, csprintf(".reqLayer%d", - defaultPortID))); + reqLayers.push_back(new ReqLayer(*bp, *this, csprintf("reqLayer%d", + defaultPortID))); snoopLayers.push_back(new SnoopRespLayer(*bp, *this, - csprintf(".snoopLayer%d", + csprintf("snoopLayer%d", defaultPortID))); } @@ -95,11 +102,9 @@ CoherentXBar::CoherentXBar(const CoherentXBarParams *p) QueuedSlavePort* bp = new CoherentXBarSlavePort(portName, *this, i); slavePorts.push_back(bp); respLayers.push_back(new RespLayer(*bp, *this, - csprintf(".respLayer%d", i))); + csprintf("respLayer%d", i))); snoopRespPorts.push_back(new SnoopRespPort(*bp, *this)); } - - clearPortCache(); } CoherentXBar::~CoherentXBar() @@ -124,8 +129,7 @@ CoherentXBar::init() for (const auto& p: slavePorts) { // check if the connected master port is snooping if (p->isSnooping()) { - DPRINTF(AddrRanges, "Adding snooping master %s\n", - p->getMasterPort().name()); + DPRINTF(AddrRanges, "Adding snooping master %s\n", p->getPeer()); snoopPorts.push_back(p); } } @@ -152,8 +156,8 @@ CoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id) // and the cache responding flag should always be the same assert(is_express_snoop == cache_responding); - // determine the destination based on the address - PortID master_port_id = findPort(pkt->getAddr()); + // determine the destination based on the destination address range + PortID master_port_id = findPort(pkt->getAddrRange()); // test if the crossbar should be considered occupied for the current // port, and exclude express snoops from the check @@ -183,11 +187,33 @@ CoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id) // determine how long to be crossbar layer is busy Tick packetFinishTime = clockEdge(Cycles(1)) + pkt->payloadDelay; + // is this the destination point for this packet? (e.g. true if + // this xbar is the PoC for a cache maintenance operation to the + // PoC) otherwise the destination is any cache that can satisfy + // the request + const bool is_destination = isDestination(pkt); + const bool snoop_caches = !system->bypassCaches() && pkt->cmd != MemCmd::WriteClean; if (snoop_caches) { assert(pkt->snoopDelay == 0); + if (pkt->isClean() && !is_destination) { + // before snooping we need to make sure that the memory + // below is not busy and the cache clean request can be + // forwarded to it + if (!masterPorts[master_port_id]->tryTiming(pkt)) { + DPRINTF(CoherentXBar, "%s: src %s packet %s RETRY\n", __func__, + src_port->name(), pkt->print()); + + // update the layer state and schedule an idle event + reqLayers[master_port_id]->failedTiming(src_port, + clockEdge(Cycles(1))); + return false; + } + } + + // the packet is a memory-mapped request and should be // broadcasted to our snoopers but the source if (snoopFilter) { @@ -243,7 +269,7 @@ CoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id) } else { // determine if we are forwarding the packet, or responding to // it - if (!pointOfCoherency || pkt->isRead() || pkt->isWrite()) { + if (forwardPacket(pkt)) { // if we are passing on, rather than sinking, a packet to // which an upstream cache has committed to responding, // the line was needs writable, and the responding only @@ -253,6 +279,13 @@ CoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id) pkt->setExpressSnoop(); } + // make sure that the write request (e.g., WriteClean) + // will stop at the memory below if this crossbar is its + // destination + if (pkt->isWrite() && is_destination) { + pkt->clearWriteThrough(); + } + // since it is a normal request, attempt to send the packet success = masterPorts[master_port_id]->sendTimingReq(pkt); } else { @@ -297,8 +330,9 @@ CoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id) outstandingSnoop.insert(pkt->req); // basic sanity check on the outstanding snoops - panic_if(outstandingSnoop.size() > 512, - "Outstanding snoop requests exceeded 512\n"); + panic_if(outstandingSnoop.size() > maxOutstandingSnoopCheck, + "%s: Outstanding snoop requests exceeded %d\n", + name(), maxOutstandingSnoopCheck); } // remember where to route the normal response to @@ -306,8 +340,9 @@ CoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id) assert(routeTo.find(pkt->req) == routeTo.end()); routeTo[pkt->req] = slave_port_id; - panic_if(routeTo.size() > 512, - "Routing table exceeds 512 packets\n"); + panic_if(routeTo.size() > maxRoutingTableSizeCheck, + "%s: Routing table exceeds %d packets\n", + name(), maxRoutingTableSizeCheck); } // update the layer state and schedule an idle event @@ -329,21 +364,77 @@ CoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id) // queue the packet for deletion pendingDelete.reset(pkt); + // normally we respond to the packet we just received if we need to + PacketPtr rsp_pkt = pkt; + PortID rsp_port_id = slave_port_id; + + // If this is the destination of the cache clean operation the + // crossbar is responsible for responding. This crossbar will + // respond when the cache clean is complete. A cache clean + // is complete either: + // * direcly, if no cache above had a dirty copy of the block + // as indicated by the satisfied flag of the packet, or + // * when the crossbar has seen both the cache clean request + // (CleanSharedReq, CleanInvalidReq) and the corresponding + // write (WriteClean) which updates the block in the memory + // below. + if (success && + ((pkt->isClean() && pkt->satisfied()) || + pkt->cmd == MemCmd::WriteClean) && + is_destination) { + PacketPtr deferred_rsp = pkt->isWrite() ? nullptr : pkt; + auto cmo_lookup = outstandingCMO.find(pkt->id); + if (cmo_lookup != outstandingCMO.end()) { + // the cache clean request has already reached this xbar + respond_directly = true; + if (pkt->isWrite()) { + rsp_pkt = cmo_lookup->second; + assert(rsp_pkt); + + // determine the destination + const auto route_lookup = routeTo.find(rsp_pkt->req); + assert(route_lookup != routeTo.end()); + rsp_port_id = route_lookup->second; + assert(rsp_port_id != InvalidPortID); + assert(rsp_port_id < respLayers.size()); + // remove the request from the routing table + routeTo.erase(route_lookup); + } + outstandingCMO.erase(cmo_lookup); + } else { + respond_directly = false; + outstandingCMO.emplace(pkt->id, deferred_rsp); + if (!pkt->isWrite()) { + assert(routeTo.find(pkt->req) == routeTo.end()); + routeTo[pkt->req] = slave_port_id; + + panic_if(routeTo.size() > maxRoutingTableSizeCheck, + "%s: Routing table exceeds %d packets\n", + name(), maxRoutingTableSizeCheck); + } + } + } + + if (respond_directly) { - assert(pkt->needsResponse()); + assert(rsp_pkt->needsResponse()); assert(success); - pkt->makeResponse(); + rsp_pkt->makeResponse(); if (snoopFilter && !system->bypassCaches()) { // let the snoop filter inspect the response and update its state - snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); + snoopFilter->updateResponse(rsp_pkt, *slavePorts[rsp_port_id]); } + // we send the response after the current packet, even if the + // response is not for this packet (e.g. cache clean operation + // where both the request and the write packet have to cross + // the destination xbar before the response is sent.) Tick response_time = clockEdge() + pkt->headerDelay; - pkt->headerDelay = 0; + rsp_pkt->headerDelay = 0; - slavePorts[slave_port_id]->schedTimingResp(pkt, response_time); + slavePorts[rsp_port_id]->schedTimingResp(rsp_pkt, response_time); } return success; @@ -468,7 +559,7 @@ CoherentXBar::recvTimingSnoopReq(PacketPtr pkt, PortID master_port_id) // device responsible for the address range something is // wrong, hence there is nothing further to do as the packet // would be going back to where it came from - assert(master_port_id == findPort(pkt->getAddr())); + assert(findPort(pkt->getAddrRange()) == master_port_id); } bool @@ -635,7 +726,8 @@ CoherentXBar::recvReqRetry(PortID master_port_id) } Tick -CoherentXBar::recvAtomic(PacketPtr pkt, PortID slave_port_id) +CoherentXBar::recvAtomicBackdoor(PacketPtr pkt, PortID slave_port_id, + MemBackdoorPtr *backdoor) { DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, slavePorts[slave_port_id]->name(), pkt->print()); @@ -646,6 +738,12 @@ CoherentXBar::recvAtomic(PacketPtr pkt, PortID slave_port_id) MemCmd snoop_response_cmd = MemCmd::InvalidCmd; Tick snoop_response_latency = 0; + // is this the destination point for this packet? (e.g. true if + // this xbar is the PoC for a cache maintenance operation to the + // PoC) otherwise the destination is any cache that can satisfy + // the request + const bool is_destination = isDestination(pkt); + const bool snoop_caches = !system->bypassCaches() && pkt->cmd != MemCmd::WriteClean; if (snoop_caches) { @@ -692,15 +790,25 @@ CoherentXBar::recvAtomic(PacketPtr pkt, PortID slave_port_id) // even if we had a snoop response, we must continue and also // perform the actual request at the destination - PortID master_port_id = findPort(pkt->getAddr()); + PortID master_port_id = findPort(pkt->getAddrRange()); if (sink_packet) { DPRINTF(CoherentXBar, "%s: Not forwarding %s\n", __func__, pkt->print()); } else { - if (!pointOfCoherency || pkt->isRead() || pkt->isWrite()) { + if (forwardPacket(pkt)) { + // make sure that the write request (e.g., WriteClean) + // will stop at the memory below if this crossbar is its + // destination + if (pkt->isWrite() && is_destination) { + pkt->clearWriteThrough(); + } + // forward the request to the appropriate destination - response_latency = masterPorts[master_port_id]->sendAtomic(pkt); + auto master = masterPorts[master_port_id]; + response_latency = backdoor ? + master->sendAtomicBackdoor(pkt, *backdoor) : + master->sendAtomic(pkt); } else { // if it does not need a response we sink the packet above assert(pkt->needsResponse()); @@ -728,6 +836,30 @@ CoherentXBar::recvAtomic(PacketPtr pkt, PortID slave_port_id) response_latency = snoop_response_latency; } + // If this is the destination of the cache clean operation the + // crossbar is responsible for responding. This crossbar will + // respond when the cache clean is complete. An atomic cache clean + // is complete when the crossbars receives the cache clean + // request (CleanSharedReq, CleanInvalidReq), as either: + // * no cache above had a dirty copy of the block as indicated by + // the satisfied flag of the packet, or + // * the crossbar has already seen the corresponding write + // (WriteClean) which updates the block in the memory below. + if (pkt->isClean() && isDestination(pkt) && pkt->satisfied()) { + auto it = outstandingCMO.find(pkt->id); + assert(it != outstandingCMO.end()); + // we are responding right away + outstandingCMO.erase(it); + } else if (pkt->cmd == MemCmd::WriteClean && isDestination(pkt)) { + // if this is the destination of the operation, the xbar + // sends the responce to the cache clean operation only + // after having encountered the cache clean request + auto M5_VAR_USED ret = outstandingCMO.emplace(pkt->id, nullptr); + // in atomic mode we know that the WriteClean packet should + // precede the clean request + assert(ret.second); + } + // add the response data if (pkt->isResponse()) { pkt_size = pkt->hasData() ? pkt->getSize() : 0; @@ -878,14 +1010,14 @@ CoherentXBar::recvFunctional(PacketPtr pkt, PortID slave_port_id) // if we find a response that has the data, then the // downstream caches/memories may be out of date, so simply stop // here - if (p->checkFunctional(pkt)) { + if (p->trySatisfyFunctional(pkt)) { if (pkt->needsResponse()) pkt->makeResponse(); return; } } - PortID dest_id = findPort(pkt->getAddr()); + PortID dest_id = findPort(pkt->getAddrRange()); masterPorts[dest_id]->sendFunctional(pkt); } @@ -901,7 +1033,7 @@ CoherentXBar::recvFunctionalSnoop(PacketPtr pkt, PortID master_port_id) } for (const auto& p : slavePorts) { - if (p->checkFunctional(pkt)) { + if (p->trySatisfyFunctional(pkt)) { if (pkt->needsResponse()) pkt->makeResponse(); return; @@ -958,33 +1090,27 @@ CoherentXBar::sinkPacket(const PacketPtr pkt) const (!pkt->needsWritable() || pkt->responderHadWritable())); } +bool +CoherentXBar::forwardPacket(const PacketPtr pkt) +{ + // we are forwarding the packet if: + // 1) this is a cache clean request to the PoU/PoC and this + // crossbar is above the PoU/PoC + // 2) this is a read or a write + // 3) this crossbar is above the point of coherency + if (pkt->isClean()) { + return !isDestination(pkt); + } + return pkt->isRead() || pkt->isWrite() || !pointOfCoherency; +} + + void CoherentXBar::regStats() { - // register the stats of the base class and our layers BaseXBar::regStats(); - for (auto l: reqLayers) - l->regStats(); - for (auto l: respLayers) - l->regStats(); - for (auto l: snoopLayers) - l->regStats(); - - snoops - .name(name() + ".snoops") - .desc("Total snoops (count)") - ; - - snoopTraffic - .name(name() + ".snoopTraffic") - .desc("Total snoop traffic (bytes)") - ; - - snoopFanout - .init(0, snoopPorts.size(), 1) - .name(name() + ".snoop_fanout") - .desc("Request fanout histogram") - ; + + snoopFanout.init(0, snoopPorts.size(), 1); } CoherentXBar *