2 * Copyright (c) 2011-2014 ARM Limited
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
14 * Copyright (c) 2006 The Regents of The University of Michigan
15 * All rights reserved.
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions are
19 * met: redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer;
21 * redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution;
24 * neither the name of the copyright holders nor the names of its
25 * contributors may be used to endorse or promote products derived from
26 * this software without specific prior written permission.
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
47 * Definition of a crossbar object.
50 #include "base/misc.hh"
51 #include "base/trace.hh"
52 #include "debug/AddrRanges.hh"
53 #include "debug/CoherentXBar.hh"
54 #include "mem/coherent_xbar.hh"
55 #include "sim/system.hh"
57 CoherentXBar::CoherentXBar(const CoherentXBarParams
*p
)
58 : BaseXBar(p
), system(p
->system
), snoopFilter(p
->snoop_filter
)
60 // create the ports based on the size of the master and slave
61 // vector ports, and the presence of the default port, the ports
62 // are enumerated starting from zero
63 for (int i
= 0; i
< p
->port_master_connection_count
; ++i
) {
64 std::string portName
= csprintf("%s.master[%d]", name(), i
);
65 MasterPort
* bp
= new CoherentXBarMasterPort(portName
, *this, i
);
66 masterPorts
.push_back(bp
);
67 reqLayers
.push_back(new ReqLayer(*bp
, *this,
68 csprintf(".reqLayer%d", i
)));
69 snoopLayers
.push_back(new SnoopRespLayer(*bp
, *this,
70 csprintf(".snoopLayer%d", i
)));
73 // see if we have a default slave device connected and if so add
74 // our corresponding master port
75 if (p
->port_default_connection_count
) {
76 defaultPortID
= masterPorts
.size();
77 std::string portName
= name() + ".default";
78 MasterPort
* bp
= new CoherentXBarMasterPort(portName
, *this,
80 masterPorts
.push_back(bp
);
81 reqLayers
.push_back(new ReqLayer(*bp
, *this, csprintf(".reqLayer%d",
83 snoopLayers
.push_back(new SnoopRespLayer(*bp
, *this,
84 csprintf(".snoopLayer%d",
88 // create the slave ports, once again starting at zero
89 for (int i
= 0; i
< p
->port_slave_connection_count
; ++i
) {
90 std::string portName
= csprintf("%s.slave[%d]", name(), i
);
91 SlavePort
* bp
= new CoherentXBarSlavePort(portName
, *this, i
);
92 slavePorts
.push_back(bp
);
93 respLayers
.push_back(new RespLayer(*bp
, *this,
94 csprintf(".respLayer%d", i
)));
95 snoopRespPorts
.push_back(new SnoopRespPort(*bp
, *this));
99 snoopFilter
->setSlavePorts(slavePorts
);
104 CoherentXBar::~CoherentXBar()
106 for (auto l
: reqLayers
)
108 for (auto l
: respLayers
)
110 for (auto l
: snoopLayers
)
112 for (auto p
: snoopRespPorts
)
119 // the base class is responsible for determining the block size
122 // iterate over our slave ports and determine which of our
123 // neighbouring master ports are snooping and add them as snoopers
124 for (const auto& p
: slavePorts
) {
125 // check if the connected master port is snooping
126 if (p
->isSnooping()) {
127 DPRINTF(AddrRanges
, "Adding snooping master %s\n",
128 p
->getMasterPort().name());
129 snoopPorts
.push_back(p
);
133 if (snoopPorts
.empty())
134 warn("CoherentXBar %s has no snooping ports attached!\n", name());
138 CoherentXBar::recvTimingReq(PacketPtr pkt
, PortID slave_port_id
)
140 // determine the source port based on the id
141 SlavePort
*src_port
= slavePorts
[slave_port_id
];
143 // remember if the packet is an express snoop
144 bool is_express_snoop
= pkt
->isExpressSnoop();
145 bool is_inhibited
= pkt
->memInhibitAsserted();
146 // for normal requests, going downstream, the express snoop flag
147 // and the inhibited flag should always be the same
148 assert(is_express_snoop
== is_inhibited
);
150 // determine the destination based on the address
151 PortID master_port_id
= findPort(pkt
->getAddr());
153 // test if the crossbar should be considered occupied for the current
154 // port, and exclude express snoops from the check
155 if (!is_express_snoop
&& !reqLayers
[master_port_id
]->tryTiming(src_port
)) {
156 DPRINTF(CoherentXBar
, "recvTimingReq: src %s %s 0x%x BUSY\n",
157 src_port
->name(), pkt
->cmdString(), pkt
->getAddr());
161 DPRINTF(CoherentXBar
, "recvTimingReq: src %s %s expr %d 0x%x\n",
162 src_port
->name(), pkt
->cmdString(), is_express_snoop
,
165 // store size and command as they might be modified when
166 // forwarding the packet
167 unsigned int pkt_size
= pkt
->hasData() ? pkt
->getSize() : 0;
168 unsigned int pkt_cmd
= pkt
->cmdToIndex();
170 calcPacketTiming(pkt
);
171 Tick packetFinishTime
= curTick() + pkt
->payloadDelay
;
173 // uncacheable requests need never be snooped
174 if (!pkt
->req
->isUncacheable() && !system
->bypassCaches()) {
175 // the packet is a memory-mapped request and should be
176 // broadcasted to our snoopers but the source
178 // check with the snoop filter where to forward this packet
179 auto sf_res
= snoopFilter
->lookupRequest(pkt
, *src_port
);
180 packetFinishTime
+= sf_res
.second
* clockPeriod();
181 DPRINTF(CoherentXBar
, "recvTimingReq: src %s %s 0x%x"\
182 " SF size: %i lat: %i\n", src_port
->name(),
183 pkt
->cmdString(), pkt
->getAddr(), sf_res
.first
.size(),
185 forwardTiming(pkt
, slave_port_id
, sf_res
.first
);
187 forwardTiming(pkt
, slave_port_id
);
191 // remember if the packet will generate a snoop response
192 const bool expect_snoop_resp
= !is_inhibited
&& pkt
->memInhibitAsserted();
193 const bool expect_response
= pkt
->needsResponse() &&
194 !pkt
->memInhibitAsserted();
196 // Note: Cannot create a copy of the full packet, here.
197 MemCmd
orig_cmd(pkt
->cmd
);
199 // since it is a normal request, attempt to send the packet
200 bool success
= masterPorts
[master_port_id
]->sendTimingReq(pkt
);
202 if (snoopFilter
&& !pkt
->req
->isUncacheable()
203 && !system
->bypassCaches()) {
204 // The packet may already be overwritten by the sendTimingReq function.
205 // The snoop filter needs to see the original request *and* the return
206 // status of the send operation, so we need to recreate the original
207 // request. Atomic mode does not have the issue, as there the send
208 // operation and the response happen instantaneously and don't need two
210 MemCmd
tmp_cmd(pkt
->cmd
);
212 // Let the snoop filter know about the success of the send operation
213 snoopFilter
->updateRequest(pkt
, *src_port
, !success
);
217 // check if we were successful in sending the packet onwards
219 // express snoops and inhibited packets should never be forced
221 assert(!is_express_snoop
);
222 assert(!pkt
->memInhibitAsserted());
224 // undo the calculation so we can check for 0 again
225 pkt
->headerDelay
= pkt
->payloadDelay
= 0;
227 DPRINTF(CoherentXBar
, "recvTimingReq: src %s %s 0x%x RETRY\n",
228 src_port
->name(), pkt
->cmdString(), pkt
->getAddr());
230 // update the layer state and schedule an idle event
231 reqLayers
[master_port_id
]->failedTiming(src_port
,
232 clockEdge(headerCycles
));
234 // express snoops currently bypass the crossbar state entirely
235 if (!is_express_snoop
) {
236 // if this particular request will generate a snoop
238 if (expect_snoop_resp
) {
239 // we should never have an exsiting request outstanding
240 assert(outstandingSnoop
.find(pkt
->req
) ==
241 outstandingSnoop
.end());
242 outstandingSnoop
.insert(pkt
->req
);
244 // basic sanity check on the outstanding snoops
245 panic_if(outstandingSnoop
.size() > 512,
246 "Outstanding snoop requests exceeded 512\n");
249 // remember where to route the normal response to
250 if (expect_response
|| expect_snoop_resp
) {
251 assert(routeTo
.find(pkt
->req
) == routeTo
.end());
252 routeTo
[pkt
->req
] = slave_port_id
;
254 panic_if(routeTo
.size() > 512,
255 "Routing table exceeds 512 packets\n");
258 // update the layer state and schedule an idle event
259 reqLayers
[master_port_id
]->succeededTiming(packetFinishTime
);
262 // stats updates only consider packets that were successfully sent
263 pktCount
[slave_port_id
][master_port_id
]++;
264 pktSize
[slave_port_id
][master_port_id
] += pkt_size
;
265 transDist
[pkt_cmd
]++;
267 if (is_express_snoop
)
275 CoherentXBar::recvTimingResp(PacketPtr pkt
, PortID master_port_id
)
277 // determine the source port based on the id
278 MasterPort
*src_port
= masterPorts
[master_port_id
];
280 // determine the destination
281 const auto route_lookup
= routeTo
.find(pkt
->req
);
282 assert(route_lookup
!= routeTo
.end());
283 const PortID slave_port_id
= route_lookup
->second
;
284 assert(slave_port_id
!= InvalidPortID
);
285 assert(slave_port_id
< respLayers
.size());
287 // test if the crossbar should be considered occupied for the
289 if (!respLayers
[slave_port_id
]->tryTiming(src_port
)) {
290 DPRINTF(CoherentXBar
, "recvTimingResp: src %s %s 0x%x BUSY\n",
291 src_port
->name(), pkt
->cmdString(), pkt
->getAddr());
295 DPRINTF(CoherentXBar
, "recvTimingResp: src %s %s 0x%x\n",
296 src_port
->name(), pkt
->cmdString(), pkt
->getAddr());
298 // store size and command as they might be modified when
299 // forwarding the packet
300 unsigned int pkt_size
= pkt
->hasData() ? pkt
->getSize() : 0;
301 unsigned int pkt_cmd
= pkt
->cmdToIndex();
303 calcPacketTiming(pkt
);
304 Tick packetFinishTime
= curTick() + pkt
->payloadDelay
;
306 if (snoopFilter
&& !pkt
->req
->isUncacheable() && !system
->bypassCaches()) {
307 // let the snoop filter inspect the response and update its state
308 snoopFilter
->updateResponse(pkt
, *slavePorts
[slave_port_id
]);
311 // send the packet through the destination slave port
312 bool success M5_VAR_USED
= slavePorts
[slave_port_id
]->sendTimingResp(pkt
);
314 // currently it is illegal to block responses... can lead to
318 // remove the request from the routing table
319 routeTo
.erase(route_lookup
);
321 respLayers
[slave_port_id
]->succeededTiming(packetFinishTime
);
324 pktCount
[slave_port_id
][master_port_id
]++;
325 pktSize
[slave_port_id
][master_port_id
] += pkt_size
;
326 transDist
[pkt_cmd
]++;
332 CoherentXBar::recvTimingSnoopReq(PacketPtr pkt
, PortID master_port_id
)
334 DPRINTF(CoherentXBar
, "recvTimingSnoopReq: src %s %s 0x%x\n",
335 masterPorts
[master_port_id
]->name(), pkt
->cmdString(),
338 // update stats here as we know the forwarding will succeed
339 transDist
[pkt
->cmdToIndex()]++;
342 // we should only see express snoops from caches
343 assert(pkt
->isExpressSnoop());
345 // remeber if the packet is inhibited so we can see if it changes
346 const bool is_inhibited
= pkt
->memInhibitAsserted();
349 // let the Snoop Filter work its magic and guide probing
350 auto sf_res
= snoopFilter
->lookupSnoop(pkt
);
351 // No timing here: packetFinishTime += sf_res.second * clockPeriod();
352 DPRINTF(CoherentXBar
, "recvTimingSnoopReq: src %s %s 0x%x"\
353 " SF size: %i lat: %i\n", masterPorts
[master_port_id
]->name(),
354 pkt
->cmdString(), pkt
->getAddr(), sf_res
.first
.size(),
357 // forward to all snoopers
358 forwardTiming(pkt
, InvalidPortID
, sf_res
.first
);
360 forwardTiming(pkt
, InvalidPortID
);
363 // if we can expect a response, remember how to route it
364 if (!is_inhibited
&& pkt
->memInhibitAsserted()) {
365 assert(routeTo
.find(pkt
->req
) == routeTo
.end());
366 routeTo
[pkt
->req
] = master_port_id
;
369 // a snoop request came from a connected slave device (one of
370 // our master ports), and if it is not coming from the slave
371 // device responsible for the address range something is
372 // wrong, hence there is nothing further to do as the packet
373 // would be going back to where it came from
374 assert(master_port_id
== findPort(pkt
->getAddr()));
378 CoherentXBar::recvTimingSnoopResp(PacketPtr pkt
, PortID slave_port_id
)
380 // determine the source port based on the id
381 SlavePort
* src_port
= slavePorts
[slave_port_id
];
383 // get the destination
384 const auto route_lookup
= routeTo
.find(pkt
->req
);
385 assert(route_lookup
!= routeTo
.end());
386 const PortID dest_port_id
= route_lookup
->second
;
387 assert(dest_port_id
!= InvalidPortID
);
389 // determine if the response is from a snoop request we
390 // created as the result of a normal request (in which case it
391 // should be in the outstandingSnoop), or if we merely forwarded
392 // someone else's snoop request
393 const bool forwardAsSnoop
= outstandingSnoop
.find(pkt
->req
) ==
394 outstandingSnoop
.end();
396 // test if the crossbar should be considered occupied for the
397 // current port, note that the check is bypassed if the response
398 // is being passed on as a normal response since this is occupying
399 // the response layer rather than the snoop response layer
400 if (forwardAsSnoop
) {
401 assert(dest_port_id
< snoopLayers
.size());
402 if (!snoopLayers
[dest_port_id
]->tryTiming(src_port
)) {
403 DPRINTF(CoherentXBar
, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n",
404 src_port
->name(), pkt
->cmdString(), pkt
->getAddr());
408 // get the master port that mirrors this slave port internally
409 MasterPort
* snoop_port
= snoopRespPorts
[slave_port_id
];
410 assert(dest_port_id
< respLayers
.size());
411 if (!respLayers
[dest_port_id
]->tryTiming(snoop_port
)) {
412 DPRINTF(CoherentXBar
, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n",
413 snoop_port
->name(), pkt
->cmdString(), pkt
->getAddr());
418 DPRINTF(CoherentXBar
, "recvTimingSnoopResp: src %s %s 0x%x\n",
419 src_port
->name(), pkt
->cmdString(), pkt
->getAddr());
421 // store size and command as they might be modified when
422 // forwarding the packet
423 unsigned int pkt_size
= pkt
->hasData() ? pkt
->getSize() : 0;
424 unsigned int pkt_cmd
= pkt
->cmdToIndex();
426 // responses are never express snoops
427 assert(!pkt
->isExpressSnoop());
429 calcPacketTiming(pkt
);
430 Tick packetFinishTime
= curTick() + pkt
->payloadDelay
;
432 // forward it either as a snoop response or a normal response
433 if (forwardAsSnoop
) {
434 // this is a snoop response to a snoop request we forwarded,
435 // e.g. coming from the L1 and going to the L2, and it should
436 // be forwarded as a snoop response
439 // update the probe filter so that it can properly track the line
440 snoopFilter
->updateSnoopForward(pkt
, *slavePorts
[slave_port_id
],
441 *masterPorts
[dest_port_id
]);
444 bool success M5_VAR_USED
=
445 masterPorts
[dest_port_id
]->sendTimingSnoopResp(pkt
);
446 pktCount
[slave_port_id
][dest_port_id
]++;
447 pktSize
[slave_port_id
][dest_port_id
] += pkt_size
;
450 snoopLayers
[dest_port_id
]->succeededTiming(packetFinishTime
);
452 // we got a snoop response on one of our slave ports,
453 // i.e. from a coherent master connected to the crossbar, and
454 // since we created the snoop request as part of recvTiming,
455 // this should now be a normal response again
456 outstandingSnoop
.erase(pkt
->req
);
458 // this is a snoop response from a coherent master, hence it
459 // should never go back to where the snoop response came from,
460 // but instead to where the original request came from
461 assert(slave_port_id
!= dest_port_id
);
464 // update the probe filter so that it can properly track the line
465 snoopFilter
->updateSnoopResponse(pkt
, *slavePorts
[slave_port_id
],
466 *slavePorts
[dest_port_id
]);
469 DPRINTF(CoherentXBar
, "recvTimingSnoopResp: src %s %s 0x%x"\
470 " FWD RESP\n", src_port
->name(), pkt
->cmdString(),
473 // as a normal response, it should go back to a master through
474 // one of our slave ports, at this point we are ignoring the
475 // fact that the response layer could be busy and do not touch
477 bool success M5_VAR_USED
=
478 slavePorts
[dest_port_id
]->sendTimingResp(pkt
);
480 // @todo Put the response in an internal FIFO and pass it on
481 // to the response layer from there
483 // currently it is illegal to block responses... can lead
487 respLayers
[dest_port_id
]->succeededTiming(packetFinishTime
);
490 // remove the request from the routing table
491 routeTo
.erase(route_lookup
);
494 transDist
[pkt_cmd
]++;
502 CoherentXBar::forwardTiming(PacketPtr pkt
, PortID exclude_slave_port_id
,
503 const std::vector
<SlavePort
*>& dests
)
505 DPRINTF(CoherentXBar
, "%s for %s address %x size %d\n", __func__
,
506 pkt
->cmdString(), pkt
->getAddr(), pkt
->getSize());
508 // snoops should only happen if the system isn't bypassing caches
509 assert(!system
->bypassCaches());
513 for (const auto& p
: dests
) {
514 // we could have gotten this request from a snooping master
515 // (corresponding to our own slave port that is also in
516 // snoopPorts) and should not send it back to where it came
518 if (exclude_slave_port_id
== InvalidPortID
||
519 p
->getId() != exclude_slave_port_id
) {
520 // cache is not allowed to refuse snoop
521 p
->sendTimingSnoopReq(pkt
);
526 // Stats for fanout of this forward operation
527 snoopFanout
.sample(fanout
);
531 CoherentXBar::recvReqRetry(PortID master_port_id
)
533 // responses and snoop responses never block on forwarding them,
534 // so the retry will always be coming from a port to which we
535 // tried to forward a request
536 reqLayers
[master_port_id
]->recvRetry();
540 CoherentXBar::recvAtomic(PacketPtr pkt
, PortID slave_port_id
)
542 DPRINTF(CoherentXBar
, "recvAtomic: packet src %s addr 0x%x cmd %s\n",
543 slavePorts
[slave_port_id
]->name(), pkt
->getAddr(),
546 unsigned int pkt_size
= pkt
->hasData() ? pkt
->getSize() : 0;
547 unsigned int pkt_cmd
= pkt
->cmdToIndex();
549 MemCmd snoop_response_cmd
= MemCmd::InvalidCmd
;
550 Tick snoop_response_latency
= 0;
552 // uncacheable requests need never be snooped
553 if (!pkt
->req
->isUncacheable() && !system
->bypassCaches()) {
554 // forward to all snoopers but the source
555 std::pair
<MemCmd
, Tick
> snoop_result
;
557 // check with the snoop filter where to forward this packet
559 snoopFilter
->lookupRequest(pkt
, *slavePorts
[slave_port_id
]);
560 snoop_response_latency
+= sf_res
.second
* clockPeriod();
561 DPRINTF(CoherentXBar
, "%s: src %s %s 0x%x"\
562 " SF size: %i lat: %i\n", __func__
,
563 slavePorts
[slave_port_id
]->name(), pkt
->cmdString(),
564 pkt
->getAddr(), sf_res
.first
.size(), sf_res
.second
);
565 snoop_result
= forwardAtomic(pkt
, slave_port_id
, InvalidPortID
,
568 snoop_result
= forwardAtomic(pkt
, slave_port_id
);
570 snoop_response_cmd
= snoop_result
.first
;
571 snoop_response_latency
+= snoop_result
.second
;
574 // even if we had a snoop response, we must continue and also
575 // perform the actual request at the destination
576 PortID master_port_id
= findPort(pkt
->getAddr());
578 // stats updates for the request
579 pktCount
[slave_port_id
][master_port_id
]++;
580 pktSize
[slave_port_id
][master_port_id
] += pkt_size
;
581 transDist
[pkt_cmd
]++;
583 // forward the request to the appropriate destination
584 Tick response_latency
= masterPorts
[master_port_id
]->sendAtomic(pkt
);
586 // Lower levels have replied, tell the snoop filter
587 if (snoopFilter
&& !pkt
->req
->isUncacheable() && !system
->bypassCaches() &&
589 snoopFilter
->updateResponse(pkt
, *slavePorts
[slave_port_id
]);
592 // if we got a response from a snooper, restore it here
593 if (snoop_response_cmd
!= MemCmd::InvalidCmd
) {
594 // no one else should have responded
595 assert(!pkt
->isResponse());
596 pkt
->cmd
= snoop_response_cmd
;
597 response_latency
= snoop_response_latency
;
600 // add the response data
601 if (pkt
->isResponse()) {
602 pkt_size
= pkt
->hasData() ? pkt
->getSize() : 0;
603 pkt_cmd
= pkt
->cmdToIndex();
606 pktCount
[slave_port_id
][master_port_id
]++;
607 pktSize
[slave_port_id
][master_port_id
] += pkt_size
;
608 transDist
[pkt_cmd
]++;
611 // @todo: Not setting header time
612 pkt
->payloadDelay
= response_latency
;
613 return response_latency
;
617 CoherentXBar::recvAtomicSnoop(PacketPtr pkt
, PortID master_port_id
)
619 DPRINTF(CoherentXBar
, "recvAtomicSnoop: packet src %s addr 0x%x cmd %s\n",
620 masterPorts
[master_port_id
]->name(), pkt
->getAddr(),
623 // add the request snoop data
626 // forward to all snoopers
627 std::pair
<MemCmd
, Tick
> snoop_result
;
628 Tick snoop_response_latency
= 0;
630 auto sf_res
= snoopFilter
->lookupSnoop(pkt
);
631 snoop_response_latency
+= sf_res
.second
* clockPeriod();
632 DPRINTF(CoherentXBar
, "%s: src %s %s 0x%x SF size: %i lat: %i\n",
633 __func__
, masterPorts
[master_port_id
]->name(), pkt
->cmdString(),
634 pkt
->getAddr(), sf_res
.first
.size(), sf_res
.second
);
635 snoop_result
= forwardAtomic(pkt
, InvalidPortID
, master_port_id
,
638 snoop_result
= forwardAtomic(pkt
, InvalidPortID
);
640 MemCmd snoop_response_cmd
= snoop_result
.first
;
641 snoop_response_latency
+= snoop_result
.second
;
643 if (snoop_response_cmd
!= MemCmd::InvalidCmd
)
644 pkt
->cmd
= snoop_response_cmd
;
646 // add the response snoop data
647 if (pkt
->isResponse()) {
651 // @todo: Not setting header time
652 pkt
->payloadDelay
= snoop_response_latency
;
653 return snoop_response_latency
;
656 std::pair
<MemCmd
, Tick
>
657 CoherentXBar::forwardAtomic(PacketPtr pkt
, PortID exclude_slave_port_id
,
658 PortID source_master_port_id
,
659 const std::vector
<SlavePort
*>& dests
)
661 // the packet may be changed on snoops, record the original
662 // command to enable us to restore it between snoops so that
663 // additional snoops can take place properly
664 MemCmd orig_cmd
= pkt
->cmd
;
665 MemCmd snoop_response_cmd
= MemCmd::InvalidCmd
;
666 Tick snoop_response_latency
= 0;
668 // snoops should only happen if the system isn't bypassing caches
669 assert(!system
->bypassCaches());
673 for (const auto& p
: dests
) {
674 // we could have gotten this request from a snooping master
675 // (corresponding to our own slave port that is also in
676 // snoopPorts) and should not send it back to where it came
678 if (exclude_slave_port_id
!= InvalidPortID
&&
679 p
->getId() == exclude_slave_port_id
)
682 Tick latency
= p
->sendAtomicSnoop(pkt
);
685 // in contrast to a functional access, we have to keep on
686 // going as all snoopers must be updated even if we get a
688 if (!pkt
->isResponse())
691 // response from snoop agent
692 assert(pkt
->cmd
!= orig_cmd
);
693 assert(pkt
->memInhibitAsserted());
694 // should only happen once
695 assert(snoop_response_cmd
== MemCmd::InvalidCmd
);
696 // save response state
697 snoop_response_cmd
= pkt
->cmd
;
698 snoop_response_latency
= latency
;
701 // Handle responses by the snoopers and differentiate between
702 // responses to requests from above and snoops from below
703 if (source_master_port_id
!= InvalidPortID
) {
704 // Getting a response for a snoop from below
705 assert(exclude_slave_port_id
== InvalidPortID
);
706 snoopFilter
->updateSnoopForward(pkt
, *p
,
707 *masterPorts
[source_master_port_id
]);
709 // Getting a response for a request from above
710 assert(source_master_port_id
== InvalidPortID
);
711 snoopFilter
->updateSnoopResponse(pkt
, *p
,
712 *slavePorts
[exclude_slave_port_id
]);
715 // restore original packet state for remaining snoopers
720 snoopFanout
.sample(fanout
);
722 // the packet is restored as part of the loop and any potential
723 // snoop response is part of the returned pair
724 return std::make_pair(snoop_response_cmd
, snoop_response_latency
);
728 CoherentXBar::recvFunctional(PacketPtr pkt
, PortID slave_port_id
)
730 if (!pkt
->isPrint()) {
731 // don't do DPRINTFs on PrintReq as it clutters up the output
732 DPRINTF(CoherentXBar
,
733 "recvFunctional: packet src %s addr 0x%x cmd %s\n",
734 slavePorts
[slave_port_id
]->name(), pkt
->getAddr(),
738 // uncacheable requests need never be snooped
739 if (!pkt
->req
->isUncacheable() && !system
->bypassCaches()) {
740 // forward to all snoopers but the source
741 forwardFunctional(pkt
, slave_port_id
);
744 // there is no need to continue if the snooping has found what we
745 // were looking for and the packet is already a response
746 if (!pkt
->isResponse()) {
747 PortID dest_id
= findPort(pkt
->getAddr());
749 masterPorts
[dest_id
]->sendFunctional(pkt
);
754 CoherentXBar::recvFunctionalSnoop(PacketPtr pkt
, PortID master_port_id
)
756 if (!pkt
->isPrint()) {
757 // don't do DPRINTFs on PrintReq as it clutters up the output
758 DPRINTF(CoherentXBar
,
759 "recvFunctionalSnoop: packet src %s addr 0x%x cmd %s\n",
760 masterPorts
[master_port_id
]->name(), pkt
->getAddr(),
764 // forward to all snoopers
765 forwardFunctional(pkt
, InvalidPortID
);
769 CoherentXBar::forwardFunctional(PacketPtr pkt
, PortID exclude_slave_port_id
)
771 // snoops should only happen if the system isn't bypassing caches
772 assert(!system
->bypassCaches());
774 for (const auto& p
: snoopPorts
) {
775 // we could have gotten this request from a snooping master
776 // (corresponding to our own slave port that is also in
777 // snoopPorts) and should not send it back to where it came
779 if (exclude_slave_port_id
== InvalidPortID
||
780 p
->getId() != exclude_slave_port_id
)
781 p
->sendFunctionalSnoop(pkt
);
783 // if we get a response we are done
784 if (pkt
->isResponse()) {
791 CoherentXBar::drain(DrainManager
*dm
)
793 // sum up the individual layers
794 unsigned int total
= 0;
795 for (auto l
: reqLayers
)
796 total
+= l
->drain(dm
);
797 for (auto l
: respLayers
)
798 total
+= l
->drain(dm
);
799 for (auto l
: snoopLayers
)
800 total
+= l
->drain(dm
);
805 CoherentXBar::regStats()
807 // register the stats of the base class and our layers
808 BaseXBar::regStats();
809 for (auto l
: reqLayers
)
811 for (auto l
: respLayers
)
813 for (auto l
: snoopLayers
)
817 .name(name() + ".snoops")
818 .desc("Total snoops (count)")
822 .init(0, snoopPorts
.size(), 1)
823 .name(name() + ".snoop_fanout")
824 .desc("Request fanout histogram")
829 CoherentXBarParams::create()
831 return new CoherentXBar(this);