2 * Copyright (c) 2013-2017,2019 ARM Limited
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 * Implementation of a snoop filter.
43 #include "mem/snoop_filter.hh"
45 #include "base/logging.hh"
46 #include "base/trace.hh"
47 #include "debug/SnoopFilter.hh"
48 #include "sim/system.hh"
50 const int SnoopFilter::SNOOP_MASK_SIZE
;
53 SnoopFilter::eraseIfNullEntry(SnoopFilterCache::iterator
& sf_it
)
55 SnoopItem
& sf_item
= sf_it
->second
;
56 if ((sf_item
.requested
| sf_item
.holder
).none()) {
57 cachedLocations
.erase(sf_it
);
58 DPRINTF(SnoopFilter
, "%s: Removed SF entry.\n",
63 std::pair
<SnoopFilter::SnoopList
, Cycles
>
64 SnoopFilter::lookupRequest(const Packet
* cpkt
, const SlavePort
& slave_port
)
66 DPRINTF(SnoopFilter
, "%s: src %s packet %s\n", __func__
,
67 slave_port
.name(), cpkt
->print());
69 // check if the packet came from a cache
70 bool allocate
= !cpkt
->req
->isUncacheable() && slave_port
.isSnooping() &&
72 Addr line_addr
= cpkt
->getBlockAddr(linesize
);
73 if (cpkt
->isSecure()) {
74 line_addr
|= LineSecure
;
76 SnoopMask req_port
= portToMask(slave_port
);
77 reqLookupResult
.it
= cachedLocations
.find(line_addr
);
78 bool is_hit
= (reqLookupResult
.it
!= cachedLocations
.end());
80 // If the snoop filter has no entry, and we should not allocate,
81 // do not create a new snoop filter entry, simply return a NULL
83 if (!is_hit
&& !allocate
)
84 return snoopDown(lookupLatency
);
86 // If no hit in snoop filter create a new element and update iterator
89 cachedLocations
.emplace(line_addr
, SnoopItem()).first
;
91 SnoopItem
& sf_item
= reqLookupResult
.it
->second
;
92 SnoopMask interested
= sf_item
.holder
| sf_item
.requested
;
94 // Store unmodified value of snoop filter item in temp storage in
95 // case we need to revert because of a send retry in
97 reqLookupResult
.retryItem
= sf_item
;
101 if (interested
.count() == 1)
107 DPRINTF(SnoopFilter
, "%s: SF value %x.%x\n",
108 __func__
, sf_item
.requested
, sf_item
.holder
);
110 // If we are not allocating, we are done
112 return snoopSelected(maskToPortList(interested
& ~req_port
),
115 if (cpkt
->needsResponse()) {
116 if (!cpkt
->cacheResponding()) {
117 // Max one request per address per port
118 panic_if((sf_item
.requested
& req_port
).any(),
119 "double request :( SF value %x.%x\n",
120 sf_item
.requested
, sf_item
.holder
);
122 // Mark in-flight requests to distinguish later on
123 sf_item
.requested
|= req_port
;
124 DPRINTF(SnoopFilter
, "%s: new SF value %x.%x\n",
125 __func__
, sf_item
.requested
, sf_item
.holder
);
127 // NOTE: The memInhibit might have been asserted by a cache closer
128 // to the CPU, already -> the response will not be seen by this
129 // filter -> we do not need to keep the in-flight request, but make
130 // sure that we know that that cluster has a copy
131 panic_if((sf_item
.holder
& req_port
).none(),
132 "Need to hold the value!");
134 "%s: not marking request. SF value %x.%x\n",
135 __func__
, sf_item
.requested
, sf_item
.holder
);
137 } else { // if (!cpkt->needsResponse())
138 assert(cpkt
->isEviction());
139 // make sure that the sender actually had the line
140 panic_if((sf_item
.holder
& req_port
).none(), "requester %x is not a " \
141 "holder :( SF value %x.%x\n", req_port
,
142 sf_item
.requested
, sf_item
.holder
);
143 // CleanEvicts and Writebacks -> the sender and all caches above
144 // it may not have the line anymore.
145 if (!cpkt
->isBlockCached()) {
146 sf_item
.holder
&= ~req_port
;
147 DPRINTF(SnoopFilter
, "%s: new SF value %x.%x\n",
148 __func__
, sf_item
.requested
, sf_item
.holder
);
152 return snoopSelected(maskToPortList(interested
& ~req_port
), lookupLatency
);
156 SnoopFilter::finishRequest(bool will_retry
, Addr addr
, bool is_secure
)
158 if (reqLookupResult
.it
!= cachedLocations
.end()) {
159 // since we rely on the caller, do a basic check to ensure
160 // that finishRequest is being called following lookupRequest
161 Addr line_addr
= (addr
& ~(Addr(linesize
- 1)));
163 line_addr
|= LineSecure
;
165 assert(reqLookupResult
.it
->first
== line_addr
);
167 SnoopItem retry_item
= reqLookupResult
.retryItem
;
168 // Undo any changes made in lookupRequest to the snoop filter
169 // entry if the request will come again. retryItem holds
170 // the previous value of the snoopfilter entry.
171 reqLookupResult
.it
->second
= retry_item
;
173 DPRINTF(SnoopFilter
, "%s: restored SF value %x.%x\n",
174 __func__
, retry_item
.requested
, retry_item
.holder
);
177 eraseIfNullEntry(reqLookupResult
.it
);
181 std::pair
<SnoopFilter::SnoopList
, Cycles
>
182 SnoopFilter::lookupSnoop(const Packet
* cpkt
)
184 DPRINTF(SnoopFilter
, "%s: packet %s\n", __func__
, cpkt
->print());
186 assert(cpkt
->isRequest());
188 Addr line_addr
= cpkt
->getBlockAddr(linesize
);
189 if (cpkt
->isSecure()) {
190 line_addr
|= LineSecure
;
192 auto sf_it
= cachedLocations
.find(line_addr
);
193 bool is_hit
= (sf_it
!= cachedLocations
.end());
195 panic_if(!is_hit
&& (cachedLocations
.size() >= maxEntryCount
),
196 "snoop filter exceeded capacity of %d cache blocks\n",
199 // If the snoop filter has no entry, simply return a NULL
200 // portlist, there is no point creating an entry only to remove it
203 return snoopDown(lookupLatency
);
205 SnoopItem
& sf_item
= sf_it
->second
;
207 SnoopMask interested
= (sf_item
.holder
| sf_item
.requested
);
211 if (interested
.count() == 1)
216 // ReadEx and Writes require both invalidation and exlusivity, while reads
217 // require neither. Writebacks on the other hand require exclusivity but
218 // not the invalidation. Previously Writebacks did not generate upward
219 // snoops so this was never an issue. Now that Writebacks generate snoops
220 // we need a special case for Writebacks. Additionally cache maintenance
221 // operations can generate snoops as they clean and/or invalidate all
222 // caches down to the specified point of reference.
223 assert(cpkt
->isWriteback() || cpkt
->req
->isUncacheable() ||
224 (cpkt
->isInvalidate() == cpkt
->needsWritable()) ||
225 cpkt
->req
->isCacheMaintenance());
226 if (cpkt
->isInvalidate() && sf_item
.requested
.none()) {
227 // Early clear of the holder, if no other request is currently going on
228 // @todo: This should possibly be updated even though we do not filter
230 DPRINTF(SnoopFilter
, "%s: old SF value %x.%x\n",
231 __func__
, sf_item
.requested
, sf_item
.holder
);
233 DPRINTF(SnoopFilter
, "%s: new SF value %x.%x\n",
234 __func__
, sf_item
.requested
, sf_item
.holder
);
235 eraseIfNullEntry(sf_it
);
238 return snoopSelected(maskToPortList(interested
), lookupLatency
);
242 SnoopFilter::updateSnoopResponse(const Packet
* cpkt
,
243 const SlavePort
& rsp_port
,
244 const SlavePort
& req_port
)
246 DPRINTF(SnoopFilter
, "%s: rsp %s req %s packet %s\n",
247 __func__
, rsp_port
.name(), req_port
.name(), cpkt
->print());
249 assert(cpkt
->isResponse());
250 assert(cpkt
->cacheResponding());
252 // if this snoop response is due to an uncacheable request, or is
253 // being turned into a normal response, there is nothing more to
255 if (cpkt
->req
->isUncacheable() || !req_port
.isSnooping()) {
259 Addr line_addr
= cpkt
->getBlockAddr(linesize
);
260 if (cpkt
->isSecure()) {
261 line_addr
|= LineSecure
;
263 SnoopMask rsp_mask
= portToMask(rsp_port
);
264 SnoopMask req_mask
= portToMask(req_port
);
265 SnoopItem
& sf_item
= cachedLocations
[line_addr
];
267 DPRINTF(SnoopFilter
, "%s: old SF value %x.%x\n",
268 __func__
, sf_item
.requested
, sf_item
.holder
);
270 // The source should have the line
271 panic_if((sf_item
.holder
& rsp_mask
).none(),
272 "SF value %x.%x does not have the line\n",
273 sf_item
.requested
, sf_item
.holder
);
275 // The destination should have had a request in
276 panic_if((sf_item
.requested
& req_mask
).none(), "SF value %x.%x missing "\
277 "the original request\n", sf_item
.requested
, sf_item
.holder
);
279 // If the snoop response has no sharers the line is passed in
280 // Modified state, and we know that there are no other copies, or
281 // they will all be invalidated imminently
282 if (!cpkt
->hasSharers()) {
284 "%s: dropping %x because non-shared snoop "
285 "response SF val: %x.%x\n", __func__
, rsp_mask
,
286 sf_item
.requested
, sf_item
.holder
);
289 assert(!cpkt
->isWriteback());
290 // @todo Deal with invalidating responses
291 sf_item
.holder
|= req_mask
;
292 sf_item
.requested
&= ~req_mask
;
293 assert((sf_item
.requested
| sf_item
.holder
).any());
294 DPRINTF(SnoopFilter
, "%s: new SF value %x.%x\n",
295 __func__
, sf_item
.requested
, sf_item
.holder
);
299 SnoopFilter::updateSnoopForward(const Packet
* cpkt
,
300 const SlavePort
& rsp_port
, const MasterPort
& req_port
)
302 DPRINTF(SnoopFilter
, "%s: rsp %s req %s packet %s\n",
303 __func__
, rsp_port
.name(), req_port
.name(), cpkt
->print());
305 assert(cpkt
->isResponse());
306 assert(cpkt
->cacheResponding());
308 Addr line_addr
= cpkt
->getBlockAddr(linesize
);
309 if (cpkt
->isSecure()) {
310 line_addr
|= LineSecure
;
312 auto sf_it
= cachedLocations
.find(line_addr
);
313 bool is_hit
= sf_it
!= cachedLocations
.end();
315 // Nothing to do if it is not a hit
319 // If the snoop response has no sharers the line is passed in
320 // Modified state, and we know that there are no other copies, or
321 // they will all be invalidated imminently
322 if (!cpkt
->hasSharers()) {
323 SnoopItem
& sf_item
= sf_it
->second
;
325 DPRINTF(SnoopFilter
, "%s: old SF value %x.%x\n",
326 __func__
, sf_item
.requested
, sf_item
.holder
);
328 DPRINTF(SnoopFilter
, "%s: new SF value %x.%x\n",
329 __func__
, sf_item
.requested
, sf_item
.holder
);
331 eraseIfNullEntry(sf_it
);
336 SnoopFilter::updateResponse(const Packet
* cpkt
, const SlavePort
& slave_port
)
338 DPRINTF(SnoopFilter
, "%s: src %s packet %s\n",
339 __func__
, slave_port
.name(), cpkt
->print());
341 assert(cpkt
->isResponse());
343 // we only allocate if the packet actually came from a cache, but
344 // start by checking if the port is snooping
345 if (cpkt
->req
->isUncacheable() || !slave_port
.isSnooping())
348 // next check if we actually allocated an entry
349 Addr line_addr
= cpkt
->getBlockAddr(linesize
);
350 if (cpkt
->isSecure()) {
351 line_addr
|= LineSecure
;
353 auto sf_it
= cachedLocations
.find(line_addr
);
354 if (sf_it
== cachedLocations
.end())
357 SnoopMask slave_mask
= portToMask(slave_port
);
358 SnoopItem
& sf_item
= sf_it
->second
;
360 DPRINTF(SnoopFilter
, "%s: old SF value %x.%x\n",
361 __func__
, sf_item
.requested
, sf_item
.holder
);
363 // Make sure we have seen the actual request, too
364 panic_if((sf_item
.requested
& slave_mask
).none(),
365 "SF value %x.%x missing request bit\n",
366 sf_item
.requested
, sf_item
.holder
);
368 sf_item
.requested
&= ~slave_mask
;
369 // Update the residency of the cache line.
371 if (cpkt
->req
->isCacheMaintenance()) {
372 // A cache clean response does not carry any data so it
373 // shouldn't change the holders, unless it is invalidating.
374 if (cpkt
->isInvalidate()) {
375 sf_item
.holder
&= ~slave_mask
;
377 eraseIfNullEntry(sf_it
);
379 // Any other response implies that a cache above will have the
381 sf_item
.holder
|= slave_mask
;
382 assert((sf_item
.holder
| sf_item
.requested
).any());
384 DPRINTF(SnoopFilter
, "%s: new SF value %x.%x\n",
385 __func__
, sf_item
.requested
, sf_item
.holder
);
389 SnoopFilter::regStats()
391 SimObject::regStats();
394 .name(name() + ".tot_requests")
395 .desc("Total number of requests made to the snoop filter.");
398 .name(name() + ".hit_single_requests")
399 .desc("Number of requests hitting in the snoop filter with a single "\
400 "holder of the requested data.");
403 .name(name() + ".hit_multi_requests")
404 .desc("Number of requests hitting in the snoop filter with multiple "\
405 "(>1) holders of the requested data.");
408 .name(name() + ".tot_snoops")
409 .desc("Total number of snoops made to the snoop filter.");
412 .name(name() + ".hit_single_snoops")
413 .desc("Number of snoops hitting in the snoop filter with a single "\
414 "holder of the requested data.");
417 .name(name() + ".hit_multi_snoops")
418 .desc("Number of snoops hitting in the snoop filter with multiple "\
419 "(>1) holders of the requested data.");
423 SnoopFilterParams::create()
425 return new SnoopFilter(this);