2 * Copyright (c) 2013-2017,2019 ARM Limited
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 * Authors: Stephan Diestelhorst
42 * Implementation of a snoop filter.
45 #include "mem/snoop_filter.hh"
47 #include "base/logging.hh"
48 #include "base/trace.hh"
49 #include "debug/SnoopFilter.hh"
50 #include "sim/system.hh"
52 const int SnoopFilter::SNOOP_MASK_SIZE
;
55 SnoopFilter::eraseIfNullEntry(SnoopFilterCache::iterator
& sf_it
)
57 SnoopItem
& sf_item
= sf_it
->second
;
58 if ((sf_item
.requested
| sf_item
.holder
).none()) {
59 cachedLocations
.erase(sf_it
);
60 DPRINTF(SnoopFilter
, "%s: Removed SF entry.\n",
65 std::pair
<SnoopFilter::SnoopList
, Cycles
>
66 SnoopFilter::lookupRequest(const Packet
* cpkt
, const SlavePort
& slave_port
)
68 DPRINTF(SnoopFilter
, "%s: src %s packet %s\n", __func__
,
69 slave_port
.name(), cpkt
->print());
71 // check if the packet came from a cache
72 bool allocate
= !cpkt
->req
->isUncacheable() && slave_port
.isSnooping() &&
74 Addr line_addr
= cpkt
->getBlockAddr(linesize
);
75 if (cpkt
->isSecure()) {
76 line_addr
|= LineSecure
;
78 SnoopMask req_port
= portToMask(slave_port
);
79 reqLookupResult
.it
= cachedLocations
.find(line_addr
);
80 bool is_hit
= (reqLookupResult
.it
!= cachedLocations
.end());
82 // If the snoop filter has no entry, and we should not allocate,
83 // do not create a new snoop filter entry, simply return a NULL
85 if (!is_hit
&& !allocate
)
86 return snoopDown(lookupLatency
);
88 // If no hit in snoop filter create a new element and update iterator
91 cachedLocations
.emplace(line_addr
, SnoopItem()).first
;
93 SnoopItem
& sf_item
= reqLookupResult
.it
->second
;
94 SnoopMask interested
= sf_item
.holder
| sf_item
.requested
;
96 // Store unmodified value of snoop filter item in temp storage in
97 // case we need to revert because of a send retry in
99 reqLookupResult
.retryItem
= sf_item
;
103 if (interested
.count() == 1)
109 DPRINTF(SnoopFilter
, "%s: SF value %x.%x\n",
110 __func__
, sf_item
.requested
, sf_item
.holder
);
112 // If we are not allocating, we are done
114 return snoopSelected(maskToPortList(interested
& ~req_port
),
117 if (cpkt
->needsResponse()) {
118 if (!cpkt
->cacheResponding()) {
119 // Max one request per address per port
120 panic_if((sf_item
.requested
& req_port
).any(),
121 "double request :( SF value %x.%x\n",
122 sf_item
.requested
, sf_item
.holder
);
124 // Mark in-flight requests to distinguish later on
125 sf_item
.requested
|= req_port
;
126 DPRINTF(SnoopFilter
, "%s: new SF value %x.%x\n",
127 __func__
, sf_item
.requested
, sf_item
.holder
);
129 // NOTE: The memInhibit might have been asserted by a cache closer
130 // to the CPU, already -> the response will not be seen by this
131 // filter -> we do not need to keep the in-flight request, but make
132 // sure that we know that that cluster has a copy
133 panic_if((sf_item
.holder
& req_port
).none(),
134 "Need to hold the value!");
136 "%s: not marking request. SF value %x.%x\n",
137 __func__
, sf_item
.requested
, sf_item
.holder
);
139 } else { // if (!cpkt->needsResponse())
140 assert(cpkt
->isEviction());
141 // make sure that the sender actually had the line
142 panic_if((sf_item
.holder
& req_port
).none(), "requester %x is not a " \
143 "holder :( SF value %x.%x\n", req_port
,
144 sf_item
.requested
, sf_item
.holder
);
145 // CleanEvicts and Writebacks -> the sender and all caches above
146 // it may not have the line anymore.
147 if (!cpkt
->isBlockCached()) {
148 sf_item
.holder
&= ~req_port
;
149 DPRINTF(SnoopFilter
, "%s: new SF value %x.%x\n",
150 __func__
, sf_item
.requested
, sf_item
.holder
);
154 return snoopSelected(maskToPortList(interested
& ~req_port
), lookupLatency
);
158 SnoopFilter::finishRequest(bool will_retry
, Addr addr
, bool is_secure
)
160 if (reqLookupResult
.it
!= cachedLocations
.end()) {
161 // since we rely on the caller, do a basic check to ensure
162 // that finishRequest is being called following lookupRequest
163 Addr line_addr
= (addr
& ~(Addr(linesize
- 1)));
165 line_addr
|= LineSecure
;
167 assert(reqLookupResult
.it
->first
== line_addr
);
169 SnoopItem retry_item
= reqLookupResult
.retryItem
;
170 // Undo any changes made in lookupRequest to the snoop filter
171 // entry if the request will come again. retryItem holds
172 // the previous value of the snoopfilter entry.
173 reqLookupResult
.it
->second
= retry_item
;
175 DPRINTF(SnoopFilter
, "%s: restored SF value %x.%x\n",
176 __func__
, retry_item
.requested
, retry_item
.holder
);
179 eraseIfNullEntry(reqLookupResult
.it
);
183 std::pair
<SnoopFilter::SnoopList
, Cycles
>
184 SnoopFilter::lookupSnoop(const Packet
* cpkt
)
186 DPRINTF(SnoopFilter
, "%s: packet %s\n", __func__
, cpkt
->print());
188 assert(cpkt
->isRequest());
190 Addr line_addr
= cpkt
->getBlockAddr(linesize
);
191 if (cpkt
->isSecure()) {
192 line_addr
|= LineSecure
;
194 auto sf_it
= cachedLocations
.find(line_addr
);
195 bool is_hit
= (sf_it
!= cachedLocations
.end());
197 panic_if(!is_hit
&& (cachedLocations
.size() >= maxEntryCount
),
198 "snoop filter exceeded capacity of %d cache blocks\n",
201 // If the snoop filter has no entry, simply return a NULL
202 // portlist, there is no point creating an entry only to remove it
205 return snoopDown(lookupLatency
);
207 SnoopItem
& sf_item
= sf_it
->second
;
209 SnoopMask interested
= (sf_item
.holder
| sf_item
.requested
);
213 if (interested
.count() == 1)
218 // ReadEx and Writes require both invalidation and exlusivity, while reads
219 // require neither. Writebacks on the other hand require exclusivity but
220 // not the invalidation. Previously Writebacks did not generate upward
221 // snoops so this was never an issue. Now that Writebacks generate snoops
222 // we need a special case for Writebacks. Additionally cache maintenance
223 // operations can generate snoops as they clean and/or invalidate all
224 // caches down to the specified point of reference.
225 assert(cpkt
->isWriteback() || cpkt
->req
->isUncacheable() ||
226 (cpkt
->isInvalidate() == cpkt
->needsWritable()) ||
227 cpkt
->req
->isCacheMaintenance());
228 if (cpkt
->isInvalidate() && sf_item
.requested
.none()) {
229 // Early clear of the holder, if no other request is currently going on
230 // @todo: This should possibly be updated even though we do not filter
232 DPRINTF(SnoopFilter
, "%s: old SF value %x.%x\n",
233 __func__
, sf_item
.requested
, sf_item
.holder
);
235 DPRINTF(SnoopFilter
, "%s: new SF value %x.%x\n",
236 __func__
, sf_item
.requested
, sf_item
.holder
);
237 eraseIfNullEntry(sf_it
);
240 return snoopSelected(maskToPortList(interested
), lookupLatency
);
244 SnoopFilter::updateSnoopResponse(const Packet
* cpkt
,
245 const SlavePort
& rsp_port
,
246 const SlavePort
& req_port
)
248 DPRINTF(SnoopFilter
, "%s: rsp %s req %s packet %s\n",
249 __func__
, rsp_port
.name(), req_port
.name(), cpkt
->print());
251 assert(cpkt
->isResponse());
252 assert(cpkt
->cacheResponding());
254 // if this snoop response is due to an uncacheable request, or is
255 // being turned into a normal response, there is nothing more to
257 if (cpkt
->req
->isUncacheable() || !req_port
.isSnooping()) {
261 Addr line_addr
= cpkt
->getBlockAddr(linesize
);
262 if (cpkt
->isSecure()) {
263 line_addr
|= LineSecure
;
265 SnoopMask rsp_mask
= portToMask(rsp_port
);
266 SnoopMask req_mask
= portToMask(req_port
);
267 SnoopItem
& sf_item
= cachedLocations
[line_addr
];
269 DPRINTF(SnoopFilter
, "%s: old SF value %x.%x\n",
270 __func__
, sf_item
.requested
, sf_item
.holder
);
272 // The source should have the line
273 panic_if((sf_item
.holder
& rsp_mask
).none(),
274 "SF value %x.%x does not have the line\n",
275 sf_item
.requested
, sf_item
.holder
);
277 // The destination should have had a request in
278 panic_if((sf_item
.requested
& req_mask
).none(), "SF value %x.%x missing "\
279 "the original request\n", sf_item
.requested
, sf_item
.holder
);
281 // If the snoop response has no sharers the line is passed in
282 // Modified state, and we know that there are no other copies, or
283 // they will all be invalidated imminently
284 if (!cpkt
->hasSharers()) {
286 "%s: dropping %x because non-shared snoop "
287 "response SF val: %x.%x\n", __func__
, rsp_mask
,
288 sf_item
.requested
, sf_item
.holder
);
291 assert(!cpkt
->isWriteback());
292 // @todo Deal with invalidating responses
293 sf_item
.holder
|= req_mask
;
294 sf_item
.requested
&= ~req_mask
;
295 assert((sf_item
.requested
| sf_item
.holder
).any());
296 DPRINTF(SnoopFilter
, "%s: new SF value %x.%x\n",
297 __func__
, sf_item
.requested
, sf_item
.holder
);
301 SnoopFilter::updateSnoopForward(const Packet
* cpkt
,
302 const SlavePort
& rsp_port
, const MasterPort
& req_port
)
304 DPRINTF(SnoopFilter
, "%s: rsp %s req %s packet %s\n",
305 __func__
, rsp_port
.name(), req_port
.name(), cpkt
->print());
307 assert(cpkt
->isResponse());
308 assert(cpkt
->cacheResponding());
310 Addr line_addr
= cpkt
->getBlockAddr(linesize
);
311 if (cpkt
->isSecure()) {
312 line_addr
|= LineSecure
;
314 auto sf_it
= cachedLocations
.find(line_addr
);
315 bool is_hit
= sf_it
!= cachedLocations
.end();
317 // Nothing to do if it is not a hit
321 // If the snoop response has no sharers the line is passed in
322 // Modified state, and we know that there are no other copies, or
323 // they will all be invalidated imminently
324 if (!cpkt
->hasSharers()) {
325 SnoopItem
& sf_item
= sf_it
->second
;
327 DPRINTF(SnoopFilter
, "%s: old SF value %x.%x\n",
328 __func__
, sf_item
.requested
, sf_item
.holder
);
330 DPRINTF(SnoopFilter
, "%s: new SF value %x.%x\n",
331 __func__
, sf_item
.requested
, sf_item
.holder
);
333 eraseIfNullEntry(sf_it
);
338 SnoopFilter::updateResponse(const Packet
* cpkt
, const SlavePort
& slave_port
)
340 DPRINTF(SnoopFilter
, "%s: src %s packet %s\n",
341 __func__
, slave_port
.name(), cpkt
->print());
343 assert(cpkt
->isResponse());
345 // we only allocate if the packet actually came from a cache, but
346 // start by checking if the port is snooping
347 if (cpkt
->req
->isUncacheable() || !slave_port
.isSnooping())
350 // next check if we actually allocated an entry
351 Addr line_addr
= cpkt
->getBlockAddr(linesize
);
352 if (cpkt
->isSecure()) {
353 line_addr
|= LineSecure
;
355 auto sf_it
= cachedLocations
.find(line_addr
);
356 if (sf_it
== cachedLocations
.end())
359 SnoopMask slave_mask
= portToMask(slave_port
);
360 SnoopItem
& sf_item
= sf_it
->second
;
362 DPRINTF(SnoopFilter
, "%s: old SF value %x.%x\n",
363 __func__
, sf_item
.requested
, sf_item
.holder
);
365 // Make sure we have seen the actual request, too
366 panic_if((sf_item
.requested
& slave_mask
).none(),
367 "SF value %x.%x missing request bit\n",
368 sf_item
.requested
, sf_item
.holder
);
370 sf_item
.requested
&= ~slave_mask
;
371 // Update the residency of the cache line.
373 if (cpkt
->req
->isCacheMaintenance()) {
374 // A cache clean response does not carry any data so it
375 // shouldn't change the holders, unless it is invalidating.
376 if (cpkt
->isInvalidate()) {
377 sf_item
.holder
&= ~slave_mask
;
379 eraseIfNullEntry(sf_it
);
381 // Any other response implies that a cache above will have the
383 sf_item
.holder
|= slave_mask
;
384 assert((sf_item
.holder
| sf_item
.requested
).any());
386 DPRINTF(SnoopFilter
, "%s: new SF value %x.%x\n",
387 __func__
, sf_item
.requested
, sf_item
.holder
);
391 SnoopFilter::regStats()
393 SimObject::regStats();
396 .name(name() + ".tot_requests")
397 .desc("Total number of requests made to the snoop filter.");
400 .name(name() + ".hit_single_requests")
401 .desc("Number of requests hitting in the snoop filter with a single "\
402 "holder of the requested data.");
405 .name(name() + ".hit_multi_requests")
406 .desc("Number of requests hitting in the snoop filter with multiple "\
407 "(>1) holders of the requested data.");
410 .name(name() + ".tot_snoops")
411 .desc("Total number of snoops made to the snoop filter.");
414 .name(name() + ".hit_single_snoops")
415 .desc("Number of snoops hitting in the snoop filter with a single "\
416 "holder of the requested data.");
419 .name(name() + ".hit_multi_snoops")
420 .desc("Number of snoops hitting in the snoop filter with multiple "\
421 "(>1) holders of the requested data.");
425 SnoopFilterParams::create()
427 return new SnoopFilter(this);