252fbb5245af81d442b3a657aa01e8f8a28690ef
2 * Copyright (c) 2013-2017 ARM Limited
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 * Authors: Stephan Diestelhorst
42 * Implementation of a snoop filter.
45 #include "mem/snoop_filter.hh"
47 #include "base/logging.hh"
48 #include "base/trace.hh"
49 #include "debug/SnoopFilter.hh"
50 #include "sim/system.hh"
53 SnoopFilter::eraseIfNullEntry(SnoopFilterCache::iterator
& sf_it
)
55 SnoopItem
& sf_item
= sf_it
->second
;
56 if (!(sf_item
.requested
| sf_item
.holder
)) {
57 cachedLocations
.erase(sf_it
);
58 DPRINTF(SnoopFilter
, "%s: Removed SF entry.\n",
63 std::pair
<SnoopFilter::SnoopList
, Cycles
>
64 SnoopFilter::lookupRequest(const Packet
* cpkt
, const SlavePort
& slave_port
)
66 DPRINTF(SnoopFilter
, "%s: src %s packet %s\n", __func__
,
67 slave_port
.name(), cpkt
->print());
69 // check if the packet came from a cache
70 bool allocate
= !cpkt
->req
->isUncacheable() && slave_port
.isSnooping() &&
72 Addr line_addr
= cpkt
->getBlockAddr(linesize
);
73 if (cpkt
->isSecure()) {
74 line_addr
|= LineSecure
;
76 SnoopMask req_port
= portToMask(slave_port
);
77 reqLookupResult
= cachedLocations
.find(line_addr
);
78 bool is_hit
= (reqLookupResult
!= cachedLocations
.end());
80 // If the snoop filter has no entry, and we should not allocate,
81 // do not create a new snoop filter entry, simply return a NULL
83 if (!is_hit
&& !allocate
)
84 return snoopDown(lookupLatency
);
86 // If no hit in snoop filter create a new element and update iterator
88 reqLookupResult
= cachedLocations
.emplace(line_addr
, SnoopItem()).first
;
89 SnoopItem
& sf_item
= reqLookupResult
->second
;
90 SnoopMask interested
= sf_item
.holder
| sf_item
.requested
;
92 // Store unmodified value of snoop filter item in temp storage in
93 // case we need to revert because of a send retry in
99 // Single bit set -> value is a power of two
100 if (isPow2(interested
))
106 DPRINTF(SnoopFilter
, "%s: SF value %x.%x\n",
107 __func__
, sf_item
.requested
, sf_item
.holder
);
109 // If we are not allocating, we are done
111 return snoopSelected(maskToPortList(interested
& ~req_port
),
114 if (cpkt
->needsResponse()) {
115 if (!cpkt
->cacheResponding()) {
116 // Max one request per address per port
117 panic_if(sf_item
.requested
& req_port
, "double request :( " \
118 "SF value %x.%x\n", sf_item
.requested
, sf_item
.holder
);
120 // Mark in-flight requests to distinguish later on
121 sf_item
.requested
|= req_port
;
122 DPRINTF(SnoopFilter
, "%s: new SF value %x.%x\n",
123 __func__
, sf_item
.requested
, sf_item
.holder
);
125 // NOTE: The memInhibit might have been asserted by a cache closer
126 // to the CPU, already -> the response will not be seen by this
127 // filter -> we do not need to keep the in-flight request, but make
128 // sure that we know that that cluster has a copy
129 panic_if(!(sf_item
.holder
& req_port
), "Need to hold the value!");
131 "%s: not marking request. SF value %x.%x\n",
132 __func__
, sf_item
.requested
, sf_item
.holder
);
134 } else { // if (!cpkt->needsResponse())
135 assert(cpkt
->isEviction());
136 // make sure that the sender actually had the line
137 panic_if(!(sf_item
.holder
& req_port
), "requester %x is not a " \
138 "holder :( SF value %x.%x\n", req_port
,
139 sf_item
.requested
, sf_item
.holder
);
140 // CleanEvicts and Writebacks -> the sender and all caches above
141 // it may not have the line anymore.
142 if (!cpkt
->isBlockCached()) {
143 sf_item
.holder
&= ~req_port
;
144 DPRINTF(SnoopFilter
, "%s: new SF value %x.%x\n",
145 __func__
, sf_item
.requested
, sf_item
.holder
);
149 return snoopSelected(maskToPortList(interested
& ~req_port
), lookupLatency
);
153 SnoopFilter::finishRequest(bool will_retry
, Addr addr
, bool is_secure
)
155 if (reqLookupResult
!= cachedLocations
.end()) {
156 // since we rely on the caller, do a basic check to ensure
157 // that finishRequest is being called following lookupRequest
158 Addr line_addr
= (addr
& ~(Addr(linesize
- 1)));
160 line_addr
|= LineSecure
;
162 assert(reqLookupResult
->first
== line_addr
);
164 // Undo any changes made in lookupRequest to the snoop filter
165 // entry if the request will come again. retryItem holds
166 // the previous value of the snoopfilter entry.
167 reqLookupResult
->second
= retryItem
;
169 DPRINTF(SnoopFilter
, "%s: restored SF value %x.%x\n",
170 __func__
, retryItem
.requested
, retryItem
.holder
);
173 eraseIfNullEntry(reqLookupResult
);
177 std::pair
<SnoopFilter::SnoopList
, Cycles
>
178 SnoopFilter::lookupSnoop(const Packet
* cpkt
)
180 DPRINTF(SnoopFilter
, "%s: packet %s\n", __func__
, cpkt
->print());
182 assert(cpkt
->isRequest());
184 Addr line_addr
= cpkt
->getBlockAddr(linesize
);
185 if (cpkt
->isSecure()) {
186 line_addr
|= LineSecure
;
188 auto sf_it
= cachedLocations
.find(line_addr
);
189 bool is_hit
= (sf_it
!= cachedLocations
.end());
191 panic_if(!is_hit
&& (cachedLocations
.size() >= maxEntryCount
),
192 "snoop filter exceeded capacity of %d cache blocks\n",
195 // If the snoop filter has no entry, simply return a NULL
196 // portlist, there is no point creating an entry only to remove it
199 return snoopDown(lookupLatency
);
201 SnoopItem
& sf_item
= sf_it
->second
;
203 DPRINTF(SnoopFilter
, "%s: old SF value %x.%x\n",
204 __func__
, sf_item
.requested
, sf_item
.holder
);
206 SnoopMask interested
= (sf_item
.holder
| sf_item
.requested
);
209 // Single bit set -> value is a power of two
210 if (isPow2(interested
))
215 // ReadEx and Writes require both invalidation and exlusivity, while reads
216 // require neither. Writebacks on the other hand require exclusivity but
217 // not the invalidation. Previously Writebacks did not generate upward
218 // snoops so this was never an issue. Now that Writebacks generate snoops
219 // we need a special case for Writebacks. Additionally cache maintenance
220 // operations can generate snoops as they clean and/or invalidate all
221 // caches down to the specified point of reference.
222 assert(cpkt
->isWriteback() || cpkt
->req
->isUncacheable() ||
223 (cpkt
->isInvalidate() == cpkt
->needsWritable()) ||
224 cpkt
->req
->isCacheMaintenance());
225 if (cpkt
->isInvalidate() && !sf_item
.requested
) {
226 // Early clear of the holder, if no other request is currently going on
227 // @todo: This should possibly be updated even though we do not filter
232 eraseIfNullEntry(sf_it
);
233 DPRINTF(SnoopFilter
, "%s: new SF value %x.%x interest: %x \n",
234 __func__
, sf_item
.requested
, sf_item
.holder
, interested
);
236 return snoopSelected(maskToPortList(interested
), lookupLatency
);
240 SnoopFilter::updateSnoopResponse(const Packet
* cpkt
,
241 const SlavePort
& rsp_port
,
242 const SlavePort
& req_port
)
244 DPRINTF(SnoopFilter
, "%s: rsp %s req %s packet %s\n",
245 __func__
, rsp_port
.name(), req_port
.name(), cpkt
->print());
247 assert(cpkt
->isResponse());
248 assert(cpkt
->cacheResponding());
250 // if this snoop response is due to an uncacheable request, or is
251 // being turned into a normal response, there is nothing more to
253 if (cpkt
->req
->isUncacheable() || !req_port
.isSnooping()) {
257 Addr line_addr
= cpkt
->getBlockAddr(linesize
);
258 if (cpkt
->isSecure()) {
259 line_addr
|= LineSecure
;
261 SnoopMask rsp_mask
= portToMask(rsp_port
);
262 SnoopMask req_mask
= portToMask(req_port
);
263 SnoopItem
& sf_item
= cachedLocations
[line_addr
];
265 DPRINTF(SnoopFilter
, "%s: old SF value %x.%x\n",
266 __func__
, sf_item
.requested
, sf_item
.holder
);
268 // The source should have the line
269 panic_if(!(sf_item
.holder
& rsp_mask
), "SF value %x.%x does not have "\
270 "the line\n", sf_item
.requested
, sf_item
.holder
);
272 // The destination should have had a request in
273 panic_if(!(sf_item
.requested
& req_mask
), "SF value %x.%x missing "\
274 "the original request\n", sf_item
.requested
, sf_item
.holder
);
276 // If the snoop response has no sharers the line is passed in
277 // Modified state, and we know that there are no other copies, or
278 // they will all be invalidated imminently
279 if (!cpkt
->hasSharers()) {
281 "%s: dropping %x because non-shared snoop "
282 "response SF val: %x.%x\n", __func__
, rsp_mask
,
283 sf_item
.requested
, sf_item
.holder
);
286 assert(!cpkt
->isWriteback());
287 // @todo Deal with invalidating responses
288 sf_item
.holder
|= req_mask
;
289 sf_item
.requested
&= ~req_mask
;
290 assert(sf_item
.requested
| sf_item
.holder
);
291 DPRINTF(SnoopFilter
, "%s: new SF value %x.%x\n",
292 __func__
, sf_item
.requested
, sf_item
.holder
);
296 SnoopFilter::updateSnoopForward(const Packet
* cpkt
,
297 const SlavePort
& rsp_port
, const MasterPort
& req_port
)
299 DPRINTF(SnoopFilter
, "%s: rsp %s req %s packet %s\n",
300 __func__
, rsp_port
.name(), req_port
.name(), cpkt
->print());
302 assert(cpkt
->isResponse());
303 assert(cpkt
->cacheResponding());
305 Addr line_addr
= cpkt
->getBlockAddr(linesize
);
306 if (cpkt
->isSecure()) {
307 line_addr
|= LineSecure
;
309 auto sf_it
= cachedLocations
.find(line_addr
);
310 bool is_hit
= sf_it
!= cachedLocations
.end();
312 // Nothing to do if it is not a hit
316 SnoopItem
& sf_item
= sf_it
->second
;
318 DPRINTF(SnoopFilter
, "%s: old SF value %x.%x\n",
319 __func__
, sf_item
.requested
, sf_item
.holder
);
321 // If the snoop response has no sharers the line is passed in
322 // Modified state, and we know that there are no other copies, or
323 // they will all be invalidated imminently
324 if (!cpkt
->hasSharers()) {
327 DPRINTF(SnoopFilter
, "%s: new SF value %x.%x\n",
328 __func__
, sf_item
.requested
, sf_item
.holder
);
329 eraseIfNullEntry(sf_it
);
334 SnoopFilter::updateResponse(const Packet
* cpkt
, const SlavePort
& slave_port
)
336 DPRINTF(SnoopFilter
, "%s: src %s packet %s\n",
337 __func__
, slave_port
.name(), cpkt
->print());
339 assert(cpkt
->isResponse());
341 // we only allocate if the packet actually came from a cache, but
342 // start by checking if the port is snooping
343 if (cpkt
->req
->isUncacheable() || !slave_port
.isSnooping())
346 // next check if we actually allocated an entry
347 Addr line_addr
= cpkt
->getBlockAddr(linesize
);
348 if (cpkt
->isSecure()) {
349 line_addr
|= LineSecure
;
351 auto sf_it
= cachedLocations
.find(line_addr
);
352 if (sf_it
== cachedLocations
.end())
355 SnoopMask slave_mask
= portToMask(slave_port
);
356 SnoopItem
& sf_item
= sf_it
->second
;
358 DPRINTF(SnoopFilter
, "%s: old SF value %x.%x\n",
359 __func__
, sf_item
.requested
, sf_item
.holder
);
361 // Make sure we have seen the actual request, too
362 panic_if(!(sf_item
.requested
& slave_mask
), "SF value %x.%x missing "\
363 "request bit\n", sf_item
.requested
, sf_item
.holder
);
365 // Update the residency of the cache line.
366 sf_item
.holder
|= slave_mask
;
367 sf_item
.requested
&= ~slave_mask
;
368 assert(sf_item
.holder
| sf_item
.requested
);
369 DPRINTF(SnoopFilter
, "%s: new SF value %x.%x\n",
370 __func__
, sf_item
.requested
, sf_item
.holder
);
374 SnoopFilter::regStats()
376 SimObject::regStats();
379 .name(name() + ".tot_requests")
380 .desc("Total number of requests made to the snoop filter.");
383 .name(name() + ".hit_single_requests")
384 .desc("Number of requests hitting in the snoop filter with a single "\
385 "holder of the requested data.");
388 .name(name() + ".hit_multi_requests")
389 .desc("Number of requests hitting in the snoop filter with multiple "\
390 "(>1) holders of the requested data.");
393 .name(name() + ".tot_snoops")
394 .desc("Total number of snoops made to the snoop filter.");
397 .name(name() + ".hit_single_snoops")
398 .desc("Number of snoops hitting in the snoop filter with a single "\
399 "holder of the requested data.");
402 .name(name() + ".hit_multi_snoops")
403 .desc("Number of snoops hitting in the snoop filter with multiple "\
404 "(>1) holders of the requested data.");
408 SnoopFilterParams::create()
410 return new SnoopFilter(this);