mem-cache: Fix increasing replacement count
[gem5.git] / src / mem / cache / base.cc
1 /*
2 * Copyright (c) 2012-2013, 2018 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Copyright (c) 2003-2005 The Regents of The University of Michigan
15 * All rights reserved.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions are
19 * met: redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer;
21 * redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution;
24 * neither the name of the copyright holders nor the names of its
25 * contributors may be used to endorse or promote products derived from
26 * this software without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 *
40 * Authors: Erik Hallnor
41 * Nikos Nikoleris
42 */
43
44 /**
45 * @file
46 * Definition of BaseCache functions.
47 */
48
49 #include "mem/cache/base.hh"
50
51 #include "base/compiler.hh"
52 #include "base/logging.hh"
53 #include "debug/Cache.hh"
54 #include "debug/CachePort.hh"
55 #include "debug/CacheRepl.hh"
56 #include "debug/CacheVerbose.hh"
57 #include "mem/cache/mshr.hh"
58 #include "mem/cache/prefetch/base.hh"
59 #include "mem/cache/queue_entry.hh"
60 #include "params/BaseCache.hh"
61 #include "params/WriteAllocator.hh"
62 #include "sim/core.hh"
63
64 class BaseMasterPort;
65 class BaseSlavePort;
66
67 using namespace std;
68
69 BaseCache::CacheSlavePort::CacheSlavePort(const std::string &_name,
70 BaseCache *_cache,
71 const std::string &_label)
72 : QueuedSlavePort(_name, _cache, queue),
73 queue(*_cache, *this, true, _label),
74 blocked(false), mustSendRetry(false),
75 sendRetryEvent([this]{ processSendRetry(); }, _name)
76 {
77 }
78
79 BaseCache::BaseCache(const BaseCacheParams *p, unsigned blk_size)
80 : MemObject(p),
81 cpuSidePort (p->name + ".cpu_side", this, "CpuSidePort"),
82 memSidePort(p->name + ".mem_side", this, "MemSidePort"),
83 mshrQueue("MSHRs", p->mshrs, 0, p->demand_mshr_reserve), // see below
84 writeBuffer("write buffer", p->write_buffers, p->mshrs), // see below
85 tags(p->tags),
86 prefetcher(p->prefetcher),
87 writeAllocator(p->write_allocator),
88 writebackClean(p->writeback_clean),
89 tempBlockWriteback(nullptr),
90 writebackTempBlockAtomicEvent([this]{ writebackTempBlockAtomic(); },
91 name(), false,
92 EventBase::Delayed_Writeback_Pri),
93 blkSize(blk_size),
94 lookupLatency(p->tag_latency),
95 dataLatency(p->data_latency),
96 forwardLatency(p->tag_latency),
97 fillLatency(p->data_latency),
98 responseLatency(p->response_latency),
99 sequentialAccess(p->sequential_access),
100 numTarget(p->tgts_per_mshr),
101 forwardSnoops(true),
102 clusivity(p->clusivity),
103 isReadOnly(p->is_read_only),
104 blocked(0),
105 order(0),
106 noTargetMSHR(nullptr),
107 missCount(p->max_miss_count),
108 addrRanges(p->addr_ranges.begin(), p->addr_ranges.end()),
109 system(p->system)
110 {
111 // the MSHR queue has no reserve entries as we check the MSHR
112 // queue on every single allocation, whereas the write queue has
113 // as many reserve entries as we have MSHRs, since every MSHR may
114 // eventually require a writeback, and we do not check the write
115 // buffer before committing to an MSHR
116
117 // forward snoops is overridden in init() once we can query
118 // whether the connected master is actually snooping or not
119
120 tempBlock = new TempCacheBlk(blkSize);
121
122 tags->tagsInit();
123 if (prefetcher)
124 prefetcher->setCache(this);
125 }
126
127 BaseCache::~BaseCache()
128 {
129 delete tempBlock;
130 }
131
132 void
133 BaseCache::CacheSlavePort::setBlocked()
134 {
135 assert(!blocked);
136 DPRINTF(CachePort, "Port is blocking new requests\n");
137 blocked = true;
138 // if we already scheduled a retry in this cycle, but it has not yet
139 // happened, cancel it
140 if (sendRetryEvent.scheduled()) {
141 owner.deschedule(sendRetryEvent);
142 DPRINTF(CachePort, "Port descheduled retry\n");
143 mustSendRetry = true;
144 }
145 }
146
147 void
148 BaseCache::CacheSlavePort::clearBlocked()
149 {
150 assert(blocked);
151 DPRINTF(CachePort, "Port is accepting new requests\n");
152 blocked = false;
153 if (mustSendRetry) {
154 // @TODO: need to find a better time (next cycle?)
155 owner.schedule(sendRetryEvent, curTick() + 1);
156 }
157 }
158
159 void
160 BaseCache::CacheSlavePort::processSendRetry()
161 {
162 DPRINTF(CachePort, "Port is sending retry\n");
163
164 // reset the flag and call retry
165 mustSendRetry = false;
166 sendRetryReq();
167 }
168
169 Addr
170 BaseCache::regenerateBlkAddr(CacheBlk* blk)
171 {
172 if (blk != tempBlock) {
173 return tags->regenerateBlkAddr(blk);
174 } else {
175 return tempBlock->getAddr();
176 }
177 }
178
179 void
180 BaseCache::init()
181 {
182 if (!cpuSidePort.isConnected() || !memSidePort.isConnected())
183 fatal("Cache ports on %s are not connected\n", name());
184 cpuSidePort.sendRangeChange();
185 forwardSnoops = cpuSidePort.isSnooping();
186 }
187
188 Port &
189 BaseCache::getPort(const std::string &if_name, PortID idx)
190 {
191 if (if_name == "mem_side") {
192 return memSidePort;
193 } else if (if_name == "cpu_side") {
194 return cpuSidePort;
195 } else {
196 return MemObject::getPort(if_name, idx);
197 }
198 }
199
200 bool
201 BaseCache::inRange(Addr addr) const
202 {
203 for (const auto& r : addrRanges) {
204 if (r.contains(addr)) {
205 return true;
206 }
207 }
208 return false;
209 }
210
211 void
212 BaseCache::handleTimingReqHit(PacketPtr pkt, CacheBlk *blk, Tick request_time)
213 {
214 if (pkt->needsResponse()) {
215 // These delays should have been consumed by now
216 assert(pkt->headerDelay == 0);
217 assert(pkt->payloadDelay == 0);
218
219 pkt->makeTimingResponse();
220
221 // In this case we are considering request_time that takes
222 // into account the delay of the xbar, if any, and just
223 // lat, neglecting responseLatency, modelling hit latency
224 // just as the value of lat overriden by access(), which calls
225 // the calculateAccessLatency() function.
226 cpuSidePort.schedTimingResp(pkt, request_time);
227 } else {
228 DPRINTF(Cache, "%s satisfied %s, no response needed\n", __func__,
229 pkt->print());
230
231 // queue the packet for deletion, as the sending cache is
232 // still relying on it; if the block is found in access(),
233 // CleanEvict and Writeback messages will be deleted
234 // here as well
235 pendingDelete.reset(pkt);
236 }
237 }
238
239 void
240 BaseCache::handleTimingReqMiss(PacketPtr pkt, MSHR *mshr, CacheBlk *blk,
241 Tick forward_time, Tick request_time)
242 {
243 if (writeAllocator &&
244 pkt && pkt->isWrite() && !pkt->req->isUncacheable()) {
245 writeAllocator->updateMode(pkt->getAddr(), pkt->getSize(),
246 pkt->getBlockAddr(blkSize));
247 }
248
249 if (mshr) {
250 /// MSHR hit
251 /// @note writebacks will be checked in getNextMSHR()
252 /// for any conflicting requests to the same block
253
254 //@todo remove hw_pf here
255
256 // Coalesce unless it was a software prefetch (see above).
257 if (pkt) {
258 assert(!pkt->isWriteback());
259 // CleanEvicts corresponding to blocks which have
260 // outstanding requests in MSHRs are simply sunk here
261 if (pkt->cmd == MemCmd::CleanEvict) {
262 pendingDelete.reset(pkt);
263 } else if (pkt->cmd == MemCmd::WriteClean) {
264 // A WriteClean should never coalesce with any
265 // outstanding cache maintenance requests.
266
267 // We use forward_time here because there is an
268 // uncached memory write, forwarded to WriteBuffer.
269 allocateWriteBuffer(pkt, forward_time);
270 } else {
271 DPRINTF(Cache, "%s coalescing MSHR for %s\n", __func__,
272 pkt->print());
273
274 assert(pkt->req->masterId() < system->maxMasters());
275 mshr_hits[pkt->cmdToIndex()][pkt->req->masterId()]++;
276
277 // We use forward_time here because it is the same
278 // considering new targets. We have multiple
279 // requests for the same address here. It
280 // specifies the latency to allocate an internal
281 // buffer and to schedule an event to the queued
282 // port and also takes into account the additional
283 // delay of the xbar.
284 mshr->allocateTarget(pkt, forward_time, order++,
285 allocOnFill(pkt->cmd));
286 if (mshr->getNumTargets() == numTarget) {
287 noTargetMSHR = mshr;
288 setBlocked(Blocked_NoTargets);
289 // need to be careful with this... if this mshr isn't
290 // ready yet (i.e. time > curTick()), we don't want to
291 // move it ahead of mshrs that are ready
292 // mshrQueue.moveToFront(mshr);
293 }
294 }
295 }
296 } else {
297 // no MSHR
298 assert(pkt->req->masterId() < system->maxMasters());
299 mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++;
300
301 if (pkt->isEviction() || pkt->cmd == MemCmd::WriteClean) {
302 // We use forward_time here because there is an
303 // writeback or writeclean, forwarded to WriteBuffer.
304 allocateWriteBuffer(pkt, forward_time);
305 } else {
306 if (blk && blk->isValid()) {
307 // If we have a write miss to a valid block, we
308 // need to mark the block non-readable. Otherwise
309 // if we allow reads while there's an outstanding
310 // write miss, the read could return stale data
311 // out of the cache block... a more aggressive
312 // system could detect the overlap (if any) and
313 // forward data out of the MSHRs, but we don't do
314 // that yet. Note that we do need to leave the
315 // block valid so that it stays in the cache, in
316 // case we get an upgrade response (and hence no
317 // new data) when the write miss completes.
318 // As long as CPUs do proper store/load forwarding
319 // internally, and have a sufficiently weak memory
320 // model, this is probably unnecessary, but at some
321 // point it must have seemed like we needed it...
322 assert((pkt->needsWritable() && !blk->isWritable()) ||
323 pkt->req->isCacheMaintenance());
324 blk->status &= ~BlkReadable;
325 }
326 // Here we are using forward_time, modelling the latency of
327 // a miss (outbound) just as forwardLatency, neglecting the
328 // lookupLatency component.
329 allocateMissBuffer(pkt, forward_time);
330 }
331 }
332 }
333
334 void
335 BaseCache::recvTimingReq(PacketPtr pkt)
336 {
337 // anything that is merely forwarded pays for the forward latency and
338 // the delay provided by the crossbar
339 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
340
341 Cycles lat;
342 CacheBlk *blk = nullptr;
343 bool satisfied = false;
344 {
345 PacketList writebacks;
346 // Note that lat is passed by reference here. The function
347 // access() will set the lat value.
348 satisfied = access(pkt, blk, lat, writebacks);
349
350 // After the evicted blocks are selected, they must be forwarded
351 // to the write buffer to ensure they logically precede anything
352 // happening below
353 doWritebacks(writebacks, clockEdge(lat + forwardLatency));
354 }
355
356 // Here we charge the headerDelay that takes into account the latencies
357 // of the bus, if the packet comes from it.
358 // The latency charged is just the value set by the access() function.
359 // In case of a hit we are neglecting response latency.
360 // In case of a miss we are neglecting forward latency.
361 Tick request_time = clockEdge(lat);
362 // Here we reset the timing of the packet.
363 pkt->headerDelay = pkt->payloadDelay = 0;
364
365 if (satisfied) {
366 // notify before anything else as later handleTimingReqHit might turn
367 // the packet in a response
368 ppHit->notify(pkt);
369
370 if (prefetcher && blk && blk->wasPrefetched()) {
371 blk->status &= ~BlkHWPrefetched;
372 }
373
374 handleTimingReqHit(pkt, blk, request_time);
375 } else {
376 handleTimingReqMiss(pkt, blk, forward_time, request_time);
377
378 ppMiss->notify(pkt);
379 }
380
381 if (prefetcher) {
382 // track time of availability of next prefetch, if any
383 Tick next_pf_time = prefetcher->nextPrefetchReadyTime();
384 if (next_pf_time != MaxTick) {
385 schedMemSideSendEvent(next_pf_time);
386 }
387 }
388 }
389
390 void
391 BaseCache::handleUncacheableWriteResp(PacketPtr pkt)
392 {
393 Tick completion_time = clockEdge(responseLatency) +
394 pkt->headerDelay + pkt->payloadDelay;
395
396 // Reset the bus additional time as it is now accounted for
397 pkt->headerDelay = pkt->payloadDelay = 0;
398
399 cpuSidePort.schedTimingResp(pkt, completion_time);
400 }
401
402 void
403 BaseCache::recvTimingResp(PacketPtr pkt)
404 {
405 assert(pkt->isResponse());
406
407 // all header delay should be paid for by the crossbar, unless
408 // this is a prefetch response from above
409 panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp,
410 "%s saw a non-zero packet delay\n", name());
411
412 const bool is_error = pkt->isError();
413
414 if (is_error) {
415 DPRINTF(Cache, "%s: Cache received %s with error\n", __func__,
416 pkt->print());
417 }
418
419 DPRINTF(Cache, "%s: Handling response %s\n", __func__,
420 pkt->print());
421
422 // if this is a write, we should be looking at an uncacheable
423 // write
424 if (pkt->isWrite()) {
425 assert(pkt->req->isUncacheable());
426 handleUncacheableWriteResp(pkt);
427 return;
428 }
429
430 // we have dealt with any (uncacheable) writes above, from here on
431 // we know we are dealing with an MSHR due to a miss or a prefetch
432 MSHR *mshr = dynamic_cast<MSHR*>(pkt->popSenderState());
433 assert(mshr);
434
435 if (mshr == noTargetMSHR) {
436 // we always clear at least one target
437 clearBlocked(Blocked_NoTargets);
438 noTargetMSHR = nullptr;
439 }
440
441 // Initial target is used just for stats
442 QueueEntry::Target *initial_tgt = mshr->getTarget();
443 int stats_cmd_idx = initial_tgt->pkt->cmdToIndex();
444 Tick miss_latency = curTick() - initial_tgt->recvTime;
445
446 if (pkt->req->isUncacheable()) {
447 assert(pkt->req->masterId() < system->maxMasters());
448 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] +=
449 miss_latency;
450 } else {
451 assert(pkt->req->masterId() < system->maxMasters());
452 mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] +=
453 miss_latency;
454 }
455
456 PacketList writebacks;
457
458 bool is_fill = !mshr->isForward &&
459 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp ||
460 mshr->wasWholeLineWrite);
461
462 // make sure that if the mshr was due to a whole line write then
463 // the response is an invalidation
464 assert(!mshr->wasWholeLineWrite || pkt->isInvalidate());
465
466 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
467
468 if (is_fill && !is_error) {
469 DPRINTF(Cache, "Block for addr %#llx being updated in Cache\n",
470 pkt->getAddr());
471
472 const bool allocate = (writeAllocator && mshr->wasWholeLineWrite) ?
473 writeAllocator->allocate() : mshr->allocOnFill();
474 blk = handleFill(pkt, blk, writebacks, allocate);
475 assert(blk != nullptr);
476 ppFill->notify(pkt);
477 }
478
479 if (blk && blk->isValid() && pkt->isClean() && !pkt->isInvalidate()) {
480 // The block was marked not readable while there was a pending
481 // cache maintenance operation, restore its flag.
482 blk->status |= BlkReadable;
483
484 // This was a cache clean operation (without invalidate)
485 // and we have a copy of the block already. Since there
486 // is no invalidation, we can promote targets that don't
487 // require a writable copy
488 mshr->promoteReadable();
489 }
490
491 if (blk && blk->isWritable() && !pkt->req->isCacheInvalidate()) {
492 // If at this point the referenced block is writable and the
493 // response is not a cache invalidate, we promote targets that
494 // were deferred as we couldn't guarrantee a writable copy
495 mshr->promoteWritable();
496 }
497
498 serviceMSHRTargets(mshr, pkt, blk);
499
500 if (mshr->promoteDeferredTargets()) {
501 // avoid later read getting stale data while write miss is
502 // outstanding.. see comment in timingAccess()
503 if (blk) {
504 blk->status &= ~BlkReadable;
505 }
506 mshrQueue.markPending(mshr);
507 schedMemSideSendEvent(clockEdge() + pkt->payloadDelay);
508 } else {
509 // while we deallocate an mshr from the queue we still have to
510 // check the isFull condition before and after as we might
511 // have been using the reserved entries already
512 const bool was_full = mshrQueue.isFull();
513 mshrQueue.deallocate(mshr);
514 if (was_full && !mshrQueue.isFull()) {
515 clearBlocked(Blocked_NoMSHRs);
516 }
517
518 // Request the bus for a prefetch if this deallocation freed enough
519 // MSHRs for a prefetch to take place
520 if (prefetcher && mshrQueue.canPrefetch()) {
521 Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(),
522 clockEdge());
523 if (next_pf_time != MaxTick)
524 schedMemSideSendEvent(next_pf_time);
525 }
526 }
527
528 // if we used temp block, check to see if its valid and then clear it out
529 if (blk == tempBlock && tempBlock->isValid()) {
530 evictBlock(blk, writebacks);
531 }
532
533 const Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
534 // copy writebacks to write buffer
535 doWritebacks(writebacks, forward_time);
536
537 DPRINTF(CacheVerbose, "%s: Leaving with %s\n", __func__, pkt->print());
538 delete pkt;
539 }
540
541
542 Tick
543 BaseCache::recvAtomic(PacketPtr pkt)
544 {
545 // should assert here that there are no outstanding MSHRs or
546 // writebacks... that would mean that someone used an atomic
547 // access in timing mode
548
549 // We use lookupLatency here because it is used to specify the latency
550 // to access.
551 Cycles lat = lookupLatency;
552
553 CacheBlk *blk = nullptr;
554 PacketList writebacks;
555 bool satisfied = access(pkt, blk, lat, writebacks);
556
557 if (pkt->isClean() && blk && blk->isDirty()) {
558 // A cache clean opearation is looking for a dirty
559 // block. If a dirty block is encountered a WriteClean
560 // will update any copies to the path to the memory
561 // until the point of reference.
562 DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n",
563 __func__, pkt->print(), blk->print());
564 PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(), pkt->id);
565 writebacks.push_back(wb_pkt);
566 pkt->setSatisfied();
567 }
568
569 // handle writebacks resulting from the access here to ensure they
570 // logically precede anything happening below
571 doWritebacksAtomic(writebacks);
572 assert(writebacks.empty());
573
574 if (!satisfied) {
575 lat += handleAtomicReqMiss(pkt, blk, writebacks);
576 }
577
578 // Note that we don't invoke the prefetcher at all in atomic mode.
579 // It's not clear how to do it properly, particularly for
580 // prefetchers that aggressively generate prefetch candidates and
581 // rely on bandwidth contention to throttle them; these will tend
582 // to pollute the cache in atomic mode since there is no bandwidth
583 // contention. If we ever do want to enable prefetching in atomic
584 // mode, though, this is the place to do it... see timingAccess()
585 // for an example (though we'd want to issue the prefetch(es)
586 // immediately rather than calling requestMemSideBus() as we do
587 // there).
588
589 // do any writebacks resulting from the response handling
590 doWritebacksAtomic(writebacks);
591
592 // if we used temp block, check to see if its valid and if so
593 // clear it out, but only do so after the call to recvAtomic is
594 // finished so that any downstream observers (such as a snoop
595 // filter), first see the fill, and only then see the eviction
596 if (blk == tempBlock && tempBlock->isValid()) {
597 // the atomic CPU calls recvAtomic for fetch and load/store
598 // sequentuially, and we may already have a tempBlock
599 // writeback from the fetch that we have not yet sent
600 if (tempBlockWriteback) {
601 // if that is the case, write the prevoius one back, and
602 // do not schedule any new event
603 writebackTempBlockAtomic();
604 } else {
605 // the writeback/clean eviction happens after the call to
606 // recvAtomic has finished (but before any successive
607 // calls), so that the response handling from the fill is
608 // allowed to happen first
609 schedule(writebackTempBlockAtomicEvent, curTick());
610 }
611
612 tempBlockWriteback = evictBlock(blk);
613 }
614
615 if (pkt->needsResponse()) {
616 pkt->makeAtomicResponse();
617 }
618
619 return lat * clockPeriod();
620 }
621
622 void
623 BaseCache::functionalAccess(PacketPtr pkt, bool from_cpu_side)
624 {
625 Addr blk_addr = pkt->getBlockAddr(blkSize);
626 bool is_secure = pkt->isSecure();
627 CacheBlk *blk = tags->findBlock(pkt->getAddr(), is_secure);
628 MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure);
629
630 pkt->pushLabel(name());
631
632 CacheBlkPrintWrapper cbpw(blk);
633
634 // Note that just because an L2/L3 has valid data doesn't mean an
635 // L1 doesn't have a more up-to-date modified copy that still
636 // needs to be found. As a result we always update the request if
637 // we have it, but only declare it satisfied if we are the owner.
638
639 // see if we have data at all (owned or otherwise)
640 bool have_data = blk && blk->isValid()
641 && pkt->trySatisfyFunctional(&cbpw, blk_addr, is_secure, blkSize,
642 blk->data);
643
644 // data we have is dirty if marked as such or if we have an
645 // in-service MSHR that is pending a modified line
646 bool have_dirty =
647 have_data && (blk->isDirty() ||
648 (mshr && mshr->inService && mshr->isPendingModified()));
649
650 bool done = have_dirty ||
651 cpuSidePort.trySatisfyFunctional(pkt) ||
652 mshrQueue.trySatisfyFunctional(pkt) ||
653 writeBuffer.trySatisfyFunctional(pkt) ||
654 memSidePort.trySatisfyFunctional(pkt);
655
656 DPRINTF(CacheVerbose, "%s: %s %s%s%s\n", __func__, pkt->print(),
657 (blk && blk->isValid()) ? "valid " : "",
658 have_data ? "data " : "", done ? "done " : "");
659
660 // We're leaving the cache, so pop cache->name() label
661 pkt->popLabel();
662
663 if (done) {
664 pkt->makeResponse();
665 } else {
666 // if it came as a request from the CPU side then make sure it
667 // continues towards the memory side
668 if (from_cpu_side) {
669 memSidePort.sendFunctional(pkt);
670 } else if (cpuSidePort.isSnooping()) {
671 // if it came from the memory side, it must be a snoop request
672 // and we should only forward it if we are forwarding snoops
673 cpuSidePort.sendFunctionalSnoop(pkt);
674 }
675 }
676 }
677
678
679 void
680 BaseCache::cmpAndSwap(CacheBlk *blk, PacketPtr pkt)
681 {
682 assert(pkt->isRequest());
683
684 uint64_t overwrite_val;
685 bool overwrite_mem;
686 uint64_t condition_val64;
687 uint32_t condition_val32;
688
689 int offset = pkt->getOffset(blkSize);
690 uint8_t *blk_data = blk->data + offset;
691
692 assert(sizeof(uint64_t) >= pkt->getSize());
693
694 overwrite_mem = true;
695 // keep a copy of our possible write value, and copy what is at the
696 // memory address into the packet
697 pkt->writeData((uint8_t *)&overwrite_val);
698 pkt->setData(blk_data);
699
700 if (pkt->req->isCondSwap()) {
701 if (pkt->getSize() == sizeof(uint64_t)) {
702 condition_val64 = pkt->req->getExtraData();
703 overwrite_mem = !std::memcmp(&condition_val64, blk_data,
704 sizeof(uint64_t));
705 } else if (pkt->getSize() == sizeof(uint32_t)) {
706 condition_val32 = (uint32_t)pkt->req->getExtraData();
707 overwrite_mem = !std::memcmp(&condition_val32, blk_data,
708 sizeof(uint32_t));
709 } else
710 panic("Invalid size for conditional read/write\n");
711 }
712
713 if (overwrite_mem) {
714 std::memcpy(blk_data, &overwrite_val, pkt->getSize());
715 blk->status |= BlkDirty;
716 }
717 }
718
719 QueueEntry*
720 BaseCache::getNextQueueEntry()
721 {
722 // Check both MSHR queue and write buffer for potential requests,
723 // note that null does not mean there is no request, it could
724 // simply be that it is not ready
725 MSHR *miss_mshr = mshrQueue.getNext();
726 WriteQueueEntry *wq_entry = writeBuffer.getNext();
727
728 // If we got a write buffer request ready, first priority is a
729 // full write buffer, otherwise we favour the miss requests
730 if (wq_entry && (writeBuffer.isFull() || !miss_mshr)) {
731 // need to search MSHR queue for conflicting earlier miss.
732 MSHR *conflict_mshr = mshrQueue.findPending(wq_entry);
733
734 if (conflict_mshr && conflict_mshr->order < wq_entry->order) {
735 // Service misses in order until conflict is cleared.
736 return conflict_mshr;
737
738 // @todo Note that we ignore the ready time of the conflict here
739 }
740
741 // No conflicts; issue write
742 return wq_entry;
743 } else if (miss_mshr) {
744 // need to check for conflicting earlier writeback
745 WriteQueueEntry *conflict_mshr = writeBuffer.findPending(miss_mshr);
746 if (conflict_mshr) {
747 // not sure why we don't check order here... it was in the
748 // original code but commented out.
749
750 // The only way this happens is if we are
751 // doing a write and we didn't have permissions
752 // then subsequently saw a writeback (owned got evicted)
753 // We need to make sure to perform the writeback first
754 // To preserve the dirty data, then we can issue the write
755
756 // should we return wq_entry here instead? I.e. do we
757 // have to flush writes in order? I don't think so... not
758 // for Alpha anyway. Maybe for x86?
759 return conflict_mshr;
760
761 // @todo Note that we ignore the ready time of the conflict here
762 }
763
764 // No conflicts; issue read
765 return miss_mshr;
766 }
767
768 // fall through... no pending requests. Try a prefetch.
769 assert(!miss_mshr && !wq_entry);
770 if (prefetcher && mshrQueue.canPrefetch()) {
771 // If we have a miss queue slot, we can try a prefetch
772 PacketPtr pkt = prefetcher->getPacket();
773 if (pkt) {
774 Addr pf_addr = pkt->getBlockAddr(blkSize);
775 if (!tags->findBlock(pf_addr, pkt->isSecure()) &&
776 !mshrQueue.findMatch(pf_addr, pkt->isSecure()) &&
777 !writeBuffer.findMatch(pf_addr, pkt->isSecure())) {
778 // Update statistic on number of prefetches issued
779 // (hwpf_mshr_misses)
780 assert(pkt->req->masterId() < system->maxMasters());
781 mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++;
782
783 // allocate an MSHR and return it, note
784 // that we send the packet straight away, so do not
785 // schedule the send
786 return allocateMissBuffer(pkt, curTick(), false);
787 } else {
788 // free the request and packet
789 delete pkt;
790 }
791 }
792 }
793
794 return nullptr;
795 }
796
797 void
798 BaseCache::satisfyRequest(PacketPtr pkt, CacheBlk *blk, bool, bool)
799 {
800 assert(pkt->isRequest());
801
802 assert(blk && blk->isValid());
803 // Occasionally this is not true... if we are a lower-level cache
804 // satisfying a string of Read and ReadEx requests from
805 // upper-level caches, a Read will mark the block as shared but we
806 // can satisfy a following ReadEx anyway since we can rely on the
807 // Read requester(s) to have buffered the ReadEx snoop and to
808 // invalidate their blocks after receiving them.
809 // assert(!pkt->needsWritable() || blk->isWritable());
810 assert(pkt->getOffset(blkSize) + pkt->getSize() <= blkSize);
811
812 // Check RMW operations first since both isRead() and
813 // isWrite() will be true for them
814 if (pkt->cmd == MemCmd::SwapReq) {
815 if (pkt->isAtomicOp()) {
816 // extract data from cache and save it into the data field in
817 // the packet as a return value from this atomic op
818 int offset = tags->extractBlkOffset(pkt->getAddr());
819 uint8_t *blk_data = blk->data + offset;
820 pkt->setData(blk_data);
821
822 // execute AMO operation
823 (*(pkt->getAtomicOp()))(blk_data);
824
825 // set block status to dirty
826 blk->status |= BlkDirty;
827 } else {
828 cmpAndSwap(blk, pkt);
829 }
830 } else if (pkt->isWrite()) {
831 // we have the block in a writable state and can go ahead,
832 // note that the line may be also be considered writable in
833 // downstream caches along the path to memory, but always
834 // Exclusive, and never Modified
835 assert(blk->isWritable());
836 // Write or WriteLine at the first cache with block in writable state
837 if (blk->checkWrite(pkt)) {
838 pkt->writeDataToBlock(blk->data, blkSize);
839 }
840 // Always mark the line as dirty (and thus transition to the
841 // Modified state) even if we are a failed StoreCond so we
842 // supply data to any snoops that have appended themselves to
843 // this cache before knowing the store will fail.
844 blk->status |= BlkDirty;
845 DPRINTF(CacheVerbose, "%s for %s (write)\n", __func__, pkt->print());
846 } else if (pkt->isRead()) {
847 if (pkt->isLLSC()) {
848 blk->trackLoadLocked(pkt);
849 }
850
851 // all read responses have a data payload
852 assert(pkt->hasRespData());
853 pkt->setDataFromBlock(blk->data, blkSize);
854 } else if (pkt->isUpgrade()) {
855 // sanity check
856 assert(!pkt->hasSharers());
857
858 if (blk->isDirty()) {
859 // we were in the Owned state, and a cache above us that
860 // has the line in Shared state needs to be made aware
861 // that the data it already has is in fact dirty
862 pkt->setCacheResponding();
863 blk->status &= ~BlkDirty;
864 }
865 } else if (pkt->isClean()) {
866 blk->status &= ~BlkDirty;
867 } else {
868 assert(pkt->isInvalidate());
869 invalidateBlock(blk);
870 DPRINTF(CacheVerbose, "%s for %s (invalidation)\n", __func__,
871 pkt->print());
872 }
873 }
874
875 /////////////////////////////////////////////////////
876 //
877 // Access path: requests coming in from the CPU side
878 //
879 /////////////////////////////////////////////////////
880 Cycles
881 BaseCache::calculateTagOnlyLatency(const uint32_t delay,
882 const Cycles lookup_lat) const
883 {
884 // A tag-only access has to wait for the packet to arrive in order to
885 // perform the tag lookup.
886 return ticksToCycles(delay) + lookup_lat;
887 }
888
889 Cycles
890 BaseCache::calculateAccessLatency(const CacheBlk* blk, const uint32_t delay,
891 const Cycles lookup_lat) const
892 {
893 Cycles lat(0);
894
895 if (blk != nullptr) {
896 // As soon as the access arrives, for sequential accesses first access
897 // tags, then the data entry. In the case of parallel accesses the
898 // latency is dictated by the slowest of tag and data latencies.
899 if (sequentialAccess) {
900 lat = ticksToCycles(delay) + lookup_lat + dataLatency;
901 } else {
902 lat = ticksToCycles(delay) + std::max(lookup_lat, dataLatency);
903 }
904
905 // Check if the block to be accessed is available. If not, apply the
906 // access latency on top of when the block is ready to be accessed.
907 const Tick tick = curTick() + delay;
908 const Tick when_ready = blk->getWhenReady();
909 if (when_ready > tick &&
910 ticksToCycles(when_ready - tick) > lat) {
911 lat += ticksToCycles(when_ready - tick);
912 }
913 } else {
914 // In case of a miss, we neglect the data access in a parallel
915 // configuration (i.e., the data access will be stopped as soon as
916 // we find out it is a miss), and use the tag-only latency.
917 lat = calculateTagOnlyLatency(delay, lookup_lat);
918 }
919
920 return lat;
921 }
922
923 bool
924 BaseCache::access(PacketPtr pkt, CacheBlk *&blk, Cycles &lat,
925 PacketList &writebacks)
926 {
927 // sanity check
928 assert(pkt->isRequest());
929
930 chatty_assert(!(isReadOnly && pkt->isWrite()),
931 "Should never see a write in a read-only cache %s\n",
932 name());
933
934 // Access block in the tags
935 Cycles tag_latency(0);
936 blk = tags->accessBlock(pkt->getAddr(), pkt->isSecure(), tag_latency);
937
938 DPRINTF(Cache, "%s for %s %s\n", __func__, pkt->print(),
939 blk ? "hit " + blk->print() : "miss");
940
941 if (pkt->req->isCacheMaintenance()) {
942 // A cache maintenance operation is always forwarded to the
943 // memory below even if the block is found in dirty state.
944
945 // We defer any changes to the state of the block until we
946 // create and mark as in service the mshr for the downstream
947 // packet.
948
949 // Calculate access latency on top of when the packet arrives. This
950 // takes into account the bus delay.
951 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency);
952
953 return false;
954 }
955
956 if (pkt->isEviction()) {
957 // We check for presence of block in above caches before issuing
958 // Writeback or CleanEvict to write buffer. Therefore the only
959 // possible cases can be of a CleanEvict packet coming from above
960 // encountering a Writeback generated in this cache peer cache and
961 // waiting in the write buffer. Cases of upper level peer caches
962 // generating CleanEvict and Writeback or simply CleanEvict and
963 // CleanEvict almost simultaneously will be caught by snoops sent out
964 // by crossbar.
965 WriteQueueEntry *wb_entry = writeBuffer.findMatch(pkt->getAddr(),
966 pkt->isSecure());
967 if (wb_entry) {
968 assert(wb_entry->getNumTargets() == 1);
969 PacketPtr wbPkt = wb_entry->getTarget()->pkt;
970 assert(wbPkt->isWriteback());
971
972 if (pkt->isCleanEviction()) {
973 // The CleanEvict and WritebackClean snoops into other
974 // peer caches of the same level while traversing the
975 // crossbar. If a copy of the block is found, the
976 // packet is deleted in the crossbar. Hence, none of
977 // the other upper level caches connected to this
978 // cache have the block, so we can clear the
979 // BLOCK_CACHED flag in the Writeback if set and
980 // discard the CleanEvict by returning true.
981 wbPkt->clearBlockCached();
982
983 // A clean evict does not need to access the data array
984 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency);
985
986 return true;
987 } else {
988 assert(pkt->cmd == MemCmd::WritebackDirty);
989 // Dirty writeback from above trumps our clean
990 // writeback... discard here
991 // Note: markInService will remove entry from writeback buffer.
992 markInService(wb_entry);
993 delete wbPkt;
994 }
995 }
996 }
997
998 // Writeback handling is special case. We can write the block into
999 // the cache without having a writeable copy (or any copy at all).
1000 if (pkt->isWriteback()) {
1001 assert(blkSize == pkt->getSize());
1002
1003 // we could get a clean writeback while we are having
1004 // outstanding accesses to a block, do the simple thing for
1005 // now and drop the clean writeback so that we do not upset
1006 // any ordering/decisions about ownership already taken
1007 if (pkt->cmd == MemCmd::WritebackClean &&
1008 mshrQueue.findMatch(pkt->getAddr(), pkt->isSecure())) {
1009 DPRINTF(Cache, "Clean writeback %#llx to block with MSHR, "
1010 "dropping\n", pkt->getAddr());
1011
1012 // A writeback searches for the block, then writes the data.
1013 // As the writeback is being dropped, the data is not touched,
1014 // and we just had to wait for the time to find a match in the
1015 // MSHR. As of now assume a mshr queue search takes as long as
1016 // a tag lookup for simplicity.
1017 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency);
1018
1019 return true;
1020 }
1021
1022 if (!blk) {
1023 // need to do a replacement
1024 blk = allocateBlock(pkt, writebacks);
1025 if (!blk) {
1026 // no replaceable block available: give up, fwd to next level.
1027 incMissCount(pkt);
1028
1029 // A writeback searches for the block, then writes the data.
1030 // As the block could not be found, it was a tag-only access.
1031 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency);
1032
1033 return false;
1034 }
1035
1036 blk->status |= BlkReadable;
1037 }
1038 // only mark the block dirty if we got a writeback command,
1039 // and leave it as is for a clean writeback
1040 if (pkt->cmd == MemCmd::WritebackDirty) {
1041 // TODO: the coherent cache can assert(!blk->isDirty());
1042 blk->status |= BlkDirty;
1043 }
1044 // if the packet does not have sharers, it is passing
1045 // writable, and we got the writeback in Modified or Exclusive
1046 // state, if not we are in the Owned or Shared state
1047 if (!pkt->hasSharers()) {
1048 blk->status |= BlkWritable;
1049 }
1050 // nothing else to do; writeback doesn't expect response
1051 assert(!pkt->needsResponse());
1052 pkt->writeDataToBlock(blk->data, blkSize);
1053 DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print());
1054 incHitCount(pkt);
1055
1056 // A writeback searches for the block, then writes the data
1057 lat = calculateAccessLatency(blk, pkt->headerDelay, tag_latency);
1058
1059 // When the packet metadata arrives, the tag lookup will be done while
1060 // the payload is arriving. Then the block will be ready to access as
1061 // soon as the fill is done
1062 blk->setWhenReady(clockEdge(fillLatency) + pkt->headerDelay +
1063 std::max(cyclesToTicks(tag_latency), (uint64_t)pkt->payloadDelay));
1064
1065 return true;
1066 } else if (pkt->cmd == MemCmd::CleanEvict) {
1067 // A CleanEvict does not need to access the data array
1068 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency);
1069
1070 if (blk) {
1071 // Found the block in the tags, need to stop CleanEvict from
1072 // propagating further down the hierarchy. Returning true will
1073 // treat the CleanEvict like a satisfied write request and delete
1074 // it.
1075 return true;
1076 }
1077 // We didn't find the block here, propagate the CleanEvict further
1078 // down the memory hierarchy. Returning false will treat the CleanEvict
1079 // like a Writeback which could not find a replaceable block so has to
1080 // go to next level.
1081 return false;
1082 } else if (pkt->cmd == MemCmd::WriteClean) {
1083 // WriteClean handling is a special case. We can allocate a
1084 // block directly if it doesn't exist and we can update the
1085 // block immediately. The WriteClean transfers the ownership
1086 // of the block as well.
1087 assert(blkSize == pkt->getSize());
1088
1089 if (!blk) {
1090 if (pkt->writeThrough()) {
1091 // A writeback searches for the block, then writes the data.
1092 // As the block could not be found, it was a tag-only access.
1093 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency);
1094
1095 // if this is a write through packet, we don't try to
1096 // allocate if the block is not present
1097 return false;
1098 } else {
1099 // a writeback that misses needs to allocate a new block
1100 blk = allocateBlock(pkt, writebacks);
1101 if (!blk) {
1102 // no replaceable block available: give up, fwd to
1103 // next level.
1104 incMissCount(pkt);
1105
1106 // A writeback searches for the block, then writes the
1107 // data. As the block could not be found, it was a tag-only
1108 // access.
1109 lat = calculateTagOnlyLatency(pkt->headerDelay,
1110 tag_latency);
1111
1112 return false;
1113 }
1114
1115 blk->status |= BlkReadable;
1116 }
1117 }
1118
1119 // at this point either this is a writeback or a write-through
1120 // write clean operation and the block is already in this
1121 // cache, we need to update the data and the block flags
1122 assert(blk);
1123 // TODO: the coherent cache can assert(!blk->isDirty());
1124 if (!pkt->writeThrough()) {
1125 blk->status |= BlkDirty;
1126 }
1127 // nothing else to do; writeback doesn't expect response
1128 assert(!pkt->needsResponse());
1129 pkt->writeDataToBlock(blk->data, blkSize);
1130 DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print());
1131
1132 incHitCount(pkt);
1133
1134 // A writeback searches for the block, then writes the data
1135 lat = calculateAccessLatency(blk, pkt->headerDelay, tag_latency);
1136
1137 // When the packet metadata arrives, the tag lookup will be done while
1138 // the payload is arriving. Then the block will be ready to access as
1139 // soon as the fill is done
1140 blk->setWhenReady(clockEdge(fillLatency) + pkt->headerDelay +
1141 std::max(cyclesToTicks(tag_latency), (uint64_t)pkt->payloadDelay));
1142
1143 // if this a write-through packet it will be sent to cache
1144 // below
1145 return !pkt->writeThrough();
1146 } else if (blk && (pkt->needsWritable() ? blk->isWritable() :
1147 blk->isReadable())) {
1148 // OK to satisfy access
1149 incHitCount(pkt);
1150
1151 // Calculate access latency based on the need to access the data array
1152 if (pkt->isRead() || pkt->isWrite()) {
1153 lat = calculateAccessLatency(blk, pkt->headerDelay, tag_latency);
1154 } else {
1155 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency);
1156 }
1157
1158 satisfyRequest(pkt, blk);
1159 maintainClusivity(pkt->fromCache(), blk);
1160
1161 return true;
1162 }
1163
1164 // Can't satisfy access normally... either no block (blk == nullptr)
1165 // or have block but need writable
1166
1167 incMissCount(pkt);
1168
1169 lat = calculateAccessLatency(blk, pkt->headerDelay, tag_latency);
1170
1171 if (!blk && pkt->isLLSC() && pkt->isWrite()) {
1172 // complete miss on store conditional... just give up now
1173 pkt->req->setExtraData(0);
1174 return true;
1175 }
1176
1177 return false;
1178 }
1179
1180 void
1181 BaseCache::maintainClusivity(bool from_cache, CacheBlk *blk)
1182 {
1183 if (from_cache && blk && blk->isValid() && !blk->isDirty() &&
1184 clusivity == Enums::mostly_excl) {
1185 // if we have responded to a cache, and our block is still
1186 // valid, but not dirty, and this cache is mostly exclusive
1187 // with respect to the cache above, drop the block
1188 invalidateBlock(blk);
1189 }
1190 }
1191
1192 CacheBlk*
1193 BaseCache::handleFill(PacketPtr pkt, CacheBlk *blk, PacketList &writebacks,
1194 bool allocate)
1195 {
1196 assert(pkt->isResponse());
1197 Addr addr = pkt->getAddr();
1198 bool is_secure = pkt->isSecure();
1199 #if TRACING_ON
1200 CacheBlk::State old_state = blk ? blk->status : 0;
1201 #endif
1202
1203 // When handling a fill, we should have no writes to this line.
1204 assert(addr == pkt->getBlockAddr(blkSize));
1205 assert(!writeBuffer.findMatch(addr, is_secure));
1206
1207 if (!blk) {
1208 // better have read new data...
1209 assert(pkt->hasData() || pkt->cmd == MemCmd::InvalidateResp);
1210
1211 // need to do a replacement if allocating, otherwise we stick
1212 // with the temporary storage
1213 blk = allocate ? allocateBlock(pkt, writebacks) : nullptr;
1214
1215 if (!blk) {
1216 // No replaceable block or a mostly exclusive
1217 // cache... just use temporary storage to complete the
1218 // current request and then get rid of it
1219 blk = tempBlock;
1220 tempBlock->insert(addr, is_secure);
1221 DPRINTF(Cache, "using temp block for %#llx (%s)\n", addr,
1222 is_secure ? "s" : "ns");
1223 }
1224 } else {
1225 // existing block... probably an upgrade
1226 // don't clear block status... if block is already dirty we
1227 // don't want to lose that
1228 }
1229
1230 // Block is guaranteed to be valid at this point
1231 assert(blk->isValid());
1232 assert(blk->isSecure() == is_secure);
1233 assert(regenerateBlkAddr(blk) == addr);
1234
1235 blk->status |= BlkReadable;
1236
1237 // sanity check for whole-line writes, which should always be
1238 // marked as writable as part of the fill, and then later marked
1239 // dirty as part of satisfyRequest
1240 if (pkt->cmd == MemCmd::InvalidateResp) {
1241 assert(!pkt->hasSharers());
1242 }
1243
1244 // here we deal with setting the appropriate state of the line,
1245 // and we start by looking at the hasSharers flag, and ignore the
1246 // cacheResponding flag (normally signalling dirty data) if the
1247 // packet has sharers, thus the line is never allocated as Owned
1248 // (dirty but not writable), and always ends up being either
1249 // Shared, Exclusive or Modified, see Packet::setCacheResponding
1250 // for more details
1251 if (!pkt->hasSharers()) {
1252 // we could get a writable line from memory (rather than a
1253 // cache) even in a read-only cache, note that we set this bit
1254 // even for a read-only cache, possibly revisit this decision
1255 blk->status |= BlkWritable;
1256
1257 // check if we got this via cache-to-cache transfer (i.e., from a
1258 // cache that had the block in Modified or Owned state)
1259 if (pkt->cacheResponding()) {
1260 // we got the block in Modified state, and invalidated the
1261 // owners copy
1262 blk->status |= BlkDirty;
1263
1264 chatty_assert(!isReadOnly, "Should never see dirty snoop response "
1265 "in read-only cache %s\n", name());
1266 }
1267 }
1268
1269 DPRINTF(Cache, "Block addr %#llx (%s) moving from state %x to %s\n",
1270 addr, is_secure ? "s" : "ns", old_state, blk->print());
1271
1272 // if we got new data, copy it in (checking for a read response
1273 // and a response that has data is the same in the end)
1274 if (pkt->isRead()) {
1275 // sanity checks
1276 assert(pkt->hasData());
1277 assert(pkt->getSize() == blkSize);
1278
1279 pkt->writeDataToBlock(blk->data, blkSize);
1280 }
1281 // The block will be ready when the payload arrives and the fill is done
1282 blk->setWhenReady(clockEdge(fillLatency) + pkt->headerDelay +
1283 pkt->payloadDelay);
1284
1285 return blk;
1286 }
1287
1288 CacheBlk*
1289 BaseCache::allocateBlock(const PacketPtr pkt, PacketList &writebacks)
1290 {
1291 // Get address
1292 const Addr addr = pkt->getAddr();
1293
1294 // Get secure bit
1295 const bool is_secure = pkt->isSecure();
1296
1297 // Find replacement victim
1298 std::vector<CacheBlk*> evict_blks;
1299 CacheBlk *victim = tags->findVictim(addr, is_secure, evict_blks);
1300
1301 // It is valid to return nullptr if there is no victim
1302 if (!victim)
1303 return nullptr;
1304
1305 // Print victim block's information
1306 DPRINTF(CacheRepl, "Replacement victim: %s\n", victim->print());
1307
1308 // Check for transient state allocations. If any of the entries listed
1309 // for eviction has a transient state, the allocation fails
1310 for (const auto& blk : evict_blks) {
1311 if (blk->isValid()) {
1312 Addr repl_addr = regenerateBlkAddr(blk);
1313 MSHR *repl_mshr = mshrQueue.findMatch(repl_addr, blk->isSecure());
1314 if (repl_mshr) {
1315 // must be an outstanding upgrade or clean request
1316 // on a block we're about to replace...
1317 assert((!blk->isWritable() && repl_mshr->needsWritable()) ||
1318 repl_mshr->isCleaning());
1319
1320 // too hard to replace block with transient state
1321 // allocation failed, block not inserted
1322 return nullptr;
1323 }
1324 }
1325 }
1326
1327 // The victim will be replaced by a new entry, so increase the replacement
1328 // counter if a valid block is being replaced
1329 if (evict_blks.size() > 0) {
1330 for (const auto& blk : evict_blks) {
1331 if (blk->isValid()) {
1332 DPRINTF(CacheRepl, "Evicting %s (%#llx) to make room for " \
1333 "%#llx (%s)\n", blk->print(), regenerateBlkAddr(blk),
1334 addr, is_secure);
1335 }
1336 }
1337
1338 replacements++;
1339 }
1340
1341 // Evict valid blocks associated to this victim block
1342 for (const auto& blk : evict_blks) {
1343 if (blk->isValid()) {
1344 if (blk->wasPrefetched()) {
1345 unusedPrefetches++;
1346 }
1347
1348 evictBlock(blk, writebacks);
1349 }
1350 }
1351
1352 // Insert new block at victimized entry
1353 tags->insertBlock(pkt, victim);
1354
1355 return victim;
1356 }
1357
1358 void
1359 BaseCache::invalidateBlock(CacheBlk *blk)
1360 {
1361 // If handling a block present in the Tags, let it do its invalidation
1362 // process, which will update stats and invalidate the block itself
1363 if (blk != tempBlock) {
1364 tags->invalidate(blk);
1365 } else {
1366 tempBlock->invalidate();
1367 }
1368 }
1369
1370 void
1371 BaseCache::evictBlock(CacheBlk *blk, PacketList &writebacks)
1372 {
1373 PacketPtr pkt = evictBlock(blk);
1374 if (pkt) {
1375 writebacks.push_back(pkt);
1376 }
1377 }
1378
1379 PacketPtr
1380 BaseCache::writebackBlk(CacheBlk *blk)
1381 {
1382 chatty_assert(!isReadOnly || writebackClean,
1383 "Writeback from read-only cache");
1384 assert(blk && blk->isValid() && (blk->isDirty() || writebackClean));
1385
1386 writebacks[Request::wbMasterId]++;
1387
1388 RequestPtr req = std::make_shared<Request>(
1389 regenerateBlkAddr(blk), blkSize, 0, Request::wbMasterId);
1390
1391 if (blk->isSecure())
1392 req->setFlags(Request::SECURE);
1393
1394 req->taskId(blk->task_id);
1395
1396 PacketPtr pkt =
1397 new Packet(req, blk->isDirty() ?
1398 MemCmd::WritebackDirty : MemCmd::WritebackClean);
1399
1400 DPRINTF(Cache, "Create Writeback %s writable: %d, dirty: %d\n",
1401 pkt->print(), blk->isWritable(), blk->isDirty());
1402
1403 if (blk->isWritable()) {
1404 // not asserting shared means we pass the block in modified
1405 // state, mark our own block non-writeable
1406 blk->status &= ~BlkWritable;
1407 } else {
1408 // we are in the Owned state, tell the receiver
1409 pkt->setHasSharers();
1410 }
1411
1412 // make sure the block is not marked dirty
1413 blk->status &= ~BlkDirty;
1414
1415 pkt->allocate();
1416 pkt->setDataFromBlock(blk->data, blkSize);
1417
1418 return pkt;
1419 }
1420
1421 PacketPtr
1422 BaseCache::writecleanBlk(CacheBlk *blk, Request::Flags dest, PacketId id)
1423 {
1424 RequestPtr req = std::make_shared<Request>(
1425 regenerateBlkAddr(blk), blkSize, 0, Request::wbMasterId);
1426
1427 if (blk->isSecure()) {
1428 req->setFlags(Request::SECURE);
1429 }
1430 req->taskId(blk->task_id);
1431
1432 PacketPtr pkt = new Packet(req, MemCmd::WriteClean, blkSize, id);
1433
1434 if (dest) {
1435 req->setFlags(dest);
1436 pkt->setWriteThrough();
1437 }
1438
1439 DPRINTF(Cache, "Create %s writable: %d, dirty: %d\n", pkt->print(),
1440 blk->isWritable(), blk->isDirty());
1441
1442 if (blk->isWritable()) {
1443 // not asserting shared means we pass the block in modified
1444 // state, mark our own block non-writeable
1445 blk->status &= ~BlkWritable;
1446 } else {
1447 // we are in the Owned state, tell the receiver
1448 pkt->setHasSharers();
1449 }
1450
1451 // make sure the block is not marked dirty
1452 blk->status &= ~BlkDirty;
1453
1454 pkt->allocate();
1455 pkt->setDataFromBlock(blk->data, blkSize);
1456
1457 return pkt;
1458 }
1459
1460
1461 void
1462 BaseCache::memWriteback()
1463 {
1464 tags->forEachBlk([this](CacheBlk &blk) { writebackVisitor(blk); });
1465 }
1466
1467 void
1468 BaseCache::memInvalidate()
1469 {
1470 tags->forEachBlk([this](CacheBlk &blk) { invalidateVisitor(blk); });
1471 }
1472
1473 bool
1474 BaseCache::isDirty() const
1475 {
1476 return tags->anyBlk([](CacheBlk &blk) { return blk.isDirty(); });
1477 }
1478
1479 bool
1480 BaseCache::coalesce() const
1481 {
1482 return writeAllocator && writeAllocator->coalesce();
1483 }
1484
1485 void
1486 BaseCache::writebackVisitor(CacheBlk &blk)
1487 {
1488 if (blk.isDirty()) {
1489 assert(blk.isValid());
1490
1491 RequestPtr request = std::make_shared<Request>(
1492 regenerateBlkAddr(&blk), blkSize, 0, Request::funcMasterId);
1493
1494 request->taskId(blk.task_id);
1495 if (blk.isSecure()) {
1496 request->setFlags(Request::SECURE);
1497 }
1498
1499 Packet packet(request, MemCmd::WriteReq);
1500 packet.dataStatic(blk.data);
1501
1502 memSidePort.sendFunctional(&packet);
1503
1504 blk.status &= ~BlkDirty;
1505 }
1506 }
1507
1508 void
1509 BaseCache::invalidateVisitor(CacheBlk &blk)
1510 {
1511 if (blk.isDirty())
1512 warn_once("Invalidating dirty cache lines. " \
1513 "Expect things to break.\n");
1514
1515 if (blk.isValid()) {
1516 assert(!blk.isDirty());
1517 invalidateBlock(&blk);
1518 }
1519 }
1520
1521 Tick
1522 BaseCache::nextQueueReadyTime() const
1523 {
1524 Tick nextReady = std::min(mshrQueue.nextReadyTime(),
1525 writeBuffer.nextReadyTime());
1526
1527 // Don't signal prefetch ready time if no MSHRs available
1528 // Will signal once enoguh MSHRs are deallocated
1529 if (prefetcher && mshrQueue.canPrefetch()) {
1530 nextReady = std::min(nextReady,
1531 prefetcher->nextPrefetchReadyTime());
1532 }
1533
1534 return nextReady;
1535 }
1536
1537
1538 bool
1539 BaseCache::sendMSHRQueuePacket(MSHR* mshr)
1540 {
1541 assert(mshr);
1542
1543 // use request from 1st target
1544 PacketPtr tgt_pkt = mshr->getTarget()->pkt;
1545
1546 DPRINTF(Cache, "%s: MSHR %s\n", __func__, tgt_pkt->print());
1547
1548 // if the cache is in write coalescing mode or (additionally) in
1549 // no allocation mode, and we have a write packet with an MSHR
1550 // that is not a whole-line write (due to incompatible flags etc),
1551 // then reset the write mode
1552 if (writeAllocator && writeAllocator->coalesce() && tgt_pkt->isWrite()) {
1553 if (!mshr->isWholeLineWrite()) {
1554 // if we are currently write coalescing, hold on the
1555 // MSHR as many cycles extra as we need to completely
1556 // write a cache line
1557 if (writeAllocator->delay(mshr->blkAddr)) {
1558 Tick delay = blkSize / tgt_pkt->getSize() * clockPeriod();
1559 DPRINTF(CacheVerbose, "Delaying pkt %s %llu ticks to allow "
1560 "for write coalescing\n", tgt_pkt->print(), delay);
1561 mshrQueue.delay(mshr, delay);
1562 return false;
1563 } else {
1564 writeAllocator->reset();
1565 }
1566 } else {
1567 writeAllocator->resetDelay(mshr->blkAddr);
1568 }
1569 }
1570
1571 CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure);
1572
1573 // either a prefetch that is not present upstream, or a normal
1574 // MSHR request, proceed to get the packet to send downstream
1575 PacketPtr pkt = createMissPacket(tgt_pkt, blk, mshr->needsWritable(),
1576 mshr->isWholeLineWrite());
1577
1578 mshr->isForward = (pkt == nullptr);
1579
1580 if (mshr->isForward) {
1581 // not a cache block request, but a response is expected
1582 // make copy of current packet to forward, keep current
1583 // copy for response handling
1584 pkt = new Packet(tgt_pkt, false, true);
1585 assert(!pkt->isWrite());
1586 }
1587
1588 // play it safe and append (rather than set) the sender state,
1589 // as forwarded packets may already have existing state
1590 pkt->pushSenderState(mshr);
1591
1592 if (pkt->isClean() && blk && blk->isDirty()) {
1593 // A cache clean opearation is looking for a dirty block. Mark
1594 // the packet so that the destination xbar can determine that
1595 // there will be a follow-up write packet as well.
1596 pkt->setSatisfied();
1597 }
1598
1599 if (!memSidePort.sendTimingReq(pkt)) {
1600 // we are awaiting a retry, but we
1601 // delete the packet and will be creating a new packet
1602 // when we get the opportunity
1603 delete pkt;
1604
1605 // note that we have now masked any requestBus and
1606 // schedSendEvent (we will wait for a retry before
1607 // doing anything), and this is so even if we do not
1608 // care about this packet and might override it before
1609 // it gets retried
1610 return true;
1611 } else {
1612 // As part of the call to sendTimingReq the packet is
1613 // forwarded to all neighbouring caches (and any caches
1614 // above them) as a snoop. Thus at this point we know if
1615 // any of the neighbouring caches are responding, and if
1616 // so, we know it is dirty, and we can determine if it is
1617 // being passed as Modified, making our MSHR the ordering
1618 // point
1619 bool pending_modified_resp = !pkt->hasSharers() &&
1620 pkt->cacheResponding();
1621 markInService(mshr, pending_modified_resp);
1622
1623 if (pkt->isClean() && blk && blk->isDirty()) {
1624 // A cache clean opearation is looking for a dirty
1625 // block. If a dirty block is encountered a WriteClean
1626 // will update any copies to the path to the memory
1627 // until the point of reference.
1628 DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n",
1629 __func__, pkt->print(), blk->print());
1630 PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(),
1631 pkt->id);
1632 PacketList writebacks;
1633 writebacks.push_back(wb_pkt);
1634 doWritebacks(writebacks, 0);
1635 }
1636
1637 return false;
1638 }
1639 }
1640
1641 bool
1642 BaseCache::sendWriteQueuePacket(WriteQueueEntry* wq_entry)
1643 {
1644 assert(wq_entry);
1645
1646 // always a single target for write queue entries
1647 PacketPtr tgt_pkt = wq_entry->getTarget()->pkt;
1648
1649 DPRINTF(Cache, "%s: write %s\n", __func__, tgt_pkt->print());
1650
1651 // forward as is, both for evictions and uncacheable writes
1652 if (!memSidePort.sendTimingReq(tgt_pkt)) {
1653 // note that we have now masked any requestBus and
1654 // schedSendEvent (we will wait for a retry before
1655 // doing anything), and this is so even if we do not
1656 // care about this packet and might override it before
1657 // it gets retried
1658 return true;
1659 } else {
1660 markInService(wq_entry);
1661 return false;
1662 }
1663 }
1664
1665 void
1666 BaseCache::serialize(CheckpointOut &cp) const
1667 {
1668 bool dirty(isDirty());
1669
1670 if (dirty) {
1671 warn("*** The cache still contains dirty data. ***\n");
1672 warn(" Make sure to drain the system using the correct flags.\n");
1673 warn(" This checkpoint will not restore correctly " \
1674 "and dirty data in the cache will be lost!\n");
1675 }
1676
1677 // Since we don't checkpoint the data in the cache, any dirty data
1678 // will be lost when restoring from a checkpoint of a system that
1679 // wasn't drained properly. Flag the checkpoint as invalid if the
1680 // cache contains dirty data.
1681 bool bad_checkpoint(dirty);
1682 SERIALIZE_SCALAR(bad_checkpoint);
1683 }
1684
1685 void
1686 BaseCache::unserialize(CheckpointIn &cp)
1687 {
1688 bool bad_checkpoint;
1689 UNSERIALIZE_SCALAR(bad_checkpoint);
1690 if (bad_checkpoint) {
1691 fatal("Restoring from checkpoints with dirty caches is not "
1692 "supported in the classic memory system. Please remove any "
1693 "caches or drain them properly before taking checkpoints.\n");
1694 }
1695 }
1696
1697 void
1698 BaseCache::regStats()
1699 {
1700 MemObject::regStats();
1701
1702 using namespace Stats;
1703
1704 // Hit statistics
1705 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) {
1706 MemCmd cmd(access_idx);
1707 const string &cstr = cmd.toString();
1708
1709 hits[access_idx]
1710 .init(system->maxMasters())
1711 .name(name() + "." + cstr + "_hits")
1712 .desc("number of " + cstr + " hits")
1713 .flags(total | nozero | nonan)
1714 ;
1715 for (int i = 0; i < system->maxMasters(); i++) {
1716 hits[access_idx].subname(i, system->getMasterName(i));
1717 }
1718 }
1719
1720 // These macros make it easier to sum the right subset of commands and
1721 // to change the subset of commands that are considered "demand" vs
1722 // "non-demand"
1723 #define SUM_DEMAND(s) \
1724 (s[MemCmd::ReadReq] + s[MemCmd::WriteReq] + s[MemCmd::WriteLineReq] + \
1725 s[MemCmd::ReadExReq] + s[MemCmd::ReadCleanReq] + s[MemCmd::ReadSharedReq])
1726
1727 // should writebacks be included here? prior code was inconsistent...
1728 #define SUM_NON_DEMAND(s) \
1729 (s[MemCmd::SoftPFReq] + s[MemCmd::HardPFReq] + s[MemCmd::SoftPFExReq])
1730
1731 demandHits
1732 .name(name() + ".demand_hits")
1733 .desc("number of demand (read+write) hits")
1734 .flags(total | nozero | nonan)
1735 ;
1736 demandHits = SUM_DEMAND(hits);
1737 for (int i = 0; i < system->maxMasters(); i++) {
1738 demandHits.subname(i, system->getMasterName(i));
1739 }
1740
1741 overallHits
1742 .name(name() + ".overall_hits")
1743 .desc("number of overall hits")
1744 .flags(total | nozero | nonan)
1745 ;
1746 overallHits = demandHits + SUM_NON_DEMAND(hits);
1747 for (int i = 0; i < system->maxMasters(); i++) {
1748 overallHits.subname(i, system->getMasterName(i));
1749 }
1750
1751 // Miss statistics
1752 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) {
1753 MemCmd cmd(access_idx);
1754 const string &cstr = cmd.toString();
1755
1756 misses[access_idx]
1757 .init(system->maxMasters())
1758 .name(name() + "." + cstr + "_misses")
1759 .desc("number of " + cstr + " misses")
1760 .flags(total | nozero | nonan)
1761 ;
1762 for (int i = 0; i < system->maxMasters(); i++) {
1763 misses[access_idx].subname(i, system->getMasterName(i));
1764 }
1765 }
1766
1767 demandMisses
1768 .name(name() + ".demand_misses")
1769 .desc("number of demand (read+write) misses")
1770 .flags(total | nozero | nonan)
1771 ;
1772 demandMisses = SUM_DEMAND(misses);
1773 for (int i = 0; i < system->maxMasters(); i++) {
1774 demandMisses.subname(i, system->getMasterName(i));
1775 }
1776
1777 overallMisses
1778 .name(name() + ".overall_misses")
1779 .desc("number of overall misses")
1780 .flags(total | nozero | nonan)
1781 ;
1782 overallMisses = demandMisses + SUM_NON_DEMAND(misses);
1783 for (int i = 0; i < system->maxMasters(); i++) {
1784 overallMisses.subname(i, system->getMasterName(i));
1785 }
1786
1787 // Miss latency statistics
1788 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) {
1789 MemCmd cmd(access_idx);
1790 const string &cstr = cmd.toString();
1791
1792 missLatency[access_idx]
1793 .init(system->maxMasters())
1794 .name(name() + "." + cstr + "_miss_latency")
1795 .desc("number of " + cstr + " miss cycles")
1796 .flags(total | nozero | nonan)
1797 ;
1798 for (int i = 0; i < system->maxMasters(); i++) {
1799 missLatency[access_idx].subname(i, system->getMasterName(i));
1800 }
1801 }
1802
1803 demandMissLatency
1804 .name(name() + ".demand_miss_latency")
1805 .desc("number of demand (read+write) miss cycles")
1806 .flags(total | nozero | nonan)
1807 ;
1808 demandMissLatency = SUM_DEMAND(missLatency);
1809 for (int i = 0; i < system->maxMasters(); i++) {
1810 demandMissLatency.subname(i, system->getMasterName(i));
1811 }
1812
1813 overallMissLatency
1814 .name(name() + ".overall_miss_latency")
1815 .desc("number of overall miss cycles")
1816 .flags(total | nozero | nonan)
1817 ;
1818 overallMissLatency = demandMissLatency + SUM_NON_DEMAND(missLatency);
1819 for (int i = 0; i < system->maxMasters(); i++) {
1820 overallMissLatency.subname(i, system->getMasterName(i));
1821 }
1822
1823 // access formulas
1824 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) {
1825 MemCmd cmd(access_idx);
1826 const string &cstr = cmd.toString();
1827
1828 accesses[access_idx]
1829 .name(name() + "." + cstr + "_accesses")
1830 .desc("number of " + cstr + " accesses(hits+misses)")
1831 .flags(total | nozero | nonan)
1832 ;
1833 accesses[access_idx] = hits[access_idx] + misses[access_idx];
1834
1835 for (int i = 0; i < system->maxMasters(); i++) {
1836 accesses[access_idx].subname(i, system->getMasterName(i));
1837 }
1838 }
1839
1840 demandAccesses
1841 .name(name() + ".demand_accesses")
1842 .desc("number of demand (read+write) accesses")
1843 .flags(total | nozero | nonan)
1844 ;
1845 demandAccesses = demandHits + demandMisses;
1846 for (int i = 0; i < system->maxMasters(); i++) {
1847 demandAccesses.subname(i, system->getMasterName(i));
1848 }
1849
1850 overallAccesses
1851 .name(name() + ".overall_accesses")
1852 .desc("number of overall (read+write) accesses")
1853 .flags(total | nozero | nonan)
1854 ;
1855 overallAccesses = overallHits + overallMisses;
1856 for (int i = 0; i < system->maxMasters(); i++) {
1857 overallAccesses.subname(i, system->getMasterName(i));
1858 }
1859
1860 // miss rate formulas
1861 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) {
1862 MemCmd cmd(access_idx);
1863 const string &cstr = cmd.toString();
1864
1865 missRate[access_idx]
1866 .name(name() + "." + cstr + "_miss_rate")
1867 .desc("miss rate for " + cstr + " accesses")
1868 .flags(total | nozero | nonan)
1869 ;
1870 missRate[access_idx] = misses[access_idx] / accesses[access_idx];
1871
1872 for (int i = 0; i < system->maxMasters(); i++) {
1873 missRate[access_idx].subname(i, system->getMasterName(i));
1874 }
1875 }
1876
1877 demandMissRate
1878 .name(name() + ".demand_miss_rate")
1879 .desc("miss rate for demand accesses")
1880 .flags(total | nozero | nonan)
1881 ;
1882 demandMissRate = demandMisses / demandAccesses;
1883 for (int i = 0; i < system->maxMasters(); i++) {
1884 demandMissRate.subname(i, system->getMasterName(i));
1885 }
1886
1887 overallMissRate
1888 .name(name() + ".overall_miss_rate")
1889 .desc("miss rate for overall accesses")
1890 .flags(total | nozero | nonan)
1891 ;
1892 overallMissRate = overallMisses / overallAccesses;
1893 for (int i = 0; i < system->maxMasters(); i++) {
1894 overallMissRate.subname(i, system->getMasterName(i));
1895 }
1896
1897 // miss latency formulas
1898 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) {
1899 MemCmd cmd(access_idx);
1900 const string &cstr = cmd.toString();
1901
1902 avgMissLatency[access_idx]
1903 .name(name() + "." + cstr + "_avg_miss_latency")
1904 .desc("average " + cstr + " miss latency")
1905 .flags(total | nozero | nonan)
1906 ;
1907 avgMissLatency[access_idx] =
1908 missLatency[access_idx] / misses[access_idx];
1909
1910 for (int i = 0; i < system->maxMasters(); i++) {
1911 avgMissLatency[access_idx].subname(i, system->getMasterName(i));
1912 }
1913 }
1914
1915 demandAvgMissLatency
1916 .name(name() + ".demand_avg_miss_latency")
1917 .desc("average overall miss latency")
1918 .flags(total | nozero | nonan)
1919 ;
1920 demandAvgMissLatency = demandMissLatency / demandMisses;
1921 for (int i = 0; i < system->maxMasters(); i++) {
1922 demandAvgMissLatency.subname(i, system->getMasterName(i));
1923 }
1924
1925 overallAvgMissLatency
1926 .name(name() + ".overall_avg_miss_latency")
1927 .desc("average overall miss latency")
1928 .flags(total | nozero | nonan)
1929 ;
1930 overallAvgMissLatency = overallMissLatency / overallMisses;
1931 for (int i = 0; i < system->maxMasters(); i++) {
1932 overallAvgMissLatency.subname(i, system->getMasterName(i));
1933 }
1934
1935 blocked_cycles.init(NUM_BLOCKED_CAUSES);
1936 blocked_cycles
1937 .name(name() + ".blocked_cycles")
1938 .desc("number of cycles access was blocked")
1939 .subname(Blocked_NoMSHRs, "no_mshrs")
1940 .subname(Blocked_NoTargets, "no_targets")
1941 ;
1942
1943
1944 blocked_causes.init(NUM_BLOCKED_CAUSES);
1945 blocked_causes
1946 .name(name() + ".blocked")
1947 .desc("number of cycles access was blocked")
1948 .subname(Blocked_NoMSHRs, "no_mshrs")
1949 .subname(Blocked_NoTargets, "no_targets")
1950 ;
1951
1952 avg_blocked
1953 .name(name() + ".avg_blocked_cycles")
1954 .desc("average number of cycles each access was blocked")
1955 .subname(Blocked_NoMSHRs, "no_mshrs")
1956 .subname(Blocked_NoTargets, "no_targets")
1957 ;
1958
1959 avg_blocked = blocked_cycles / blocked_causes;
1960
1961 unusedPrefetches
1962 .name(name() + ".unused_prefetches")
1963 .desc("number of HardPF blocks evicted w/o reference")
1964 .flags(nozero)
1965 ;
1966
1967 writebacks
1968 .init(system->maxMasters())
1969 .name(name() + ".writebacks")
1970 .desc("number of writebacks")
1971 .flags(total | nozero | nonan)
1972 ;
1973 for (int i = 0; i < system->maxMasters(); i++) {
1974 writebacks.subname(i, system->getMasterName(i));
1975 }
1976
1977 // MSHR statistics
1978 // MSHR hit statistics
1979 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) {
1980 MemCmd cmd(access_idx);
1981 const string &cstr = cmd.toString();
1982
1983 mshr_hits[access_idx]
1984 .init(system->maxMasters())
1985 .name(name() + "." + cstr + "_mshr_hits")
1986 .desc("number of " + cstr + " MSHR hits")
1987 .flags(total | nozero | nonan)
1988 ;
1989 for (int i = 0; i < system->maxMasters(); i++) {
1990 mshr_hits[access_idx].subname(i, system->getMasterName(i));
1991 }
1992 }
1993
1994 demandMshrHits
1995 .name(name() + ".demand_mshr_hits")
1996 .desc("number of demand (read+write) MSHR hits")
1997 .flags(total | nozero | nonan)
1998 ;
1999 demandMshrHits = SUM_DEMAND(mshr_hits);
2000 for (int i = 0; i < system->maxMasters(); i++) {
2001 demandMshrHits.subname(i, system->getMasterName(i));
2002 }
2003
2004 overallMshrHits
2005 .name(name() + ".overall_mshr_hits")
2006 .desc("number of overall MSHR hits")
2007 .flags(total | nozero | nonan)
2008 ;
2009 overallMshrHits = demandMshrHits + SUM_NON_DEMAND(mshr_hits);
2010 for (int i = 0; i < system->maxMasters(); i++) {
2011 overallMshrHits.subname(i, system->getMasterName(i));
2012 }
2013
2014 // MSHR miss statistics
2015 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) {
2016 MemCmd cmd(access_idx);
2017 const string &cstr = cmd.toString();
2018
2019 mshr_misses[access_idx]
2020 .init(system->maxMasters())
2021 .name(name() + "." + cstr + "_mshr_misses")
2022 .desc("number of " + cstr + " MSHR misses")
2023 .flags(total | nozero | nonan)
2024 ;
2025 for (int i = 0; i < system->maxMasters(); i++) {
2026 mshr_misses[access_idx].subname(i, system->getMasterName(i));
2027 }
2028 }
2029
2030 demandMshrMisses
2031 .name(name() + ".demand_mshr_misses")
2032 .desc("number of demand (read+write) MSHR misses")
2033 .flags(total | nozero | nonan)
2034 ;
2035 demandMshrMisses = SUM_DEMAND(mshr_misses);
2036 for (int i = 0; i < system->maxMasters(); i++) {
2037 demandMshrMisses.subname(i, system->getMasterName(i));
2038 }
2039
2040 overallMshrMisses
2041 .name(name() + ".overall_mshr_misses")
2042 .desc("number of overall MSHR misses")
2043 .flags(total | nozero | nonan)
2044 ;
2045 overallMshrMisses = demandMshrMisses + SUM_NON_DEMAND(mshr_misses);
2046 for (int i = 0; i < system->maxMasters(); i++) {
2047 overallMshrMisses.subname(i, system->getMasterName(i));
2048 }
2049
2050 // MSHR miss latency statistics
2051 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) {
2052 MemCmd cmd(access_idx);
2053 const string &cstr = cmd.toString();
2054
2055 mshr_miss_latency[access_idx]
2056 .init(system->maxMasters())
2057 .name(name() + "." + cstr + "_mshr_miss_latency")
2058 .desc("number of " + cstr + " MSHR miss cycles")
2059 .flags(total | nozero | nonan)
2060 ;
2061 for (int i = 0; i < system->maxMasters(); i++) {
2062 mshr_miss_latency[access_idx].subname(i, system->getMasterName(i));
2063 }
2064 }
2065
2066 demandMshrMissLatency
2067 .name(name() + ".demand_mshr_miss_latency")
2068 .desc("number of demand (read+write) MSHR miss cycles")
2069 .flags(total | nozero | nonan)
2070 ;
2071 demandMshrMissLatency = SUM_DEMAND(mshr_miss_latency);
2072 for (int i = 0; i < system->maxMasters(); i++) {
2073 demandMshrMissLatency.subname(i, system->getMasterName(i));
2074 }
2075
2076 overallMshrMissLatency
2077 .name(name() + ".overall_mshr_miss_latency")
2078 .desc("number of overall MSHR miss cycles")
2079 .flags(total | nozero | nonan)
2080 ;
2081 overallMshrMissLatency =
2082 demandMshrMissLatency + SUM_NON_DEMAND(mshr_miss_latency);
2083 for (int i = 0; i < system->maxMasters(); i++) {
2084 overallMshrMissLatency.subname(i, system->getMasterName(i));
2085 }
2086
2087 // MSHR uncacheable statistics
2088 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) {
2089 MemCmd cmd(access_idx);
2090 const string &cstr = cmd.toString();
2091
2092 mshr_uncacheable[access_idx]
2093 .init(system->maxMasters())
2094 .name(name() + "." + cstr + "_mshr_uncacheable")
2095 .desc("number of " + cstr + " MSHR uncacheable")
2096 .flags(total | nozero | nonan)
2097 ;
2098 for (int i = 0; i < system->maxMasters(); i++) {
2099 mshr_uncacheable[access_idx].subname(i, system->getMasterName(i));
2100 }
2101 }
2102
2103 overallMshrUncacheable
2104 .name(name() + ".overall_mshr_uncacheable_misses")
2105 .desc("number of overall MSHR uncacheable misses")
2106 .flags(total | nozero | nonan)
2107 ;
2108 overallMshrUncacheable =
2109 SUM_DEMAND(mshr_uncacheable) + SUM_NON_DEMAND(mshr_uncacheable);
2110 for (int i = 0; i < system->maxMasters(); i++) {
2111 overallMshrUncacheable.subname(i, system->getMasterName(i));
2112 }
2113
2114 // MSHR miss latency statistics
2115 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) {
2116 MemCmd cmd(access_idx);
2117 const string &cstr = cmd.toString();
2118
2119 mshr_uncacheable_lat[access_idx]
2120 .init(system->maxMasters())
2121 .name(name() + "." + cstr + "_mshr_uncacheable_latency")
2122 .desc("number of " + cstr + " MSHR uncacheable cycles")
2123 .flags(total | nozero | nonan)
2124 ;
2125 for (int i = 0; i < system->maxMasters(); i++) {
2126 mshr_uncacheable_lat[access_idx].subname(
2127 i, system->getMasterName(i));
2128 }
2129 }
2130
2131 overallMshrUncacheableLatency
2132 .name(name() + ".overall_mshr_uncacheable_latency")
2133 .desc("number of overall MSHR uncacheable cycles")
2134 .flags(total | nozero | nonan)
2135 ;
2136 overallMshrUncacheableLatency =
2137 SUM_DEMAND(mshr_uncacheable_lat) +
2138 SUM_NON_DEMAND(mshr_uncacheable_lat);
2139 for (int i = 0; i < system->maxMasters(); i++) {
2140 overallMshrUncacheableLatency.subname(i, system->getMasterName(i));
2141 }
2142
2143 #if 0
2144 // MSHR access formulas
2145 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) {
2146 MemCmd cmd(access_idx);
2147 const string &cstr = cmd.toString();
2148
2149 mshrAccesses[access_idx]
2150 .name(name() + "." + cstr + "_mshr_accesses")
2151 .desc("number of " + cstr + " mshr accesses(hits+misses)")
2152 .flags(total | nozero | nonan)
2153 ;
2154 mshrAccesses[access_idx] =
2155 mshr_hits[access_idx] + mshr_misses[access_idx]
2156 + mshr_uncacheable[access_idx];
2157 }
2158
2159 demandMshrAccesses
2160 .name(name() + ".demand_mshr_accesses")
2161 .desc("number of demand (read+write) mshr accesses")
2162 .flags(total | nozero | nonan)
2163 ;
2164 demandMshrAccesses = demandMshrHits + demandMshrMisses;
2165
2166 overallMshrAccesses
2167 .name(name() + ".overall_mshr_accesses")
2168 .desc("number of overall (read+write) mshr accesses")
2169 .flags(total | nozero | nonan)
2170 ;
2171 overallMshrAccesses = overallMshrHits + overallMshrMisses
2172 + overallMshrUncacheable;
2173 #endif
2174
2175 // MSHR miss rate formulas
2176 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) {
2177 MemCmd cmd(access_idx);
2178 const string &cstr = cmd.toString();
2179
2180 mshrMissRate[access_idx]
2181 .name(name() + "." + cstr + "_mshr_miss_rate")
2182 .desc("mshr miss rate for " + cstr + " accesses")
2183 .flags(total | nozero | nonan)
2184 ;
2185 mshrMissRate[access_idx] =
2186 mshr_misses[access_idx] / accesses[access_idx];
2187
2188 for (int i = 0; i < system->maxMasters(); i++) {
2189 mshrMissRate[access_idx].subname(i, system->getMasterName(i));
2190 }
2191 }
2192
2193 demandMshrMissRate
2194 .name(name() + ".demand_mshr_miss_rate")
2195 .desc("mshr miss rate for demand accesses")
2196 .flags(total | nozero | nonan)
2197 ;
2198 demandMshrMissRate = demandMshrMisses / demandAccesses;
2199 for (int i = 0; i < system->maxMasters(); i++) {
2200 demandMshrMissRate.subname(i, system->getMasterName(i));
2201 }
2202
2203 overallMshrMissRate
2204 .name(name() + ".overall_mshr_miss_rate")
2205 .desc("mshr miss rate for overall accesses")
2206 .flags(total | nozero | nonan)
2207 ;
2208 overallMshrMissRate = overallMshrMisses / overallAccesses;
2209 for (int i = 0; i < system->maxMasters(); i++) {
2210 overallMshrMissRate.subname(i, system->getMasterName(i));
2211 }
2212
2213 // mshrMiss latency formulas
2214 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) {
2215 MemCmd cmd(access_idx);
2216 const string &cstr = cmd.toString();
2217
2218 avgMshrMissLatency[access_idx]
2219 .name(name() + "." + cstr + "_avg_mshr_miss_latency")
2220 .desc("average " + cstr + " mshr miss latency")
2221 .flags(total | nozero | nonan)
2222 ;
2223 avgMshrMissLatency[access_idx] =
2224 mshr_miss_latency[access_idx] / mshr_misses[access_idx];
2225
2226 for (int i = 0; i < system->maxMasters(); i++) {
2227 avgMshrMissLatency[access_idx].subname(
2228 i, system->getMasterName(i));
2229 }
2230 }
2231
2232 demandAvgMshrMissLatency
2233 .name(name() + ".demand_avg_mshr_miss_latency")
2234 .desc("average overall mshr miss latency")
2235 .flags(total | nozero | nonan)
2236 ;
2237 demandAvgMshrMissLatency = demandMshrMissLatency / demandMshrMisses;
2238 for (int i = 0; i < system->maxMasters(); i++) {
2239 demandAvgMshrMissLatency.subname(i, system->getMasterName(i));
2240 }
2241
2242 overallAvgMshrMissLatency
2243 .name(name() + ".overall_avg_mshr_miss_latency")
2244 .desc("average overall mshr miss latency")
2245 .flags(total | nozero | nonan)
2246 ;
2247 overallAvgMshrMissLatency = overallMshrMissLatency / overallMshrMisses;
2248 for (int i = 0; i < system->maxMasters(); i++) {
2249 overallAvgMshrMissLatency.subname(i, system->getMasterName(i));
2250 }
2251
2252 // mshrUncacheable latency formulas
2253 for (int access_idx = 0; access_idx < MemCmd::NUM_MEM_CMDS; ++access_idx) {
2254 MemCmd cmd(access_idx);
2255 const string &cstr = cmd.toString();
2256
2257 avgMshrUncacheableLatency[access_idx]
2258 .name(name() + "." + cstr + "_avg_mshr_uncacheable_latency")
2259 .desc("average " + cstr + " mshr uncacheable latency")
2260 .flags(total | nozero | nonan)
2261 ;
2262 avgMshrUncacheableLatency[access_idx] =
2263 mshr_uncacheable_lat[access_idx] / mshr_uncacheable[access_idx];
2264
2265 for (int i = 0; i < system->maxMasters(); i++) {
2266 avgMshrUncacheableLatency[access_idx].subname(
2267 i, system->getMasterName(i));
2268 }
2269 }
2270
2271 overallAvgMshrUncacheableLatency
2272 .name(name() + ".overall_avg_mshr_uncacheable_latency")
2273 .desc("average overall mshr uncacheable latency")
2274 .flags(total | nozero | nonan)
2275 ;
2276 overallAvgMshrUncacheableLatency =
2277 overallMshrUncacheableLatency / overallMshrUncacheable;
2278 for (int i = 0; i < system->maxMasters(); i++) {
2279 overallAvgMshrUncacheableLatency.subname(i, system->getMasterName(i));
2280 }
2281
2282 replacements
2283 .name(name() + ".replacements")
2284 .desc("number of replacements")
2285 ;
2286 }
2287
2288 void
2289 BaseCache::regProbePoints()
2290 {
2291 ppHit = new ProbePointArg<PacketPtr>(this->getProbeManager(), "Hit");
2292 ppMiss = new ProbePointArg<PacketPtr>(this->getProbeManager(), "Miss");
2293 ppFill = new ProbePointArg<PacketPtr>(this->getProbeManager(), "Fill");
2294 }
2295
2296 ///////////////
2297 //
2298 // CpuSidePort
2299 //
2300 ///////////////
2301 bool
2302 BaseCache::CpuSidePort::recvTimingSnoopResp(PacketPtr pkt)
2303 {
2304 // Snoops shouldn't happen when bypassing caches
2305 assert(!cache->system->bypassCaches());
2306
2307 assert(pkt->isResponse());
2308
2309 // Express snoop responses from master to slave, e.g., from L1 to L2
2310 cache->recvTimingSnoopResp(pkt);
2311 return true;
2312 }
2313
2314
2315 bool
2316 BaseCache::CpuSidePort::tryTiming(PacketPtr pkt)
2317 {
2318 if (cache->system->bypassCaches() || pkt->isExpressSnoop()) {
2319 // always let express snoop packets through even if blocked
2320 return true;
2321 } else if (blocked || mustSendRetry) {
2322 // either already committed to send a retry, or blocked
2323 mustSendRetry = true;
2324 return false;
2325 }
2326 mustSendRetry = false;
2327 return true;
2328 }
2329
2330 bool
2331 BaseCache::CpuSidePort::recvTimingReq(PacketPtr pkt)
2332 {
2333 assert(pkt->isRequest());
2334
2335 if (cache->system->bypassCaches()) {
2336 // Just forward the packet if caches are disabled.
2337 // @todo This should really enqueue the packet rather
2338 bool M5_VAR_USED success = cache->memSidePort.sendTimingReq(pkt);
2339 assert(success);
2340 return true;
2341 } else if (tryTiming(pkt)) {
2342 cache->recvTimingReq(pkt);
2343 return true;
2344 }
2345 return false;
2346 }
2347
2348 Tick
2349 BaseCache::CpuSidePort::recvAtomic(PacketPtr pkt)
2350 {
2351 if (cache->system->bypassCaches()) {
2352 // Forward the request if the system is in cache bypass mode.
2353 return cache->memSidePort.sendAtomic(pkt);
2354 } else {
2355 return cache->recvAtomic(pkt);
2356 }
2357 }
2358
2359 void
2360 BaseCache::CpuSidePort::recvFunctional(PacketPtr pkt)
2361 {
2362 if (cache->system->bypassCaches()) {
2363 // The cache should be flushed if we are in cache bypass mode,
2364 // so we don't need to check if we need to update anything.
2365 cache->memSidePort.sendFunctional(pkt);
2366 return;
2367 }
2368
2369 // functional request
2370 cache->functionalAccess(pkt, true);
2371 }
2372
2373 AddrRangeList
2374 BaseCache::CpuSidePort::getAddrRanges() const
2375 {
2376 return cache->getAddrRanges();
2377 }
2378
2379
2380 BaseCache::
2381 CpuSidePort::CpuSidePort(const std::string &_name, BaseCache *_cache,
2382 const std::string &_label)
2383 : CacheSlavePort(_name, _cache, _label), cache(_cache)
2384 {
2385 }
2386
2387 ///////////////
2388 //
2389 // MemSidePort
2390 //
2391 ///////////////
2392 bool
2393 BaseCache::MemSidePort::recvTimingResp(PacketPtr pkt)
2394 {
2395 cache->recvTimingResp(pkt);
2396 return true;
2397 }
2398
2399 // Express snooping requests to memside port
2400 void
2401 BaseCache::MemSidePort::recvTimingSnoopReq(PacketPtr pkt)
2402 {
2403 // Snoops shouldn't happen when bypassing caches
2404 assert(!cache->system->bypassCaches());
2405
2406 // handle snooping requests
2407 cache->recvTimingSnoopReq(pkt);
2408 }
2409
2410 Tick
2411 BaseCache::MemSidePort::recvAtomicSnoop(PacketPtr pkt)
2412 {
2413 // Snoops shouldn't happen when bypassing caches
2414 assert(!cache->system->bypassCaches());
2415
2416 return cache->recvAtomicSnoop(pkt);
2417 }
2418
2419 void
2420 BaseCache::MemSidePort::recvFunctionalSnoop(PacketPtr pkt)
2421 {
2422 // Snoops shouldn't happen when bypassing caches
2423 assert(!cache->system->bypassCaches());
2424
2425 // functional snoop (note that in contrast to atomic we don't have
2426 // a specific functionalSnoop method, as they have the same
2427 // behaviour regardless)
2428 cache->functionalAccess(pkt, false);
2429 }
2430
2431 void
2432 BaseCache::CacheReqPacketQueue::sendDeferredPacket()
2433 {
2434 // sanity check
2435 assert(!waitingOnRetry);
2436
2437 // there should never be any deferred request packets in the
2438 // queue, instead we resly on the cache to provide the packets
2439 // from the MSHR queue or write queue
2440 assert(deferredPacketReadyTime() == MaxTick);
2441
2442 // check for request packets (requests & writebacks)
2443 QueueEntry* entry = cache.getNextQueueEntry();
2444
2445 if (!entry) {
2446 // can happen if e.g. we attempt a writeback and fail, but
2447 // before the retry, the writeback is eliminated because
2448 // we snoop another cache's ReadEx.
2449 } else {
2450 // let our snoop responses go first if there are responses to
2451 // the same addresses
2452 if (checkConflictingSnoop(entry->getTarget()->pkt)) {
2453 return;
2454 }
2455 waitingOnRetry = entry->sendPacket(cache);
2456 }
2457
2458 // if we succeeded and are not waiting for a retry, schedule the
2459 // next send considering when the next queue is ready, note that
2460 // snoop responses have their own packet queue and thus schedule
2461 // their own events
2462 if (!waitingOnRetry) {
2463 schedSendEvent(cache.nextQueueReadyTime());
2464 }
2465 }
2466
2467 BaseCache::MemSidePort::MemSidePort(const std::string &_name,
2468 BaseCache *_cache,
2469 const std::string &_label)
2470 : CacheMasterPort(_name, _cache, _reqQueue, _snoopRespQueue),
2471 _reqQueue(*_cache, *this, _snoopRespQueue, _label),
2472 _snoopRespQueue(*_cache, *this, true, _label), cache(_cache)
2473 {
2474 }
2475
2476 void
2477 WriteAllocator::updateMode(Addr write_addr, unsigned write_size,
2478 Addr blk_addr)
2479 {
2480 // check if we are continuing where the last write ended
2481 if (nextAddr == write_addr) {
2482 delayCtr[blk_addr] = delayThreshold;
2483 // stop if we have already saturated
2484 if (mode != WriteMode::NO_ALLOCATE) {
2485 byteCount += write_size;
2486 // switch to streaming mode if we have passed the lower
2487 // threshold
2488 if (mode == WriteMode::ALLOCATE &&
2489 byteCount > coalesceLimit) {
2490 mode = WriteMode::COALESCE;
2491 DPRINTF(Cache, "Switched to write coalescing\n");
2492 } else if (mode == WriteMode::COALESCE &&
2493 byteCount > noAllocateLimit) {
2494 // and continue and switch to non-allocating mode if we
2495 // pass the upper threshold
2496 mode = WriteMode::NO_ALLOCATE;
2497 DPRINTF(Cache, "Switched to write-no-allocate\n");
2498 }
2499 }
2500 } else {
2501 // we did not see a write matching the previous one, start
2502 // over again
2503 byteCount = write_size;
2504 mode = WriteMode::ALLOCATE;
2505 resetDelay(blk_addr);
2506 }
2507 nextAddr = write_addr + write_size;
2508 }
2509
2510 WriteAllocator*
2511 WriteAllocatorParams::create()
2512 {
2513 return new WriteAllocator(this);
2514 }