2 * Copyright (c) 2010-2014 ARM Limited
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
14 * Copyright (c) 2002-2005 The Regents of The University of Michigan
15 * Copyright (c) 2010 Advanced Micro Devices, Inc.
16 * All rights reserved.
18 * Redistribution and use in source and binary forms, with or without
19 * modification, are permitted provided that the following conditions are
20 * met: redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer;
22 * redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution;
25 * neither the name of the copyright holders nor the names of its
26 * contributors may be used to endorse or promote products derived from
27 * this software without specific prior written permission.
29 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
30 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
31 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
32 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
33 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
34 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
35 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
36 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
37 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
38 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
39 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
41 * Authors: Erik Hallnor
49 #ifndef __MEM_CACHE_CACHE_IMPL_HH__
50 #define __MEM_CACHE_CACHE_IMPL_HH__
57 #include "base/misc.hh"
58 #include "base/types.hh"
59 #include "debug/Cache.hh"
60 #include "debug/CachePort.hh"
61 #include "debug/CacheTags.hh"
62 #include "mem/cache/prefetch/base.hh"
63 #include "mem/cache/blk.hh"
64 #include "mem/cache/cache.hh"
65 #include "mem/cache/mshr.hh"
66 #include "sim/sim_exit.hh"
68 template<class TagStore>
69 Cache<TagStore>::Cache(const Params *p)
71 tags(dynamic_cast<TagStore*>(p->tags)),
72 prefetcher(p->prefetcher),
74 prefetchOnAccess(p->prefetch_on_access)
76 tempBlock = new BlkType();
77 tempBlock->data = new uint8_t[blkSize];
79 cpuSidePort = new CpuSidePort(p->name + ".cpu_side", this,
81 memSidePort = new MemSidePort(p->name + ".mem_side", this,
86 prefetcher->setCache(this);
89 template<class TagStore>
90 Cache<TagStore>::~Cache()
92 delete [] tempBlock->data;
99 template<class TagStore>
101 Cache<TagStore>::regStats()
103 BaseCache::regStats();
106 template<class TagStore>
108 Cache<TagStore>::cmpAndSwap(BlkType *blk, PacketPtr pkt)
110 uint64_t overwrite_val;
112 uint64_t condition_val64;
113 uint32_t condition_val32;
115 int offset = tags->extractBlkOffset(pkt->getAddr());
116 uint8_t *blk_data = blk->data + offset;
118 assert(sizeof(uint64_t) >= pkt->getSize());
120 overwrite_mem = true;
121 // keep a copy of our possible write value, and copy what is at the
122 // memory address into the packet
123 pkt->writeData((uint8_t *)&overwrite_val);
124 pkt->setData(blk_data);
126 if (pkt->req->isCondSwap()) {
127 if (pkt->getSize() == sizeof(uint64_t)) {
128 condition_val64 = pkt->req->getExtraData();
129 overwrite_mem = !std::memcmp(&condition_val64, blk_data,
131 } else if (pkt->getSize() == sizeof(uint32_t)) {
132 condition_val32 = (uint32_t)pkt->req->getExtraData();
133 overwrite_mem = !std::memcmp(&condition_val32, blk_data,
136 panic("Invalid size for conditional read/write\n");
140 std::memcpy(blk_data, &overwrite_val, pkt->getSize());
141 blk->status |= BlkDirty;
146 template<class TagStore>
148 Cache<TagStore>::satisfyCpuSideRequest(PacketPtr pkt, BlkType *blk,
149 bool deferred_response,
150 bool pending_downgrade)
152 assert(blk && blk->isValid());
153 // Occasionally this is not true... if we are a lower-level cache
154 // satisfying a string of Read and ReadEx requests from
155 // upper-level caches, a Read will mark the block as shared but we
156 // can satisfy a following ReadEx anyway since we can rely on the
157 // Read requester(s) to have buffered the ReadEx snoop and to
158 // invalidate their blocks after receiving them.
159 // assert(!pkt->needsExclusive() || blk->isWritable());
160 assert(pkt->getOffset(blkSize) + pkt->getSize() <= blkSize);
162 // Check RMW operations first since both isRead() and
163 // isWrite() will be true for them
164 if (pkt->cmd == MemCmd::SwapReq) {
165 cmpAndSwap(blk, pkt);
166 } else if (pkt->isWrite()) {
167 if (blk->checkWrite(pkt)) {
168 pkt->writeDataToBlock(blk->data, blkSize);
170 // Always mark the line as dirty even if we are a failed
171 // StoreCond so we supply data to any snoops that have
172 // appended themselves to this cache before knowing the store
174 blk->status |= BlkDirty;
175 } else if (pkt->isRead()) {
177 blk->trackLoadLocked(pkt);
179 pkt->setDataFromBlock(blk->data, blkSize);
180 if (pkt->getSize() == blkSize) {
181 // special handling for coherent block requests from
182 // upper-level caches
183 if (pkt->needsExclusive()) {
184 // if we have a dirty copy, make sure the recipient
185 // keeps it marked dirty
186 if (blk->isDirty()) {
187 pkt->assertMemInhibit();
189 // on ReadExReq we give up our copy unconditionally
190 if (blk != tempBlock)
191 tags->invalidate(blk);
193 } else if (blk->isWritable() && !pending_downgrade
194 && !pkt->sharedAsserted() && !pkt->req->isInstFetch()) {
195 // we can give the requester an exclusive copy (by not
196 // asserting shared line) on a read request if:
197 // - we have an exclusive copy at this level (& below)
198 // - we don't have a pending snoop from below
199 // signaling another read request
200 // - no other cache above has a copy (otherwise it
201 // would have asseretd shared line on request)
202 // - we are not satisfying an instruction fetch (this
203 // prevents dirty data in the i-cache)
205 if (blk->isDirty()) {
206 // special considerations if we're owner:
207 if (!deferred_response && !isTopLevel) {
208 // if we are responding immediately and can
209 // signal that we're transferring ownership
210 // along with exclusivity, do so
211 pkt->assertMemInhibit();
212 blk->status &= ~BlkDirty;
214 // if we're responding after our own miss,
215 // there's a window where the recipient didn't
216 // know it was getting ownership and may not
217 // have responded to snoops correctly, so we
218 // can't pass off ownership *or* exclusivity
223 // otherwise only respond with a shared copy
228 // Not a read or write... must be an upgrade. it's OK
229 // to just ack those as long as we have an exclusive
230 // copy at this level.
231 assert(pkt->isUpgrade());
232 assert(blk != tempBlock);
233 tags->invalidate(blk);
239 /////////////////////////////////////////////////////
241 // MSHR helper functions
243 /////////////////////////////////////////////////////
246 template<class TagStore>
248 Cache<TagStore>::markInService(MSHR *mshr, PacketPtr pkt)
250 markInServiceInternal(mshr, pkt);
252 if (mshr->originalCmd == MemCmd::HardPFReq) {
253 DPRINTF(HWPrefetch, "%s:Marking a HW_PF in service\n",
255 //Also clear pending if need be
256 if (!prefetcher->havePending())
258 deassertMemSideBusRequest(Request_PF);
265 template<class TagStore>
267 Cache<TagStore>::squash(int threadNum)
269 bool unblock = false;
270 BlockedCause cause = NUM_BLOCKED_CAUSES;
272 if (noTargetMSHR && noTargetMSHR->threadNum == threadNum) {
275 cause = Blocked_NoTargets;
277 if (mshrQueue.isFull()) {
279 cause = Blocked_NoMSHRs;
281 mshrQueue.squash(threadNum);
282 if (unblock && !mshrQueue.isFull()) {
287 /////////////////////////////////////////////////////
289 // Access path: requests coming in from the CPU side
291 /////////////////////////////////////////////////////
293 template<class TagStore>
295 Cache<TagStore>::access(PacketPtr pkt, BlkType *&blk,
296 Cycles &lat, PacketList &writebacks)
298 DPRINTF(Cache, "%s for %s address %x size %d\n", __func__,
299 pkt->cmdString(), pkt->getAddr(), pkt->getSize());
300 if (pkt->req->isUncacheable()) {
301 uncacheableFlush(pkt);
307 int id = pkt->req->hasContextId() ? pkt->req->contextId() : -1;
308 blk = tags->accessBlock(pkt->getAddr(), pkt->isSecure(), lat, id);
310 DPRINTF(Cache, "%s%s %x (%s) %s %s\n", pkt->cmdString(),
311 pkt->req->isInstFetch() ? " (ifetch)" : "",
312 pkt->getAddr(), pkt->isSecure() ? "s" : "ns",
313 blk ? "hit" : "miss", blk ? blk->print() : "");
315 // Writeback handling is special case. We can write the block
316 // into the cache without having a writeable copy (or any copy at
317 // all). Like writebacks, we write into the cache upon initial
318 // receipt of a write-invalidate packets as well.
319 if ((pkt->cmd == MemCmd::Writeback) ||
320 ((pkt->cmd == MemCmd::WriteInvalidateReq) && isTopLevel)) {
321 assert(blkSize == pkt->getSize());
323 // need to do a replacement
324 blk = allocateBlock(pkt->getAddr(), pkt->isSecure(), writebacks);
326 // no replaceable block available, give up.
327 // Writeback will be forwarded to next level,
328 // WriteInvalidate will be retried.
332 tags->insertBlock(pkt, blk);
334 blk->status = (BlkValid | BlkReadable);
335 if (pkt->isSecure()) {
336 blk->status |= BlkSecure;
339 std::memcpy(blk->data, pkt->getPtr<uint8_t>(), blkSize);
340 if (pkt->cmd == MemCmd::Writeback) {
341 blk->status |= BlkDirty;
342 if (pkt->isSupplyExclusive()) {
343 blk->status |= BlkWritable;
345 // nothing else to do; writeback doesn't expect response
346 assert(!pkt->needsResponse());
347 } else if (pkt->cmd == MemCmd::WriteInvalidateReq) {
348 assert(blk->isReadable()); // implicitly checks for Valid bit also
349 blk->status |= (BlkDirty | BlkCanGoExclusive);
350 blk->status &= ~BlkWritable;
353 DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print());
356 } else if ((pkt->cmd == MemCmd::WriteInvalidateReq) && !isTopLevel) {
358 assert(blk != tempBlock);
359 tags->invalidate(blk);
363 } else if ((blk != NULL) &&
364 (pkt->needsExclusive() ? blk->isWritable()
365 : blk->isReadable())) {
366 // OK to satisfy access
368 satisfyCpuSideRequest(pkt, blk);
372 // Can't satisfy access normally... either no block (blk == NULL)
373 // or have block but need exclusive & only have shared.
377 if (blk == NULL && pkt->isLLSC() && pkt->isWrite()) {
378 // complete miss on store conditional... just give up now
379 pkt->req->setExtraData(0);
387 class ForwardResponseRecord : public Packet::SenderState
393 ForwardResponseRecord(PortID prev_src) : prevSrc(prev_src)
397 template<class TagStore>
399 Cache<TagStore>::recvTimingSnoopResp(PacketPtr pkt)
401 DPRINTF(Cache, "%s for %s address %x size %d\n", __func__,
402 pkt->cmdString(), pkt->getAddr(), pkt->getSize());
403 Tick time = clockEdge(hitLatency);
405 assert(pkt->isResponse());
407 // must be cache-to-cache response from upper to lower level
408 ForwardResponseRecord *rec =
409 dynamic_cast<ForwardResponseRecord *>(pkt->senderState);
410 assert(!system->bypassCaches());
413 assert(pkt->cmd == MemCmd::HardPFResp);
414 // Check if it's a prefetch response and handle it. We shouldn't
415 // get any other kinds of responses without FRRs.
416 DPRINTF(Cache, "Got prefetch response from above for addr %#x (%s)\n",
417 pkt->getAddr(), pkt->isSecure() ? "s" : "ns");
422 pkt->popSenderState();
423 pkt->setDest(rec->prevSrc);
425 // @todo someone should pay for this
426 pkt->busFirstWordDelay = pkt->busLastWordDelay = 0;
427 memSidePort->schedTimingSnoopResp(pkt, time);
430 template<class TagStore>
432 Cache<TagStore>::promoteWholeLineWrites(PacketPtr pkt)
434 // Cache line clearing instructions
435 if (doFastWrites && (pkt->cmd == MemCmd::WriteReq) &&
436 (pkt->getSize() == blkSize) && (pkt->getOffset(blkSize) == 0)) {
437 pkt->cmd = MemCmd::WriteInvalidateReq;
438 DPRINTF(Cache, "packet promoted from Write to WriteInvalidate\n");
439 assert(isTopLevel); // should only happen at L1 or I/O cache
443 template<class TagStore>
445 Cache<TagStore>::recvTimingReq(PacketPtr pkt)
447 DPRINTF(CacheTags, "%s tags: %s\n", __func__, tags->print());
448 //@todo Add back in MemDebug Calls
449 // MemDebug::cacheAccess(pkt);
452 /// @todo temporary hack to deal with memory corruption issue until
453 /// 4-phase transactions are complete
454 for (int x = 0; x < pendingDelete.size(); x++)
455 delete pendingDelete[x];
456 pendingDelete.clear();
458 // we charge hitLatency for doing just about anything here
459 Tick time = clockEdge(hitLatency);
461 assert(pkt->isRequest());
463 // Just forward the packet if caches are disabled.
464 if (system->bypassCaches()) {
465 // @todo This should really enqueue the packet rather
466 bool M5_VAR_USED success = memSidePort->sendTimingReq(pkt);
471 promoteWholeLineWrites(pkt);
473 if (pkt->memInhibitAsserted()) {
474 DPRINTF(Cache, "mem inhibited on 0x%x (%s): not responding\n",
475 pkt->getAddr(), pkt->isSecure() ? "s" : "ns");
476 assert(!pkt->req->isUncacheable());
477 // Special tweak for multilevel coherence: snoop downward here
478 // on invalidates since there may be other caches below here
479 // that have shared copies. Not necessary if we know that
480 // supplier had exclusive copy to begin with.
481 if (pkt->needsExclusive() && !pkt->isSupplyExclusive()) {
482 Packet *snoopPkt = new Packet(pkt, true); // clear flags
483 // also reset the bus time that the original packet has
485 snoopPkt->busFirstWordDelay = snoopPkt->busLastWordDelay = 0;
486 snoopPkt->setExpressSnoop();
487 snoopPkt->assertMemInhibit();
488 bool M5_VAR_USED success = memSidePort->sendTimingReq(snoopPkt);
489 // the packet is marked inhibited and will thus bypass any
492 // main memory will delete snoopPkt
494 // since we're the official target but we aren't responding,
495 // delete the packet now.
497 /// @todo nominally we should just delete the packet here,
498 /// however, until 4-phase stuff we can't because sending
499 /// cache is still relying on it
500 pendingDelete.push_back(pkt);
504 if (pkt->req->isUncacheable()) {
505 uncacheableFlush(pkt);
507 // @todo: someone should pay for this
508 pkt->busFirstWordDelay = pkt->busLastWordDelay = 0;
510 // writes go in write buffer, reads use MSHR,
511 // prefetches are acknowledged (responded to) and dropped
512 if (pkt->cmd.isPrefetch()) {
513 // prefetching (cache loading) uncacheable data is nonsensical
514 pkt->makeTimingResponse();
515 std::memset(pkt->getPtr<uint8_t>(), 0xFF, pkt->getSize());
516 cpuSidePort->schedTimingResp(pkt, clockEdge(hitLatency));
518 } else if (pkt->isWrite() && !pkt->isRead()) {
519 allocateWriteBuffer(pkt, time, true);
521 allocateUncachedReadBuffer(pkt, time, true);
523 assert(pkt->needsResponse()); // else we should delete it here??
527 Cycles lat = hitLatency;
529 PacketList writebacks;
531 bool satisfied = access(pkt, blk, lat, writebacks);
533 // track time of availability of next prefetch, if any
534 Tick next_pf_time = 0;
536 bool needsResponse = pkt->needsResponse();
538 if (pkt->cmd == MemCmd::WriteInvalidateReq) {
539 if (!satisfied && isTopLevel) {
540 // access() tried to allocate a block but it could not; abort.
541 setBlocked(Blocked_PendingWriteInvalidate);
545 // we need to take the miss path (allocate MSHR, etc.) for
546 // WriteInvalidates because they always need to propagate
547 // throughout the memory system
551 // hit (for all other request types)
553 if (prefetcher && (prefetchOnAccess || (blk && blk->wasPrefetched()))) {
555 blk->status &= ~BlkHWPrefetched;
557 // Don't notify on SWPrefetch
558 if (!pkt->cmd.isSWPrefetch())
559 next_pf_time = prefetcher->notify(pkt, time);
563 pkt->makeTimingResponse();
564 // @todo: Make someone pay for this
565 pkt->busFirstWordDelay = pkt->busLastWordDelay = 0;
566 cpuSidePort->schedTimingResp(pkt, clockEdge(lat));
568 /// @todo nominally we should just delete the packet here,
569 /// however, until 4-phase stuff we can't because sending
570 /// cache is still relying on it
571 pendingDelete.push_back(pkt);
576 // @todo: Make someone pay for this
577 pkt->busFirstWordDelay = pkt->busLastWordDelay = 0;
579 if (blk && blk->isValid() && (blk->status & BlkCanGoExclusive) &&
580 pkt->isWrite() && (pkt->cmd != MemCmd::WriteInvalidateReq)) {
581 // Packet is a Write (needs exclusive) should be delayed because
582 // a WriteInvalidate is pending. Instead of going the MSHR route,
583 // the Packet should be replayed, since if the block transitions
584 // to Exclusive the write can complete immediately.
585 setBlocked(Blocked_PendingWriteInvalidate);
589 Addr blk_addr = blockAlign(pkt->getAddr());
590 MSHR *mshr = mshrQueue.findMatch(blk_addr, pkt->isSecure());
592 // Software prefetch handling:
593 // To keep the core from waiting on data it won't look at
594 // anyway, send back a response with dummy data. Miss handling
595 // will continue asynchronously. Unfortunately, the core will
596 // insist upon freeing original Packet/Request, so we have to
597 // create a new pair with a different lifecycle. Note that this
598 // processing happens before any MSHR munging on the behalf of
599 // this request because this new Request will be the one stored
600 // into the MSHRs, not the original.
601 if (pkt->cmd.isSWPrefetch() && isTopLevel) {
602 assert(needsResponse);
603 assert(pkt->req->hasPaddr());
605 // There's no reason to add a prefetch as an additional target
606 // to an existing MSHR. If an outstanding request is already
607 // in progress, there is nothing for the prefetch to do.
608 // If this is the case, we don't even create a request at all.
609 PacketPtr pf = mshr ? NULL : new Packet(pkt);
612 pf->req = new Request(pkt->req->getPaddr(),
614 pkt->req->getFlags(),
615 pkt->req->masterId());
616 // The core will clean up prior senderState; we need our own.
617 pf->senderState = NULL;
620 pkt->makeTimingResponse();
621 // for debugging, set all the bits in the response data
622 // (also keeps valgrind from complaining when debugging settings
623 // print out instruction results)
624 std::memset(pkt->getPtr<uint8_t>(), 0xFF, pkt->getSize());
625 cpuSidePort->schedTimingResp(pkt, clockEdge(lat));
632 /// @note writebacks will be checked in getNextMSHR()
633 /// for any conflicting requests to the same block
635 //@todo remove hw_pf here
637 // Coalesce unless it was a software prefetch (see above).
639 assert(pkt->req->masterId() < system->maxMasters());
640 mshr_hits[pkt->cmdToIndex()][pkt->req->masterId()]++;
641 if (mshr->threadNum != 0/*pkt->req->threadId()*/) {
642 mshr->threadNum = -1;
644 mshr->allocateTarget(pkt, time, order++);
645 if (mshr->getNumTargets() == numTarget) {
647 setBlocked(Blocked_NoTargets);
648 // need to be careful with this... if this mshr isn't
649 // ready yet (i.e. time > curTick()), we don't want to
650 // move it ahead of mshrs that are ready
651 // mshrQueue.moveToFront(mshr);
654 // We should call the prefetcher reguardless if the request is
655 // satisfied or not, reguardless if the request is in the MSHR or
656 // not. The request could be a ReadReq hit, but still not
657 // satisfied (potentially because of a prior write to the same
658 // cache line. So, even when not satisfied, tehre is an MSHR
659 // already allocated for this, we need to let the prefetcher know
662 // Don't notify on SWPrefetch
663 if (!pkt->cmd.isSWPrefetch())
664 next_pf_time = prefetcher->notify(pkt, time);
669 assert(pkt->req->masterId() < system->maxMasters());
670 mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++;
671 // always mark as cache fill for now... if we implement
672 // no-write-allocate or bypass accesses this will have to
674 if (pkt->cmd == MemCmd::Writeback) {
675 allocateWriteBuffer(pkt, time, true);
677 if (pkt->cmd == MemCmd::WriteInvalidateReq) {
678 // a WriteInvalidate is not a normal write miss;
679 // the assertions below are not applicable.
680 } else if (blk && blk->isValid()) {
681 // If we have a write miss to a valid block, we
682 // need to mark the block non-readable. Otherwise
683 // if we allow reads while there's an outstanding
684 // write miss, the read could return stale data
685 // out of the cache block... a more aggressive
686 // system could detect the overlap (if any) and
687 // forward data out of the MSHRs, but we don't do
688 // that yet. Note that we do need to leave the
689 // block valid so that it stays in the cache, in
690 // case we get an upgrade response (and hence no
691 // new data) when the write miss completes.
692 // As long as CPUs do proper store/load forwarding
693 // internally, and have a sufficiently weak memory
694 // model, this is probably unnecessary, but at some
695 // point it must have seemed like we needed it...
696 assert(pkt->needsExclusive());
697 assert(!blk->isWritable());
698 blk->status &= ~BlkReadable;
701 allocateMissBuffer(pkt, time, true);
705 // Don't notify on SWPrefetch
706 if (!pkt->cmd.isSWPrefetch())
707 next_pf_time = prefetcher->notify(pkt, time);
712 if (next_pf_time != 0)
713 requestMemSideBus(Request_PF, std::max(time, next_pf_time));
715 // copy writebacks to write buffer
716 while (!writebacks.empty()) {
717 PacketPtr wbPkt = writebacks.front();
718 allocateWriteBuffer(wbPkt, time, true);
719 writebacks.pop_front();
726 // See comment in cache.hh.
727 template<class TagStore>
729 Cache<TagStore>::getBusPacket(PacketPtr cpu_pkt, BlkType *blk,
730 bool needsExclusive) const
732 bool blkValid = blk && blk->isValid();
734 if (cpu_pkt->req->isUncacheable()) {
735 //assert(blk == NULL);
739 // WriteInvalidates for cache line clearing instructions don't
740 // require a read; just send directly to the bus.
741 if (cpu_pkt->cmd == MemCmd::WriteInvalidateReq) {
746 (cpu_pkt->cmd == MemCmd::Writeback || cpu_pkt->isUpgrade())) {
747 // Writebacks that weren't allocated in access() and upgrades
748 // from upper-level caches that missed completely just go
753 assert(cpu_pkt->needsResponse());
756 // @TODO make useUpgrades a parameter.
757 // Note that ownership protocols require upgrade, otherwise a
758 // write miss on a shared owned block will generate a ReadExcl,
759 // which will clobber the owned copy.
760 const bool useUpgrades = true;
761 if (blkValid && useUpgrades) {
762 // only reason to be here is that blk is shared
763 // (read-only) and we need exclusive
764 assert(needsExclusive);
765 assert(!blk->isWritable());
766 cmd = cpu_pkt->isLLSC() ? MemCmd::SCUpgradeReq : MemCmd::UpgradeReq;
767 } else if (cpu_pkt->cmd == MemCmd::SCUpgradeFailReq ||
768 cpu_pkt->cmd == MemCmd::StoreCondFailReq) {
769 // Even though this SC will fail, we still need to send out the
770 // request and get the data to supply it to other snoopers in the case
771 // where the determination the StoreCond fails is delayed due to
772 // all caches not being on the same local bus.
773 cmd = MemCmd::SCUpgradeFailReq;
776 cmd = needsExclusive ? MemCmd::ReadExReq : MemCmd::ReadReq;
778 PacketPtr pkt = new Packet(cpu_pkt->req, cmd, blkSize);
781 DPRINTF(Cache, "%s created %s address %x size %d\n",
782 __func__, pkt->cmdString(), pkt->getAddr(), pkt->getSize());
787 template<class TagStore>
789 Cache<TagStore>::recvAtomic(PacketPtr pkt)
791 Cycles lat = hitLatency;
793 // @TODO: make this a parameter
794 bool last_level_cache = false;
796 // Forward the request if the system is in cache bypass mode.
797 if (system->bypassCaches())
798 return ticksToCycles(memSidePort->sendAtomic(pkt));
800 promoteWholeLineWrites(pkt);
802 if (pkt->memInhibitAsserted()) {
803 assert(!pkt->req->isUncacheable());
804 // have to invalidate ourselves and any lower caches even if
805 // upper cache will be responding
806 if (pkt->isInvalidate()) {
807 BlkType *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
808 if (blk && blk->isValid()) {
809 tags->invalidate(blk);
811 DPRINTF(Cache, "rcvd mem-inhibited %s on 0x%x (%s):"
813 pkt->cmdString(), pkt->getAddr(),
814 pkt->isSecure() ? "s" : "ns");
816 if (!last_level_cache) {
817 DPRINTF(Cache, "forwarding mem-inhibited %s on 0x%x (%s)\n",
818 pkt->cmdString(), pkt->getAddr(),
819 pkt->isSecure() ? "s" : "ns");
820 lat += ticksToCycles(memSidePort->sendAtomic(pkt));
823 DPRINTF(Cache, "rcvd mem-inhibited %s on 0x%x: not responding\n",
824 pkt->cmdString(), pkt->getAddr());
827 return lat * clockPeriod();
830 // should assert here that there are no outstanding MSHRs or
831 // writebacks... that would mean that someone used an atomic
832 // access in timing mode
835 PacketList writebacks;
837 if (!access(pkt, blk, lat, writebacks)) {
840 // WriteInvalidates should never fail an access() in Atomic mode
841 assert(pkt->cmd != MemCmd::WriteInvalidateReq);
843 PacketPtr bus_pkt = getBusPacket(pkt, blk, pkt->needsExclusive());
845 bool is_forward = (bus_pkt == NULL);
848 // just forwarding the same request to the next level
849 // no local cache operation involved
853 DPRINTF(Cache, "Sending an atomic %s for %x (%s)\n",
854 bus_pkt->cmdString(), bus_pkt->getAddr(),
855 bus_pkt->isSecure() ? "s" : "ns");
858 CacheBlk::State old_state = blk ? blk->status : 0;
861 lat += ticksToCycles(memSidePort->sendAtomic(bus_pkt));
863 DPRINTF(Cache, "Receive response: %s for addr %x (%s) in state %i\n",
864 bus_pkt->cmdString(), bus_pkt->getAddr(),
865 bus_pkt->isSecure() ? "s" : "ns",
868 // If packet was a forward, the response (if any) is already
869 // in place in the bus_pkt == pkt structure, so we don't need
870 // to do anything. Otherwise, use the separate bus_pkt to
871 // generate response to pkt and then delete it.
873 if (pkt->needsResponse()) {
874 assert(bus_pkt->isResponse());
875 if (bus_pkt->isError()) {
876 pkt->makeAtomicResponse();
877 pkt->copyError(bus_pkt);
878 } else if (bus_pkt->isRead() ||
879 bus_pkt->cmd == MemCmd::UpgradeResp) {
880 // we're updating cache state to allow us to
881 // satisfy the upstream request from the cache
882 blk = handleFill(bus_pkt, blk, writebacks);
883 satisfyCpuSideRequest(pkt, blk);
885 // we're satisfying the upstream request without
886 // modifying cache state, e.g., a write-through
887 pkt->makeAtomicResponse();
894 // Note that we don't invoke the prefetcher at all in atomic mode.
895 // It's not clear how to do it properly, particularly for
896 // prefetchers that aggressively generate prefetch candidates and
897 // rely on bandwidth contention to throttle them; these will tend
898 // to pollute the cache in atomic mode since there is no bandwidth
899 // contention. If we ever do want to enable prefetching in atomic
900 // mode, though, this is the place to do it... see timingAccess()
901 // for an example (though we'd want to issue the prefetch(es)
902 // immediately rather than calling requestMemSideBus() as we do
905 // Handle writebacks if needed
906 while (!writebacks.empty()){
907 PacketPtr wbPkt = writebacks.front();
908 memSidePort->sendAtomic(wbPkt);
909 writebacks.pop_front();
913 // We now have the block one way or another (hit or completed miss),
914 // except for Request types that perform an invalidate, where the point
915 // is to make sure there is no block.
917 if (pkt->cmd == MemCmd::WriteInvalidateReq) {
918 memSidePort->sendAtomic(pkt); // complete writeback
920 // top level caches allocate and write the data
921 assert(blk->isDirty());
922 assert(!blk->isWritable());
923 assert(blk->status & BlkCanGoExclusive);
924 blk->status &= ~(BlkDirty | BlkCanGoExclusive); // and mark clean
925 blk->status |= BlkWritable; // i.e. O(+cgE) -> E
927 // other caches invalidate.
928 // if the block was found, it was invalidated.
929 assert(!blk || !blk->isValid());
933 if (pkt->needsResponse()) {
934 pkt->makeAtomicResponse();
937 return lat * clockPeriod();
941 template<class TagStore>
943 Cache<TagStore>::functionalAccess(PacketPtr pkt, bool fromCpuSide)
945 if (system->bypassCaches()) {
946 // Packets from the memory side are snoop request and
947 // shouldn't happen in bypass mode.
950 // The cache should be flushed if we are in cache bypass mode,
951 // so we don't need to check if we need to update anything.
952 memSidePort->sendFunctional(pkt);
956 Addr blk_addr = blockAlign(pkt->getAddr());
957 bool is_secure = pkt->isSecure();
958 BlkType *blk = tags->findBlock(pkt->getAddr(), is_secure);
959 MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure);
961 pkt->pushLabel(name());
963 CacheBlkPrintWrapper cbpw(blk);
965 // Note that just because an L2/L3 has valid data doesn't mean an
966 // L1 doesn't have a more up-to-date modified copy that still
967 // needs to be found. As a result we always update the request if
968 // we have it, but only declare it satisfied if we are the owner.
970 // see if we have data at all (owned or otherwise)
971 bool have_data = blk && blk->isValid()
972 && pkt->checkFunctional(&cbpw, blk_addr, is_secure, blkSize,
975 // data we have is dirty if marked as such or if valid & ownership
976 // pending due to outstanding UpgradeReq
978 have_data && (blk->isDirty() ||
979 (mshr && mshr->inService && mshr->isPendingDirty()));
981 bool done = have_dirty
982 || cpuSidePort->checkFunctional(pkt)
983 || mshrQueue.checkFunctional(pkt, blk_addr)
984 || writeBuffer.checkFunctional(pkt, blk_addr)
985 || memSidePort->checkFunctional(pkt);
987 DPRINTF(Cache, "functional %s %x (%s) %s%s%s\n",
988 pkt->cmdString(), pkt->getAddr(), is_secure ? "s" : "ns",
989 (blk && blk->isValid()) ? "valid " : "",
990 have_data ? "data " : "", done ? "done " : "");
992 // We're leaving the cache, so pop cache->name() label
998 // if it came as a request from the CPU side then make sure it
999 // continues towards the memory side
1001 memSidePort->sendFunctional(pkt);
1002 } else if (forwardSnoops && cpuSidePort->isSnooping()) {
1003 // if it came from the memory side, it must be a snoop request
1004 // and we should only forward it if we are forwarding snoops
1005 cpuSidePort->sendFunctionalSnoop(pkt);
1011 /////////////////////////////////////////////////////
1013 // Response handling: responses from the memory side
1015 /////////////////////////////////////////////////////
1018 template<class TagStore>
1020 Cache<TagStore>::recvTimingResp(PacketPtr pkt)
1022 assert(pkt->isResponse());
1024 Tick time = clockEdge(hitLatency);
1025 MSHR *mshr = dynamic_cast<MSHR*>(pkt->senderState);
1026 bool is_error = pkt->isError();
1031 DPRINTF(Cache, "Cache received packet with error for address %x (%s), "
1032 "cmd: %s\n", pkt->getAddr(), pkt->isSecure() ? "s" : "ns",
1036 DPRINTF(Cache, "Handling response to %s for address %x (%s)\n",
1037 pkt->cmdString(), pkt->getAddr(), pkt->isSecure() ? "s" : "ns");
1039 MSHRQueue *mq = mshr->queue;
1040 bool wasFull = mq->isFull();
1042 if (mshr == noTargetMSHR) {
1043 // we always clear at least one target
1044 clearBlocked(Blocked_NoTargets);
1045 noTargetMSHR = NULL;
1048 // Initial target is used just for stats
1049 MSHR::Target *initial_tgt = mshr->getTarget();
1050 BlkType *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
1051 int stats_cmd_idx = initial_tgt->pkt->cmdToIndex();
1052 Tick miss_latency = curTick() - initial_tgt->recvTime;
1053 PacketList writebacks;
1055 if (pkt->req->isUncacheable()) {
1056 assert(pkt->req->masterId() < system->maxMasters());
1057 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] +=
1060 assert(pkt->req->masterId() < system->maxMasters());
1061 mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] +=
1065 bool is_fill = !mshr->isForward &&
1066 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);
1068 if (is_fill && !is_error) {
1069 DPRINTF(Cache, "Block for addr %x being updated in Cache\n",
1072 // give mshr a chance to do some dirty work
1073 mshr->handleFill(pkt, blk);
1075 blk = handleFill(pkt, blk, writebacks);
1076 assert(blk != NULL);
1079 // First offset for critical word first calculations
1080 int initial_offset = 0;
1082 if (mshr->hasTargets()) {
1083 initial_offset = mshr->getTarget()->pkt->getOffset(blkSize);
1086 while (mshr->hasTargets()) {
1087 MSHR::Target *target = mshr->getTarget();
1089 switch (target->source) {
1090 case MSHR::Target::FromCPU:
1091 Tick completion_time;
1093 // Software prefetch handling for cache closest to core
1094 if (target->pkt->cmd.isSWPrefetch() && isTopLevel) {
1095 // a software prefetch would have already been ack'd immediately
1096 // with dummy data so the core would be able to retire it.
1097 // this request completes right here, so we deallocate it.
1098 delete target->pkt->req;
1100 break; // skip response
1104 satisfyCpuSideRequest(target->pkt, blk,
1105 true, mshr->hasPostDowngrade());
1106 // How many bytes past the first request is this one
1107 int transfer_offset =
1108 target->pkt->getOffset(blkSize) - initial_offset;
1109 if (transfer_offset < 0) {
1110 transfer_offset += blkSize;
1113 // If critical word (no offset) return first word time.
1114 // responseLatency is the latency of the return path
1115 // from lower level caches/memory to an upper level cache or
1117 completion_time = clockEdge(responseLatency) +
1118 (transfer_offset ? pkt->busLastWordDelay :
1119 pkt->busFirstWordDelay);
1121 assert(!target->pkt->req->isUncacheable());
1123 assert(target->pkt->req->masterId() < system->maxMasters());
1124 missLatency[target->pkt->cmdToIndex()][target->pkt->req->masterId()] +=
1125 completion_time - target->recvTime;
1126 } else if (pkt->cmd == MemCmd::UpgradeFailResp) {
1127 // failed StoreCond upgrade
1128 assert(target->pkt->cmd == MemCmd::StoreCondReq ||
1129 target->pkt->cmd == MemCmd::StoreCondFailReq ||
1130 target->pkt->cmd == MemCmd::SCUpgradeFailReq);
1131 // responseLatency is the latency of the return path
1132 // from lower level caches/memory to an upper level cache or
1134 completion_time = clockEdge(responseLatency) +
1135 pkt->busLastWordDelay;
1136 target->pkt->req->setExtraData(0);
1137 } else if (pkt->cmd == MemCmd::WriteInvalidateResp) {
1139 assert(blk->isDirty() && !blk->isWritable());
1140 // The block, having been written back, is no longer dirty,
1141 // nor do we have any reason to see if it was snooped in the
1142 // meantime (which CanGoExclusive tracks). If it can go
1143 // exclusive, we put it in that state, and otherwise S.
1144 // In short: O(+cgE) -> E, O(-cgE) -> S
1145 if (blk->status & BlkCanGoExclusive) {
1146 blk->status |= BlkWritable;
1148 blk->status &= ~(BlkDirty | BlkCanGoExclusive);
1151 // makeTimingResponse() will turn it into a WriteResp
1152 target->pkt->cmd = MemCmd::WriteReq;
1153 // Writes may have been blocked - quite rare case, but
1154 // it does happen. Prevent deadlock by telling the core
1155 if (isBlocked()) { // to retry.
1156 clearBlocked(Blocked_PendingWriteInvalidate);
1159 // If the block managed to get evicted before its own
1160 // writeback (e.g. by a Read/Upgrade (from O(-cgE)/S to
1161 // I/E) or ReadExclusive (direct to I/E); either way a
1162 // cache-to-cache ownership transfer) completed, that's
1163 // OK, we just ignore this response. If the new owner
1164 // doesn't actually modify it, a superfluous writeback
1165 // will occur for its impatience (since it will think it
1166 // has dirty data), but it really can't be helped.
1167 completion_time = clockEdge(responseLatency) +
1168 pkt->busLastWordDelay;
1170 // not a cache fill, just forwarding response
1171 // responseLatency is the latency of the return path
1172 // from lower level cahces/memory to the core.
1173 completion_time = clockEdge(responseLatency) +
1174 pkt->busLastWordDelay;
1175 if (pkt->isRead() && !is_error) {
1176 target->pkt->setData(pkt->getPtr<uint8_t>());
1179 target->pkt->makeTimingResponse();
1180 // if this packet is an error copy that to the new packet
1182 target->pkt->copyError(pkt);
1183 if (target->pkt->cmd == MemCmd::ReadResp &&
1184 (pkt->isInvalidate() || mshr->hasPostInvalidate())) {
1185 // If intermediate cache got ReadRespWithInvalidate,
1186 // propagate that. Response should not have
1187 // isInvalidate() set otherwise.
1188 target->pkt->cmd = MemCmd::ReadRespWithInvalidate;
1189 DPRINTF(Cache, "%s updated cmd to %s for address %x\n",
1190 __func__, target->pkt->cmdString(),
1191 target->pkt->getAddr());
1193 // reset the bus additional time as it is now accounted for
1194 target->pkt->busFirstWordDelay = target->pkt->busLastWordDelay = 0;
1195 cpuSidePort->schedTimingResp(target->pkt, completion_time);
1198 case MSHR::Target::FromPrefetcher:
1199 assert(target->pkt->cmd == MemCmd::HardPFReq);
1201 blk->status |= BlkHWPrefetched;
1202 delete target->pkt->req;
1206 case MSHR::Target::FromSnoop:
1207 // I don't believe that a snoop can be in an error state
1209 // response to snoop request
1210 DPRINTF(Cache, "processing deferred snoop...\n");
1211 assert(!(pkt->isInvalidate() && !mshr->hasPostInvalidate()));
1212 handleSnoop(target->pkt, blk, true, true,
1213 mshr->hasPostInvalidate());
1217 panic("Illegal target->source enum %d\n", target->source);
1223 if (blk && blk->isValid()) {
1224 if (pkt->isInvalidate() || mshr->hasPostInvalidate()) {
1225 assert(blk != tempBlock);
1226 tags->invalidate(blk);
1228 } else if (mshr->hasPostDowngrade()) {
1229 blk->status &= ~BlkWritable;
1233 if (mshr->promoteDeferredTargets()) {
1234 // avoid later read getting stale data while write miss is
1235 // outstanding.. see comment in timingAccess()
1237 blk->status &= ~BlkReadable;
1240 mq->markPending(mshr);
1241 requestMemSideBus((RequestCause)mq->index, clockEdge() +
1242 pkt->busLastWordDelay);
1244 mq->deallocate(mshr);
1245 if (wasFull && !mq->isFull()) {
1246 clearBlocked((BlockedCause)mq->index);
1250 // copy writebacks to write buffer
1251 while (!writebacks.empty()) {
1252 PacketPtr wbPkt = writebacks.front();
1253 allocateWriteBuffer(wbPkt, time, true);
1254 writebacks.pop_front();
1256 // if we used temp block, clear it out
1257 if (blk == tempBlock) {
1258 if (blk->isDirty()) {
1259 allocateWriteBuffer(writebackBlk(blk), time, true);
1264 DPRINTF(Cache, "Leaving %s with %s for address %x\n", __func__,
1265 pkt->cmdString(), pkt->getAddr());
1272 template<class TagStore>
1274 Cache<TagStore>::writebackBlk(BlkType *blk)
1276 assert(blk && blk->isValid() && blk->isDirty());
1278 writebacks[Request::wbMasterId]++;
1280 Request *writebackReq =
1281 new Request(tags->regenerateBlkAddr(blk->tag, blk->set), blkSize, 0,
1282 Request::wbMasterId);
1283 if (blk->isSecure())
1284 writebackReq->setFlags(Request::SECURE);
1286 writebackReq->taskId(blk->task_id);
1287 blk->task_id= ContextSwitchTaskId::Unknown;
1288 blk->tickInserted = curTick();
1290 PacketPtr writeback = new Packet(writebackReq, MemCmd::Writeback);
1291 if (blk->isWritable()) {
1292 writeback->setSupplyExclusive();
1294 writeback->allocate();
1295 std::memcpy(writeback->getPtr<uint8_t>(), blk->data, blkSize);
1297 blk->status &= ~BlkDirty;
1301 template<class TagStore>
1303 Cache<TagStore>::memWriteback()
1305 WrappedBlkVisitor visitor(*this, &Cache<TagStore>::writebackVisitor);
1306 tags->forEachBlk(visitor);
1309 template<class TagStore>
1311 Cache<TagStore>::memInvalidate()
1313 WrappedBlkVisitor visitor(*this, &Cache<TagStore>::invalidateVisitor);
1314 tags->forEachBlk(visitor);
1317 template<class TagStore>
1319 Cache<TagStore>::isDirty() const
1321 CacheBlkIsDirtyVisitor<BlkType> visitor;
1322 tags->forEachBlk(visitor);
1324 return visitor.isDirty();
1327 template<class TagStore>
1329 Cache<TagStore>::writebackVisitor(BlkType &blk)
1331 if (blk.isDirty()) {
1332 assert(blk.isValid());
1334 Request request(tags->regenerateBlkAddr(blk.tag, blk.set),
1335 blkSize, 0, Request::funcMasterId);
1336 request.taskId(blk.task_id);
1338 Packet packet(&request, MemCmd::WriteReq);
1339 packet.dataStatic(blk.data);
1341 memSidePort->sendFunctional(&packet);
1343 blk.status &= ~BlkDirty;
1349 template<class TagStore>
1351 Cache<TagStore>::invalidateVisitor(BlkType &blk)
1355 warn_once("Invalidating dirty cache lines. Expect things to break.\n");
1357 if (blk.isValid()) {
1358 assert(!blk.isDirty());
1359 tags->invalidate(dynamic_cast< BlkType *>(&blk));
1366 template<class TagStore>
1368 Cache<TagStore>::uncacheableFlush(PacketPtr pkt)
1370 DPRINTF(Cache, "%s%s %x uncacheable\n", pkt->cmdString(),
1371 pkt->req->isInstFetch() ? " (ifetch)" : "",
1374 if (pkt->req->isClearLL())
1377 BlkType *blk(tags->findBlock(pkt->getAddr(), pkt->isSecure()));
1379 writebackVisitor(*blk);
1380 invalidateVisitor(*blk);
1385 template<class TagStore>
1386 typename Cache<TagStore>::BlkType*
1387 Cache<TagStore>::allocateBlock(Addr addr, bool is_secure,
1388 PacketList &writebacks)
1390 BlkType *blk = tags->findVictim(addr);
1392 if (blk->isValid()) {
1393 Addr repl_addr = tags->regenerateBlkAddr(blk->tag, blk->set);
1394 MSHR *repl_mshr = mshrQueue.findMatch(repl_addr, blk->isSecure());
1396 // must be an outstanding upgrade request (common case)
1397 // or WriteInvalidate pending writeback (very uncommon case)
1398 // on a block we're about to replace...
1399 assert(!blk->isWritable() || blk->isDirty());
1400 assert(repl_mshr->needsExclusive());
1401 // too hard to replace block with transient state
1402 // allocation failed, block not inserted
1405 DPRINTF(Cache, "replacement: replacing %x (%s) with %x (%s): %s\n",
1406 repl_addr, blk->isSecure() ? "s" : "ns",
1407 addr, is_secure ? "s" : "ns",
1408 blk->isDirty() ? "writeback" : "clean");
1410 if (blk->isDirty()) {
1411 // Save writeback packet for handling by caller
1412 writebacks.push_back(writebackBlk(blk));
1421 // Note that the reason we return a list of writebacks rather than
1422 // inserting them directly in the write buffer is that this function
1423 // is called by both atomic and timing-mode accesses, and in atomic
1424 // mode we don't mess with the write buffer (we just perform the
1425 // writebacks atomically once the original request is complete).
1426 template<class TagStore>
1427 typename Cache<TagStore>::BlkType*
1428 Cache<TagStore>::handleFill(PacketPtr pkt, BlkType *blk,
1429 PacketList &writebacks)
1431 Addr addr = pkt->getAddr();
1432 bool is_secure = pkt->isSecure();
1434 CacheBlk::State old_state = blk ? blk->status : 0;
1438 // better have read new data...
1439 assert(pkt->hasData());
1440 // need to do a replacement
1441 blk = allocateBlock(addr, is_secure, writebacks);
1443 // No replaceable block... just use temporary storage to
1444 // complete the current request and then get rid of it
1445 assert(!tempBlock->isValid());
1447 tempBlock->set = tags->extractSet(addr);
1448 tempBlock->tag = tags->extractTag(addr);
1449 // @todo: set security state as well...
1450 DPRINTF(Cache, "using temp block for %x (%s)\n", addr,
1451 is_secure ? "s" : "ns");
1453 tags->insertBlock(pkt, blk);
1456 // we should never be overwriting a valid block
1457 assert(!blk->isValid());
1459 // existing block... probably an upgrade
1460 assert(blk->tag == tags->extractTag(addr));
1461 // either we're getting new data or the block should already be valid
1462 assert(pkt->hasData() || blk->isValid());
1463 // don't clear block status... if block is already dirty we
1464 // don't want to lose that
1468 blk->status |= BlkSecure;
1469 blk->status |= BlkValid | BlkReadable;
1471 if (!pkt->sharedAsserted()) {
1472 blk->status |= BlkWritable;
1473 // If we got this via cache-to-cache transfer (i.e., from a
1474 // cache that was an owner) and took away that owner's copy,
1475 // then we need to write it back. Normally this happens
1476 // anyway as a side effect of getting a copy to write it, but
1477 // there are cases (such as failed store conditionals or
1478 // compare-and-swaps) where we'll demand an exclusive copy but
1479 // end up not writing it.
1480 if (pkt->memInhibitAsserted())
1481 blk->status |= BlkDirty;
1484 if (pkt->cmd == MemCmd::WriteInvalidateReq) {
1485 // a block written immediately, all at once, pre-writeback is dirty
1486 blk->status |= BlkDirty;
1489 DPRINTF(Cache, "Block addr %x (%s) moving from state %x to %s\n",
1490 addr, is_secure ? "s" : "ns", old_state, blk->print());
1492 // if we got new data, copy it in
1493 if (pkt->isRead()) {
1494 std::memcpy(blk->data, pkt->getPtr<uint8_t>(), blkSize);
1497 blk->whenReady = clockEdge() + responseLatency * clockPeriod() +
1498 pkt->busLastWordDelay;
1504 /////////////////////////////////////////////////////
1506 // Snoop path: requests coming in from the memory side
1508 /////////////////////////////////////////////////////
1510 template<class TagStore>
1513 doTimingSupplyResponse(PacketPtr req_pkt, uint8_t *blk_data,
1514 bool already_copied, bool pending_inval)
1516 DPRINTF(Cache, "%s for %s address %x size %d\n", __func__,
1517 req_pkt->cmdString(), req_pkt->getAddr(), req_pkt->getSize());
1518 // timing-mode snoop responses require a new packet, unless we
1519 // already made a copy...
1520 PacketPtr pkt = already_copied ? req_pkt : new Packet(req_pkt);
1521 assert(req_pkt->isInvalidate() || pkt->sharedAsserted());
1523 pkt->makeTimingResponse();
1524 // @todo Make someone pay for this
1525 pkt->busFirstWordDelay = pkt->busLastWordDelay = 0;
1526 if (pkt->isRead()) {
1527 pkt->setDataFromBlock(blk_data, blkSize);
1529 if (pkt->cmd == MemCmd::ReadResp && pending_inval) {
1530 // Assume we defer a response to a read from a far-away cache
1531 // A, then later defer a ReadExcl from a cache B on the same
1532 // bus as us. We'll assert MemInhibit in both cases, but in
1533 // the latter case MemInhibit will keep the invalidation from
1534 // reaching cache A. This special response tells cache A that
1535 // it gets the block to satisfy its read, but must immediately
1537 pkt->cmd = MemCmd::ReadRespWithInvalidate;
1539 DPRINTF(Cache, "%s created response: %s address %x size %d\n",
1540 __func__, pkt->cmdString(), pkt->getAddr(), pkt->getSize());
1541 memSidePort->schedTimingSnoopResp(pkt, clockEdge(hitLatency));
1544 template<class TagStore>
1546 Cache<TagStore>::handleSnoop(PacketPtr pkt, BlkType *blk,
1547 bool is_timing, bool is_deferred,
1550 DPRINTF(Cache, "%s for %s address %x size %d\n", __func__,
1551 pkt->cmdString(), pkt->getAddr(), pkt->getSize());
1552 // deferred snoops can only happen in timing mode
1553 assert(!(is_deferred && !is_timing));
1554 // pending_inval only makes sense on deferred snoops
1555 assert(!(pending_inval && !is_deferred));
1556 assert(pkt->isRequest());
1558 // the packet may get modified if we or a forwarded snooper
1559 // responds in atomic mode, so remember a few things about the
1560 // original packet up front
1561 bool invalidate = pkt->isInvalidate();
1562 bool M5_VAR_USED needs_exclusive = pkt->needsExclusive();
1564 if (forwardSnoops) {
1565 // first propagate snoop upward to see if anyone above us wants to
1566 // handle it. save & restore packet src since it will get
1567 // rewritten to be relative to cpu-side bus (if any)
1568 bool alreadyResponded = pkt->memInhibitAsserted();
1570 Packet snoopPkt(pkt, true); // clear flags
1571 snoopPkt.setExpressSnoop();
1572 snoopPkt.pushSenderState(new ForwardResponseRecord(pkt->getSrc()));
1573 // the snoop packet does not need to wait any additional
1575 snoopPkt.busFirstWordDelay = snoopPkt.busLastWordDelay = 0;
1576 cpuSidePort->sendTimingSnoopReq(&snoopPkt);
1577 if (snoopPkt.memInhibitAsserted()) {
1578 // cache-to-cache response from some upper cache
1579 assert(!alreadyResponded);
1580 pkt->assertMemInhibit();
1582 delete snoopPkt.popSenderState();
1584 if (snoopPkt.sharedAsserted()) {
1585 pkt->assertShared();
1587 // If this request is a prefetch and an
1588 // upper level squashes the prefetch request,
1589 // make sure to propogate the squash to the requester.
1590 if (snoopPkt.prefetchSquashed()) {
1591 pkt->setPrefetchSquashed();
1594 cpuSidePort->sendAtomicSnoop(pkt);
1595 if (!alreadyResponded && pkt->memInhibitAsserted()) {
1596 // cache-to-cache response from some upper cache:
1597 // forward response to original requester
1598 assert(pkt->isResponse());
1603 if (!blk || !blk->isValid()) {
1604 DPRINTF(Cache, "%s snoop miss for %s address %x size %d\n",
1605 __func__, pkt->cmdString(), pkt->getAddr(), pkt->getSize());
1608 DPRINTF(Cache, "%s snoop hit for %s for address %x size %d, "
1609 "old state is %s\n", __func__, pkt->cmdString(),
1610 pkt->getAddr(), pkt->getSize(), blk->print());
1613 // we may end up modifying both the block state and the packet (if
1614 // we respond in atomic mode), so just figure out what to do now
1615 // and then do it later. If we find dirty data while snooping for a
1616 // WriteInvalidate, we don't care, since no merging needs to take place.
1617 // We need the eviction to happen as normal, but the data needn't be
1618 // sent anywhere, nor should the writeback be inhibited at the memory
1619 // controller for any reason.
1620 bool respond = blk->isDirty() && pkt->needsResponse()
1621 && (pkt->cmd != MemCmd::WriteInvalidateReq);
1622 bool have_exclusive = blk->isWritable();
1624 // Invalidate any prefetch's from below that would strip write permissions
1625 // MemCmd::HardPFReq is only observed by upstream caches. After missing
1626 // above and in it's own cache, a new MemCmd::ReadReq is created that
1627 // downstream caches observe.
1628 if (pkt->cmd == MemCmd::HardPFReq) {
1629 DPRINTF(Cache, "Squashing prefetch from lower cache %#x\n",
1631 pkt->setPrefetchSquashed();
1635 if (pkt->isRead() && !invalidate) {
1636 assert(!needs_exclusive);
1637 pkt->assertShared();
1638 int bits_to_clear = BlkWritable | BlkCanGoExclusive;
1639 const bool haveOwnershipState = true; // for now
1640 if (!haveOwnershipState) {
1641 // if we don't support pure ownership (dirty && !writable),
1642 // have to clear dirty bit here, assume memory snarfs data
1643 // on cache-to-cache xfer
1644 bits_to_clear |= BlkDirty;
1646 blk->status &= ~bits_to_clear;
1650 assert(!pkt->memInhibitAsserted());
1651 pkt->assertMemInhibit();
1652 if (have_exclusive) {
1653 pkt->setSupplyExclusive();
1656 doTimingSupplyResponse(pkt, blk->data, is_deferred, pending_inval);
1658 pkt->makeAtomicResponse();
1659 pkt->setDataFromBlock(blk->data, blkSize);
1661 } else if (is_timing && is_deferred) {
1662 // if it's a deferred timing snoop then we've made a copy of
1663 // the packet, and so if we're not using that copy to respond
1664 // then we need to delete it here.
1668 // Do this last in case it deallocates block data or something
1671 if (blk != tempBlock)
1672 tags->invalidate(blk);
1676 DPRINTF(Cache, "new state is %s\n", blk->print());
1680 template<class TagStore>
1682 Cache<TagStore>::recvTimingSnoopReq(PacketPtr pkt)
1684 DPRINTF(Cache, "%s for %s address %x size %d\n", __func__,
1685 pkt->cmdString(), pkt->getAddr(), pkt->getSize());
1687 // Snoops shouldn't happen when bypassing caches
1688 assert(!system->bypassCaches());
1690 // check if the packet is for an address range covered by this
1691 // cache, partly to not waste time looking for it, but also to
1692 // ensure that we only forward the snoop upwards if it is within
1693 // our address ranges
1694 bool in_range = false;
1695 for (AddrRangeList::const_iterator r = addrRanges.begin();
1696 r != addrRanges.end(); ++r) {
1697 if (r->contains(pkt->getAddr())) {
1703 // Note that some deferred snoops don't have requests, since the
1704 // original access may have already completed
1705 if ((pkt->req && pkt->req->isUncacheable()) ||
1706 pkt->cmd == MemCmd::Writeback || !in_range) {
1707 //Can't get a hit on an uncacheable address
1708 //Revisit this for multi level coherence
1712 bool is_secure = pkt->isSecure();
1713 BlkType *blk = tags->findBlock(pkt->getAddr(), is_secure);
1715 Addr blk_addr = blockAlign(pkt->getAddr());
1716 MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure);
1718 // Squash any prefetch requests from below on MSHR hits
1719 if (mshr && pkt->cmd == MemCmd::HardPFReq) {
1720 DPRINTF(Cache, "Squashing prefetch from lower cache on mshr hit %#x\n",
1722 pkt->setPrefetchSquashed();
1726 // Let the MSHR itself track the snoop and decide whether we want
1727 // to go ahead and do the regular cache snoop
1728 if (mshr && mshr->handleSnoop(pkt, order++)) {
1729 DPRINTF(Cache, "Deferring snoop on in-service MSHR to blk %x (%s)."
1730 "mshrs: %s\n", blk_addr, is_secure ? "s" : "ns",
1733 if (mshr->getNumTargets() > numTarget)
1734 warn("allocating bonus target for snoop"); //handle later
1738 //We also need to check the writeback buffers and handle those
1739 std::vector<MSHR *> writebacks;
1740 if (writeBuffer.findMatches(blk_addr, is_secure, writebacks)) {
1741 DPRINTF(Cache, "Snoop hit in writeback to addr: %x (%s)\n",
1742 pkt->getAddr(), is_secure ? "s" : "ns");
1744 //Look through writebacks for any non-uncachable writes, use that
1745 if (writebacks.size()) {
1746 // We should only ever find a single match
1747 assert(writebacks.size() == 1);
1748 mshr = writebacks[0];
1749 assert(!mshr->isUncacheable());
1750 assert(mshr->getNumTargets() == 1);
1751 PacketPtr wb_pkt = mshr->getTarget()->pkt;
1752 assert(wb_pkt->cmd == MemCmd::Writeback);
1754 assert(!pkt->memInhibitAsserted());
1755 pkt->assertMemInhibit();
1756 if (!pkt->needsExclusive()) {
1757 pkt->assertShared();
1758 // the writeback is no longer the exclusive copy in the system
1759 wb_pkt->clearSupplyExclusive();
1761 // if we're not asserting the shared line, we need to
1762 // invalidate our copy. we'll do that below as long as
1763 // the packet's invalidate flag is set...
1764 assert(pkt->isInvalidate());
1766 doTimingSupplyResponse(pkt, wb_pkt->getPtr<uint8_t>(),
1769 if (pkt->isInvalidate()) {
1770 // Invalidation trumps our writeback... discard here
1771 markInService(mshr);
1774 } // writebacks.size()
1777 // If this was a shared writeback, there may still be
1778 // other shared copies above that require invalidation.
1779 // We could be more selective and return here if the
1780 // request is non-exclusive or if the writeback is
1782 handleSnoop(pkt, blk, true, false, false);
1785 template<class TagStore>
1787 Cache<TagStore>::CpuSidePort::recvTimingSnoopResp(PacketPtr pkt)
1789 // Express snoop responses from master to slave, e.g., from L1 to L2
1790 cache->recvTimingSnoopResp(pkt);
1794 template<class TagStore>
1796 Cache<TagStore>::recvAtomicSnoop(PacketPtr pkt)
1798 // Snoops shouldn't happen when bypassing caches
1799 assert(!system->bypassCaches());
1801 if (pkt->req->isUncacheable() || pkt->cmd == MemCmd::Writeback) {
1802 // Can't get a hit on an uncacheable address
1803 // Revisit this for multi level coherence
1807 BlkType *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
1808 handleSnoop(pkt, blk, false, false, false);
1809 return hitLatency * clockPeriod();
1813 template<class TagStore>
1815 Cache<TagStore>::getNextMSHR()
1817 // Check both MSHR queue and write buffer for potential requests
1818 MSHR *miss_mshr = mshrQueue.getNextMSHR();
1819 MSHR *write_mshr = writeBuffer.getNextMSHR();
1821 // Now figure out which one to send... some cases are easy
1822 if (miss_mshr && !write_mshr) {
1825 if (write_mshr && !miss_mshr) {
1829 if (miss_mshr && write_mshr) {
1830 // We have one of each... normally we favor the miss request
1831 // unless the write buffer is full
1832 if (writeBuffer.isFull() && writeBuffer.inServiceEntries == 0) {
1833 // Write buffer is full, so we'd like to issue a write;
1834 // need to search MSHR queue for conflicting earlier miss.
1835 MSHR *conflict_mshr =
1836 mshrQueue.findPending(write_mshr->addr, write_mshr->size,
1837 write_mshr->isSecure);
1839 if (conflict_mshr && conflict_mshr->order < write_mshr->order) {
1840 // Service misses in order until conflict is cleared.
1841 return conflict_mshr;
1844 // No conflicts; issue write
1848 // Write buffer isn't full, but need to check it for
1849 // conflicting earlier writeback
1850 MSHR *conflict_mshr =
1851 writeBuffer.findPending(miss_mshr->addr, miss_mshr->size,
1852 miss_mshr->isSecure);
1853 if (conflict_mshr) {
1854 // not sure why we don't check order here... it was in the
1855 // original code but commented out.
1857 // The only way this happens is if we are
1858 // doing a write and we didn't have permissions
1859 // then subsequently saw a writeback (owned got evicted)
1860 // We need to make sure to perform the writeback first
1861 // To preserve the dirty data, then we can issue the write
1863 // should we return write_mshr here instead? I.e. do we
1864 // have to flush writes in order? I don't think so... not
1865 // for Alpha anyway. Maybe for x86?
1866 return conflict_mshr;
1869 // No conflicts; issue read
1873 // fall through... no pending requests. Try a prefetch.
1874 assert(!miss_mshr && !write_mshr);
1875 if (prefetcher && !mshrQueue.isFull()) {
1876 // If we have a miss queue slot, we can try a prefetch
1877 PacketPtr pkt = prefetcher->getPacket();
1879 Addr pf_addr = blockAlign(pkt->getAddr());
1880 if (!tags->findBlock(pf_addr, pkt->isSecure()) &&
1881 !mshrQueue.findMatch(pf_addr, pkt->isSecure()) &&
1882 !writeBuffer.findMatch(pf_addr, pkt->isSecure())) {
1883 // Update statistic on number of prefetches issued
1884 // (hwpf_mshr_misses)
1885 assert(pkt->req->masterId() < system->maxMasters());
1886 mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++;
1887 // Don't request bus, since we already have it
1888 return allocateMissBuffer(pkt, curTick(), false);
1890 // free the request and packet
1901 template<class TagStore>
1903 Cache<TagStore>::getTimingPacket()
1905 MSHR *mshr = getNextMSHR();
1911 // use request from 1st target
1912 PacketPtr tgt_pkt = mshr->getTarget()->pkt;
1913 PacketPtr pkt = NULL;
1915 DPRINTF(CachePort, "%s %s for address %x size %d\n", __func__,
1916 tgt_pkt->cmdString(), tgt_pkt->getAddr(), tgt_pkt->getSize());
1918 if (mshr->isForwardNoResponse()) {
1919 // no response expected, just forward packet as it is
1920 assert(tags->findBlock(mshr->addr, mshr->isSecure) == NULL);
1923 BlkType *blk = tags->findBlock(mshr->addr, mshr->isSecure);
1925 if (tgt_pkt->cmd == MemCmd::HardPFReq) {
1926 // It might be possible for a writeback to arrive between
1927 // the time the prefetch is placed in the MSHRs and when
1928 // it's selected to send... if so, this assert will catch
1929 // that, and then we'll have to figure out what to do.
1930 assert(blk == NULL);
1932 // We need to check the caches above us to verify that
1933 // they don't have a copy of this block in the dirty state
1934 // at the moment. Without this check we could get a stale
1935 // copy from memory that might get used in place of the
1937 Packet snoop_pkt(tgt_pkt, true);
1938 snoop_pkt.setExpressSnoop();
1939 snoop_pkt.senderState = mshr;
1940 cpuSidePort->sendTimingSnoopReq(&snoop_pkt);
1942 // Check to see if the prefetch was squashed by an upper cache
1943 if (snoop_pkt.prefetchSquashed()) {
1944 DPRINTF(Cache, "Prefetch squashed by upper cache. "
1945 "Deallocating mshr target %#x.\n", mshr->addr);
1947 // Deallocate the mshr target
1948 if (mshr->queue->forceDeallocateTarget(mshr)) {
1949 // Clear block if this deallocation resulted freed an
1950 // mshr when all had previously been utilized
1951 clearBlocked((BlockedCause)(mshr->queue->index));
1956 if (snoop_pkt.memInhibitAsserted()) {
1957 markInService(mshr, &snoop_pkt);
1958 DPRINTF(Cache, "Upward snoop of prefetch for addr"
1960 tgt_pkt->getAddr(), tgt_pkt->isSecure()? "s": "ns");
1965 pkt = getBusPacket(tgt_pkt, blk, mshr->needsExclusive());
1967 mshr->isForward = (pkt == NULL);
1969 if (mshr->isForward) {
1970 // not a cache block request, but a response is expected
1971 // make copy of current packet to forward, keep current
1972 // copy for response handling
1973 pkt = new Packet(tgt_pkt);
1975 if (pkt->isWrite()) {
1976 pkt->setData(tgt_pkt->getPtr<uint8_t>());
1981 assert(pkt != NULL);
1982 pkt->senderState = mshr;
1987 template<class TagStore>
1989 Cache<TagStore>::nextMSHRReadyTime() const
1991 Tick nextReady = std::min(mshrQueue.nextMSHRReadyTime(),
1992 writeBuffer.nextMSHRReadyTime());
1995 nextReady = std::min(nextReady,
1996 prefetcher->nextPrefetchReadyTime());
2002 template<class TagStore>
2004 Cache<TagStore>::serialize(std::ostream &os)
2006 bool dirty(isDirty());
2009 warn("*** The cache still contains dirty data. ***\n");
2010 warn(" Make sure to drain the system using the correct flags.\n");
2011 warn(" This checkpoint will not restore correctly and dirty data in "
2012 "the cache will be lost!\n");
2015 // Since we don't checkpoint the data in the cache, any dirty data
2016 // will be lost when restoring from a checkpoint of a system that
2017 // wasn't drained properly. Flag the checkpoint as invalid if the
2018 // cache contains dirty data.
2019 bool bad_checkpoint(dirty);
2020 SERIALIZE_SCALAR(bad_checkpoint);
2023 template<class TagStore>
2025 Cache<TagStore>::unserialize(Checkpoint *cp, const std::string §ion)
2027 bool bad_checkpoint;
2028 UNSERIALIZE_SCALAR(bad_checkpoint);
2029 if (bad_checkpoint) {
2030 fatal("Restoring from checkpoints with dirty caches is not supported "
2031 "in the classic memory system. Please remove any caches or "
2032 " drain them properly before taking checkpoints.\n");
2042 template<class TagStore>
2044 Cache<TagStore>::CpuSidePort::getAddrRanges() const
2046 return cache->getAddrRanges();
2049 template<class TagStore>
2051 Cache<TagStore>::CpuSidePort::recvTimingReq(PacketPtr pkt)
2053 assert(!cache->system->bypassCaches());
2055 bool success = false;
2057 // always let inhibited requests through, even if blocked
2058 if (pkt->memInhibitAsserted()) {
2059 // this should always succeed
2060 success = cache->recvTimingReq(pkt);
2062 } else if (blocked || mustSendRetry) {
2063 // either already committed to send a retry, or blocked
2066 // pass it on to the cache, and let the cache decide if we
2067 // have to retry or not
2068 success = cache->recvTimingReq(pkt);
2071 // remember if we have to retry
2072 mustSendRetry = !success;
2076 template<class TagStore>
2078 Cache<TagStore>::CpuSidePort::recvAtomic(PacketPtr pkt)
2080 return cache->recvAtomic(pkt);
2083 template<class TagStore>
2085 Cache<TagStore>::CpuSidePort::recvFunctional(PacketPtr pkt)
2087 // functional request
2088 cache->functionalAccess(pkt, true);
2091 template<class TagStore>
2093 CpuSidePort::CpuSidePort(const std::string &_name, Cache<TagStore> *_cache,
2094 const std::string &_label)
2095 : BaseCache::CacheSlavePort(_name, _cache, _label), cache(_cache)
2105 template<class TagStore>
2107 Cache<TagStore>::MemSidePort::recvTimingResp(PacketPtr pkt)
2109 cache->recvTimingResp(pkt);
2113 // Express snooping requests to memside port
2114 template<class TagStore>
2116 Cache<TagStore>::MemSidePort::recvTimingSnoopReq(PacketPtr pkt)
2118 // handle snooping requests
2119 cache->recvTimingSnoopReq(pkt);
2122 template<class TagStore>
2124 Cache<TagStore>::MemSidePort::recvAtomicSnoop(PacketPtr pkt)
2126 return cache->recvAtomicSnoop(pkt);
2129 template<class TagStore>
2131 Cache<TagStore>::MemSidePort::recvFunctionalSnoop(PacketPtr pkt)
2133 // functional snoop (note that in contrast to atomic we don't have
2134 // a specific functionalSnoop method, as they have the same
2135 // behaviour regardless)
2136 cache->functionalAccess(pkt, false);
2139 template<class TagStore>
2141 Cache<TagStore>::MemSidePacketQueue::sendDeferredPacket()
2143 // if we have a response packet waiting we have to start with that
2144 if (deferredPacketReady()) {
2145 // use the normal approach from the timing port
2148 // check for request packets (requests & writebacks)
2149 PacketPtr pkt = cache.getTimingPacket();
2151 // can happen if e.g. we attempt a writeback and fail, but
2152 // before the retry, the writeback is eliminated because
2153 // we snoop another cache's ReadEx.
2154 waitingOnRetry = false;
2156 MSHR *mshr = dynamic_cast<MSHR*>(pkt->senderState);
2158 waitingOnRetry = !masterPort.sendTimingReq(pkt);
2160 if (waitingOnRetry) {
2161 DPRINTF(CachePort, "now waiting on a retry\n");
2162 if (!mshr->isForwardNoResponse()) {
2163 // we are awaiting a retry, but we
2164 // delete the packet and will be creating a new packet
2165 // when we get the opportunity
2168 // note that we have now masked any requestBus and
2169 // schedSendEvent (we will wait for a retry before
2170 // doing anything), and this is so even if we do not
2171 // care about this packet and might override it before
2174 cache.markInService(mshr, pkt);
2179 // if we succeeded and are not waiting for a retry, schedule the
2180 // next send, not only looking at the response transmit list, but
2181 // also considering when the next MSHR is ready
2182 if (!waitingOnRetry) {
2183 scheduleSend(cache.nextMSHRReadyTime());
2187 template<class TagStore>
2189 MemSidePort::MemSidePort(const std::string &_name, Cache<TagStore> *_cache,
2190 const std::string &_label)
2191 : BaseCache::CacheMasterPort(_name, _cache, _queue),
2192 _queue(*_cache, *this, _label), cache(_cache)
2196 #endif//__MEM_CACHE_CACHE_IMPL_HH__