2 * Copyright (c) 2010-2011 ARM Limited
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
14 * Copyright (c) 2004-2005 The Regents of The University of Michigan
15 * All rights reserved.
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions are
19 * met: redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer;
21 * redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution;
24 * neither the name of the copyright holders nor the names of its
25 * contributors may be used to endorse or promote products derived from
26 * this software without specific prior written permission.
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
44 #include "arch/generic/debugfaults.hh"
45 #include "arch/locked_mem.hh"
46 #include "base/str.hh"
47 #include "config/the_isa.hh"
48 #include "cpu/checker/cpu.hh"
49 #include "cpu/o3/lsq.hh"
50 #include "cpu/o3/lsq_unit.hh"
51 #include "debug/Activity.hh"
52 #include "debug/IEW.hh"
53 #include "debug/LSQUnit.hh"
54 #include "mem/packet.hh"
55 #include "mem/request.hh"
58 LSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt,
60 : Event(Default_Pri, AutoDelete),
61 inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
67 LSQUnit<Impl>::WritebackEvent::process()
69 if (!lsqPtr->isSwitchedOut()) {
70 lsqPtr->writeback(inst, pkt);
74 delete pkt->senderState;
82 LSQUnit<Impl>::WritebackEvent::description() const
84 return "Store writeback";
89 LSQUnit<Impl>::completeDataAccess(PacketPtr pkt)
91 LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState);
92 DynInstPtr inst = state->inst;
93 DPRINTF(IEW, "Writeback event [sn:%lli].\n", inst->seqNum);
94 DPRINTF(Activity, "Activity: Writeback event [sn:%lli].\n", inst->seqNum);
96 //iewStage->ldstQueue.removeMSHR(inst->threadNumber,inst->seqNum);
98 // If this is a split access, wait until all packets are received.
99 if (TheISA::HasUnalignedMemAcc && !state->complete()) {
105 if (isSwitchedOut() || inst->isSquashed()) {
106 iewStage->decrWb(inst->seqNum);
109 if (!TheISA::HasUnalignedMemAcc || !state->isSplit ||
111 writeback(inst, pkt);
113 writeback(inst, state->mainPkt);
117 if (inst->isStore()) {
118 completeStore(state->idx);
122 if (TheISA::HasUnalignedMemAcc && state->isSplit && state->isLoad) {
123 delete state->mainPkt->req;
124 delete state->mainPkt;
131 template <class Impl>
132 LSQUnit<Impl>::LSQUnit()
133 : loads(0), stores(0), storesToWB(0), cacheBlockMask(0), stalled(false),
134 isStoreBlocked(false), isLoadBlocked(false),
135 loadBlockedHandled(false), storeInFlight(false), hasPendingPkt(false)
141 LSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params,
142 LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries,
148 DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id);
158 // Add 1 for the sentinel entry (they are circular queues).
159 LQEntries = maxLQEntries + 1;
160 SQEntries = maxSQEntries + 1;
162 loadQueue.resize(LQEntries);
163 storeQueue.resize(SQEntries);
165 depCheckShift = params->LSQDepCheckShift;
166 checkLoads = params->LSQCheckLoads;
168 loadHead = loadTail = 0;
170 storeHead = storeWBIdx = storeTail = 0;
173 cachePorts = params->cachePorts;
176 memDepViolator = NULL;
178 blockedLoadSeqNum = 0;
179 needsTSO = params->needsTSO;
184 LSQUnit<Impl>::name() const
186 if (Impl::MaxThreads == 1) {
187 return iewStage->name() + ".lsq";
189 return iewStage->name() + ".lsq.thread" + to_string(lsqID);
195 LSQUnit<Impl>::regStats()
198 .name(name() + ".forwLoads")
199 .desc("Number of loads that had data forwarded from stores");
202 .name(name() + ".invAddrLoads")
203 .desc("Number of loads ignored due to an invalid address");
206 .name(name() + ".squashedLoads")
207 .desc("Number of loads squashed");
210 .name(name() + ".ignoredResponses")
211 .desc("Number of memory responses ignored because the instruction is squashed");
214 .name(name() + ".memOrderViolation")
215 .desc("Number of memory ordering violations");
218 .name(name() + ".squashedStores")
219 .desc("Number of stores squashed");
222 .name(name() + ".invAddrSwpfs")
223 .desc("Number of software prefetches ignored due to an invalid address");
226 .name(name() + ".blockedLoads")
227 .desc("Number of blocked loads due to partial load-store forwarding");
230 .name(name() + ".rescheduledLoads")
231 .desc("Number of loads that were rescheduled");
234 .name(name() + ".cacheBlocked")
235 .desc("Number of times an access to memory failed due to the cache being blocked");
240 LSQUnit<Impl>::setDcachePort(MasterPort *dcache_port)
242 dcachePort = dcache_port;
247 LSQUnit<Impl>::clearLQ()
254 LSQUnit<Impl>::clearSQ()
261 LSQUnit<Impl>::switchOut()
264 for (int i = 0; i < loadQueue.size(); ++i) {
265 assert(!loadQueue[i]);
269 assert(storesToWB == 0);
274 LSQUnit<Impl>::takeOverFrom()
277 loads = stores = storesToWB = 0;
279 loadHead = loadTail = 0;
281 storeHead = storeWBIdx = storeTail = 0;
285 memDepViolator = NULL;
287 blockedLoadSeqNum = 0;
290 isLoadBlocked = false;
291 loadBlockedHandled = false;
293 // Just incase the memory system changed out from under us
299 LSQUnit<Impl>::resizeLQ(unsigned size)
301 unsigned size_plus_sentinel = size + 1;
302 assert(size_plus_sentinel >= LQEntries);
304 if (size_plus_sentinel > LQEntries) {
305 while (size_plus_sentinel > loadQueue.size()) {
307 loadQueue.push_back(dummy);
311 LQEntries = size_plus_sentinel;
318 LSQUnit<Impl>::resizeSQ(unsigned size)
320 unsigned size_plus_sentinel = size + 1;
321 if (size_plus_sentinel > SQEntries) {
322 while (size_plus_sentinel > storeQueue.size()) {
324 storeQueue.push_back(dummy);
328 SQEntries = size_plus_sentinel;
332 template <class Impl>
334 LSQUnit<Impl>::insert(DynInstPtr &inst)
336 assert(inst->isMemRef());
338 assert(inst->isLoad() || inst->isStore());
340 if (inst->isLoad()) {
349 template <class Impl>
351 LSQUnit<Impl>::insertLoad(DynInstPtr &load_inst)
353 assert((loadTail + 1) % LQEntries != loadHead);
354 assert(loads < LQEntries);
356 DPRINTF(LSQUnit, "Inserting load PC %s, idx:%i [sn:%lli]\n",
357 load_inst->pcState(), loadTail, load_inst->seqNum);
359 load_inst->lqIdx = loadTail;
362 load_inst->sqIdx = -1;
364 load_inst->sqIdx = storeTail;
367 loadQueue[loadTail] = load_inst;
374 template <class Impl>
376 LSQUnit<Impl>::insertStore(DynInstPtr &store_inst)
378 // Make sure it is not full before inserting an instruction.
379 assert((storeTail + 1) % SQEntries != storeHead);
380 assert(stores < SQEntries);
382 DPRINTF(LSQUnit, "Inserting store PC %s, idx:%i [sn:%lli]\n",
383 store_inst->pcState(), storeTail, store_inst->seqNum);
385 store_inst->sqIdx = storeTail;
386 store_inst->lqIdx = loadTail;
388 storeQueue[storeTail] = SQEntry(store_inst);
390 incrStIdx(storeTail);
395 template <class Impl>
396 typename Impl::DynInstPtr
397 LSQUnit<Impl>::getMemDepViolator()
399 DynInstPtr temp = memDepViolator;
401 memDepViolator = NULL;
406 template <class Impl>
408 LSQUnit<Impl>::numFreeEntries()
410 unsigned free_lq_entries = LQEntries - loads;
411 unsigned free_sq_entries = SQEntries - stores;
413 // Both the LQ and SQ entries have an extra dummy entry to differentiate
414 // empty/full conditions. Subtract 1 from the free entries.
415 if (free_lq_entries < free_sq_entries) {
416 return free_lq_entries - 1;
418 return free_sq_entries - 1;
422 template <class Impl>
424 LSQUnit<Impl>::numLoadsReady()
426 int load_idx = loadHead;
429 while (load_idx != loadTail) {
430 assert(loadQueue[load_idx]);
432 if (loadQueue[load_idx]->readyToIssue()) {
440 template <class Impl>
442 LSQUnit<Impl>::checkSnoop(PacketPtr pkt)
444 int load_idx = loadHead;
446 if (!cacheBlockMask) {
448 Addr bs = dcachePort->peerBlockSize();
450 // Make sure we actually got a size
453 cacheBlockMask = ~(bs - 1);
456 // If this is the only load in the LSQ we don't care
457 if (load_idx == loadTail)
461 DPRINTF(LSQUnit, "Got snoop for address %#x\n", pkt->getAddr());
462 Addr invalidate_addr = pkt->getAddr() & cacheBlockMask;
463 while (load_idx != loadTail) {
464 DynInstPtr ld_inst = loadQueue[load_idx];
466 if (!ld_inst->effAddrValid() || ld_inst->uncacheable()) {
471 Addr load_addr = ld_inst->physEffAddr & cacheBlockMask;
472 DPRINTF(LSQUnit, "-- inst [sn:%lli] load_addr: %#x to pktAddr:%#x\n",
473 ld_inst->seqNum, load_addr, invalidate_addr);
475 if (load_addr == invalidate_addr) {
476 if (ld_inst->possibleLoadViolation()) {
477 DPRINTF(LSQUnit, "Conflicting load at addr %#x [sn:%lli]\n",
478 ld_inst->physEffAddr, pkt->getAddr(), ld_inst->seqNum);
480 // Mark the load for re-execution
481 ld_inst->fault = new ReExec;
483 // If a older load checks this and it's true
484 // then we might have missed the snoop
485 // in which case we need to invalidate to be sure
486 ld_inst->hitExternalSnoop(true);
494 template <class Impl>
496 LSQUnit<Impl>::checkViolations(int load_idx, DynInstPtr &inst)
498 Addr inst_eff_addr1 = inst->effAddr >> depCheckShift;
499 Addr inst_eff_addr2 = (inst->effAddr + inst->effSize - 1) >> depCheckShift;
501 /** @todo in theory you only need to check an instruction that has executed
502 * however, there isn't a good way in the pipeline at the moment to check
503 * all instructions that will execute before the store writes back. Thus,
504 * like the implementation that came before it, we're overly conservative.
506 while (load_idx != loadTail) {
507 DynInstPtr ld_inst = loadQueue[load_idx];
508 if (!ld_inst->effAddrValid() || ld_inst->uncacheable()) {
513 Addr ld_eff_addr1 = ld_inst->effAddr >> depCheckShift;
515 (ld_inst->effAddr + ld_inst->effSize - 1) >> depCheckShift;
517 if (inst_eff_addr2 >= ld_eff_addr1 && inst_eff_addr1 <= ld_eff_addr2) {
518 if (inst->isLoad()) {
519 // If this load is to the same block as an external snoop
520 // invalidate that we've observed then the load needs to be
521 // squashed as it could have newer data
522 if (ld_inst->hitExternalSnoop()) {
523 if (!memDepViolator ||
524 ld_inst->seqNum < memDepViolator->seqNum) {
525 DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] "
526 "and [sn:%lli] at address %#x\n",
527 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
528 memDepViolator = ld_inst;
530 ++lsqMemOrderViolation;
532 return new GenericISA::M5PanicFault(
533 "Detected fault with inst [sn:%lli] and "
534 "[sn:%lli] at address %#x\n",
535 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
539 // Otherwise, mark the load has a possible load violation
540 // and if we see a snoop before it's commited, we need to squash
541 ld_inst->possibleLoadViolation(true);
542 DPRINTF(LSQUnit, "Found possible load violaiton at addr: %#x"
543 " between instructions [sn:%lli] and [sn:%lli]\n",
544 inst_eff_addr1, inst->seqNum, ld_inst->seqNum);
546 // A load/store incorrectly passed this store.
547 // Check if we already have a violator, or if it's newer
548 // squash and refetch.
549 if (memDepViolator && ld_inst->seqNum > memDepViolator->seqNum)
552 DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] and "
553 "[sn:%lli] at address %#x\n",
554 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
555 memDepViolator = ld_inst;
557 ++lsqMemOrderViolation;
559 return new GenericISA::M5PanicFault("Detected fault with "
560 "inst [sn:%lli] and [sn:%lli] at address %#x\n",
561 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
573 template <class Impl>
575 LSQUnit<Impl>::executeLoad(DynInstPtr &inst)
577 using namespace TheISA;
578 // Execute a specific load.
579 Fault load_fault = NoFault;
581 DPRINTF(LSQUnit, "Executing load PC %s, [sn:%lli]\n",
582 inst->pcState(), inst->seqNum);
584 assert(!inst->isSquashed());
586 load_fault = inst->initiateAcc();
588 if (inst->isTranslationDelayed() &&
589 load_fault == NoFault)
592 // If the instruction faulted or predicated false, then we need to send it
593 // along to commit without the instruction completing.
594 if (load_fault != NoFault || inst->readPredicate() == false) {
595 // Send this instruction to commit, also make sure iew stage
596 // realizes there is activity.
597 // Mark it as executed unless it is an uncached load that
598 // needs to hit the head of commit.
599 if (inst->readPredicate() == false)
600 inst->forwardOldRegs();
601 DPRINTF(LSQUnit, "Load [sn:%lli] not executed from %s\n",
603 (load_fault != NoFault ? "fault" : "predication"));
604 if (!(inst->hasRequest() && inst->uncacheable()) ||
605 inst->isAtCommit()) {
608 iewStage->instToCommit(inst);
609 iewStage->activityThisCycle();
610 } else if (!loadBlocked()) {
611 assert(inst->effAddrValid());
612 int load_idx = inst->lqIdx;
616 return checkViolations(load_idx, inst);
622 template <class Impl>
624 LSQUnit<Impl>::executeStore(DynInstPtr &store_inst)
626 using namespace TheISA;
627 // Make sure that a store exists.
630 int store_idx = store_inst->sqIdx;
632 DPRINTF(LSQUnit, "Executing store PC %s [sn:%lli]\n",
633 store_inst->pcState(), store_inst->seqNum);
635 assert(!store_inst->isSquashed());
637 // Check the recently completed loads to see if any match this store's
638 // address. If so, then we have a memory ordering violation.
639 int load_idx = store_inst->lqIdx;
641 Fault store_fault = store_inst->initiateAcc();
643 if (store_inst->isTranslationDelayed() &&
644 store_fault == NoFault)
647 if (store_inst->readPredicate() == false)
648 store_inst->forwardOldRegs();
650 if (storeQueue[store_idx].size == 0) {
651 DPRINTF(LSQUnit,"Fault on Store PC %s, [sn:%lli], Size = 0\n",
652 store_inst->pcState(), store_inst->seqNum);
655 } else if (store_inst->readPredicate() == false) {
656 DPRINTF(LSQUnit, "Store [sn:%lli] not executed from predication\n",
661 assert(store_fault == NoFault);
663 if (store_inst->isStoreConditional()) {
664 // Store conditionals need to set themselves as able to
665 // writeback if we haven't had a fault by here.
666 storeQueue[store_idx].canWB = true;
671 return checkViolations(load_idx, store_inst);
675 template <class Impl>
677 LSQUnit<Impl>::commitLoad()
679 assert(loadQueue[loadHead]);
681 DPRINTF(LSQUnit, "Committing head load instruction, PC %s\n",
682 loadQueue[loadHead]->pcState());
684 loadQueue[loadHead] = NULL;
691 template <class Impl>
693 LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst)
695 assert(loads == 0 || loadQueue[loadHead]);
697 while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
702 template <class Impl>
704 LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst)
706 assert(stores == 0 || storeQueue[storeHead].inst);
708 int store_idx = storeHead;
710 while (store_idx != storeTail) {
711 assert(storeQueue[store_idx].inst);
712 // Mark any stores that are now committed and have not yet
713 // been marked as able to write back.
714 if (!storeQueue[store_idx].canWB) {
715 if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
718 DPRINTF(LSQUnit, "Marking store as able to write back, PC "
720 storeQueue[store_idx].inst->pcState(),
721 storeQueue[store_idx].inst->seqNum);
723 storeQueue[store_idx].canWB = true;
728 incrStIdx(store_idx);
732 template <class Impl>
734 LSQUnit<Impl>::writebackPendingStore()
737 assert(pendingPkt != NULL);
739 // If the cache is blocked, this will store the packet for retry.
740 if (sendStore(pendingPkt)) {
741 storePostSend(pendingPkt);
744 hasPendingPkt = false;
748 template <class Impl>
750 LSQUnit<Impl>::writebackStores()
752 // First writeback the second packet from any split store that didn't
753 // complete last cycle because there weren't enough cache ports available.
754 if (TheISA::HasUnalignedMemAcc) {
755 writebackPendingStore();
758 while (storesToWB > 0 &&
759 storeWBIdx != storeTail &&
760 storeQueue[storeWBIdx].inst &&
761 storeQueue[storeWBIdx].canWB &&
762 ((!needsTSO) || (!storeInFlight)) &&
763 usedPorts < cachePorts) {
765 if (isStoreBlocked || lsq->cacheBlocked()) {
766 DPRINTF(LSQUnit, "Unable to write back any more stores, cache"
771 // Store didn't write any data so no need to write it back to
773 if (storeQueue[storeWBIdx].size == 0) {
774 completeStore(storeWBIdx);
776 incrStIdx(storeWBIdx);
783 if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
784 incrStIdx(storeWBIdx);
789 assert(storeQueue[storeWBIdx].req);
790 assert(!storeQueue[storeWBIdx].committed);
792 if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) {
793 assert(storeQueue[storeWBIdx].sreqLow);
794 assert(storeQueue[storeWBIdx].sreqHigh);
797 DynInstPtr inst = storeQueue[storeWBIdx].inst;
799 Request *req = storeQueue[storeWBIdx].req;
800 RequestPtr sreqLow = storeQueue[storeWBIdx].sreqLow;
801 RequestPtr sreqHigh = storeQueue[storeWBIdx].sreqHigh;
803 storeQueue[storeWBIdx].committed = true;
805 assert(!inst->memData);
806 inst->memData = new uint8_t[64];
808 memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize());
811 req->isSwap() ? MemCmd::SwapReq :
812 (req->isLLSC() ? MemCmd::StoreCondReq : MemCmd::WriteReq);
814 PacketPtr snd_data_pkt = NULL;
816 LSQSenderState *state = new LSQSenderState;
817 state->isLoad = false;
818 state->idx = storeWBIdx;
821 if (!TheISA::HasUnalignedMemAcc || !storeQueue[storeWBIdx].isSplit) {
823 // Build a single data packet if the store isn't split.
824 data_pkt = new Packet(req, command);
825 data_pkt->dataStatic(inst->memData);
826 data_pkt->senderState = state;
828 // Create two packets if the store is split in two.
829 data_pkt = new Packet(sreqLow, command);
830 snd_data_pkt = new Packet(sreqHigh, command);
832 data_pkt->dataStatic(inst->memData);
833 snd_data_pkt->dataStatic(inst->memData + sreqLow->getSize());
835 data_pkt->senderState = state;
836 snd_data_pkt->senderState = state;
838 state->isSplit = true;
839 state->outstanding = 2;
841 // Can delete the main request now.
846 DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%s "
847 "to Addr:%#x, data:%#x [sn:%lli]\n",
848 storeWBIdx, inst->pcState(),
849 req->getPaddr(), (int)*(inst->memData),
852 // @todo: Remove this SC hack once the memory system handles it.
853 if (inst->isStoreConditional()) {
854 assert(!storeQueue[storeWBIdx].isSplit);
855 // Disable recording the result temporarily. Writing to
856 // misc regs normally updates the result, but this is not
857 // the desired behavior when handling store conditionals.
858 inst->recordResult(false);
859 bool success = TheISA::handleLockedWrite(inst.get(), req);
860 inst->recordResult(true);
863 // Instantly complete this store.
864 DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed. "
865 "Instantly completing it.\n",
867 WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
868 cpu->schedule(wb, curTick() + 1);
870 // Make sure to set the LLSC data for verification
871 // if checker is loaded
872 inst->reqToVerify->setExtraData(0);
873 inst->completeAcc(data_pkt);
875 completeStore(storeWBIdx);
876 incrStIdx(storeWBIdx);
880 // Non-store conditionals do not need a writeback.
885 TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit;
887 ThreadContext *thread = cpu->tcBase(lsqID);
889 if (req->isMmappedIpr()) {
890 assert(!inst->isStoreConditional());
891 TheISA::handleIprWrite(thread, data_pkt);
894 assert(snd_data_pkt->req->isMmappedIpr());
895 TheISA::handleIprWrite(thread, snd_data_pkt);
902 completeStore(storeWBIdx);
903 incrStIdx(storeWBIdx);
904 } else if (!sendStore(data_pkt)) {
905 DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will"
909 // Need to store the second packet, if split.
911 state->pktToSend = true;
912 state->pendingPacket = snd_data_pkt;
916 // If split, try to send the second packet too
918 assert(snd_data_pkt);
920 // Ensure there are enough ports to use.
921 if (usedPorts < cachePorts) {
923 if (sendStore(snd_data_pkt)) {
924 storePostSend(snd_data_pkt);
926 DPRINTF(IEW, "D-Cache became blocked when writing"
927 " [sn:%lli] second packet, will retry later\n",
932 // Store the packet for when there's free ports.
933 assert(pendingPkt == NULL);
934 pendingPkt = snd_data_pkt;
935 hasPendingPkt = true;
939 // Not a split store.
940 storePostSend(data_pkt);
945 // Not sure this should set it to 0.
948 assert(stores >= 0 && storesToWB >= 0);
951 /*template <class Impl>
953 LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum)
955 list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
959 if (mshr_it != mshrSeqNums.end()) {
960 mshrSeqNums.erase(mshr_it);
961 DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size());
965 template <class Impl>
967 LSQUnit<Impl>::squash(const InstSeqNum &squashed_num)
969 DPRINTF(LSQUnit, "Squashing until [sn:%lli]!"
970 "(Loads:%i Stores:%i)\n", squashed_num, loads, stores);
972 int load_idx = loadTail;
975 while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
976 DPRINTF(LSQUnit,"Load Instruction PC %s squashed, "
978 loadQueue[load_idx]->pcState(),
979 loadQueue[load_idx]->seqNum);
981 if (isStalled() && load_idx == stallingLoadIdx) {
983 stallingStoreIsn = 0;
987 // Clear the smart pointer to make sure it is decremented.
988 loadQueue[load_idx]->setSquashed();
989 loadQueue[load_idx] = NULL;
1000 if (squashed_num < blockedLoadSeqNum) {
1001 isLoadBlocked = false;
1002 loadBlockedHandled = false;
1003 blockedLoadSeqNum = 0;
1007 if (memDepViolator && squashed_num < memDepViolator->seqNum) {
1008 memDepViolator = NULL;
1011 int store_idx = storeTail;
1012 decrStIdx(store_idx);
1014 while (stores != 0 &&
1015 storeQueue[store_idx].inst->seqNum > squashed_num) {
1016 // Instructions marked as can WB are already committed.
1017 if (storeQueue[store_idx].canWB) {
1021 DPRINTF(LSQUnit,"Store Instruction PC %s squashed, "
1022 "idx:%i [sn:%lli]\n",
1023 storeQueue[store_idx].inst->pcState(),
1024 store_idx, storeQueue[store_idx].inst->seqNum);
1026 // I don't think this can happen. It should have been cleared
1027 // by the stalling load.
1029 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
1030 panic("Is stalled should have been cleared by stalling load!\n");
1032 stallingStoreIsn = 0;
1035 // Clear the smart pointer to make sure it is decremented.
1036 storeQueue[store_idx].inst->setSquashed();
1037 storeQueue[store_idx].inst = NULL;
1038 storeQueue[store_idx].canWB = 0;
1040 // Must delete request now that it wasn't handed off to
1041 // memory. This is quite ugly. @todo: Figure out the proper
1042 // place to really handle request deletes.
1043 delete storeQueue[store_idx].req;
1044 if (TheISA::HasUnalignedMemAcc && storeQueue[store_idx].isSplit) {
1045 delete storeQueue[store_idx].sreqLow;
1046 delete storeQueue[store_idx].sreqHigh;
1048 storeQueue[store_idx].sreqLow = NULL;
1049 storeQueue[store_idx].sreqHigh = NULL;
1052 storeQueue[store_idx].req = NULL;
1056 storeTail = store_idx;
1058 decrStIdx(store_idx);
1059 ++lsqSquashedStores;
1063 template <class Impl>
1065 LSQUnit<Impl>::storePostSend(PacketPtr pkt)
1068 storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
1069 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1071 stallingStoreIsn, stallingLoadIdx);
1073 stallingStoreIsn = 0;
1074 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
1077 if (!storeQueue[storeWBIdx].inst->isStoreConditional()) {
1078 // The store is basically completed at this time. This
1079 // only works so long as the checker doesn't try to
1080 // verify the value in memory for stores.
1081 storeQueue[storeWBIdx].inst->setCompleted();
1084 cpu->checker->verify(storeQueue[storeWBIdx].inst);
1089 storeInFlight = true;
1092 incrStIdx(storeWBIdx);
1095 template <class Impl>
1097 LSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt)
1099 iewStage->wakeCPU();
1101 // Squashed instructions do not need to complete their access.
1102 if (inst->isSquashed()) {
1103 iewStage->decrWb(inst->seqNum);
1104 assert(!inst->isStore());
1105 ++lsqIgnoredResponses;
1109 if (!inst->isExecuted()) {
1110 inst->setExecuted();
1112 // Complete access to copy data to proper place.
1113 inst->completeAcc(pkt);
1116 // Need to insert instruction into queue to commit
1117 iewStage->instToCommit(inst);
1119 iewStage->activityThisCycle();
1121 // see if this load changed the PC
1122 iewStage->checkMisprediction(inst);
1125 template <class Impl>
1127 LSQUnit<Impl>::completeStore(int store_idx)
1129 assert(storeQueue[store_idx].inst);
1130 storeQueue[store_idx].completed = true;
1132 // A bit conservative because a store completion may not free up entries,
1133 // but hopefully avoids two store completions in one cycle from making
1134 // the CPU tick twice.
1136 cpu->activityThisCycle();
1138 if (store_idx == storeHead) {
1140 incrStIdx(storeHead);
1143 } while (storeQueue[storeHead].completed &&
1144 storeHead != storeTail);
1146 iewStage->updateLSQNextCycle = true;
1149 DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head "
1151 storeQueue[store_idx].inst->seqNum, store_idx, storeHead);
1154 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
1155 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1157 stallingStoreIsn, stallingLoadIdx);
1159 stallingStoreIsn = 0;
1160 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
1163 storeQueue[store_idx].inst->setCompleted();
1166 storeInFlight = false;
1169 // Tell the checker we've completed this instruction. Some stores
1170 // may get reported twice to the checker, but the checker can
1171 // handle that case.
1173 cpu->checker->verify(storeQueue[store_idx].inst);
1177 template <class Impl>
1179 LSQUnit<Impl>::sendStore(PacketPtr data_pkt)
1181 if (!dcachePort->sendTimingReq(data_pkt)) {
1182 // Need to handle becoming blocked on a store.
1183 isStoreBlocked = true;
1185 assert(retryPkt == NULL);
1186 retryPkt = data_pkt;
1187 lsq->setRetryTid(lsqID);
1193 template <class Impl>
1195 LSQUnit<Impl>::recvRetry()
1197 if (isStoreBlocked) {
1198 DPRINTF(LSQUnit, "Receiving retry: store blocked\n");
1199 assert(retryPkt != NULL);
1201 LSQSenderState *state =
1202 dynamic_cast<LSQSenderState *>(retryPkt->senderState);
1204 if (dcachePort->sendTimingReq(retryPkt)) {
1205 // Don't finish the store unless this is the last packet.
1206 if (!TheISA::HasUnalignedMemAcc || !state->pktToSend ||
1207 state->pendingPacket == retryPkt) {
1208 state->pktToSend = false;
1209 storePostSend(retryPkt);
1212 isStoreBlocked = false;
1213 lsq->setRetryTid(InvalidThreadID);
1215 // Send any outstanding packet.
1216 if (TheISA::HasUnalignedMemAcc && state->pktToSend) {
1217 assert(state->pendingPacket);
1218 if (sendStore(state->pendingPacket)) {
1219 storePostSend(state->pendingPacket);
1225 lsq->setRetryTid(lsqID);
1227 } else if (isLoadBlocked) {
1228 DPRINTF(LSQUnit, "Loads squash themselves and all younger insts, "
1229 "no need to resend packet.\n");
1231 DPRINTF(LSQUnit, "Retry received but LSQ is no longer blocked.\n");
1235 template <class Impl>
1237 LSQUnit<Impl>::incrStIdx(int &store_idx)
1239 if (++store_idx >= SQEntries)
1243 template <class Impl>
1245 LSQUnit<Impl>::decrStIdx(int &store_idx)
1247 if (--store_idx < 0)
1248 store_idx += SQEntries;
1251 template <class Impl>
1253 LSQUnit<Impl>::incrLdIdx(int &load_idx)
1255 if (++load_idx >= LQEntries)
1259 template <class Impl>
1261 LSQUnit<Impl>::decrLdIdx(int &load_idx)
1264 load_idx += LQEntries;
1267 template <class Impl>
1269 LSQUnit<Impl>::dumpInsts()
1271 cprintf("Load store queue: Dumping instructions.\n");
1272 cprintf("Load queue size: %i\n", loads);
1273 cprintf("Load queue: ");
1275 int load_idx = loadHead;
1277 while (load_idx != loadTail && loadQueue[load_idx]) {
1278 cprintf("%s ", loadQueue[load_idx]->pcState());
1280 incrLdIdx(load_idx);
1283 cprintf("Store queue size: %i\n", stores);
1284 cprintf("Store queue: ");
1286 int store_idx = storeHead;
1288 while (store_idx != storeTail && storeQueue[store_idx].inst) {
1289 cprintf("%s ", storeQueue[store_idx].inst->pcState());
1291 incrStIdx(store_idx);