2 * Copyright (c) 2010-2011 ARM Limited
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
14 * Copyright (c) 2004-2005 The Regents of The University of Michigan
15 * All rights reserved.
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions are
19 * met: redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer;
21 * redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution;
24 * neither the name of the copyright holders nor the names of its
25 * contributors may be used to endorse or promote products derived from
26 * this software without specific prior written permission.
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
44 #include "arch/generic/debugfaults.hh"
45 #include "arch/locked_mem.hh"
46 #include "base/str.hh"
47 #include "config/the_isa.hh"
48 #include "cpu/checker/cpu.hh"
49 #include "cpu/o3/lsq.hh"
50 #include "cpu/o3/lsq_unit.hh"
51 #include "debug/Activity.hh"
52 #include "debug/IEW.hh"
53 #include "debug/LSQUnit.hh"
54 #include "mem/packet.hh"
55 #include "mem/request.hh"
58 LSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt,
60 : Event(Default_Pri, AutoDelete),
61 inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
67 LSQUnit<Impl>::WritebackEvent::process()
69 if (!lsqPtr->isSwitchedOut()) {
70 lsqPtr->writeback(inst, pkt);
74 delete pkt->senderState;
82 LSQUnit<Impl>::WritebackEvent::description() const
84 return "Store writeback";
89 LSQUnit<Impl>::completeDataAccess(PacketPtr pkt)
91 LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState);
92 DynInstPtr inst = state->inst;
93 DPRINTF(IEW, "Writeback event [sn:%lli].\n", inst->seqNum);
94 DPRINTF(Activity, "Activity: Writeback event [sn:%lli].\n", inst->seqNum);
96 //iewStage->ldstQueue.removeMSHR(inst->threadNumber,inst->seqNum);
98 // If this is a split access, wait until all packets are received.
99 if (TheISA::HasUnalignedMemAcc && !state->complete()) {
105 if (isSwitchedOut() || inst->isSquashed()) {
106 iewStage->decrWb(inst->seqNum);
109 if (!TheISA::HasUnalignedMemAcc || !state->isSplit ||
111 writeback(inst, pkt);
113 writeback(inst, state->mainPkt);
117 if (inst->isStore()) {
118 completeStore(state->idx);
122 if (TheISA::HasUnalignedMemAcc && state->isSplit && state->isLoad) {
123 delete state->mainPkt->req;
124 delete state->mainPkt;
131 template <class Impl>
132 LSQUnit<Impl>::LSQUnit()
133 : loads(0), stores(0), storesToWB(0), cacheBlockMask(0), stalled(false),
134 isStoreBlocked(false), isLoadBlocked(false),
135 loadBlockedHandled(false), storeInFlight(false), hasPendingPkt(false)
141 LSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params,
142 LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries,
148 DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id);
158 // Add 1 for the sentinel entry (they are circular queues).
159 LQEntries = maxLQEntries + 1;
160 SQEntries = maxSQEntries + 1;
162 loadQueue.resize(LQEntries);
163 storeQueue.resize(SQEntries);
165 depCheckShift = params->LSQDepCheckShift;
166 checkLoads = params->LSQCheckLoads;
168 loadHead = loadTail = 0;
170 storeHead = storeWBIdx = storeTail = 0;
173 cachePorts = params->cachePorts;
176 memDepViolator = NULL;
178 blockedLoadSeqNum = 0;
179 needsTSO = params->needsTSO;
184 LSQUnit<Impl>::name() const
186 if (Impl::MaxThreads == 1) {
187 return iewStage->name() + ".lsq";
189 return iewStage->name() + ".lsq.thread" + to_string(lsqID);
195 LSQUnit<Impl>::regStats()
198 .name(name() + ".forwLoads")
199 .desc("Number of loads that had data forwarded from stores");
202 .name(name() + ".invAddrLoads")
203 .desc("Number of loads ignored due to an invalid address");
206 .name(name() + ".squashedLoads")
207 .desc("Number of loads squashed");
210 .name(name() + ".ignoredResponses")
211 .desc("Number of memory responses ignored because the instruction is squashed");
214 .name(name() + ".memOrderViolation")
215 .desc("Number of memory ordering violations");
218 .name(name() + ".squashedStores")
219 .desc("Number of stores squashed");
222 .name(name() + ".invAddrSwpfs")
223 .desc("Number of software prefetches ignored due to an invalid address");
226 .name(name() + ".blockedLoads")
227 .desc("Number of blocked loads due to partial load-store forwarding");
230 .name(name() + ".rescheduledLoads")
231 .desc("Number of loads that were rescheduled");
234 .name(name() + ".cacheBlocked")
235 .desc("Number of times an access to memory failed due to the cache being blocked");
240 LSQUnit<Impl>::setDcachePort(MasterPort *dcache_port)
242 dcachePort = dcache_port;
247 LSQUnit<Impl>::clearLQ()
254 LSQUnit<Impl>::clearSQ()
261 LSQUnit<Impl>::switchOut()
264 for (int i = 0; i < loadQueue.size(); ++i) {
265 assert(!loadQueue[i]);
269 assert(storesToWB == 0);
274 LSQUnit<Impl>::takeOverFrom()
277 loads = stores = storesToWB = 0;
279 loadHead = loadTail = 0;
281 storeHead = storeWBIdx = storeTail = 0;
285 memDepViolator = NULL;
287 blockedLoadSeqNum = 0;
290 isLoadBlocked = false;
291 loadBlockedHandled = false;
293 // Just incase the memory system changed out from under us
299 LSQUnit<Impl>::resizeLQ(unsigned size)
301 unsigned size_plus_sentinel = size + 1;
302 assert(size_plus_sentinel >= LQEntries);
304 if (size_plus_sentinel > LQEntries) {
305 while (size_plus_sentinel > loadQueue.size()) {
307 loadQueue.push_back(dummy);
311 LQEntries = size_plus_sentinel;
318 LSQUnit<Impl>::resizeSQ(unsigned size)
320 unsigned size_plus_sentinel = size + 1;
321 if (size_plus_sentinel > SQEntries) {
322 while (size_plus_sentinel > storeQueue.size()) {
324 storeQueue.push_back(dummy);
328 SQEntries = size_plus_sentinel;
332 template <class Impl>
334 LSQUnit<Impl>::insert(DynInstPtr &inst)
336 assert(inst->isMemRef());
338 assert(inst->isLoad() || inst->isStore());
340 if (inst->isLoad()) {
349 template <class Impl>
351 LSQUnit<Impl>::insertLoad(DynInstPtr &load_inst)
353 assert((loadTail + 1) % LQEntries != loadHead);
354 assert(loads < LQEntries);
356 DPRINTF(LSQUnit, "Inserting load PC %s, idx:%i [sn:%lli]\n",
357 load_inst->pcState(), loadTail, load_inst->seqNum);
359 load_inst->lqIdx = loadTail;
362 load_inst->sqIdx = -1;
364 load_inst->sqIdx = storeTail;
367 loadQueue[loadTail] = load_inst;
374 template <class Impl>
376 LSQUnit<Impl>::insertStore(DynInstPtr &store_inst)
378 // Make sure it is not full before inserting an instruction.
379 assert((storeTail + 1) % SQEntries != storeHead);
380 assert(stores < SQEntries);
382 DPRINTF(LSQUnit, "Inserting store PC %s, idx:%i [sn:%lli]\n",
383 store_inst->pcState(), storeTail, store_inst->seqNum);
385 store_inst->sqIdx = storeTail;
386 store_inst->lqIdx = loadTail;
388 storeQueue[storeTail] = SQEntry(store_inst);
390 incrStIdx(storeTail);
395 template <class Impl>
396 typename Impl::DynInstPtr
397 LSQUnit<Impl>::getMemDepViolator()
399 DynInstPtr temp = memDepViolator;
401 memDepViolator = NULL;
406 template <class Impl>
408 LSQUnit<Impl>::numFreeEntries()
410 unsigned free_lq_entries = LQEntries - loads;
411 unsigned free_sq_entries = SQEntries - stores;
413 // Both the LQ and SQ entries have an extra dummy entry to differentiate
414 // empty/full conditions. Subtract 1 from the free entries.
415 if (free_lq_entries < free_sq_entries) {
416 return free_lq_entries - 1;
418 return free_sq_entries - 1;
422 template <class Impl>
424 LSQUnit<Impl>::checkSnoop(PacketPtr pkt)
426 int load_idx = loadHead;
428 if (!cacheBlockMask) {
430 Addr bs = dcachePort->peerBlockSize();
432 // Make sure we actually got a size
435 cacheBlockMask = ~(bs - 1);
438 // If this is the only load in the LSQ we don't care
439 if (load_idx == loadTail)
443 DPRINTF(LSQUnit, "Got snoop for address %#x\n", pkt->getAddr());
444 Addr invalidate_addr = pkt->getAddr() & cacheBlockMask;
445 while (load_idx != loadTail) {
446 DynInstPtr ld_inst = loadQueue[load_idx];
448 if (!ld_inst->effAddrValid() || ld_inst->uncacheable()) {
453 Addr load_addr = ld_inst->physEffAddr & cacheBlockMask;
454 DPRINTF(LSQUnit, "-- inst [sn:%lli] load_addr: %#x to pktAddr:%#x\n",
455 ld_inst->seqNum, load_addr, invalidate_addr);
457 if (load_addr == invalidate_addr) {
458 if (ld_inst->possibleLoadViolation()) {
459 DPRINTF(LSQUnit, "Conflicting load at addr %#x [sn:%lli]\n",
460 ld_inst->physEffAddr, pkt->getAddr(), ld_inst->seqNum);
462 // Mark the load for re-execution
463 ld_inst->fault = new ReExec;
465 // If a older load checks this and it's true
466 // then we might have missed the snoop
467 // in which case we need to invalidate to be sure
468 ld_inst->hitExternalSnoop(true);
476 template <class Impl>
478 LSQUnit<Impl>::checkViolations(int load_idx, DynInstPtr &inst)
480 Addr inst_eff_addr1 = inst->effAddr >> depCheckShift;
481 Addr inst_eff_addr2 = (inst->effAddr + inst->effSize - 1) >> depCheckShift;
483 /** @todo in theory you only need to check an instruction that has executed
484 * however, there isn't a good way in the pipeline at the moment to check
485 * all instructions that will execute before the store writes back. Thus,
486 * like the implementation that came before it, we're overly conservative.
488 while (load_idx != loadTail) {
489 DynInstPtr ld_inst = loadQueue[load_idx];
490 if (!ld_inst->effAddrValid() || ld_inst->uncacheable()) {
495 Addr ld_eff_addr1 = ld_inst->effAddr >> depCheckShift;
497 (ld_inst->effAddr + ld_inst->effSize - 1) >> depCheckShift;
499 if (inst_eff_addr2 >= ld_eff_addr1 && inst_eff_addr1 <= ld_eff_addr2) {
500 if (inst->isLoad()) {
501 // If this load is to the same block as an external snoop
502 // invalidate that we've observed then the load needs to be
503 // squashed as it could have newer data
504 if (ld_inst->hitExternalSnoop()) {
505 if (!memDepViolator ||
506 ld_inst->seqNum < memDepViolator->seqNum) {
507 DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] "
508 "and [sn:%lli] at address %#x\n",
509 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
510 memDepViolator = ld_inst;
512 ++lsqMemOrderViolation;
514 return new GenericISA::M5PanicFault(
515 "Detected fault with inst [sn:%lli] and "
516 "[sn:%lli] at address %#x\n",
517 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
521 // Otherwise, mark the load has a possible load violation
522 // and if we see a snoop before it's commited, we need to squash
523 ld_inst->possibleLoadViolation(true);
524 DPRINTF(LSQUnit, "Found possible load violaiton at addr: %#x"
525 " between instructions [sn:%lli] and [sn:%lli]\n",
526 inst_eff_addr1, inst->seqNum, ld_inst->seqNum);
528 // A load/store incorrectly passed this store.
529 // Check if we already have a violator, or if it's newer
530 // squash and refetch.
531 if (memDepViolator && ld_inst->seqNum > memDepViolator->seqNum)
534 DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] and "
535 "[sn:%lli] at address %#x\n",
536 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
537 memDepViolator = ld_inst;
539 ++lsqMemOrderViolation;
541 return new GenericISA::M5PanicFault("Detected fault with "
542 "inst [sn:%lli] and [sn:%lli] at address %#x\n",
543 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
555 template <class Impl>
557 LSQUnit<Impl>::executeLoad(DynInstPtr &inst)
559 using namespace TheISA;
560 // Execute a specific load.
561 Fault load_fault = NoFault;
563 DPRINTF(LSQUnit, "Executing load PC %s, [sn:%lli]\n",
564 inst->pcState(), inst->seqNum);
566 assert(!inst->isSquashed());
568 load_fault = inst->initiateAcc();
570 if (inst->isTranslationDelayed() &&
571 load_fault == NoFault)
574 // If the instruction faulted or predicated false, then we need to send it
575 // along to commit without the instruction completing.
576 if (load_fault != NoFault || inst->readPredicate() == false) {
577 // Send this instruction to commit, also make sure iew stage
578 // realizes there is activity.
579 // Mark it as executed unless it is an uncached load that
580 // needs to hit the head of commit.
581 if (inst->readPredicate() == false)
582 inst->forwardOldRegs();
583 DPRINTF(LSQUnit, "Load [sn:%lli] not executed from %s\n",
585 (load_fault != NoFault ? "fault" : "predication"));
586 if (!(inst->hasRequest() && inst->uncacheable()) ||
587 inst->isAtCommit()) {
590 iewStage->instToCommit(inst);
591 iewStage->activityThisCycle();
592 } else if (!loadBlocked()) {
593 assert(inst->effAddrValid());
594 int load_idx = inst->lqIdx;
598 return checkViolations(load_idx, inst);
604 template <class Impl>
606 LSQUnit<Impl>::executeStore(DynInstPtr &store_inst)
608 using namespace TheISA;
609 // Make sure that a store exists.
612 int store_idx = store_inst->sqIdx;
614 DPRINTF(LSQUnit, "Executing store PC %s [sn:%lli]\n",
615 store_inst->pcState(), store_inst->seqNum);
617 assert(!store_inst->isSquashed());
619 // Check the recently completed loads to see if any match this store's
620 // address. If so, then we have a memory ordering violation.
621 int load_idx = store_inst->lqIdx;
623 Fault store_fault = store_inst->initiateAcc();
625 if (store_inst->isTranslationDelayed() &&
626 store_fault == NoFault)
629 if (store_inst->readPredicate() == false)
630 store_inst->forwardOldRegs();
632 if (storeQueue[store_idx].size == 0) {
633 DPRINTF(LSQUnit,"Fault on Store PC %s, [sn:%lli], Size = 0\n",
634 store_inst->pcState(), store_inst->seqNum);
637 } else if (store_inst->readPredicate() == false) {
638 DPRINTF(LSQUnit, "Store [sn:%lli] not executed from predication\n",
643 assert(store_fault == NoFault);
645 if (store_inst->isStoreConditional()) {
646 // Store conditionals need to set themselves as able to
647 // writeback if we haven't had a fault by here.
648 storeQueue[store_idx].canWB = true;
653 return checkViolations(load_idx, store_inst);
657 template <class Impl>
659 LSQUnit<Impl>::commitLoad()
661 assert(loadQueue[loadHead]);
663 DPRINTF(LSQUnit, "Committing head load instruction, PC %s\n",
664 loadQueue[loadHead]->pcState());
666 loadQueue[loadHead] = NULL;
673 template <class Impl>
675 LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst)
677 assert(loads == 0 || loadQueue[loadHead]);
679 while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
684 template <class Impl>
686 LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst)
688 assert(stores == 0 || storeQueue[storeHead].inst);
690 int store_idx = storeHead;
692 while (store_idx != storeTail) {
693 assert(storeQueue[store_idx].inst);
694 // Mark any stores that are now committed and have not yet
695 // been marked as able to write back.
696 if (!storeQueue[store_idx].canWB) {
697 if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
700 DPRINTF(LSQUnit, "Marking store as able to write back, PC "
702 storeQueue[store_idx].inst->pcState(),
703 storeQueue[store_idx].inst->seqNum);
705 storeQueue[store_idx].canWB = true;
710 incrStIdx(store_idx);
714 template <class Impl>
716 LSQUnit<Impl>::writebackPendingStore()
719 assert(pendingPkt != NULL);
721 // If the cache is blocked, this will store the packet for retry.
722 if (sendStore(pendingPkt)) {
723 storePostSend(pendingPkt);
726 hasPendingPkt = false;
730 template <class Impl>
732 LSQUnit<Impl>::writebackStores()
734 // First writeback the second packet from any split store that didn't
735 // complete last cycle because there weren't enough cache ports available.
736 if (TheISA::HasUnalignedMemAcc) {
737 writebackPendingStore();
740 while (storesToWB > 0 &&
741 storeWBIdx != storeTail &&
742 storeQueue[storeWBIdx].inst &&
743 storeQueue[storeWBIdx].canWB &&
744 ((!needsTSO) || (!storeInFlight)) &&
745 usedPorts < cachePorts) {
747 if (isStoreBlocked || lsq->cacheBlocked()) {
748 DPRINTF(LSQUnit, "Unable to write back any more stores, cache"
753 // Store didn't write any data so no need to write it back to
755 if (storeQueue[storeWBIdx].size == 0) {
756 completeStore(storeWBIdx);
758 incrStIdx(storeWBIdx);
765 if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
766 incrStIdx(storeWBIdx);
771 assert(storeQueue[storeWBIdx].req);
772 assert(!storeQueue[storeWBIdx].committed);
774 if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) {
775 assert(storeQueue[storeWBIdx].sreqLow);
776 assert(storeQueue[storeWBIdx].sreqHigh);
779 DynInstPtr inst = storeQueue[storeWBIdx].inst;
781 Request *req = storeQueue[storeWBIdx].req;
782 RequestPtr sreqLow = storeQueue[storeWBIdx].sreqLow;
783 RequestPtr sreqHigh = storeQueue[storeWBIdx].sreqHigh;
785 storeQueue[storeWBIdx].committed = true;
787 assert(!inst->memData);
788 inst->memData = new uint8_t[64];
790 memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize());
793 req->isSwap() ? MemCmd::SwapReq :
794 (req->isLLSC() ? MemCmd::StoreCondReq : MemCmd::WriteReq);
796 PacketPtr snd_data_pkt = NULL;
798 LSQSenderState *state = new LSQSenderState;
799 state->isLoad = false;
800 state->idx = storeWBIdx;
803 if (!TheISA::HasUnalignedMemAcc || !storeQueue[storeWBIdx].isSplit) {
805 // Build a single data packet if the store isn't split.
806 data_pkt = new Packet(req, command);
807 data_pkt->dataStatic(inst->memData);
808 data_pkt->senderState = state;
810 // Create two packets if the store is split in two.
811 data_pkt = new Packet(sreqLow, command);
812 snd_data_pkt = new Packet(sreqHigh, command);
814 data_pkt->dataStatic(inst->memData);
815 snd_data_pkt->dataStatic(inst->memData + sreqLow->getSize());
817 data_pkt->senderState = state;
818 snd_data_pkt->senderState = state;
820 state->isSplit = true;
821 state->outstanding = 2;
823 // Can delete the main request now.
828 DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%s "
829 "to Addr:%#x, data:%#x [sn:%lli]\n",
830 storeWBIdx, inst->pcState(),
831 req->getPaddr(), (int)*(inst->memData),
834 // @todo: Remove this SC hack once the memory system handles it.
835 if (inst->isStoreConditional()) {
836 assert(!storeQueue[storeWBIdx].isSplit);
837 // Disable recording the result temporarily. Writing to
838 // misc regs normally updates the result, but this is not
839 // the desired behavior when handling store conditionals.
840 inst->recordResult(false);
841 bool success = TheISA::handleLockedWrite(inst.get(), req);
842 inst->recordResult(true);
845 // Instantly complete this store.
846 DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed. "
847 "Instantly completing it.\n",
849 WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
850 cpu->schedule(wb, curTick() + 1);
852 // Make sure to set the LLSC data for verification
853 // if checker is loaded
854 inst->reqToVerify->setExtraData(0);
855 inst->completeAcc(data_pkt);
857 completeStore(storeWBIdx);
858 incrStIdx(storeWBIdx);
862 // Non-store conditionals do not need a writeback.
867 TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit;
869 ThreadContext *thread = cpu->tcBase(lsqID);
871 if (req->isMmappedIpr()) {
872 assert(!inst->isStoreConditional());
873 TheISA::handleIprWrite(thread, data_pkt);
876 assert(snd_data_pkt->req->isMmappedIpr());
877 TheISA::handleIprWrite(thread, snd_data_pkt);
884 completeStore(storeWBIdx);
885 incrStIdx(storeWBIdx);
886 } else if (!sendStore(data_pkt)) {
887 DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will"
891 // Need to store the second packet, if split.
893 state->pktToSend = true;
894 state->pendingPacket = snd_data_pkt;
898 // If split, try to send the second packet too
900 assert(snd_data_pkt);
902 // Ensure there are enough ports to use.
903 if (usedPorts < cachePorts) {
905 if (sendStore(snd_data_pkt)) {
906 storePostSend(snd_data_pkt);
908 DPRINTF(IEW, "D-Cache became blocked when writing"
909 " [sn:%lli] second packet, will retry later\n",
914 // Store the packet for when there's free ports.
915 assert(pendingPkt == NULL);
916 pendingPkt = snd_data_pkt;
917 hasPendingPkt = true;
921 // Not a split store.
922 storePostSend(data_pkt);
927 // Not sure this should set it to 0.
930 assert(stores >= 0 && storesToWB >= 0);
933 /*template <class Impl>
935 LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum)
937 list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
941 if (mshr_it != mshrSeqNums.end()) {
942 mshrSeqNums.erase(mshr_it);
943 DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size());
947 template <class Impl>
949 LSQUnit<Impl>::squash(const InstSeqNum &squashed_num)
951 DPRINTF(LSQUnit, "Squashing until [sn:%lli]!"
952 "(Loads:%i Stores:%i)\n", squashed_num, loads, stores);
954 int load_idx = loadTail;
957 while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
958 DPRINTF(LSQUnit,"Load Instruction PC %s squashed, "
960 loadQueue[load_idx]->pcState(),
961 loadQueue[load_idx]->seqNum);
963 if (isStalled() && load_idx == stallingLoadIdx) {
965 stallingStoreIsn = 0;
969 // Clear the smart pointer to make sure it is decremented.
970 loadQueue[load_idx]->setSquashed();
971 loadQueue[load_idx] = NULL;
982 if (squashed_num < blockedLoadSeqNum) {
983 isLoadBlocked = false;
984 loadBlockedHandled = false;
985 blockedLoadSeqNum = 0;
989 if (memDepViolator && squashed_num < memDepViolator->seqNum) {
990 memDepViolator = NULL;
993 int store_idx = storeTail;
994 decrStIdx(store_idx);
996 while (stores != 0 &&
997 storeQueue[store_idx].inst->seqNum > squashed_num) {
998 // Instructions marked as can WB are already committed.
999 if (storeQueue[store_idx].canWB) {
1003 DPRINTF(LSQUnit,"Store Instruction PC %s squashed, "
1004 "idx:%i [sn:%lli]\n",
1005 storeQueue[store_idx].inst->pcState(),
1006 store_idx, storeQueue[store_idx].inst->seqNum);
1008 // I don't think this can happen. It should have been cleared
1009 // by the stalling load.
1011 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
1012 panic("Is stalled should have been cleared by stalling load!\n");
1014 stallingStoreIsn = 0;
1017 // Clear the smart pointer to make sure it is decremented.
1018 storeQueue[store_idx].inst->setSquashed();
1019 storeQueue[store_idx].inst = NULL;
1020 storeQueue[store_idx].canWB = 0;
1022 // Must delete request now that it wasn't handed off to
1023 // memory. This is quite ugly. @todo: Figure out the proper
1024 // place to really handle request deletes.
1025 delete storeQueue[store_idx].req;
1026 if (TheISA::HasUnalignedMemAcc && storeQueue[store_idx].isSplit) {
1027 delete storeQueue[store_idx].sreqLow;
1028 delete storeQueue[store_idx].sreqHigh;
1030 storeQueue[store_idx].sreqLow = NULL;
1031 storeQueue[store_idx].sreqHigh = NULL;
1034 storeQueue[store_idx].req = NULL;
1038 storeTail = store_idx;
1040 decrStIdx(store_idx);
1041 ++lsqSquashedStores;
1045 template <class Impl>
1047 LSQUnit<Impl>::storePostSend(PacketPtr pkt)
1050 storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
1051 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1053 stallingStoreIsn, stallingLoadIdx);
1055 stallingStoreIsn = 0;
1056 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
1059 if (!storeQueue[storeWBIdx].inst->isStoreConditional()) {
1060 // The store is basically completed at this time. This
1061 // only works so long as the checker doesn't try to
1062 // verify the value in memory for stores.
1063 storeQueue[storeWBIdx].inst->setCompleted();
1066 cpu->checker->verify(storeQueue[storeWBIdx].inst);
1071 storeInFlight = true;
1074 incrStIdx(storeWBIdx);
1077 template <class Impl>
1079 LSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt)
1081 iewStage->wakeCPU();
1083 // Squashed instructions do not need to complete their access.
1084 if (inst->isSquashed()) {
1085 iewStage->decrWb(inst->seqNum);
1086 assert(!inst->isStore());
1087 ++lsqIgnoredResponses;
1091 if (!inst->isExecuted()) {
1092 inst->setExecuted();
1094 // Complete access to copy data to proper place.
1095 inst->completeAcc(pkt);
1098 // Need to insert instruction into queue to commit
1099 iewStage->instToCommit(inst);
1101 iewStage->activityThisCycle();
1103 // see if this load changed the PC
1104 iewStage->checkMisprediction(inst);
1107 template <class Impl>
1109 LSQUnit<Impl>::completeStore(int store_idx)
1111 assert(storeQueue[store_idx].inst);
1112 storeQueue[store_idx].completed = true;
1114 // A bit conservative because a store completion may not free up entries,
1115 // but hopefully avoids two store completions in one cycle from making
1116 // the CPU tick twice.
1118 cpu->activityThisCycle();
1120 if (store_idx == storeHead) {
1122 incrStIdx(storeHead);
1125 } while (storeQueue[storeHead].completed &&
1126 storeHead != storeTail);
1128 iewStage->updateLSQNextCycle = true;
1131 DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head "
1133 storeQueue[store_idx].inst->seqNum, store_idx, storeHead);
1136 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
1137 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1139 stallingStoreIsn, stallingLoadIdx);
1141 stallingStoreIsn = 0;
1142 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
1145 storeQueue[store_idx].inst->setCompleted();
1148 storeInFlight = false;
1151 // Tell the checker we've completed this instruction. Some stores
1152 // may get reported twice to the checker, but the checker can
1153 // handle that case.
1155 cpu->checker->verify(storeQueue[store_idx].inst);
1159 template <class Impl>
1161 LSQUnit<Impl>::sendStore(PacketPtr data_pkt)
1163 if (!dcachePort->sendTimingReq(data_pkt)) {
1164 // Need to handle becoming blocked on a store.
1165 isStoreBlocked = true;
1167 assert(retryPkt == NULL);
1168 retryPkt = data_pkt;
1169 lsq->setRetryTid(lsqID);
1175 template <class Impl>
1177 LSQUnit<Impl>::recvRetry()
1179 if (isStoreBlocked) {
1180 DPRINTF(LSQUnit, "Receiving retry: store blocked\n");
1181 assert(retryPkt != NULL);
1183 LSQSenderState *state =
1184 dynamic_cast<LSQSenderState *>(retryPkt->senderState);
1186 if (dcachePort->sendTimingReq(retryPkt)) {
1187 // Don't finish the store unless this is the last packet.
1188 if (!TheISA::HasUnalignedMemAcc || !state->pktToSend ||
1189 state->pendingPacket == retryPkt) {
1190 state->pktToSend = false;
1191 storePostSend(retryPkt);
1194 isStoreBlocked = false;
1195 lsq->setRetryTid(InvalidThreadID);
1197 // Send any outstanding packet.
1198 if (TheISA::HasUnalignedMemAcc && state->pktToSend) {
1199 assert(state->pendingPacket);
1200 if (sendStore(state->pendingPacket)) {
1201 storePostSend(state->pendingPacket);
1207 lsq->setRetryTid(lsqID);
1209 } else if (isLoadBlocked) {
1210 DPRINTF(LSQUnit, "Loads squash themselves and all younger insts, "
1211 "no need to resend packet.\n");
1213 DPRINTF(LSQUnit, "Retry received but LSQ is no longer blocked.\n");
1217 template <class Impl>
1219 LSQUnit<Impl>::incrStIdx(int &store_idx)
1221 if (++store_idx >= SQEntries)
1225 template <class Impl>
1227 LSQUnit<Impl>::decrStIdx(int &store_idx)
1229 if (--store_idx < 0)
1230 store_idx += SQEntries;
1233 template <class Impl>
1235 LSQUnit<Impl>::incrLdIdx(int &load_idx)
1237 if (++load_idx >= LQEntries)
1241 template <class Impl>
1243 LSQUnit<Impl>::decrLdIdx(int &load_idx)
1246 load_idx += LQEntries;
1249 template <class Impl>
1251 LSQUnit<Impl>::dumpInsts()
1253 cprintf("Load store queue: Dumping instructions.\n");
1254 cprintf("Load queue size: %i\n", loads);
1255 cprintf("Load queue: ");
1257 int load_idx = loadHead;
1259 while (load_idx != loadTail && loadQueue[load_idx]) {
1260 cprintf("%s ", loadQueue[load_idx]->pcState());
1262 incrLdIdx(load_idx);
1265 cprintf("Store queue size: %i\n", stores);
1266 cprintf("Store queue: ");
1268 int store_idx = storeHead;
1270 while (store_idx != storeTail && storeQueue[store_idx].inst) {
1271 cprintf("%s ", storeQueue[store_idx].inst->pcState());
1273 incrStIdx(store_idx);