2 * Copyright (c) 2010 ARM Limited
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
14 * Copyright (c) 2004-2005 The Regents of The University of Michigan
15 * All rights reserved.
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions are
19 * met: redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer;
21 * redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution;
24 * neither the name of the copyright holders nor the names of its
25 * contributors may be used to endorse or promote products derived from
26 * this software without specific prior written permission.
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
44 #include "arch/locked_mem.hh"
45 #include "base/str.hh"
46 #include "config/the_isa.hh"
47 #include "config/use_checker.hh"
48 #include "cpu/o3/lsq.hh"
49 #include "cpu/o3/lsq_unit.hh"
50 #include "mem/packet.hh"
51 #include "mem/request.hh"
54 #include "cpu/checker/cpu.hh"
58 LSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt,
60 : inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
62 this->setFlags(Event::AutoDelete);
67 LSQUnit<Impl>::WritebackEvent::process()
69 if (!lsqPtr->isSwitchedOut()) {
70 lsqPtr->writeback(inst, pkt);
74 delete pkt->senderState;
82 LSQUnit<Impl>::WritebackEvent::description() const
84 return "Store writeback";
89 LSQUnit<Impl>::completeDataAccess(PacketPtr pkt)
91 LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState);
92 DynInstPtr inst = state->inst;
93 DPRINTF(IEW, "Writeback event [sn:%lli].\n", inst->seqNum);
94 DPRINTF(Activity, "Activity: Writeback event [sn:%lli].\n", inst->seqNum);
96 //iewStage->ldstQueue.removeMSHR(inst->threadNumber,inst->seqNum);
98 assert(!pkt->wasNacked());
100 // If this is a split access, wait until all packets are received.
101 if (TheISA::HasUnalignedMemAcc && !state->complete()) {
107 if (isSwitchedOut() || inst->isSquashed()) {
108 iewStage->decrWb(inst->seqNum);
111 if (!TheISA::HasUnalignedMemAcc || !state->isSplit ||
113 writeback(inst, pkt);
115 writeback(inst, state->mainPkt);
119 if (inst->isStore()) {
120 completeStore(state->idx);
124 if (TheISA::HasUnalignedMemAcc && state->isSplit && state->isLoad) {
125 delete state->mainPkt->req;
126 delete state->mainPkt;
133 template <class Impl>
134 LSQUnit<Impl>::LSQUnit()
135 : loads(0), stores(0), storesToWB(0), stalled(false),
136 isStoreBlocked(false), isLoadBlocked(false),
137 loadBlockedHandled(false), hasPendingPkt(false)
143 LSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params,
144 LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries,
150 DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id);
158 // Add 1 for the sentinel entry (they are circular queues).
159 LQEntries = maxLQEntries + 1;
160 SQEntries = maxSQEntries + 1;
162 loadQueue.resize(LQEntries);
163 storeQueue.resize(SQEntries);
165 depCheckShift = params->LSQDepCheckShift;
166 checkLoads = params->LSQCheckLoads;
168 loadHead = loadTail = 0;
170 storeHead = storeWBIdx = storeTail = 0;
173 cachePorts = params->cachePorts;
176 memDepViolator = NULL;
178 blockedLoadSeqNum = 0;
183 LSQUnit<Impl>::name() const
185 if (Impl::MaxThreads == 1) {
186 return iewStage->name() + ".lsq";
188 return iewStage->name() + ".lsq.thread." + to_string(lsqID);
194 LSQUnit<Impl>::regStats()
197 .name(name() + ".forwLoads")
198 .desc("Number of loads that had data forwarded from stores");
201 .name(name() + ".invAddrLoads")
202 .desc("Number of loads ignored due to an invalid address");
205 .name(name() + ".squashedLoads")
206 .desc("Number of loads squashed");
209 .name(name() + ".ignoredResponses")
210 .desc("Number of memory responses ignored because the instruction is squashed");
213 .name(name() + ".memOrderViolation")
214 .desc("Number of memory ordering violations");
217 .name(name() + ".squashedStores")
218 .desc("Number of stores squashed");
221 .name(name() + ".invAddrSwpfs")
222 .desc("Number of software prefetches ignored due to an invalid address");
225 .name(name() + ".blockedLoads")
226 .desc("Number of blocked loads due to partial load-store forwarding");
229 .name(name() + ".rescheduledLoads")
230 .desc("Number of loads that were rescheduled");
233 .name(name() + ".cacheBlocked")
234 .desc("Number of times an access to memory failed due to the cache being blocked");
239 LSQUnit<Impl>::setDcachePort(Port *dcache_port)
241 dcachePort = dcache_port;
245 cpu->checker->setDcachePort(dcachePort);
252 LSQUnit<Impl>::clearLQ()
259 LSQUnit<Impl>::clearSQ()
266 LSQUnit<Impl>::switchOut()
269 for (int i = 0; i < loadQueue.size(); ++i) {
270 assert(!loadQueue[i]);
274 assert(storesToWB == 0);
279 LSQUnit<Impl>::takeOverFrom()
282 loads = stores = storesToWB = 0;
284 loadHead = loadTail = 0;
286 storeHead = storeWBIdx = storeTail = 0;
290 memDepViolator = NULL;
292 blockedLoadSeqNum = 0;
295 isLoadBlocked = false;
296 loadBlockedHandled = false;
301 LSQUnit<Impl>::resizeLQ(unsigned size)
303 unsigned size_plus_sentinel = size + 1;
304 assert(size_plus_sentinel >= LQEntries);
306 if (size_plus_sentinel > LQEntries) {
307 while (size_plus_sentinel > loadQueue.size()) {
309 loadQueue.push_back(dummy);
313 LQEntries = size_plus_sentinel;
320 LSQUnit<Impl>::resizeSQ(unsigned size)
322 unsigned size_plus_sentinel = size + 1;
323 if (size_plus_sentinel > SQEntries) {
324 while (size_plus_sentinel > storeQueue.size()) {
326 storeQueue.push_back(dummy);
330 SQEntries = size_plus_sentinel;
334 template <class Impl>
336 LSQUnit<Impl>::insert(DynInstPtr &inst)
338 assert(inst->isMemRef());
340 assert(inst->isLoad() || inst->isStore());
342 if (inst->isLoad()) {
351 template <class Impl>
353 LSQUnit<Impl>::insertLoad(DynInstPtr &load_inst)
355 assert((loadTail + 1) % LQEntries != loadHead);
356 assert(loads < LQEntries);
358 DPRINTF(LSQUnit, "Inserting load PC %s, idx:%i [sn:%lli]\n",
359 load_inst->pcState(), loadTail, load_inst->seqNum);
361 load_inst->lqIdx = loadTail;
364 load_inst->sqIdx = -1;
366 load_inst->sqIdx = storeTail;
369 loadQueue[loadTail] = load_inst;
376 template <class Impl>
378 LSQUnit<Impl>::insertStore(DynInstPtr &store_inst)
380 // Make sure it is not full before inserting an instruction.
381 assert((storeTail + 1) % SQEntries != storeHead);
382 assert(stores < SQEntries);
384 DPRINTF(LSQUnit, "Inserting store PC %s, idx:%i [sn:%lli]\n",
385 store_inst->pcState(), storeTail, store_inst->seqNum);
387 store_inst->sqIdx = storeTail;
388 store_inst->lqIdx = loadTail;
390 storeQueue[storeTail] = SQEntry(store_inst);
392 incrStIdx(storeTail);
397 template <class Impl>
398 typename Impl::DynInstPtr
399 LSQUnit<Impl>::getMemDepViolator()
401 DynInstPtr temp = memDepViolator;
403 memDepViolator = NULL;
408 template <class Impl>
410 LSQUnit<Impl>::numFreeEntries()
412 unsigned free_lq_entries = LQEntries - loads;
413 unsigned free_sq_entries = SQEntries - stores;
415 // Both the LQ and SQ entries have an extra dummy entry to differentiate
416 // empty/full conditions. Subtract 1 from the free entries.
417 if (free_lq_entries < free_sq_entries) {
418 return free_lq_entries - 1;
420 return free_sq_entries - 1;
424 template <class Impl>
426 LSQUnit<Impl>::numLoadsReady()
428 int load_idx = loadHead;
431 while (load_idx != loadTail) {
432 assert(loadQueue[load_idx]);
434 if (loadQueue[load_idx]->readyToIssue()) {
442 template <class Impl>
444 LSQUnit<Impl>::checkViolations(int load_idx, DynInstPtr &inst)
446 Addr inst_eff_addr1 = inst->effAddr >> depCheckShift;
447 Addr inst_eff_addr2 = (inst->effAddr + inst->effSize - 1) >> depCheckShift;
449 /** @todo in theory you only need to check an instruction that has executed
450 * however, there isn't a good way in the pipeline at the moment to check
451 * all instructions that will execute before the store writes back. Thus,
452 * like the implementation that came before it, we're overly conservative.
454 while (load_idx != loadTail) {
455 DynInstPtr ld_inst = loadQueue[load_idx];
456 if (!ld_inst->effAddrValid || ld_inst->uncacheable()) {
461 Addr ld_eff_addr1 = ld_inst->effAddr >> depCheckShift;
463 (ld_inst->effAddr + ld_inst->effSize - 1) >> depCheckShift;
465 if ((inst_eff_addr2 > ld_eff_addr1 && inst_eff_addr1 < ld_eff_addr2) ||
466 inst_eff_addr1 == ld_eff_addr1) {
467 // A load/store incorrectly passed this load/store.
468 // Check if we already have a violator, or if it's newer
469 // squash and refetch.
470 if (memDepViolator && ld_inst->seqNum > memDepViolator->seqNum)
473 DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] and [sn:%lli]"
474 " at address %#x\n", inst->seqNum, ld_inst->seqNum,
476 memDepViolator = ld_inst;
478 ++lsqMemOrderViolation;
480 return TheISA::genMachineCheckFault();
491 template <class Impl>
493 LSQUnit<Impl>::executeLoad(DynInstPtr &inst)
495 using namespace TheISA;
496 // Execute a specific load.
497 Fault load_fault = NoFault;
499 DPRINTF(LSQUnit, "Executing load PC %s, [sn:%lli]\n",
500 inst->pcState(), inst->seqNum);
502 assert(!inst->isSquashed());
504 load_fault = inst->initiateAcc();
506 if (inst->isTranslationDelayed() &&
507 load_fault == NoFault)
510 // If the instruction faulted or predicated false, then we need to send it
511 // along to commit without the instruction completing.
512 if (load_fault != NoFault || inst->readPredicate() == false) {
513 // Send this instruction to commit, also make sure iew stage
514 // realizes there is activity.
515 // Mark it as executed unless it is an uncached load that
516 // needs to hit the head of commit.
517 if (inst->readPredicate() == false)
518 inst->forwardOldRegs();
519 DPRINTF(LSQUnit, "Load [sn:%lli] not executed from %s\n",
521 (load_fault != NoFault ? "fault" : "predication"));
522 if (!(inst->hasRequest() && inst->uncacheable()) ||
523 inst->isAtCommit()) {
526 iewStage->instToCommit(inst);
527 iewStage->activityThisCycle();
528 } else if (!loadBlocked()) {
529 assert(inst->effAddrValid);
530 int load_idx = inst->lqIdx;
534 return checkViolations(load_idx, inst);
540 template <class Impl>
542 LSQUnit<Impl>::executeStore(DynInstPtr &store_inst)
544 using namespace TheISA;
545 // Make sure that a store exists.
548 int store_idx = store_inst->sqIdx;
550 DPRINTF(LSQUnit, "Executing store PC %s [sn:%lli]\n",
551 store_inst->pcState(), store_inst->seqNum);
553 assert(!store_inst->isSquashed());
555 // Check the recently completed loads to see if any match this store's
556 // address. If so, then we have a memory ordering violation.
557 int load_idx = store_inst->lqIdx;
559 Fault store_fault = store_inst->initiateAcc();
561 if (store_inst->isTranslationDelayed() &&
562 store_fault == NoFault)
565 if (store_inst->readPredicate() == false)
566 store_inst->forwardOldRegs();
568 if (storeQueue[store_idx].size == 0) {
569 DPRINTF(LSQUnit,"Fault on Store PC %s, [sn:%lli], Size = 0\n",
570 store_inst->pcState(), store_inst->seqNum);
573 } else if (store_inst->readPredicate() == false) {
574 DPRINTF(LSQUnit, "Store [sn:%lli] not executed from predication\n",
579 assert(store_fault == NoFault);
581 if (store_inst->isStoreConditional()) {
582 // Store conditionals need to set themselves as able to
583 // writeback if we haven't had a fault by here.
584 storeQueue[store_idx].canWB = true;
589 return checkViolations(load_idx, store_inst);
593 template <class Impl>
595 LSQUnit<Impl>::commitLoad()
597 assert(loadQueue[loadHead]);
599 DPRINTF(LSQUnit, "Committing head load instruction, PC %s\n",
600 loadQueue[loadHead]->pcState());
602 loadQueue[loadHead] = NULL;
609 template <class Impl>
611 LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst)
613 assert(loads == 0 || loadQueue[loadHead]);
615 while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
620 template <class Impl>
622 LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst)
624 assert(stores == 0 || storeQueue[storeHead].inst);
626 int store_idx = storeHead;
628 while (store_idx != storeTail) {
629 assert(storeQueue[store_idx].inst);
630 // Mark any stores that are now committed and have not yet
631 // been marked as able to write back.
632 if (!storeQueue[store_idx].canWB) {
633 if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
636 DPRINTF(LSQUnit, "Marking store as able to write back, PC "
638 storeQueue[store_idx].inst->pcState(),
639 storeQueue[store_idx].inst->seqNum);
641 storeQueue[store_idx].canWB = true;
646 incrStIdx(store_idx);
650 template <class Impl>
652 LSQUnit<Impl>::writebackPendingStore()
655 assert(pendingPkt != NULL);
657 // If the cache is blocked, this will store the packet for retry.
658 if (sendStore(pendingPkt)) {
659 storePostSend(pendingPkt);
662 hasPendingPkt = false;
666 template <class Impl>
668 LSQUnit<Impl>::writebackStores()
670 // First writeback the second packet from any split store that didn't
671 // complete last cycle because there weren't enough cache ports available.
672 if (TheISA::HasUnalignedMemAcc) {
673 writebackPendingStore();
676 while (storesToWB > 0 &&
677 storeWBIdx != storeTail &&
678 storeQueue[storeWBIdx].inst &&
679 storeQueue[storeWBIdx].canWB &&
680 usedPorts < cachePorts) {
682 if (isStoreBlocked || lsq->cacheBlocked()) {
683 DPRINTF(LSQUnit, "Unable to write back any more stores, cache"
688 // Store didn't write any data so no need to write it back to
690 if (storeQueue[storeWBIdx].size == 0) {
691 completeStore(storeWBIdx);
693 incrStIdx(storeWBIdx);
700 if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
701 incrStIdx(storeWBIdx);
706 assert(storeQueue[storeWBIdx].req);
707 assert(!storeQueue[storeWBIdx].committed);
709 if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) {
710 assert(storeQueue[storeWBIdx].sreqLow);
711 assert(storeQueue[storeWBIdx].sreqHigh);
714 DynInstPtr inst = storeQueue[storeWBIdx].inst;
716 Request *req = storeQueue[storeWBIdx].req;
717 storeQueue[storeWBIdx].committed = true;
719 assert(!inst->memData);
720 inst->memData = new uint8_t[64];
722 memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize());
725 req->isSwap() ? MemCmd::SwapReq :
726 (req->isLLSC() ? MemCmd::StoreCondReq : MemCmd::WriteReq);
728 PacketPtr snd_data_pkt = NULL;
730 LSQSenderState *state = new LSQSenderState;
731 state->isLoad = false;
732 state->idx = storeWBIdx;
735 if (!TheISA::HasUnalignedMemAcc || !storeQueue[storeWBIdx].isSplit) {
737 // Build a single data packet if the store isn't split.
738 data_pkt = new Packet(req, command, Packet::Broadcast);
739 data_pkt->dataStatic(inst->memData);
740 data_pkt->senderState = state;
742 RequestPtr sreqLow = storeQueue[storeWBIdx].sreqLow;
743 RequestPtr sreqHigh = storeQueue[storeWBIdx].sreqHigh;
745 // Create two packets if the store is split in two.
746 data_pkt = new Packet(sreqLow, command, Packet::Broadcast);
747 snd_data_pkt = new Packet(sreqHigh, command, Packet::Broadcast);
749 data_pkt->dataStatic(inst->memData);
750 snd_data_pkt->dataStatic(inst->memData + sreqLow->getSize());
752 data_pkt->senderState = state;
753 snd_data_pkt->senderState = state;
755 state->isSplit = true;
756 state->outstanding = 2;
758 // Can delete the main request now.
763 DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%s "
764 "to Addr:%#x, data:%#x [sn:%lli]\n",
765 storeWBIdx, inst->pcState(),
766 req->getPaddr(), (int)*(inst->memData),
769 // @todo: Remove this SC hack once the memory system handles it.
770 if (inst->isStoreConditional()) {
771 assert(!storeQueue[storeWBIdx].isSplit);
772 // Disable recording the result temporarily. Writing to
773 // misc regs normally updates the result, but this is not
774 // the desired behavior when handling store conditionals.
775 inst->recordResult = false;
776 bool success = TheISA::handleLockedWrite(inst.get(), req);
777 inst->recordResult = true;
780 // Instantly complete this store.
781 DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed. "
782 "Instantly completing it.\n",
784 WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
785 cpu->schedule(wb, curTick() + 1);
786 completeStore(storeWBIdx);
787 incrStIdx(storeWBIdx);
791 // Non-store conditionals do not need a writeback.
795 if (!sendStore(data_pkt)) {
796 DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will"
800 // Need to store the second packet, if split.
801 if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) {
802 state->pktToSend = true;
803 state->pendingPacket = snd_data_pkt;
807 // If split, try to send the second packet too
808 if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) {
809 assert(snd_data_pkt);
811 // Ensure there are enough ports to use.
812 if (usedPorts < cachePorts) {
814 if (sendStore(snd_data_pkt)) {
815 storePostSend(snd_data_pkt);
817 DPRINTF(IEW, "D-Cache became blocked when writing"
818 " [sn:%lli] second packet, will retry later\n",
823 // Store the packet for when there's free ports.
824 assert(pendingPkt == NULL);
825 pendingPkt = snd_data_pkt;
826 hasPendingPkt = true;
830 // Not a split store.
831 storePostSend(data_pkt);
836 // Not sure this should set it to 0.
839 assert(stores >= 0 && storesToWB >= 0);
842 /*template <class Impl>
844 LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum)
846 list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
850 if (mshr_it != mshrSeqNums.end()) {
851 mshrSeqNums.erase(mshr_it);
852 DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size());
856 template <class Impl>
858 LSQUnit<Impl>::squash(const InstSeqNum &squashed_num)
860 DPRINTF(LSQUnit, "Squashing until [sn:%lli]!"
861 "(Loads:%i Stores:%i)\n", squashed_num, loads, stores);
863 int load_idx = loadTail;
866 while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
867 DPRINTF(LSQUnit,"Load Instruction PC %s squashed, "
869 loadQueue[load_idx]->pcState(),
870 loadQueue[load_idx]->seqNum);
872 if (isStalled() && load_idx == stallingLoadIdx) {
874 stallingStoreIsn = 0;
878 // Clear the smart pointer to make sure it is decremented.
879 loadQueue[load_idx]->setSquashed();
880 loadQueue[load_idx] = NULL;
891 if (squashed_num < blockedLoadSeqNum) {
892 isLoadBlocked = false;
893 loadBlockedHandled = false;
894 blockedLoadSeqNum = 0;
898 if (memDepViolator && squashed_num < memDepViolator->seqNum) {
899 memDepViolator = NULL;
902 int store_idx = storeTail;
903 decrStIdx(store_idx);
905 while (stores != 0 &&
906 storeQueue[store_idx].inst->seqNum > squashed_num) {
907 // Instructions marked as can WB are already committed.
908 if (storeQueue[store_idx].canWB) {
912 DPRINTF(LSQUnit,"Store Instruction PC %s squashed, "
913 "idx:%i [sn:%lli]\n",
914 storeQueue[store_idx].inst->pcState(),
915 store_idx, storeQueue[store_idx].inst->seqNum);
917 // I don't think this can happen. It should have been cleared
918 // by the stalling load.
920 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
921 panic("Is stalled should have been cleared by stalling load!\n");
923 stallingStoreIsn = 0;
926 // Clear the smart pointer to make sure it is decremented.
927 storeQueue[store_idx].inst->setSquashed();
928 storeQueue[store_idx].inst = NULL;
929 storeQueue[store_idx].canWB = 0;
931 // Must delete request now that it wasn't handed off to
932 // memory. This is quite ugly. @todo: Figure out the proper
933 // place to really handle request deletes.
934 delete storeQueue[store_idx].req;
935 if (TheISA::HasUnalignedMemAcc && storeQueue[store_idx].isSplit) {
936 delete storeQueue[store_idx].sreqLow;
937 delete storeQueue[store_idx].sreqHigh;
939 storeQueue[store_idx].sreqLow = NULL;
940 storeQueue[store_idx].sreqHigh = NULL;
943 storeQueue[store_idx].req = NULL;
947 storeTail = store_idx;
949 decrStIdx(store_idx);
954 template <class Impl>
956 LSQUnit<Impl>::storePostSend(PacketPtr pkt)
959 storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
960 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
962 stallingStoreIsn, stallingLoadIdx);
964 stallingStoreIsn = 0;
965 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
968 if (!storeQueue[storeWBIdx].inst->isStoreConditional()) {
969 // The store is basically completed at this time. This
970 // only works so long as the checker doesn't try to
971 // verify the value in memory for stores.
972 storeQueue[storeWBIdx].inst->setCompleted();
975 cpu->checker->verify(storeQueue[storeWBIdx].inst);
980 incrStIdx(storeWBIdx);
983 template <class Impl>
985 LSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt)
989 // Squashed instructions do not need to complete their access.
990 if (inst->isSquashed()) {
991 iewStage->decrWb(inst->seqNum);
992 assert(!inst->isStore());
993 ++lsqIgnoredResponses;
997 if (!inst->isExecuted()) {
1000 // Complete access to copy data to proper place.
1001 inst->completeAcc(pkt);
1004 // Need to insert instruction into queue to commit
1005 iewStage->instToCommit(inst);
1007 iewStage->activityThisCycle();
1009 // see if this load changed the PC
1010 iewStage->checkMisprediction(inst);
1013 template <class Impl>
1015 LSQUnit<Impl>::completeStore(int store_idx)
1017 assert(storeQueue[store_idx].inst);
1018 storeQueue[store_idx].completed = true;
1020 // A bit conservative because a store completion may not free up entries,
1021 // but hopefully avoids two store completions in one cycle from making
1022 // the CPU tick twice.
1024 cpu->activityThisCycle();
1026 if (store_idx == storeHead) {
1028 incrStIdx(storeHead);
1031 } while (storeQueue[storeHead].completed &&
1032 storeHead != storeTail);
1034 iewStage->updateLSQNextCycle = true;
1037 DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head "
1039 storeQueue[store_idx].inst->seqNum, store_idx, storeHead);
1042 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
1043 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1045 stallingStoreIsn, stallingLoadIdx);
1047 stallingStoreIsn = 0;
1048 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
1051 storeQueue[store_idx].inst->setCompleted();
1053 // Tell the checker we've completed this instruction. Some stores
1054 // may get reported twice to the checker, but the checker can
1055 // handle that case.
1058 cpu->checker->verify(storeQueue[store_idx].inst);
1063 template <class Impl>
1065 LSQUnit<Impl>::sendStore(PacketPtr data_pkt)
1067 if (!dcachePort->sendTiming(data_pkt)) {
1068 // Need to handle becoming blocked on a store.
1069 isStoreBlocked = true;
1071 assert(retryPkt == NULL);
1072 retryPkt = data_pkt;
1073 lsq->setRetryTid(lsqID);
1079 template <class Impl>
1081 LSQUnit<Impl>::recvRetry()
1083 if (isStoreBlocked) {
1084 DPRINTF(LSQUnit, "Receiving retry: store blocked\n");
1085 assert(retryPkt != NULL);
1087 if (dcachePort->sendTiming(retryPkt)) {
1088 LSQSenderState *state =
1089 dynamic_cast<LSQSenderState *>(retryPkt->senderState);
1091 // Don't finish the store unless this is the last packet.
1092 if (!TheISA::HasUnalignedMemAcc || !state->pktToSend ||
1093 state->pendingPacket == retryPkt) {
1094 state->pktToSend = false;
1095 storePostSend(retryPkt);
1098 isStoreBlocked = false;
1099 lsq->setRetryTid(InvalidThreadID);
1101 // Send any outstanding packet.
1102 if (TheISA::HasUnalignedMemAcc && state->pktToSend) {
1103 assert(state->pendingPacket);
1104 if (sendStore(state->pendingPacket)) {
1105 storePostSend(state->pendingPacket);
1111 lsq->setRetryTid(lsqID);
1113 } else if (isLoadBlocked) {
1114 DPRINTF(LSQUnit, "Loads squash themselves and all younger insts, "
1115 "no need to resend packet.\n");
1117 DPRINTF(LSQUnit, "Retry received but LSQ is no longer blocked.\n");
1121 template <class Impl>
1123 LSQUnit<Impl>::incrStIdx(int &store_idx)
1125 if (++store_idx >= SQEntries)
1129 template <class Impl>
1131 LSQUnit<Impl>::decrStIdx(int &store_idx)
1133 if (--store_idx < 0)
1134 store_idx += SQEntries;
1137 template <class Impl>
1139 LSQUnit<Impl>::incrLdIdx(int &load_idx)
1141 if (++load_idx >= LQEntries)
1145 template <class Impl>
1147 LSQUnit<Impl>::decrLdIdx(int &load_idx)
1150 load_idx += LQEntries;
1153 template <class Impl>
1155 LSQUnit<Impl>::dumpInsts()
1157 cprintf("Load store queue: Dumping instructions.\n");
1158 cprintf("Load queue size: %i\n", loads);
1159 cprintf("Load queue: ");
1161 int load_idx = loadHead;
1163 while (load_idx != loadTail && loadQueue[load_idx]) {
1164 cprintf("%s ", loadQueue[load_idx]->pcState());
1166 incrLdIdx(load_idx);
1169 cprintf("Store queue size: %i\n", stores);
1170 cprintf("Store queue: ");
1172 int store_idx = storeHead;
1174 while (store_idx != storeTail && storeQueue[store_idx].inst) {
1175 cprintf("%s ", storeQueue[store_idx].inst->pcState());
1177 incrStIdx(store_idx);