2 * Copyright (c) 2006 The Regents of The University of Michigan
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #include "base/str.hh"
32 #include "config/the_isa.hh"
33 #include "cpu/checker/cpu.hh"
34 #include "cpu/ozone/lw_lsq.hh"
35 #include "sim/fault_fwd.hh"
38 OzoneLWLSQ<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt,
40 : Event(&mainEventQueue), inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
42 this->setFlags(Event::AutoDelete);
47 OzoneLWLSQ<Impl>::WritebackEvent::process()
49 if (!lsqPtr->isSwitchedOut()) {
50 lsqPtr->writeback(inst, pkt);
57 OzoneLWLSQ<Impl>::WritebackEvent::description() const
59 return "Store writeback";
64 OzoneLWLSQ<Impl>::DcachePort::recvAtomic(PacketPtr pkt)
66 panic("O3CPU model does not work with atomic mode!");
72 OzoneLWLSQ<Impl>::DcachePort::recvFunctional(PacketPtr pkt)
74 warn("O3CPU doesn't update things on a recvFunctional");
79 OzoneLWLSQ<Impl>::DcachePort::recvTiming(PacketPtr pkt)
81 lsq->completeDataAccess(pkt);
87 OzoneLWLSQ<Impl>::DcachePort::recvRetry()
94 OzoneLWLSQ<Impl>::completeDataAccess(PacketPtr pkt)
96 LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState);
97 DynInstPtr inst = state->inst;
98 DPRINTF(IEW, "Writeback event [sn:%lli]\n", inst->seqNum);
99 DPRINTF(Activity, "Activity: Writeback event [sn:%lli]\n", inst->seqNum);
101 //iewStage->ldstQueue.removeMSHR(inst->threadNumber,inst->seqNum);
103 if (isSwitchedOut() || inst->isSquashed()) {
109 writeback(inst, pkt);
112 if (inst->isStore()) {
121 template <class Impl>
122 OzoneLWLSQ<Impl>::OzoneLWLSQ()
123 : switchedOut(false), dcachePort(this), loads(0), stores(0),
124 storesToWB(0), storesInFlight(0), stalled(false), isStoreBlocked(false),
125 isLoadBlocked(false), loadBlockedHandled(false)
131 OzoneLWLSQ<Impl>::init(Params *params, unsigned maxLQEntries,
132 unsigned maxSQEntries, unsigned id)
134 DPRINTF(OzoneLSQ, "Creating OzoneLWLSQ%i object.\n",id);
138 LQEntries = maxLQEntries;
139 SQEntries = maxSQEntries;
141 for (int i = 0; i < LQEntries * 2; i++) {
147 cachePorts = params->cachePorts;
149 loadFaultInst = storeFaultInst = memDepViolator = NULL;
151 blockedLoadSeqNum = 0;
156 OzoneLWLSQ<Impl>::name() const
163 OzoneLWLSQ<Impl>::regStats()
166 .name(name() + ".memOrderViolation")
167 .desc("Number of memory ordering violations");
172 OzoneLWLSQ<Impl>::setCPU(OzoneCPU *cpu_ptr)
175 dcachePort.setName(this->name() + "-dport");
178 cpu->checker->setDcachePort(&dcachePort);
184 OzoneLWLSQ<Impl>::clearLQ()
191 OzoneLWLSQ<Impl>::clearSQ()
198 OzoneLWLSQ<Impl>::setPageTable(PageTable *pt_ptr)
200 DPRINTF(OzoneLSQ, "Setting the page table pointer.\n");
206 OzoneLWLSQ<Impl>::resizeLQ(unsigned size)
208 assert( size >= LQEntries);
210 if (size > LQEntries) {
211 while (size > loadQueue.size()) {
213 loadQueue.push_back(dummy);
224 OzoneLWLSQ<Impl>::resizeSQ(unsigned size)
226 if (size > SQEntries) {
227 while (size > storeQueue.size()) {
229 storeQueue.push_back(dummy);
237 template <class Impl>
239 OzoneLWLSQ<Impl>::insert(DynInstPtr &inst)
241 // Make sure we really have a memory reference.
242 assert(inst->isMemRef());
244 // Make sure it's one of the two classes of memory references.
245 assert(inst->isLoad() || inst->isStore());
247 if (inst->isLoad()) {
254 template <class Impl>
256 OzoneLWLSQ<Impl>::insertLoad(DynInstPtr &load_inst)
258 assert(loads < LQEntries * 2);
259 assert(!LQIndices.empty());
260 int load_index = LQIndices.front();
263 DPRINTF(OzoneLSQ, "Inserting load PC %#x, idx:%i [sn:%lli]\n",
264 load_inst->readPC(), load_index, load_inst->seqNum);
266 load_inst->lqIdx = load_index;
268 loadQueue.push_front(load_inst);
269 LQItHash[load_index] = loadQueue.begin();
274 template <class Impl>
276 OzoneLWLSQ<Impl>::insertStore(DynInstPtr &store_inst)
278 // Make sure it is not full before inserting an instruction.
279 assert(stores - storesToWB < SQEntries);
281 assert(!SQIndices.empty());
282 int store_index = SQIndices.front();
285 DPRINTF(OzoneLSQ, "Inserting store PC %#x, idx:%i [sn:%lli]\n",
286 store_inst->readPC(), store_index, store_inst->seqNum);
288 store_inst->sqIdx = store_index;
289 SQEntry entry(store_inst);
290 if (loadQueue.empty()) {
291 entry.lqIt = loadQueue.end();
293 entry.lqIt = loadQueue.begin();
295 storeQueue.push_front(entry);
297 SQItHash[store_index] = storeQueue.begin();
302 template <class Impl>
303 typename Impl::DynInstPtr
304 OzoneLWLSQ<Impl>::getMemDepViolator()
306 DynInstPtr temp = memDepViolator;
308 memDepViolator = NULL;
313 template <class Impl>
315 OzoneLWLSQ<Impl>::numFreeEntries()
317 unsigned free_lq_entries = LQEntries - loads;
318 unsigned free_sq_entries = SQEntries - (stores + storesInFlight);
320 // Both the LQ and SQ entries have an extra dummy entry to differentiate
321 // empty/full conditions. Subtract 1 from the free entries.
322 if (free_lq_entries < free_sq_entries) {
323 return free_lq_entries - 1;
325 return free_sq_entries - 1;
329 template <class Impl>
331 OzoneLWLSQ<Impl>::numLoadsReady()
334 LQIt lq_it = loadQueue.begin();
335 LQIt end_it = loadQueue.end();
337 while (lq_it != end_it) {
338 if ((*lq_it)->readyToIssue()) {
346 template <class Impl>
348 OzoneLWLSQ<Impl>::executeLoad(DynInstPtr &inst)
350 // Execute a specific load.
351 Fault load_fault = NoFault;
353 DPRINTF(OzoneLSQ, "Executing load PC %#x, [sn:%lli]\n",
354 inst->readPC(),inst->seqNum);
356 // Make sure it's really in the list.
357 // Normally it should always be in the list. However,
358 /* due to a syscall it may not be the list.
362 if (i == loadTail && !find(inst)) {
363 assert(0 && "Load not in the queue!");
364 } else if (loadQueue[i] == inst) {
369 if (i >= LQEntries) {
375 load_fault = inst->initiateAcc();
377 // Might want to make sure that I'm not overwriting a previously faulting
378 // instruction that hasn't been checked yet.
379 // Actually probably want the oldest faulting load
380 if (load_fault != NoFault) {
381 DPRINTF(OzoneLSQ, "Load [sn:%lli] has a fault\n", inst->seqNum);
382 if (!(inst->req->isUncacheable() && !inst->isAtCommit())) {
385 // Maybe just set it as can commit here, although that might cause
386 // some other problems with sending traps to the ROB too quickly.
387 be->instToCommit(inst);
388 // iewStage->activityThisCycle();
394 template <class Impl>
396 OzoneLWLSQ<Impl>::executeStore(DynInstPtr &store_inst)
398 // Make sure that a store exists.
401 int store_idx = store_inst->sqIdx;
402 SQHashIt sq_hash_it = SQItHash.find(store_idx);
403 assert(sq_hash_it != SQItHash.end());
404 DPRINTF(OzoneLSQ, "Executing store PC %#x [sn:%lli]\n",
405 store_inst->readPC(), store_inst->seqNum);
407 SQIt sq_it = (*sq_hash_it).second;
409 Fault store_fault = store_inst->initiateAcc();
411 // Store size should now be available. Use it to get proper offset for
413 int size = (*sq_it).size;
416 DPRINTF(OzoneLSQ,"Fault on Store PC %#x, [sn:%lli],Size = 0\n",
417 store_inst->readPC(),store_inst->seqNum);
422 assert(store_fault == NoFault);
424 if (!storeFaultInst) {
425 if (store_fault != NoFault) {
426 panic("Fault in a store instruction!");
427 storeFaultInst = store_inst;
428 } else if (store_inst->isStoreConditional()) {
429 // Store conditionals need to set themselves as able to
430 // writeback if we haven't had a fault by here.
431 (*sq_it).canWB = true;
434 DPRINTF(OzoneLSQ, "Nonspeculative store! storesToWB:%i\n",
439 LQIt lq_it = --(loadQueue.end());
441 if (!memDepViolator) {
442 while (lq_it != loadQueue.end()) {
443 if ((*lq_it)->seqNum < store_inst->seqNum) {
447 // Actually should only check loads that have actually executed
448 // Might be safe because effAddr is set to InvalAddr when the
449 // dyn inst is created.
451 // Must actually check all addrs in the proper size range
452 // Which is more correct than needs to be. What if for now we just
453 // assume all loads are quad-word loads, and do the addr based
455 // @todo: Fix this, magic number being used here
456 if (((*lq_it)->effAddr >> 8) ==
457 (store_inst->effAddr >> 8)) {
458 // A load incorrectly passed this store. Squash and refetch.
459 // For now return a fault to show that it was unsuccessful.
460 memDepViolator = (*lq_it);
461 ++lsqMemOrderViolation;
463 return TheISA::genMachineCheckFault();
469 // If we've reached this point, there was no violation.
470 memDepViolator = NULL;
476 template <class Impl>
478 OzoneLWLSQ<Impl>::commitLoad()
480 assert(!loadQueue.empty());
482 DPRINTF(OzoneLSQ, "[sn:%lli] Committing head load instruction, PC %#x\n",
483 loadQueue.back()->seqNum, loadQueue.back()->readPC());
485 LQIndices.push(loadQueue.back()->lqIdx);
486 LQItHash.erase(loadQueue.back()->lqIdx);
488 loadQueue.pop_back();
493 template <class Impl>
495 OzoneLWLSQ<Impl>::commitLoads(InstSeqNum &youngest_inst)
497 assert(loads == 0 || !loadQueue.empty());
500 loadQueue.back()->seqNum <= youngest_inst) {
505 template <class Impl>
507 OzoneLWLSQ<Impl>::commitStores(InstSeqNum &youngest_inst)
509 assert(stores == 0 || !storeQueue.empty());
511 SQIt sq_it = --(storeQueue.end());
512 while (!storeQueue.empty() && sq_it != storeQueue.end()) {
513 assert((*sq_it).inst);
514 if (!(*sq_it).canWB) {
515 if ((*sq_it).inst->seqNum > youngest_inst) {
520 DPRINTF(OzoneLSQ, "Marking store as able to write back, PC "
521 "%#x [sn:%lli], storesToWB:%i\n",
522 (*sq_it).inst->readPC(),
523 (*sq_it).inst->seqNum,
526 (*sq_it).canWB = true;
533 template <class Impl>
535 OzoneLWLSQ<Impl>::writebackStores()
537 SQIt sq_it = --(storeQueue.end());
538 while (storesToWB > 0 &&
539 sq_it != storeQueue.end() &&
542 usedPorts < cachePorts) {
544 if (isStoreBlocked) {
545 DPRINTF(OzoneLSQ, "Unable to write back any more stores, cache"
550 DynInstPtr inst = (*sq_it).inst;
552 if ((*sq_it).size == 0 && !(*sq_it).completed) {
554 removeStore(inst->sqIdx);
559 if (inst->isDataPrefetch() || (*sq_it).committed) {
566 assert((*sq_it).req);
567 assert(!(*sq_it).committed);
569 Request *req = (*sq_it).req;
570 (*sq_it).committed = true;
572 assert(!inst->memData);
573 inst->memData = new uint8_t[64];
574 memcpy(inst->memData, (uint8_t *)&(*sq_it).data,
578 req->isSwap() ? MemCmd::SwapReq :
579 (req->isLLSC() ? MemCmd::WriteReq : MemCmd::StoreCondReq);
580 PacketPtr data_pkt = new Packet(req, command);
581 data_pkt->dataStatic(inst->memData);
583 LSQSenderState *state = new LSQSenderState;
584 state->isLoad = false;
585 state->idx = inst->sqIdx;
587 data_pkt->senderState = state;
589 DPRINTF(OzoneLSQ, "D-Cache: Writing back store PC:%#x "
590 "to Addr:%#x, data:%#x [sn:%lli]\n",
591 (*sq_it).inst->readPC(),
592 req->getPaddr(), *(inst->memData),
595 // @todo: Remove this SC hack once the memory system handles it.
597 if (req->isUncacheable()) {
598 req->setExtraData(2);
601 req->setExtraData(1);
603 req->setExtraData(0);
604 // Hack: Instantly complete this store.
605 completeDataAccess(data_pkt);
611 // Non-store conditionals do not need a writeback.
615 if (!dcachePort.sendTiming(data_pkt)) {
616 // Need to handle becoming blocked on a store.
617 isStoreBlocked = true;
618 assert(retryPkt == NULL);
621 storePostSend(data_pkt, inst);
625 DPRINTF(OzoneLSQ, "D-Cache: Writing back store idx:%i PC:%#x "
626 "to Addr:%#x, data:%#x [sn:%lli]\n",
627 inst->sqIdx,inst->readPC(),
628 req->paddr, *(req->data),
630 DPRINTF(OzoneLSQ, "StoresInFlight: %i\n",
633 if (dcacheInterface) {
634 assert(!req->completionEvent);
635 StoreCompletionEvent *store_event = new
636 StoreCompletionEvent(inst, be, NULL, this);
637 req->completionEvent = store_event;
639 MemAccessResult result = dcacheInterface->access(req);
642 inst->seqNum == stallingStoreIsn) {
643 DPRINTF(OzoneLSQ, "Unstalling, stalling store [sn:%lli] "
645 stallingStoreIsn, (*stallingLoad)->seqNum);
647 stallingStoreIsn = 0;
648 be->replayMemInst((*stallingLoad));
651 if (result != MA_HIT && dcacheInterface->doEvents()) {
652 store_event->miss = true;
653 typename BackEnd::LdWritebackEvent *wb = NULL;
655 wb = new typename BackEnd::LdWritebackEvent(inst,
657 store_event->wbEvent = wb;
660 DPRINTF(OzoneLSQ,"D-Cache Write Miss!\n");
662 // DPRINTF(Activity, "Active st accessing mem miss [sn:%lli]\n",
665 be->addDcacheMiss(inst);
667 lastDcacheStall = curTick();
669 _status = DcacheMissStall;
671 // Increment stat here or something
675 DPRINTF(OzoneLSQ,"D-Cache: Write Hit on idx:%i !\n",
678 // DPRINTF(Activity, "Active st accessing mem hit [sn:%lli]\n",
682 // Stx_C does not generate a system port
683 // transaction in the 21264, but that might be
684 // hard to accomplish in this model.
686 typename BackEnd::LdWritebackEvent *wb =
687 new typename BackEnd::LdWritebackEvent(inst,
689 store_event->wbEvent = wb;
694 // removeStore(inst->sqIdx);
696 panic("Must HAVE DCACHE!!!!!\n");
701 // Not sure this should set it to 0.
704 assert(stores >= 0 && storesToWB >= 0);
707 template <class Impl>
709 OzoneLWLSQ<Impl>::squash(const InstSeqNum &squashed_num)
711 DPRINTF(OzoneLSQ, "Squashing until [sn:%lli]!"
712 "(Loads:%i Stores:%i)\n",squashed_num,loads,stores+storesInFlight);
715 LQIt lq_it = loadQueue.begin();
717 while (loads != 0 && (*lq_it)->seqNum > squashed_num) {
718 assert(!loadQueue.empty());
719 // Clear the smart pointer to make sure it is decremented.
720 DPRINTF(OzoneLSQ,"Load Instruction PC %#x squashed, "
725 if (isStalled() && lq_it == stallingLoad) {
727 stallingStoreIsn = 0;
734 LQHashIt lq_hash_it = LQItHash.find((*lq_it)->lqIdx);
735 assert(lq_hash_it != LQItHash.end());
736 LQItHash.erase(lq_hash_it);
737 LQIndices.push((*lq_it)->lqIdx);
738 loadQueue.erase(lq_it++);
742 if (squashed_num < blockedLoadSeqNum) {
743 isLoadBlocked = false;
744 loadBlockedHandled = false;
745 blockedLoadSeqNum = 0;
749 SQIt sq_it = storeQueue.begin();
751 while (stores != 0 && (*sq_it).inst->seqNum > squashed_num) {
752 assert(!storeQueue.empty());
754 if ((*sq_it).canWB) {
758 // Clear the smart pointer to make sure it is decremented.
759 DPRINTF(OzoneLSQ,"Store Instruction PC %#x idx:%i squashed [sn:%lli]\n",
760 (*sq_it).inst->readPC(), (*sq_it).inst->sqIdx,
761 (*sq_it).inst->seqNum);
763 // I don't think this can happen. It should have been cleared by the
766 (*sq_it).inst->seqNum == stallingStoreIsn) {
767 panic("Is stalled should have been cleared by stalling load!\n");
769 stallingStoreIsn = 0;
772 SQHashIt sq_hash_it = SQItHash.find((*sq_it).inst->sqIdx);
773 assert(sq_hash_it != SQItHash.end());
774 SQItHash.erase(sq_hash_it);
775 SQIndices.push((*sq_it).inst->sqIdx);
776 (*sq_it).inst = NULL;
780 storeQueue.erase(sq_it++);
784 template <class Impl>
786 OzoneLWLSQ<Impl>::dumpInsts()
788 cprintf("Load store queue: Dumping instructions.\n");
789 cprintf("Load queue size: %i\n", loads);
790 cprintf("Load queue: ");
792 LQIt lq_it = --(loadQueue.end());
794 while (lq_it != loadQueue.end() && (*lq_it)) {
795 cprintf("[sn:%lli] %#x ", (*lq_it)->seqNum,
801 cprintf("\nStore queue size: %i\n", stores);
802 cprintf("Store queue: ");
804 SQIt sq_it = --(storeQueue.end());
806 while (sq_it != storeQueue.end() && (*sq_it).inst) {
807 cprintf("[sn:%lli]\nPC:%#x\nSize:%i\nCommitted:%i\nCompleted:%i\ncanWB:%i\n",
808 (*sq_it).inst->seqNum,
809 (*sq_it).inst->readPC(),
821 template <class Impl>
823 OzoneLWLSQ<Impl>::storePostSend(PacketPtr pkt, DynInstPtr &inst)
826 inst->seqNum == stallingStoreIsn) {
827 DPRINTF(OzoneLSQ, "Unstalling, stalling store [sn:%lli] "
829 stallingStoreIsn, (*stallingLoad)->seqNum);
831 stallingStoreIsn = 0;
832 be->replayMemInst((*stallingLoad));
835 if (!inst->isStoreConditional()) {
836 // The store is basically completed at this time. This
837 // only works so long as the checker doesn't try to
838 // verify the value in memory for stores.
839 inst->setCompleted();
841 cpu->checker->verify(inst);
846 template <class Impl>
848 OzoneLWLSQ<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt)
850 // Squashed instructions do not need to complete their access.
851 if (inst->isSquashed()) {
852 assert(!inst->isStore());
856 if (!inst->isExecuted()) {
859 // Complete access to copy data to proper place.
860 inst->completeAcc(pkt);
863 // Need to insert instruction into queue to commit
864 be->instToCommit(inst);
867 template <class Impl>
869 OzoneLWLSQ<Impl>::removeStore(int store_idx)
871 SQHashIt sq_hash_it = SQItHash.find(store_idx);
872 assert(sq_hash_it != SQItHash.end());
873 SQIt sq_it = (*sq_hash_it).second;
875 assert((*sq_it).inst);
876 (*sq_it).completed = true;
877 DynInstPtr inst = (*sq_it).inst;
880 inst->seqNum == stallingStoreIsn) {
881 DPRINTF(OzoneLSQ, "Unstalling, stalling store [sn:%lli] "
883 stallingStoreIsn, (*stallingLoad)->seqNum);
885 stallingStoreIsn = 0;
886 be->replayMemInst((*stallingLoad));
889 DPRINTF(OzoneLSQ, "Completing store idx:%i [sn:%lli], storesToWB:%i\n",
890 inst->sqIdx, inst->seqNum, storesToWB);
892 assert(!storeQueue.empty());
893 SQItHash.erase(sq_hash_it);
894 SQIndices.push(inst->sqIdx);
895 storeQueue.erase(sq_it);
898 template <class Impl>
900 OzoneLWLSQ<Impl>::completeStore(DynInstPtr &inst)
905 inst->setCompleted();
907 cpu->checker->verify(inst);
911 template <class Impl>
913 OzoneLWLSQ<Impl>::recvRetry()
915 panic("Unimplemented!");
918 template <class Impl>
920 OzoneLWLSQ<Impl>::switchOut()
922 assert(storesToWB == 0);
925 // Clear the queue to free up resources
927 assert(storeQueue.empty());
929 assert(loadQueue.empty());
930 assert(storesInFlight == 0);
933 loads = stores = storesToWB = storesInFlight = 0;
936 template <class Impl>
938 OzoneLWLSQ<Impl>::takeOverFrom(ThreadContext *old_tc)
940 // Clear out any old state. May be redundant if this is the first time
941 // the CPU is being used.
943 isLoadBlocked = false;
944 loadBlockedHandled = false;
947 // Could do simple checks here to see if indices are on twice
948 while (!LQIndices.empty())
950 while (!SQIndices.empty())
953 for (int i = 0; i < LQEntries * 2; i++) {
960 loadFaultInst = storeFaultInst = memDepViolator = NULL;
962 blockedLoadSeqNum = 0;