2 * Copyright (c) 2006 The Regents of The University of Michigan
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #include "arch/isa_traits.hh"
32 #include "base/str.hh"
33 #include "cpu/ozone/lw_lsq.hh"
34 #include "cpu/checker/cpu.hh"
37 OzoneLWLSQ<Impl>::StoreCompletionEvent::StoreCompletionEvent(DynInstPtr &_inst,
40 OzoneLWLSQ<Impl> *lsq_ptr)
41 : Event(&mainEventQueue),
48 this->setFlags(Event::AutoDelete);
53 OzoneLWLSQ<Impl>::StoreCompletionEvent::process()
55 DPRINTF(OzoneLSQ, "Cache miss complete for store [sn:%lli]\n",
58 //lsqPtr->removeMSHR(lsqPtr->storeQueue[storeIdx].inst->seqNum);
60 // lsqPtr->cpu->wakeCPU();
61 if (lsqPtr->isSwitchedOut()) {
73 lsqPtr->completeStore(inst->sqIdx);
75 be->removeDcacheMiss(inst);
80 OzoneLWLSQ<Impl>::StoreCompletionEvent::description()
82 return "LSQ store completion event";
86 OzoneLWLSQ<Impl>::OzoneLWLSQ()
87 : loads(0), stores(0), storesToWB(0), stalled(false), isLoadBlocked(false),
88 loadBlockedHandled(false)
94 OzoneLWLSQ<Impl>::init(Params *params, unsigned maxLQEntries,
95 unsigned maxSQEntries, unsigned id)
97 DPRINTF(OzoneLSQ, "Creating OzoneLWLSQ%i object.\n",id);
101 LQEntries = maxLQEntries;
102 SQEntries = maxSQEntries;
104 for (int i = 0; i < LQEntries * 2; i++) {
110 cachePorts = params->cachePorts;
112 dcacheInterface = params->dcacheInterface;
114 loadFaultInst = storeFaultInst = memDepViolator = NULL;
116 blockedLoadSeqNum = 0;
121 OzoneLWLSQ<Impl>::name() const
128 OzoneLWLSQ<Impl>::clearLQ()
135 OzoneLWLSQ<Impl>::clearSQ()
142 OzoneLWLSQ<Impl>::setPageTable(PageTable *pt_ptr)
144 DPRINTF(OzoneLSQ, "Setting the page table pointer.\n");
150 OzoneLWLSQ<Impl>::resizeLQ(unsigned size)
152 assert( size >= LQEntries);
154 if (size > LQEntries) {
155 while (size > loadQueue.size()) {
157 loadQueue.push_back(dummy);
168 OzoneLWLSQ<Impl>::resizeSQ(unsigned size)
170 if (size > SQEntries) {
171 while (size > storeQueue.size()) {
173 storeQueue.push_back(dummy);
181 template <class Impl>
183 OzoneLWLSQ<Impl>::insert(DynInstPtr &inst)
185 // Make sure we really have a memory reference.
186 assert(inst->isMemRef());
188 // Make sure it's one of the two classes of memory references.
189 assert(inst->isLoad() || inst->isStore());
191 if (inst->isLoad()) {
198 template <class Impl>
200 OzoneLWLSQ<Impl>::insertLoad(DynInstPtr &load_inst)
202 assert(loads < LQEntries * 2);
203 assert(!LQIndices.empty());
204 int load_index = LQIndices.front();
207 DPRINTF(OzoneLSQ, "Inserting load PC %#x, idx:%i [sn:%lli]\n",
208 load_inst->readPC(), load_index, load_inst->seqNum);
210 load_inst->lqIdx = load_index;
212 loadQueue.push_front(load_inst);
213 LQItHash[load_index] = loadQueue.begin();
218 template <class Impl>
220 OzoneLWLSQ<Impl>::insertStore(DynInstPtr &store_inst)
222 // Make sure it is not full before inserting an instruction.
223 assert(stores - storesToWB < SQEntries);
225 assert(!SQIndices.empty());
226 int store_index = SQIndices.front();
229 DPRINTF(OzoneLSQ, "Inserting store PC %#x, idx:%i [sn:%lli]\n",
230 store_inst->readPC(), store_index, store_inst->seqNum);
232 store_inst->sqIdx = store_index;
233 SQEntry entry(store_inst);
234 if (loadQueue.empty()) {
235 entry.lqIt = loadQueue.end();
237 entry.lqIt = loadQueue.begin();
239 storeQueue.push_front(entry);
241 SQItHash[store_index] = storeQueue.begin();
246 template <class Impl>
247 typename Impl::DynInstPtr
248 OzoneLWLSQ<Impl>::getMemDepViolator()
250 DynInstPtr temp = memDepViolator;
252 memDepViolator = NULL;
257 template <class Impl>
259 OzoneLWLSQ<Impl>::numFreeEntries()
261 unsigned free_lq_entries = LQEntries - loads;
262 unsigned free_sq_entries = SQEntries - stores;
264 // Both the LQ and SQ entries have an extra dummy entry to differentiate
265 // empty/full conditions. Subtract 1 from the free entries.
266 if (free_lq_entries < free_sq_entries) {
267 return free_lq_entries - 1;
269 return free_sq_entries - 1;
273 template <class Impl>
275 OzoneLWLSQ<Impl>::numLoadsReady()
278 LQIt lq_it = loadQueue.begin();
279 LQIt end_it = loadQueue.end();
281 while (lq_it != end_it) {
282 if ((*lq_it)->readyToIssue()) {
290 template <class Impl>
292 OzoneLWLSQ<Impl>::executeLoad(DynInstPtr &inst)
294 // Execute a specific load.
295 Fault load_fault = NoFault;
297 DPRINTF(OzoneLSQ, "Executing load PC %#x, [sn:%lli]\n",
298 inst->readPC(),inst->seqNum);
300 // Make sure it's really in the list.
301 // Normally it should always be in the list. However,
302 /* due to a syscall it may not be the list.
306 if (i == loadTail && !find(inst)) {
307 assert(0 && "Load not in the queue!");
308 } else if (loadQueue[i] == inst) {
313 if (i >= LQEntries) {
319 load_fault = inst->initiateAcc();
321 // Might want to make sure that I'm not overwriting a previously faulting
322 // instruction that hasn't been checked yet.
323 // Actually probably want the oldest faulting load
324 if (load_fault != NoFault) {
325 DPRINTF(OzoneLSQ, "Load [sn:%lli] has a fault\n", inst->seqNum);
326 // Maybe just set it as can commit here, although that might cause
327 // some other problems with sending traps to the ROB too quickly.
328 be->instToCommit(inst);
329 // iewStage->activityThisCycle();
335 template <class Impl>
337 OzoneLWLSQ<Impl>::executeStore(DynInstPtr &store_inst)
339 // Make sure that a store exists.
342 int store_idx = store_inst->sqIdx;
343 SQHashIt sq_hash_it = SQItHash.find(store_idx);
344 assert(sq_hash_it != SQItHash.end());
345 DPRINTF(OzoneLSQ, "Executing store PC %#x [sn:%lli]\n",
346 store_inst->readPC(), store_inst->seqNum);
348 SQIt sq_it = (*sq_hash_it).second;
350 Fault store_fault = store_inst->initiateAcc();
352 // Store size should now be available. Use it to get proper offset for
354 int size = (*sq_it).size;
357 DPRINTF(OzoneLSQ,"Fault on Store PC %#x, [sn:%lli],Size = 0\n",
358 store_inst->readPC(),store_inst->seqNum);
363 assert(store_fault == NoFault);
365 if (!storeFaultInst) {
366 if (store_fault != NoFault) {
367 panic("Fault in a store instruction!");
368 storeFaultInst = store_inst;
369 } else if (store_inst->isStoreConditional()) {
370 // Store conditionals need to set themselves as able to
371 // writeback if we haven't had a fault by here.
372 (*sq_it).canWB = true;
375 DPRINTF(OzoneLSQ, "Nonspeculative store! storesToWB:%i\n",
380 LQIt lq_it = --(loadQueue.end());
382 if (!memDepViolator) {
383 while (lq_it != loadQueue.end()) {
384 if ((*lq_it)->seqNum < store_inst->seqNum) {
388 // Actually should only check loads that have actually executed
389 // Might be safe because effAddr is set to InvalAddr when the
390 // dyn inst is created.
392 // Must actually check all addrs in the proper size range
393 // Which is more correct than needs to be. What if for now we just
394 // assume all loads are quad-word loads, and do the addr based
396 // @todo: Fix this, magic number being used here
397 if (((*lq_it)->effAddr >> 8) ==
398 (store_inst->effAddr >> 8)) {
399 // A load incorrectly passed this store. Squash and refetch.
400 // For now return a fault to show that it was unsuccessful.
401 memDepViolator = (*lq_it);
403 return TheISA::genMachineCheckFault();
409 // If we've reached this point, there was no violation.
410 memDepViolator = NULL;
416 template <class Impl>
418 OzoneLWLSQ<Impl>::commitLoad()
420 assert(!loadQueue.empty());
422 DPRINTF(OzoneLSQ, "[sn:%lli] Committing head load instruction, PC %#x\n",
423 loadQueue.back()->seqNum, loadQueue.back()->readPC());
425 LQIndices.push(loadQueue.back()->lqIdx);
426 LQItHash.erase(loadQueue.back()->lqIdx);
428 loadQueue.pop_back();
433 template <class Impl>
435 OzoneLWLSQ<Impl>::commitLoads(InstSeqNum &youngest_inst)
437 assert(loads == 0 || !loadQueue.empty());
440 loadQueue.back()->seqNum <= youngest_inst) {
445 template <class Impl>
447 OzoneLWLSQ<Impl>::commitStores(InstSeqNum &youngest_inst)
449 assert(stores == 0 || !storeQueue.empty());
451 SQIt sq_it = --(storeQueue.end());
452 while (!storeQueue.empty() && sq_it != storeQueue.end()) {
453 assert((*sq_it).inst);
454 if (!(*sq_it).canWB) {
455 if ((*sq_it).inst->seqNum > youngest_inst) {
460 DPRINTF(OzoneLSQ, "Marking store as able to write back, PC "
461 "%#x [sn:%lli], storesToWB:%i\n",
462 (*sq_it).inst->readPC(),
463 (*sq_it).inst->seqNum,
466 (*sq_it).canWB = true;
473 template <class Impl>
475 OzoneLWLSQ<Impl>::writebackStores()
477 SQIt sq_it = --(storeQueue.end());
478 while (storesToWB > 0 &&
479 sq_it != storeQueue.end() &&
482 usedPorts < cachePorts) {
484 DynInstPtr inst = (*sq_it).inst;
486 if ((*sq_it).size == 0 && !(*sq_it).completed) {
488 completeStore(inst->sqIdx);
493 if (inst->isDataPrefetch() || (*sq_it).committed) {
498 if (dcacheInterface && dcacheInterface->isBlocked()) {
499 DPRINTF(OzoneLSQ, "Unable to write back any more stores, cache"
506 assert((*sq_it).req);
507 assert(!(*sq_it).committed);
509 (*sq_it).committed = true;
511 MemReqPtr req = (*sq_it).req;
514 req->completionEvent = NULL;
517 switch((*sq_it).size) {
519 cpu->write(req, (uint8_t &)(*sq_it).data);
522 cpu->write(req, (uint16_t &)(*sq_it).data);
525 cpu->write(req, (uint32_t &)(*sq_it).data);
528 cpu->write(req, (uint64_t &)(*sq_it).data);
531 panic("Unexpected store size!\n");
533 if (!(req->flags & LOCKED)) {
534 (*sq_it).inst->setCompleted();
536 cpu->checker->tick((*sq_it).inst);
540 DPRINTF(OzoneLSQ, "D-Cache: Writing back store idx:%i PC:%#x "
541 "to Addr:%#x, data:%#x [sn:%lli]\n",
542 inst->sqIdx,inst->readPC(),
543 req->paddr, *(req->data),
546 if (dcacheInterface) {
547 assert(!req->completionEvent);
548 StoreCompletionEvent *store_event = new
549 StoreCompletionEvent(inst, be, NULL, this);
550 req->completionEvent = store_event;
552 MemAccessResult result = dcacheInterface->access(req);
555 inst->seqNum == stallingStoreIsn) {
556 DPRINTF(OzoneLSQ, "Unstalling, stalling store [sn:%lli] "
558 stallingStoreIsn, (*stallingLoad)->seqNum);
560 stallingStoreIsn = 0;
561 be->replayMemInst((*stallingLoad));
564 if (result != MA_HIT && dcacheInterface->doEvents()) {
565 store_event->miss = true;
566 typename BackEnd::LdWritebackEvent *wb = NULL;
567 if (req->flags & LOCKED) {
568 wb = new typename BackEnd::LdWritebackEvent(inst,
570 store_event->wbEvent = wb;
573 DPRINTF(OzoneLSQ,"D-Cache Write Miss!\n");
575 // DPRINTF(Activity, "Active st accessing mem miss [sn:%lli]\n",
578 be->addDcacheMiss(inst);
580 lastDcacheStall = curTick;
582 _status = DcacheMissStall;
584 // Increment stat here or something
588 DPRINTF(OzoneLSQ,"D-Cache: Write Hit on idx:%i !\n",
591 // DPRINTF(Activity, "Active st accessing mem hit [sn:%lli]\n",
594 if (req->flags & LOCKED) {
595 // Stx_C does not generate a system port
596 // transaction in the 21264, but that might be
597 // hard to accomplish in this model.
599 typename BackEnd::LdWritebackEvent *wb =
600 new typename BackEnd::LdWritebackEvent(inst,
602 store_event->wbEvent = wb;
607 panic("Must HAVE DCACHE!!!!!\n");
611 // Not sure this should set it to 0.
614 assert(stores >= 0 && storesToWB >= 0);
617 template <class Impl>
619 OzoneLWLSQ<Impl>::squash(const InstSeqNum &squashed_num)
621 DPRINTF(OzoneLSQ, "Squashing until [sn:%lli]!"
622 "(Loads:%i Stores:%i)\n",squashed_num,loads,stores);
625 LQIt lq_it = loadQueue.begin();
627 while (loads != 0 && (*lq_it)->seqNum > squashed_num) {
628 assert(!loadQueue.empty());
629 // Clear the smart pointer to make sure it is decremented.
630 DPRINTF(OzoneLSQ,"Load Instruction PC %#x squashed, "
635 if (isStalled() && lq_it == stallingLoad) {
637 stallingStoreIsn = 0;
644 LQHashIt lq_hash_it = LQItHash.find((*lq_it)->lqIdx);
645 assert(lq_hash_it != LQItHash.end());
646 LQItHash.erase(lq_hash_it);
647 LQIndices.push((*lq_it)->lqIdx);
648 loadQueue.erase(lq_it++);
652 if (squashed_num < blockedLoadSeqNum) {
653 isLoadBlocked = false;
654 loadBlockedHandled = false;
655 blockedLoadSeqNum = 0;
659 SQIt sq_it = storeQueue.begin();
661 while (stores != 0 && (*sq_it).inst->seqNum > squashed_num) {
662 assert(!storeQueue.empty());
664 if ((*sq_it).canWB) {
668 // Clear the smart pointer to make sure it is decremented.
669 DPRINTF(OzoneLSQ,"Store Instruction PC %#x idx:%i squashed [sn:%lli]\n",
670 (*sq_it).inst->readPC(), (*sq_it).inst->sqIdx,
671 (*sq_it).inst->seqNum);
673 // I don't think this can happen. It should have been cleared by the
676 (*sq_it).inst->seqNum == stallingStoreIsn) {
677 panic("Is stalled should have been cleared by stalling load!\n");
679 stallingStoreIsn = 0;
682 SQHashIt sq_hash_it = SQItHash.find((*sq_it).inst->sqIdx);
683 assert(sq_hash_it != SQItHash.end());
684 SQItHash.erase(sq_hash_it);
685 SQIndices.push((*sq_it).inst->sqIdx);
686 (*sq_it).inst = NULL;
690 assert(!(*sq_it).req->completionEvent);
694 storeQueue.erase(sq_it++);
698 template <class Impl>
700 OzoneLWLSQ<Impl>::dumpInsts()
702 cprintf("Load store queue: Dumping instructions.\n");
703 cprintf("Load queue size: %i\n", loads);
704 cprintf("Load queue: ");
706 LQIt lq_it = --(loadQueue.end());
708 while (lq_it != loadQueue.end() && (*lq_it)) {
709 cprintf("[sn:%lli] %#x ", (*lq_it)->seqNum,
715 cprintf("\nStore queue size: %i\n", stores);
716 cprintf("Store queue: ");
718 SQIt sq_it = --(storeQueue.end());
720 while (sq_it != storeQueue.end() && (*sq_it).inst) {
721 cprintf("[sn:%lli]\nPC:%#x\nSize:%i\nCommitted:%i\nCompleted:%i\ncanWB:%i\n",
722 (*sq_it).inst->seqNum,
723 (*sq_it).inst->readPC(),
735 template <class Impl>
737 OzoneLWLSQ<Impl>::completeStore(int store_idx)
739 SQHashIt sq_hash_it = SQItHash.find(store_idx);
740 assert(sq_hash_it != SQItHash.end());
741 SQIt sq_it = (*sq_hash_it).second;
743 assert((*sq_it).inst);
744 (*sq_it).completed = true;
745 DynInstPtr inst = (*sq_it).inst;
750 inst->seqNum == stallingStoreIsn) {
751 DPRINTF(OzoneLSQ, "Unstalling, stalling store [sn:%lli] "
753 stallingStoreIsn, (*stallingLoad)->seqNum);
755 stallingStoreIsn = 0;
756 be->replayMemInst((*stallingLoad));
759 DPRINTF(OzoneLSQ, "Completing store idx:%i [sn:%lli], storesToWB:%i\n",
760 inst->sqIdx, inst->seqNum, storesToWB);
762 assert(!storeQueue.empty());
763 SQItHash.erase(sq_hash_it);
764 SQIndices.push(inst->sqIdx);
765 storeQueue.erase(sq_it);
768 inst->setCompleted();
770 cpu->checker->tick(inst);
774 template <class Impl>
776 OzoneLWLSQ<Impl>::switchOut()
778 assert(storesToWB == 0);
780 SQIt sq_it = --(storeQueue.end());
781 while (storesToWB > 0 &&
782 sq_it != storeQueue.end() &&
786 DynInstPtr inst = (*sq_it).inst;
788 if ((*sq_it).size == 0 && !(*sq_it).completed) {
793 // Store conditionals don't complete until *after* they have written
794 // back. If it's here and not yet sent to memory, then don't bother
795 // as it's not part of committed state.
796 if (inst->isDataPrefetch() || (*sq_it).committed) {
799 } else if ((*sq_it).req->flags & LOCKED) {
801 assert(!(*sq_it).canWB ||
802 ((*sq_it).canWB && (*sq_it).req->flags & LOCKED));
806 assert((*sq_it).req);
807 assert(!(*sq_it).committed);
809 MemReqPtr req = (*sq_it).req;
810 (*sq_it).committed = true;
813 req->completionEvent = NULL;
816 req->data = new uint8_t[64];
817 memcpy(req->data, (uint8_t *)&(*sq_it).data, req->size);
819 DPRINTF(OzoneLSQ, "Switching out : Writing back store idx:%i PC:%#x "
820 "to Addr:%#x, data:%#x directly to memory [sn:%lli]\n",
821 inst->sqIdx,inst->readPC(),
822 req->paddr, *(req->data),
825 switch((*sq_it).size) {
827 cpu->write(req, (uint8_t &)(*sq_it).data);
830 cpu->write(req, (uint16_t &)(*sq_it).data);
833 cpu->write(req, (uint32_t &)(*sq_it).data);
836 cpu->write(req, (uint64_t &)(*sq_it).data);
839 panic("Unexpected store size!\n");
843 // Clear the queue to free up resources
846 loads = stores = storesToWB = 0;
849 template <class Impl>
851 OzoneLWLSQ<Impl>::takeOverFrom(ThreadContext *old_tc)
853 // Clear out any old state. May be redundant if this is the first time
854 // the CPU is being used.
856 isLoadBlocked = false;
857 loadBlockedHandled = false;
860 // Could do simple checks here to see if indices are on twice
861 while (!LQIndices.empty())
863 while (!SQIndices.empty())
866 for (int i = 0; i < LQEntries * 2; i++) {
873 loadFaultInst = storeFaultInst = memDepViolator = NULL;
875 blockedLoadSeqNum = 0;