2 * Copyright (c) 2004-2006 The Regents of The University of Michigan
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #include "arch/faults.hh"
32 #include "base/str.hh"
33 #include "cpu/ozone/lsq_unit.hh"
36 OzoneLSQ<Impl>::StoreCompletionEvent::StoreCompletionEvent(int store_idx,
38 OzoneLSQ<Impl> *lsq_ptr)
39 : Event(&mainEventQueue),
44 this->setFlags(Event::AutoDelete);
49 OzoneLSQ<Impl>::StoreCompletionEvent::process()
51 DPRINTF(OzoneLSQ, "Cache miss complete for store idx:%i\n", storeIdx);
53 //lsqPtr->removeMSHR(lsqPtr->storeQueue[storeIdx].inst->seqNum);
55 // lsqPtr->cpu->wakeCPU();
58 lsqPtr->completeStore(storeIdx);
63 OzoneLSQ<Impl>::StoreCompletionEvent::description()
65 return "LSQ store completion event";
69 OzoneLSQ<Impl>::OzoneLSQ()
70 : loads(0), stores(0), storesToWB(0), stalled(false), isLoadBlocked(false)
76 OzoneLSQ<Impl>::init(Params *params, unsigned maxLQEntries,
77 unsigned maxSQEntries, unsigned id)
80 DPRINTF(OzoneLSQ, "Creating OzoneLSQ%i object.\n",id);
84 LQEntries = maxLQEntries;
85 SQEntries = maxSQEntries;
87 loadQueue.resize(LQEntries);
88 storeQueue.resize(SQEntries);
91 // May want to initialize these entries to NULL
93 loadHead = loadTail = 0;
95 storeHead = storeWBIdx = storeTail = 0;
98 cachePorts = params->cachePorts;
100 dcacheInterface = params->dcacheInterface;
102 loadFaultInst = storeFaultInst = memDepViolator = NULL;
107 OzoneLSQ<Impl>::name() const
114 OzoneLSQ<Impl>::clearLQ()
121 OzoneLSQ<Impl>::clearSQ()
128 OzoneLSQ<Impl>::resizeLQ(unsigned size)
130 assert( size >= LQEntries);
132 if (size > LQEntries) {
133 while (size > loadQueue.size()) {
135 loadQueue.push_back(dummy);
146 OzoneLSQ<Impl>::resizeSQ(unsigned size)
148 if (size > SQEntries) {
149 while (size > storeQueue.size()) {
151 storeQueue.push_back(dummy);
159 template <class Impl>
161 OzoneLSQ<Impl>::insert(DynInstPtr &inst)
163 // Make sure we really have a memory reference.
164 assert(inst->isMemRef());
166 // Make sure it's one of the two classes of memory references.
167 assert(inst->isLoad() || inst->isStore());
169 if (inst->isLoad()) {
178 template <class Impl>
180 OzoneLSQ<Impl>::insertLoad(DynInstPtr &load_inst)
182 assert((loadTail + 1) % LQEntries != loadHead && loads < LQEntries);
184 DPRINTF(OzoneLSQ, "Inserting load PC %#x, idx:%i [sn:%lli]\n",
185 load_inst->readPC(), loadTail, load_inst->seqNum);
187 load_inst->lqIdx = loadTail;
190 load_inst->sqIdx = -1;
192 load_inst->sqIdx = storeTail;
195 loadQueue[loadTail] = load_inst;
202 template <class Impl>
204 OzoneLSQ<Impl>::insertStore(DynInstPtr &store_inst)
206 // Make sure it is not full before inserting an instruction.
207 assert((storeTail + 1) % SQEntries != storeHead);
208 assert(stores < SQEntries);
210 DPRINTF(OzoneLSQ, "Inserting store PC %#x, idx:%i [sn:%lli]\n",
211 store_inst->readPC(), storeTail, store_inst->seqNum);
213 store_inst->sqIdx = storeTail;
214 store_inst->lqIdx = loadTail;
216 storeQueue[storeTail] = SQEntry(store_inst);
218 incrStIdx(storeTail);
224 template <class Impl>
225 typename Impl::DynInstPtr
226 OzoneLSQ<Impl>::getMemDepViolator()
228 DynInstPtr temp = memDepViolator;
230 memDepViolator = NULL;
235 template <class Impl>
237 OzoneLSQ<Impl>::numFreeEntries()
239 unsigned free_lq_entries = LQEntries - loads;
240 unsigned free_sq_entries = SQEntries - stores;
242 // Both the LQ and SQ entries have an extra dummy entry to differentiate
243 // empty/full conditions. Subtract 1 from the free entries.
244 if (free_lq_entries < free_sq_entries) {
245 return free_lq_entries - 1;
247 return free_sq_entries - 1;
251 template <class Impl>
253 OzoneLSQ<Impl>::numLoadsReady()
255 int load_idx = loadHead;
258 while (load_idx != loadTail) {
259 assert(loadQueue[load_idx]);
261 if (loadQueue[load_idx]->readyToIssue()) {
270 template <class Impl>
272 OzoneLSQ<Impl>::executeLoad()
274 Fault load_fault = NoFault;
275 DynInstPtr load_inst;
277 assert(readyLoads.size() != 0);
279 // Execute a ready load.
280 LdMapIt ready_it = readyLoads.begin();
282 load_inst = (*ready_it).second;
284 // Execute the instruction, which is held in the data portion of the
286 load_fault = load_inst->execute();
288 // If it executed successfully, then switch it over to the executed
290 if (load_fault == NoFault) {
291 executedLoads[load_inst->seqNum] = load_inst;
293 readyLoads.erase(ready_it);
295 loadFaultInst = load_inst;
302 template <class Impl>
304 OzoneLSQ<Impl>::executeLoad(DynInstPtr &inst)
306 // Execute a specific load.
307 Fault load_fault = NoFault;
309 DPRINTF(OzoneLSQ, "Executing load PC %#x, [sn:%lli]\n",
310 inst->readPC(),inst->seqNum);
312 // Make sure it's really in the list.
313 // Normally it should always be in the list. However,
314 /* due to a syscall it may not be the list.
318 if (i == loadTail && !find(inst)) {
319 assert(0 && "Load not in the queue!");
320 } else if (loadQueue[i] == inst) {
325 if (i >= LQEntries) {
331 load_fault = inst->initiateAcc();
333 // Might want to make sure that I'm not overwriting a previously faulting
334 // instruction that hasn't been checked yet.
335 // Actually probably want the oldest faulting load
336 if (load_fault != NoFault) {
337 // Maybe just set it as can commit here, although that might cause
338 // some other problems with sending traps to the ROB too quickly.
339 // iewStage->instToCommit(inst);
340 // iewStage->activityThisCycle();
346 template <class Impl>
348 OzoneLSQ<Impl>::executeLoad(int lq_idx)
350 // Very hackish. Not sure the best way to check that this
351 // instruction is at the head of the ROB. I should have some sort
352 // of extra information here so that I'm not overloading the
353 // canCommit signal for 15 different things.
354 loadQueue[lq_idx]->setCanCommit();
355 Fault ret_fault = executeLoad(loadQueue[lq_idx]);
356 loadQueue[lq_idx]->clearCanCommit();
360 template <class Impl>
362 OzoneLSQ<Impl>::executeStore(DynInstPtr &store_inst)
364 // Make sure that a store exists.
367 int store_idx = store_inst->sqIdx;
369 DPRINTF(OzoneLSQ, "Executing store PC %#x [sn:%lli]\n",
370 store_inst->readPC(), store_inst->seqNum);
372 // Check the recently completed loads to see if any match this store's
373 // address. If so, then we have a memory ordering violation.
374 int load_idx = store_inst->lqIdx;
376 Fault store_fault = store_inst->initiateAcc();
378 // Store size should now be available. Use it to get proper offset for
380 int size = storeQueue[store_idx].size;
383 DPRINTF(OzoneLSQ,"Fault on Store PC %#x, [sn:%lli],Size = 0\n",
384 store_inst->readPC(),store_inst->seqNum);
389 assert(store_fault == NoFault);
391 if (!storeFaultInst) {
392 if (store_fault != NoFault) {
393 panic("Fault in a store instruction!");
394 storeFaultInst = store_inst;
395 } else if (store_inst->isNonSpeculative()) {
396 // Nonspeculative accesses (namely store conditionals)
397 // need to set themselves as able to writeback if we
398 // haven't had a fault by here.
399 storeQueue[store_idx].canWB = true;
405 if (!memDepViolator) {
406 while (load_idx != loadTail) {
407 // Actually should only check loads that have actually executed
408 // Might be safe because effAddr is set to InvalAddr when the
409 // dyn inst is created.
411 // Must actually check all addrs in the proper size range
412 // Which is more correct than needs to be. What if for now we just
413 // assume all loads are quad-word loads, and do the addr based
415 // @todo: Fix this, magic number being used here
416 if ((loadQueue[load_idx]->effAddr >> 8) ==
417 (store_inst->effAddr >> 8)) {
418 // A load incorrectly passed this store. Squash and refetch.
419 // For now return a fault to show that it was unsuccessful.
420 memDepViolator = loadQueue[load_idx];
422 return TheISA::genMachineCheckFault();
428 // If we've reached this point, there was no violation.
429 memDepViolator = NULL;
435 template <class Impl>
437 OzoneLSQ<Impl>::commitLoad()
439 assert(loadQueue[loadHead]);
441 DPRINTF(OzoneLSQ, "[sn:%lli] Committing head load instruction, PC %#x\n",
442 loadQueue[loadHead]->seqNum, loadQueue[loadHead]->readPC());
445 loadQueue[loadHead] = NULL;
452 template <class Impl>
454 OzoneLSQ<Impl>::commitLoad(InstSeqNum &inst)
456 // Hopefully I don't use this function too much
457 panic("Don't use this function!");
462 assert(0 && "Load not in the queue!");
463 } else if (loadQueue[i]->seqNum == inst) {
468 if (i >= LQEntries) {
473 // loadQueue[i]->removeInLSQ();
478 template <class Impl>
480 OzoneLSQ<Impl>::commitLoads(InstSeqNum &youngest_inst)
482 assert(loads == 0 || loadQueue[loadHead]);
484 while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
489 template <class Impl>
491 OzoneLSQ<Impl>::commitStores(InstSeqNum &youngest_inst)
493 assert(stores == 0 || storeQueue[storeHead].inst);
495 int store_idx = storeHead;
497 while (store_idx != storeTail) {
498 assert(storeQueue[store_idx].inst);
499 if (!storeQueue[store_idx].canWB) {
500 if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
503 DPRINTF(OzoneLSQ, "Marking store as able to write back, PC "
505 storeQueue[store_idx].inst->readPC(),
506 storeQueue[store_idx].inst->seqNum);
508 storeQueue[store_idx].canWB = true;
514 incrStIdx(store_idx);
518 template <class Impl>
520 OzoneLSQ<Impl>::writebackStores()
522 while (storesToWB > 0 &&
523 storeWBIdx != storeTail &&
524 storeQueue[storeWBIdx].inst &&
525 storeQueue[storeWBIdx].canWB &&
526 usedPorts < cachePorts) {
528 if (storeQueue[storeWBIdx].size == 0) {
529 completeStore(storeWBIdx);
531 incrStIdx(storeWBIdx);
536 if (dcacheInterface && dcacheInterface->isBlocked()) {
537 DPRINTF(OzoneLSQ, "Unable to write back any more stores, cache"
544 if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
545 incrStIdx(storeWBIdx);
550 assert(storeQueue[storeWBIdx].req);
551 assert(!storeQueue[storeWBIdx].committed);
553 MemReqPtr req = storeQueue[storeWBIdx].req;
554 storeQueue[storeWBIdx].committed = true;
556 // Fault fault = cpu->translateDataReadReq(req);
558 req->completionEvent = NULL;
561 req->data = new uint8_t[64];
562 memcpy(req->data, (uint8_t *)&storeQueue[storeWBIdx].data, req->size);
564 DPRINTF(OzoneLSQ, "D-Cache: Writing back store idx:%i PC:%#x "
565 "to Addr:%#x, data:%#x [sn:%lli]\n",
566 storeWBIdx,storeQueue[storeWBIdx].inst->readPC(),
567 req->paddr, *(req->data),
568 storeQueue[storeWBIdx].inst->seqNum);
570 // if (fault != NoFault) {
571 //What should we do if there is a fault???
573 // panic("Page Table Fault!!!!!\n");
576 if (dcacheInterface) {
577 MemAccessResult result = dcacheInterface->access(req);
579 //@todo temp fix for LL/SC (works fine for 1 CPU)
580 if (req->flags & LOCKED) {
582 panic("LL/SC! oh no no support!!!");
586 storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
587 DPRINTF(OzoneLSQ, "Unstalling, stalling store [sn:%lli] "
589 stallingStoreIsn, stallingLoadIdx);
591 stallingStoreIsn = 0;
592 be->replayMemInst(loadQueue[stallingLoadIdx]);
595 if (result != MA_HIT && dcacheInterface->doEvents()) {
598 typename IEW::LdWritebackEvent *wb = NULL;
599 if (req->flags & LOCKED) {
600 // Stx_C does not generate a system port transaction.
602 wb = new typename IEW::LdWritebackEvent(storeQueue[storeWBIdx].inst,
606 DPRINTF(OzoneLSQ,"D-Cache Write Miss!\n");
608 // DPRINTF(Activity, "Active st accessing mem miss [sn:%lli]\n",
609 // storeQueue[storeWBIdx].inst->seqNum);
611 // Will stores need their own kind of writeback events?
612 // Do stores even need writeback events?
613 assert(!req->completionEvent);
614 req->completionEvent = new
615 StoreCompletionEvent(storeWBIdx, wb, this);
617 lastDcacheStall = curTick;
619 _status = DcacheMissStall;
621 //mshrSeqNums.push_back(storeQueue[storeWBIdx].inst->seqNum);
623 //DPRINTF(OzoneLSQ, "Added MSHR. count = %i\n",mshrSeqNums.size());
625 // Increment stat here or something
627 DPRINTF(OzoneLSQ,"D-Cache: Write Hit on idx:%i !\n",
630 // DPRINTF(Activity, "Active st accessing mem hit [sn:%lli]\n",
631 // storeQueue[storeWBIdx].inst->seqNum);
633 if (req->flags & LOCKED) {
634 // Stx_C does not generate a system port transaction.
636 typename BackEnd::LdWritebackEvent *wb =
637 new typename BackEnd::LdWritebackEvent(storeQueue[storeWBIdx].inst,
639 wb->schedule(curTick);
642 completeStore(storeWBIdx);
645 incrStIdx(storeWBIdx);
647 panic("Must HAVE DCACHE!!!!!\n");
651 // Not sure this should set it to 0.
654 assert(stores >= 0 && storesToWB >= 0);
657 /*template <class Impl>
659 OzoneLSQ<Impl>::removeMSHR(InstSeqNum seqNum)
661 list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
665 if (mshr_it != mshrSeqNums.end()) {
666 mshrSeqNums.erase(mshr_it);
667 DPRINTF(OzoneLSQ, "Removing MSHR. count = %i\n",mshrSeqNums.size());
671 template <class Impl>
673 OzoneLSQ<Impl>::squash(const InstSeqNum &squashed_num)
675 DPRINTF(OzoneLSQ, "Squashing until [sn:%lli]!"
676 "(Loads:%i Stores:%i)\n",squashed_num,loads,stores);
678 int load_idx = loadTail;
681 while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
683 // Clear the smart pointer to make sure it is decremented.
684 DPRINTF(OzoneLSQ,"Load Instruction PC %#x squashed, "
686 loadQueue[load_idx]->readPC(),
687 loadQueue[load_idx]->seqNum);
689 if (isStalled() && load_idx == stallingLoadIdx) {
691 stallingStoreIsn = 0;
695 // loadQueue[load_idx]->squashed = true;
696 loadQueue[load_idx] = NULL;
705 int store_idx = storeTail;
706 decrStIdx(store_idx);
708 while (stores != 0 && storeQueue[store_idx].inst->seqNum > squashed_num) {
710 // Clear the smart pointer to make sure it is decremented.
711 DPRINTF(OzoneLSQ,"Store Instruction PC %#x squashed, "
712 "idx:%i [sn:%lli]\n",
713 storeQueue[store_idx].inst->readPC(),
714 store_idx, storeQueue[store_idx].inst->seqNum);
716 // I don't think this can happen. It should have been cleared by the
719 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
720 panic("Is stalled should have been cleared by stalling load!\n");
722 stallingStoreIsn = 0;
725 // storeQueue[store_idx].inst->squashed = true;
726 storeQueue[store_idx].inst = NULL;
727 storeQueue[store_idx].canWB = 0;
729 if (storeQueue[store_idx].req) {
730 assert(!storeQueue[store_idx].req->completionEvent);
732 storeQueue[store_idx].req = NULL;
736 storeTail = store_idx;
738 decrStIdx(store_idx);
742 template <class Impl>
744 OzoneLSQ<Impl>::dumpInsts()
746 cprintf("Load store queue: Dumping instructions.\n");
747 cprintf("Load queue size: %i\n", loads);
748 cprintf("Load queue: ");
750 int load_idx = loadHead;
752 while (load_idx != loadTail && loadQueue[load_idx]) {
753 cprintf("[sn:%lli] %#x ", loadQueue[load_idx]->seqNum,
754 loadQueue[load_idx]->readPC());
759 cprintf("\nStore queue size: %i\n", stores);
760 cprintf("Store queue: ");
762 int store_idx = storeHead;
764 while (store_idx != storeTail && storeQueue[store_idx].inst) {
765 cprintf("[sn:%lli] %#x ", storeQueue[store_idx].inst->seqNum,
766 storeQueue[store_idx].inst->readPC());
768 incrStIdx(store_idx);
774 template <class Impl>
776 OzoneLSQ<Impl>::completeStore(int store_idx)
778 assert(storeQueue[store_idx].inst);
779 storeQueue[store_idx].completed = true;
781 // A bit conservative because a store completion may not free up entries,
782 // but hopefully avoids two store completions in one cycle from making
783 // the CPU tick twice.
784 // cpu->activityThisCycle();
786 if (store_idx == storeHead) {
788 incrStIdx(storeHead);
791 } while (storeQueue[storeHead].completed &&
792 storeHead != storeTail);
794 // be->updateLSQNextCycle = true;
797 DPRINTF(OzoneLSQ, "Store head idx:%i\n", storeHead);
800 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
801 DPRINTF(OzoneLSQ, "Unstalling, stalling store [sn:%lli] "
803 stallingStoreIsn, stallingLoadIdx);
805 stallingStoreIsn = 0;
806 be->replayMemInst(loadQueue[stallingLoadIdx]);
810 template <class Impl>
812 OzoneLSQ<Impl>::incrStIdx(int &store_idx)
814 if (++store_idx >= SQEntries)
818 template <class Impl>
820 OzoneLSQ<Impl>::decrStIdx(int &store_idx)
823 store_idx += SQEntries;
826 template <class Impl>
828 OzoneLSQ<Impl>::incrLdIdx(int &load_idx)
830 if (++load_idx >= LQEntries)
834 template <class Impl>
836 OzoneLSQ<Impl>::decrLdIdx(int &load_idx)
839 load_idx += LQEntries;