2 * Copyright (c) 2004-2006 The Regents of The University of Michigan
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #include "base/str.hh"
32 #include "config/the_isa.hh"
33 #include "cpu/ozone/lsq_unit.hh"
34 #include "sim/fault_fwd.hh"
37 OzoneLSQ<Impl>::StoreCompletionEvent::StoreCompletionEvent(int store_idx,
39 OzoneLSQ<Impl> *lsq_ptr)
40 : Event(&mainEventQueue),
45 this->setFlags(Event::AutoDelete);
50 OzoneLSQ<Impl>::StoreCompletionEvent::process()
52 DPRINTF(OzoneLSQ, "Cache miss complete for store idx:%i\n", storeIdx);
54 //lsqPtr->removeMSHR(lsqPtr->storeQueue[storeIdx].inst->seqNum);
56 // lsqPtr->cpu->wakeCPU();
59 lsqPtr->completeStore(storeIdx);
64 OzoneLSQ<Impl>::StoreCompletionEvent::description() const
66 return "LSQ store completion";
70 OzoneLSQ<Impl>::OzoneLSQ()
71 : loads(0), stores(0), storesToWB(0), stalled(false), isLoadBlocked(false)
77 OzoneLSQ<Impl>::init(Params *params, unsigned maxLQEntries,
78 unsigned maxSQEntries, unsigned id)
81 DPRINTF(OzoneLSQ, "Creating OzoneLSQ%i object.\n",id);
85 LQEntries = maxLQEntries;
86 SQEntries = maxSQEntries;
88 loadQueue.resize(LQEntries);
89 storeQueue.resize(SQEntries);
92 // May want to initialize these entries to NULL
94 loadHead = loadTail = 0;
96 storeHead = storeWBIdx = storeTail = 0;
99 cachePorts = params->cachePorts;
101 dcacheInterface = params->dcacheInterface;
103 loadFaultInst = storeFaultInst = memDepViolator = NULL;
108 OzoneLSQ<Impl>::name() const
115 OzoneLSQ<Impl>::clearLQ()
122 OzoneLSQ<Impl>::clearSQ()
129 OzoneLSQ<Impl>::resizeLQ(unsigned size)
131 assert( size >= LQEntries);
133 if (size > LQEntries) {
134 while (size > loadQueue.size()) {
136 loadQueue.push_back(dummy);
147 OzoneLSQ<Impl>::resizeSQ(unsigned size)
149 if (size > SQEntries) {
150 while (size > storeQueue.size()) {
152 storeQueue.push_back(dummy);
160 template <class Impl>
162 OzoneLSQ<Impl>::insert(DynInstPtr &inst)
164 // Make sure we really have a memory reference.
165 assert(inst->isMemRef());
167 // Make sure it's one of the two classes of memory references.
168 assert(inst->isLoad() || inst->isStore());
170 if (inst->isLoad()) {
179 template <class Impl>
181 OzoneLSQ<Impl>::insertLoad(DynInstPtr &load_inst)
183 assert((loadTail + 1) % LQEntries != loadHead && loads < LQEntries);
185 DPRINTF(OzoneLSQ, "Inserting load PC %#x, idx:%i [sn:%lli]\n",
186 load_inst->readPC(), loadTail, load_inst->seqNum);
188 load_inst->lqIdx = loadTail;
191 load_inst->sqIdx = -1;
193 load_inst->sqIdx = storeTail;
196 loadQueue[loadTail] = load_inst;
203 template <class Impl>
205 OzoneLSQ<Impl>::insertStore(DynInstPtr &store_inst)
207 // Make sure it is not full before inserting an instruction.
208 assert((storeTail + 1) % SQEntries != storeHead);
209 assert(stores < SQEntries);
211 DPRINTF(OzoneLSQ, "Inserting store PC %#x, idx:%i [sn:%lli]\n",
212 store_inst->readPC(), storeTail, store_inst->seqNum);
214 store_inst->sqIdx = storeTail;
215 store_inst->lqIdx = loadTail;
217 storeQueue[storeTail] = SQEntry(store_inst);
219 incrStIdx(storeTail);
225 template <class Impl>
226 typename Impl::DynInstPtr
227 OzoneLSQ<Impl>::getMemDepViolator()
229 DynInstPtr temp = memDepViolator;
231 memDepViolator = NULL;
236 template <class Impl>
238 OzoneLSQ<Impl>::numFreeEntries()
240 unsigned free_lq_entries = LQEntries - loads;
241 unsigned free_sq_entries = SQEntries - stores;
243 // Both the LQ and SQ entries have an extra dummy entry to differentiate
244 // empty/full conditions. Subtract 1 from the free entries.
245 if (free_lq_entries < free_sq_entries) {
246 return free_lq_entries - 1;
248 return free_sq_entries - 1;
252 template <class Impl>
254 OzoneLSQ<Impl>::numLoadsReady()
256 int load_idx = loadHead;
259 while (load_idx != loadTail) {
260 assert(loadQueue[load_idx]);
262 if (loadQueue[load_idx]->readyToIssue()) {
271 template <class Impl>
273 OzoneLSQ<Impl>::executeLoad()
275 Fault load_fault = NoFault;
276 DynInstPtr load_inst;
278 assert(readyLoads.size() != 0);
280 // Execute a ready load.
281 LdMapIt ready_it = readyLoads.begin();
283 load_inst = (*ready_it).second;
285 // Execute the instruction, which is held in the data portion of the
287 load_fault = load_inst->execute();
289 // If it executed successfully, then switch it over to the executed
291 if (load_fault == NoFault) {
292 executedLoads[load_inst->seqNum] = load_inst;
294 readyLoads.erase(ready_it);
296 loadFaultInst = load_inst;
303 template <class Impl>
305 OzoneLSQ<Impl>::executeLoad(DynInstPtr &inst)
307 // Execute a specific load.
308 Fault load_fault = NoFault;
310 DPRINTF(OzoneLSQ, "Executing load PC %#x, [sn:%lli]\n",
311 inst->readPC(),inst->seqNum);
313 // Make sure it's really in the list.
314 // Normally it should always be in the list. However,
315 /* due to a syscall it may not be the list.
319 if (i == loadTail && !find(inst)) {
320 assert(0 && "Load not in the queue!");
321 } else if (loadQueue[i] == inst) {
326 if (i >= LQEntries) {
332 load_fault = inst->initiateAcc();
334 // Might want to make sure that I'm not overwriting a previously faulting
335 // instruction that hasn't been checked yet.
336 // Actually probably want the oldest faulting load
337 if (load_fault != NoFault) {
338 // Maybe just set it as can commit here, although that might cause
339 // some other problems with sending traps to the ROB too quickly.
340 // iewStage->instToCommit(inst);
341 // iewStage->activityThisCycle();
347 template <class Impl>
349 OzoneLSQ<Impl>::executeLoad(int lq_idx)
351 // Very hackish. Not sure the best way to check that this
352 // instruction is at the head of the ROB. I should have some sort
353 // of extra information here so that I'm not overloading the
354 // canCommit signal for 15 different things.
355 loadQueue[lq_idx]->setCanCommit();
356 Fault ret_fault = executeLoad(loadQueue[lq_idx]);
357 loadQueue[lq_idx]->clearCanCommit();
361 template <class Impl>
363 OzoneLSQ<Impl>::executeStore(DynInstPtr &store_inst)
365 // Make sure that a store exists.
368 int store_idx = store_inst->sqIdx;
370 DPRINTF(OzoneLSQ, "Executing store PC %#x [sn:%lli]\n",
371 store_inst->readPC(), store_inst->seqNum);
373 // Check the recently completed loads to see if any match this store's
374 // address. If so, then we have a memory ordering violation.
375 int load_idx = store_inst->lqIdx;
377 Fault store_fault = store_inst->initiateAcc();
379 // Store size should now be available. Use it to get proper offset for
381 int size = storeQueue[store_idx].size;
384 DPRINTF(OzoneLSQ,"Fault on Store PC %#x, [sn:%lli],Size = 0\n",
385 store_inst->readPC(),store_inst->seqNum);
390 assert(store_fault == NoFault);
392 if (!storeFaultInst) {
393 if (store_fault != NoFault) {
394 panic("Fault in a store instruction!");
395 storeFaultInst = store_inst;
396 } else if (store_inst->isNonSpeculative()) {
397 // Nonspeculative accesses (namely store conditionals)
398 // need to set themselves as able to writeback if we
399 // haven't had a fault by here.
400 storeQueue[store_idx].canWB = true;
406 if (!memDepViolator) {
407 while (load_idx != loadTail) {
408 // Actually should only check loads that have actually executed
409 // Might be safe because effAddr is set to InvalAddr when the
410 // dyn inst is created.
412 // Must actually check all addrs in the proper size range
413 // Which is more correct than needs to be. What if for now we just
414 // assume all loads are quad-word loads, and do the addr based
416 // @todo: Fix this, magic number being used here
417 if ((loadQueue[load_idx]->effAddr >> 8) ==
418 (store_inst->effAddr >> 8)) {
419 // A load incorrectly passed this store. Squash and refetch.
420 // For now return a fault to show that it was unsuccessful.
421 memDepViolator = loadQueue[load_idx];
423 return TheISA::genMachineCheckFault();
429 // If we've reached this point, there was no violation.
430 memDepViolator = NULL;
436 template <class Impl>
438 OzoneLSQ<Impl>::commitLoad()
440 assert(loadQueue[loadHead]);
442 DPRINTF(OzoneLSQ, "[sn:%lli] Committing head load instruction, PC %#x\n",
443 loadQueue[loadHead]->seqNum, loadQueue[loadHead]->readPC());
446 loadQueue[loadHead] = NULL;
453 template <class Impl>
455 OzoneLSQ<Impl>::commitLoad(InstSeqNum &inst)
457 // Hopefully I don't use this function too much
458 panic("Don't use this function!");
463 assert(0 && "Load not in the queue!");
464 } else if (loadQueue[i]->seqNum == inst) {
469 if (i >= LQEntries) {
474 // loadQueue[i]->removeInLSQ();
479 template <class Impl>
481 OzoneLSQ<Impl>::commitLoads(InstSeqNum &youngest_inst)
483 assert(loads == 0 || loadQueue[loadHead]);
485 while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
490 template <class Impl>
492 OzoneLSQ<Impl>::commitStores(InstSeqNum &youngest_inst)
494 assert(stores == 0 || storeQueue[storeHead].inst);
496 int store_idx = storeHead;
498 while (store_idx != storeTail) {
499 assert(storeQueue[store_idx].inst);
500 if (!storeQueue[store_idx].canWB) {
501 if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
504 DPRINTF(OzoneLSQ, "Marking store as able to write back, PC "
506 storeQueue[store_idx].inst->readPC(),
507 storeQueue[store_idx].inst->seqNum);
509 storeQueue[store_idx].canWB = true;
515 incrStIdx(store_idx);
519 template <class Impl>
521 OzoneLSQ<Impl>::writebackStores()
523 while (storesToWB > 0 &&
524 storeWBIdx != storeTail &&
525 storeQueue[storeWBIdx].inst &&
526 storeQueue[storeWBIdx].canWB &&
527 usedPorts < cachePorts) {
529 if (storeQueue[storeWBIdx].size == 0) {
530 completeStore(storeWBIdx);
532 incrStIdx(storeWBIdx);
537 if (dcacheInterface && dcacheInterface->isBlocked()) {
538 DPRINTF(OzoneLSQ, "Unable to write back any more stores, cache"
545 if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
546 incrStIdx(storeWBIdx);
551 assert(storeQueue[storeWBIdx].req);
552 assert(!storeQueue[storeWBIdx].committed);
554 MemReqPtr req = storeQueue[storeWBIdx].req;
555 storeQueue[storeWBIdx].committed = true;
557 // Fault fault = cpu->translateDataReadReq(req);
559 req->completionEvent = NULL;
560 req->time = curTick();
562 req->data = new uint8_t[64];
563 memcpy(req->data, (uint8_t *)&storeQueue[storeWBIdx].data, req->size);
565 DPRINTF(OzoneLSQ, "D-Cache: Writing back store idx:%i PC:%#x "
566 "to Addr:%#x, data:%#x [sn:%lli]\n",
567 storeWBIdx,storeQueue[storeWBIdx].inst->readPC(),
568 req->paddr, *(req->data),
569 storeQueue[storeWBIdx].inst->seqNum);
571 // if (fault != NoFault) {
572 //What should we do if there is a fault???
574 // panic("Page Table Fault!!!!!\n");
577 if (dcacheInterface) {
578 MemAccessResult result = dcacheInterface->access(req);
580 //@todo temp fix for LL/SC (works fine for 1 CPU)
583 panic("LL/SC! oh no no support!!!");
587 storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
588 DPRINTF(OzoneLSQ, "Unstalling, stalling store [sn:%lli] "
590 stallingStoreIsn, stallingLoadIdx);
592 stallingStoreIsn = 0;
593 be->replayMemInst(loadQueue[stallingLoadIdx]);
596 if (result != MA_HIT && dcacheInterface->doEvents()) {
599 typename IEW::LdWritebackEvent *wb = NULL;
601 // Stx_C does not generate a system port transaction.
603 wb = new typename IEW::LdWritebackEvent(storeQueue[storeWBIdx].inst,
607 DPRINTF(OzoneLSQ,"D-Cache Write Miss!\n");
609 // DPRINTF(Activity, "Active st accessing mem miss [sn:%lli]\n",
610 // storeQueue[storeWBIdx].inst->seqNum);
612 // Will stores need their own kind of writeback events?
613 // Do stores even need writeback events?
614 assert(!req->completionEvent);
615 req->completionEvent = new
616 StoreCompletionEvent(storeWBIdx, wb, this);
618 lastDcacheStall = curTick();
620 _status = DcacheMissStall;
622 //mshrSeqNums.push_back(storeQueue[storeWBIdx].inst->seqNum);
624 //DPRINTF(OzoneLSQ, "Added MSHR. count = %i\n",mshrSeqNums.size());
626 // Increment stat here or something
628 DPRINTF(OzoneLSQ,"D-Cache: Write Hit on idx:%i !\n",
631 // DPRINTF(Activity, "Active st accessing mem hit [sn:%lli]\n",
632 // storeQueue[storeWBIdx].inst->seqNum);
635 // Stx_C does not generate a system port transaction.
637 typename BackEnd::LdWritebackEvent *wb =
638 new typename BackEnd::LdWritebackEvent(storeQueue[storeWBIdx].inst,
640 wb->schedule(curTick());
643 completeStore(storeWBIdx);
646 incrStIdx(storeWBIdx);
648 panic("Must HAVE DCACHE!!!!!\n");
652 // Not sure this should set it to 0.
655 assert(stores >= 0 && storesToWB >= 0);
658 /*template <class Impl>
660 OzoneLSQ<Impl>::removeMSHR(InstSeqNum seqNum)
662 list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
666 if (mshr_it != mshrSeqNums.end()) {
667 mshrSeqNums.erase(mshr_it);
668 DPRINTF(OzoneLSQ, "Removing MSHR. count = %i\n",mshrSeqNums.size());
672 template <class Impl>
674 OzoneLSQ<Impl>::squash(const InstSeqNum &squashed_num)
676 DPRINTF(OzoneLSQ, "Squashing until [sn:%lli]!"
677 "(Loads:%i Stores:%i)\n",squashed_num,loads,stores);
679 int load_idx = loadTail;
682 while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
684 // Clear the smart pointer to make sure it is decremented.
685 DPRINTF(OzoneLSQ,"Load Instruction PC %#x squashed, "
687 loadQueue[load_idx]->readPC(),
688 loadQueue[load_idx]->seqNum);
690 if (isStalled() && load_idx == stallingLoadIdx) {
692 stallingStoreIsn = 0;
696 // loadQueue[load_idx]->squashed = true;
697 loadQueue[load_idx] = NULL;
706 int store_idx = storeTail;
707 decrStIdx(store_idx);
709 while (stores != 0 && storeQueue[store_idx].inst->seqNum > squashed_num) {
711 // Clear the smart pointer to make sure it is decremented.
712 DPRINTF(OzoneLSQ,"Store Instruction PC %#x squashed, "
713 "idx:%i [sn:%lli]\n",
714 storeQueue[store_idx].inst->readPC(),
715 store_idx, storeQueue[store_idx].inst->seqNum);
717 // I don't think this can happen. It should have been cleared by the
720 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
721 panic("Is stalled should have been cleared by stalling load!\n");
723 stallingStoreIsn = 0;
726 // storeQueue[store_idx].inst->squashed = true;
727 storeQueue[store_idx].inst = NULL;
728 storeQueue[store_idx].canWB = 0;
730 if (storeQueue[store_idx].req) {
731 assert(!storeQueue[store_idx].req->completionEvent);
733 storeQueue[store_idx].req = NULL;
737 storeTail = store_idx;
739 decrStIdx(store_idx);
743 template <class Impl>
745 OzoneLSQ<Impl>::dumpInsts()
747 cprintf("Load store queue: Dumping instructions.\n");
748 cprintf("Load queue size: %i\n", loads);
749 cprintf("Load queue: ");
751 int load_idx = loadHead;
753 while (load_idx != loadTail && loadQueue[load_idx]) {
754 cprintf("[sn:%lli] %#x ", loadQueue[load_idx]->seqNum,
755 loadQueue[load_idx]->readPC());
760 cprintf("\nStore queue size: %i\n", stores);
761 cprintf("Store queue: ");
763 int store_idx = storeHead;
765 while (store_idx != storeTail && storeQueue[store_idx].inst) {
766 cprintf("[sn:%lli] %#x ", storeQueue[store_idx].inst->seqNum,
767 storeQueue[store_idx].inst->readPC());
769 incrStIdx(store_idx);
775 template <class Impl>
777 OzoneLSQ<Impl>::completeStore(int store_idx)
779 assert(storeQueue[store_idx].inst);
780 storeQueue[store_idx].completed = true;
782 // A bit conservative because a store completion may not free up entries,
783 // but hopefully avoids two store completions in one cycle from making
784 // the CPU tick twice.
785 // cpu->activityThisCycle();
787 if (store_idx == storeHead) {
789 incrStIdx(storeHead);
792 } while (storeQueue[storeHead].completed &&
793 storeHead != storeTail);
795 // be->updateLSQNextCycle = true;
798 DPRINTF(OzoneLSQ, "Store head idx:%i\n", storeHead);
801 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
802 DPRINTF(OzoneLSQ, "Unstalling, stalling store [sn:%lli] "
804 stallingStoreIsn, stallingLoadIdx);
806 stallingStoreIsn = 0;
807 be->replayMemInst(loadQueue[stallingLoadIdx]);
811 template <class Impl>
813 OzoneLSQ<Impl>::incrStIdx(int &store_idx)
815 if (++store_idx >= SQEntries)
819 template <class Impl>
821 OzoneLSQ<Impl>::decrStIdx(int &store_idx)
824 store_idx += SQEntries;
827 template <class Impl>
829 OzoneLSQ<Impl>::incrLdIdx(int &load_idx)
831 if (++load_idx >= LQEntries)
835 template <class Impl>
837 OzoneLSQ<Impl>::decrLdIdx(int &load_idx)
840 load_idx += LQEntries;