src/cpu/o3/lsq_unit_impl.hh

   1 /*
   2  * Copyright (c) 2004-2005 The Regents of The University of Michigan
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are
   7  * met: redistributions of source code must retain the above copyright
   8  * notice, this list of conditions and the following disclaimer;
   9  * redistributions in binary form must reproduce the above copyright
  10  * notice, this list of conditions and the following disclaimer in the
  11  * documentation and/or other materials provided with the distribution;
  12  * neither the name of the copyright holders nor the names of its
  13  * contributors may be used to endorse or promote products derived from
  14  * this software without specific prior written permission.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  17  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  18  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  19  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  20  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  26  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * Authors: Kevin Lim
  29  *          Korey Sewell
  30  */
  31
  32 #include "arch/locked_mem.hh"
  33 #include "config/use_checker.hh"
  34
  35 #include "cpu/o3/lsq.hh"
  36 #include "cpu/o3/lsq_unit.hh"
  37 #include "base/str.hh"
  38 #include "mem/packet.hh"
  39 #include "mem/request.hh"
  40
  41 #if USE_CHECKER
  42 #include "cpu/checker/cpu.hh"
  43 #endif
  44
  45 template<class Impl>
  46 LSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt,
  47                                               LSQUnit *lsq_ptr)
  48     : Event(&mainEventQueue), inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
  49 {
  50     this->setFlags(Event::AutoDelete);
  51 }
  52
  53 template<class Impl>
  54 void
  55 LSQUnit<Impl>::WritebackEvent::process()
  56 {
  57     if (!lsqPtr->isSwitchedOut()) {
  58         lsqPtr->writeback(inst, pkt);
  59     }
  60
  61     if (pkt->senderState)
  62         delete pkt->senderState;
  63
  64     delete pkt->req;
  65     delete pkt;
  66 }
  67
  68 template<class Impl>
  69 const char *
  70 LSQUnit<Impl>::WritebackEvent::description()
  71 {
  72     return "Store writeback event";
  73 }
  74
  75 template<class Impl>
  76 void
  77 LSQUnit<Impl>::completeDataAccess(PacketPtr pkt)
  78 {
  79     LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState);
  80     DynInstPtr inst = state->inst;
  81     DPRINTF(IEW, "Writeback event [sn:%lli]\n", inst->seqNum);
  82     DPRINTF(Activity, "Activity: Writeback event [sn:%lli]\n", inst->seqNum);
  83
  84     //iewStage->ldstQueue.removeMSHR(inst->threadNumber,inst->seqNum);
  85
  86     if (isSwitchedOut() || inst->isSquashed()) {
  87         iewStage->decrWb(inst->seqNum);
  88     } else {
  89         if (!state->noWB) {
  90             writeback(inst, pkt);
  91         }
  92
  93         if (inst->isStore()) {
  94             completeStore(state->idx);
  95         }
  96     }
  97
  98     delete state;
  99     delete pkt->req;
 100     delete pkt;
 101 }
 102
 103 template <class Impl>
 104 LSQUnit<Impl>::LSQUnit()
 105     : loads(0), stores(0), storesToWB(0), stalled(false),
 106       isStoreBlocked(false), isLoadBlocked(false),
 107       loadBlockedHandled(false)
 108 {
 109 }
 110
 111 template<class Impl>
 112 void
 113 LSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, Params *params, LSQ *lsq_ptr,
 114                     unsigned maxLQEntries, unsigned maxSQEntries, unsigned id)
 115 {
 116     cpu = cpu_ptr;
 117     iewStage = iew_ptr;
 118
 119     DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id);
 120
 121     switchedOut = false;
 122
 123     lsq = lsq_ptr;
 124
 125     lsqID = id;
 126
 127     // Add 1 for the sentinel entry (they are circular queues).
 128     LQEntries = maxLQEntries + 1;
 129     SQEntries = maxSQEntries + 1;
 130
 131     loadQueue.resize(LQEntries);
 132     storeQueue.resize(SQEntries);
 133
 134     loadHead = loadTail = 0;
 135
 136     storeHead = storeWBIdx = storeTail = 0;
 137
 138     usedPorts = 0;
 139     cachePorts = params->cachePorts;
 140
 141     retryPkt = NULL;
 142     memDepViolator = NULL;
 143
 144     blockedLoadSeqNum = 0;
 145 }
 146
 147 template<class Impl>
 148 std::string
 149 LSQUnit<Impl>::name() const
 150 {
 151     if (Impl::MaxThreads == 1) {
 152         return iewStage->name() + ".lsq";
 153     } else {
 154         return iewStage->name() + ".lsq.thread." + to_string(lsqID);
 155     }
 156 }
 157
 158 template<class Impl>
 159 void
 160 LSQUnit<Impl>::regStats()
 161 {
 162     lsqForwLoads
 163         .name(name() + ".forwLoads")
 164         .desc("Number of loads that had data forwarded from stores");
 165
 166     invAddrLoads
 167         .name(name() + ".invAddrLoads")
 168         .desc("Number of loads ignored due to an invalid address");
 169
 170     lsqSquashedLoads
 171         .name(name() + ".squashedLoads")
 172         .desc("Number of loads squashed");
 173
 174     lsqIgnoredResponses
 175         .name(name() + ".ignoredResponses")
 176         .desc("Number of memory responses ignored because the instruction is squashed");
 177
 178     lsqMemOrderViolation
 179         .name(name() + ".memOrderViolation")
 180         .desc("Number of memory ordering violations");
 181
 182     lsqSquashedStores
 183         .name(name() + ".squashedStores")
 184         .desc("Number of stores squashed");
 185
 186     invAddrSwpfs
 187         .name(name() + ".invAddrSwpfs")
 188         .desc("Number of software prefetches ignored due to an invalid address");
 189
 190     lsqBlockedLoads
 191         .name(name() + ".blockedLoads")
 192         .desc("Number of blocked loads due to partial load-store forwarding");
 193
 194     lsqRescheduledLoads
 195         .name(name() + ".rescheduledLoads")
 196         .desc("Number of loads that were rescheduled");
 197
 198     lsqCacheBlocked
 199         .name(name() + ".cacheBlocked")
 200         .desc("Number of times an access to memory failed due to the cache being blocked");
 201 }
 202
 203 template<class Impl>
 204 void
 205 LSQUnit<Impl>::setDcachePort(Port *dcache_port)
 206 {
 207     dcachePort = dcache_port;
 208
 209 #if USE_CHECKER
 210     if (cpu->checker) {
 211         cpu->checker->setDcachePort(dcachePort);
 212     }
 213 #endif
 214 }
 215
 216 template<class Impl>
 217 void
 218 LSQUnit<Impl>::clearLQ()
 219 {
 220     loadQueue.clear();
 221 }
 222
 223 template<class Impl>
 224 void
 225 LSQUnit<Impl>::clearSQ()
 226 {
 227     storeQueue.clear();
 228 }
 229
 230 template<class Impl>
 231 void
 232 LSQUnit<Impl>::switchOut()
 233 {
 234     switchedOut = true;
 235     for (int i = 0; i < loadQueue.size(); ++i) {
 236         assert(!loadQueue[i]);
 237         loadQueue[i] = NULL;
 238     }
 239
 240     assert(storesToWB == 0);
 241 }
 242
 243 template<class Impl>
 244 void
 245 LSQUnit<Impl>::takeOverFrom()
 246 {
 247     switchedOut = false;
 248     loads = stores = storesToWB = 0;
 249
 250     loadHead = loadTail = 0;
 251
 252     storeHead = storeWBIdx = storeTail = 0;
 253
 254     usedPorts = 0;
 255
 256     memDepViolator = NULL;
 257
 258     blockedLoadSeqNum = 0;
 259
 260     stalled = false;
 261     isLoadBlocked = false;
 262     loadBlockedHandled = false;
 263 }
 264
 265 template<class Impl>
 266 void
 267 LSQUnit<Impl>::resizeLQ(unsigned size)
 268 {
 269     unsigned size_plus_sentinel = size + 1;
 270     assert(size_plus_sentinel >= LQEntries);
 271
 272     if (size_plus_sentinel > LQEntries) {
 273         while (size_plus_sentinel > loadQueue.size()) {
 274             DynInstPtr dummy;
 275             loadQueue.push_back(dummy);
 276             LQEntries++;
 277         }
 278     } else {
 279         LQEntries = size_plus_sentinel;
 280     }
 281
 282 }
 283
 284 template<class Impl>
 285 void
 286 LSQUnit<Impl>::resizeSQ(unsigned size)
 287 {
 288     unsigned size_plus_sentinel = size + 1;
 289     if (size_plus_sentinel > SQEntries) {
 290         while (size_plus_sentinel > storeQueue.size()) {
 291             SQEntry dummy;
 292             storeQueue.push_back(dummy);
 293             SQEntries++;
 294         }
 295     } else {
 296         SQEntries = size_plus_sentinel;
 297     }
 298 }
 299
 300 template <class Impl>
 301 void
 302 LSQUnit<Impl>::insert(DynInstPtr &inst)
 303 {
 304     assert(inst->isMemRef());
 305
 306     assert(inst->isLoad() || inst->isStore());
 307
 308     if (inst->isLoad()) {
 309         insertLoad(inst);
 310     } else {
 311         insertStore(inst);
 312     }
 313
 314     inst->setInLSQ();
 315 }
 316
 317 template <class Impl>
 318 void
 319 LSQUnit<Impl>::insertLoad(DynInstPtr &load_inst)
 320 {
 321     assert((loadTail + 1) % LQEntries != loadHead);
 322     assert(loads < LQEntries);
 323
 324     DPRINTF(LSQUnit, "Inserting load PC %#x, idx:%i [sn:%lli]\n",
 325             load_inst->readPC(), loadTail, load_inst->seqNum);
 326
 327     load_inst->lqIdx = loadTail;
 328
 329     if (stores == 0) {
 330         load_inst->sqIdx = -1;
 331     } else {
 332         load_inst->sqIdx = storeTail;
 333     }
 334
 335     loadQueue[loadTail] = load_inst;
 336
 337     incrLdIdx(loadTail);
 338
 339     ++loads;
 340 }
 341
 342 template <class Impl>
 343 void
 344 LSQUnit<Impl>::insertStore(DynInstPtr &store_inst)
 345 {
 346     // Make sure it is not full before inserting an instruction.
 347     assert((storeTail + 1) % SQEntries != storeHead);
 348     assert(stores < SQEntries);
 349
 350     DPRINTF(LSQUnit, "Inserting store PC %#x, idx:%i [sn:%lli]\n",
 351             store_inst->readPC(), storeTail, store_inst->seqNum);
 352
 353     store_inst->sqIdx = storeTail;
 354     store_inst->lqIdx = loadTail;
 355
 356     storeQueue[storeTail] = SQEntry(store_inst);
 357
 358     incrStIdx(storeTail);
 359
 360     ++stores;
 361 }
 362
 363 template <class Impl>
 364 typename Impl::DynInstPtr
 365 LSQUnit<Impl>::getMemDepViolator()
 366 {
 367     DynInstPtr temp = memDepViolator;
 368
 369     memDepViolator = NULL;
 370
 371     return temp;
 372 }
 373
 374 template <class Impl>
 375 unsigned
 376 LSQUnit<Impl>::numFreeEntries()
 377 {
 378     unsigned free_lq_entries = LQEntries - loads;
 379     unsigned free_sq_entries = SQEntries - stores;
 380
 381     // Both the LQ and SQ entries have an extra dummy entry to differentiate
 382     // empty/full conditions.  Subtract 1 from the free entries.
 383     if (free_lq_entries < free_sq_entries) {
 384         return free_lq_entries - 1;
 385     } else {
 386         return free_sq_entries - 1;
 387     }
 388 }
 389
 390 template <class Impl>
 391 int
 392 LSQUnit<Impl>::numLoadsReady()
 393 {
 394     int load_idx = loadHead;
 395     int retval = 0;
 396
 397     while (load_idx != loadTail) {
 398         assert(loadQueue[load_idx]);
 399
 400         if (loadQueue[load_idx]->readyToIssue()) {
 401             ++retval;
 402         }
 403     }
 404
 405     return retval;
 406 }
 407
 408 template <class Impl>
 409 Fault
 410 LSQUnit<Impl>::executeLoad(DynInstPtr &inst)
 411 {
 412     using namespace TheISA;
 413     // Execute a specific load.
 414     Fault load_fault = NoFault;
 415
 416     DPRINTF(LSQUnit, "Executing load PC %#x, [sn:%lli]\n",
 417             inst->readPC(),inst->seqNum);
 418
 419     assert(!inst->isSquashed());
 420
 421     load_fault = inst->initiateAcc();
 422
 423     // If the instruction faulted, then we need to send it along to commit
 424     // without the instruction completing.
 425     if (load_fault != NoFault) {
 426         // Send this instruction to commit, also make sure iew stage
 427         // realizes there is activity.
 428         // Mark it as executed unless it is an uncached load that
 429         // needs to hit the head of commit.
 430         if (!(inst->hasRequest() && inst->uncacheable()) ||
 431             inst->isAtCommit()) {
 432             inst->setExecuted();
 433         }
 434         iewStage->instToCommit(inst);
 435         iewStage->activityThisCycle();
 436     } else if (!loadBlocked()) {
 437         assert(inst->effAddrValid);
 438         int load_idx = inst->lqIdx;
 439         incrLdIdx(load_idx);
 440         while (load_idx != loadTail) {
 441             // Really only need to check loads that have actually executed
 442
 443             // @todo: For now this is extra conservative, detecting a
 444             // violation if the addresses match assuming all accesses
 445             // are quad word accesses.
 446
 447             // @todo: Fix this, magic number being used here
 448             if (loadQueue[load_idx]->effAddrValid &&
 449                 (loadQueue[load_idx]->effAddr >> 8) ==
 450                 (inst->effAddr >> 8)) {
 451                 // A load incorrectly passed this load.  Squash and refetch.
 452                 // For now return a fault to show that it was unsuccessful.
 453                 DynInstPtr violator = loadQueue[load_idx];
 454                 if (!memDepViolator ||
 455                     (violator->seqNum < memDepViolator->seqNum)) {
 456                     memDepViolator = violator;
 457                 } else {
 458                     break;
 459                 }
 460
 461                 ++lsqMemOrderViolation;
 462
 463                 return genMachineCheckFault();
 464             }
 465
 466             incrLdIdx(load_idx);
 467         }
 468     }
 469
 470     return load_fault;
 471 }
 472
 473 template <class Impl>
 474 Fault
 475 LSQUnit<Impl>::executeStore(DynInstPtr &store_inst)
 476 {
 477     using namespace TheISA;
 478     // Make sure that a store exists.
 479     assert(stores != 0);
 480
 481     int store_idx = store_inst->sqIdx;
 482
 483     DPRINTF(LSQUnit, "Executing store PC %#x [sn:%lli]\n",
 484             store_inst->readPC(), store_inst->seqNum);
 485
 486     assert(!store_inst->isSquashed());
 487
 488     // Check the recently completed loads to see if any match this store's
 489     // address.  If so, then we have a memory ordering violation.
 490     int load_idx = store_inst->lqIdx;
 491
 492     Fault store_fault = store_inst->initiateAcc();
 493
 494     if (storeQueue[store_idx].size == 0) {
 495         DPRINTF(LSQUnit,"Fault on Store PC %#x, [sn:%lli],Size = 0\n",
 496                 store_inst->readPC(),store_inst->seqNum);
 497
 498         return store_fault;
 499     }
 500
 501     assert(store_fault == NoFault);
 502
 503     if (store_inst->isStoreConditional()) {
 504         // Store conditionals need to set themselves as able to
 505         // writeback if we haven't had a fault by here.
 506         storeQueue[store_idx].canWB = true;
 507
 508         ++storesToWB;
 509     }
 510
 511     assert(store_inst->effAddrValid);
 512     while (load_idx != loadTail) {
 513         // Really only need to check loads that have actually executed
 514         // It's safe to check all loads because effAddr is set to
 515         // InvalAddr when the dyn inst is created.
 516
 517         // @todo: For now this is extra conservative, detecting a
 518         // violation if the addresses match assuming all accesses
 519         // are quad word accesses.
 520
 521         // @todo: Fix this, magic number being used here
 522         if (loadQueue[load_idx]->effAddrValid &&
 523             (loadQueue[load_idx]->effAddr >> 8) ==
 524             (store_inst->effAddr >> 8)) {
 525             // A load incorrectly passed this store.  Squash and refetch.
 526             // For now return a fault to show that it was unsuccessful.
 527             DynInstPtr violator = loadQueue[load_idx];
 528             if (!memDepViolator ||
 529                 (violator->seqNum < memDepViolator->seqNum)) {
 530                 memDepViolator = violator;
 531             } else {
 532                 break;
 533             }
 534
 535             ++lsqMemOrderViolation;
 536
 537             return genMachineCheckFault();
 538         }
 539
 540         incrLdIdx(load_idx);
 541     }
 542
 543     return store_fault;
 544 }
 545
 546 template <class Impl>
 547 void
 548 LSQUnit<Impl>::commitLoad()
 549 {
 550     assert(loadQueue[loadHead]);
 551
 552     DPRINTF(LSQUnit, "Committing head load instruction, PC %#x\n",
 553             loadQueue[loadHead]->readPC());
 554
 555     loadQueue[loadHead] = NULL;
 556
 557     incrLdIdx(loadHead);
 558
 559     --loads;
 560 }
 561
 562 template <class Impl>
 563 void
 564 LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst)
 565 {
 566     assert(loads == 0 || loadQueue[loadHead]);
 567
 568     while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
 569         commitLoad();
 570     }
 571 }
 572
 573 template <class Impl>
 574 void
 575 LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst)
 576 {
 577     assert(stores == 0 || storeQueue[storeHead].inst);
 578
 579     int store_idx = storeHead;
 580
 581     while (store_idx != storeTail) {
 582         assert(storeQueue[store_idx].inst);
 583         // Mark any stores that are now committed and have not yet
 584         // been marked as able to write back.
 585         if (!storeQueue[store_idx].canWB) {
 586             if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
 587                 break;
 588             }
 589             DPRINTF(LSQUnit, "Marking store as able to write back, PC "
 590                     "%#x [sn:%lli]\n",
 591                     storeQueue[store_idx].inst->readPC(),
 592                     storeQueue[store_idx].inst->seqNum);
 593
 594             storeQueue[store_idx].canWB = true;
 595
 596             ++storesToWB;
 597         }
 598
 599         incrStIdx(store_idx);
 600     }
 601 }
 602
 603 template <class Impl>
 604 void
 605 LSQUnit<Impl>::writebackStores()
 606 {
 607     while (storesToWB > 0 &&
 608            storeWBIdx != storeTail &&
 609            storeQueue[storeWBIdx].inst &&
 610            storeQueue[storeWBIdx].canWB &&
 611            usedPorts < cachePorts) {
 612
 613         if (isStoreBlocked || lsq->cacheBlocked()) {
 614             DPRINTF(LSQUnit, "Unable to write back any more stores, cache"
 615                     " is blocked!\n");
 616             break;
 617         }
 618
 619         // Store didn't write any data so no need to write it back to
 620         // memory.
 621         if (storeQueue[storeWBIdx].size == 0) {
 622             completeStore(storeWBIdx);
 623
 624             incrStIdx(storeWBIdx);
 625
 626             continue;
 627         }
 628
 629         ++usedPorts;
 630
 631         if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
 632             incrStIdx(storeWBIdx);
 633
 634             continue;
 635         }
 636
 637         assert(storeQueue[storeWBIdx].req);
 638         assert(!storeQueue[storeWBIdx].committed);
 639
 640         DynInstPtr inst = storeQueue[storeWBIdx].inst;
 641
 642         Request *req = storeQueue[storeWBIdx].req;
 643         storeQueue[storeWBIdx].committed = true;
 644
 645         assert(!inst->memData);
 646         inst->memData = new uint8_t[64];
 647
 648         memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize());
 649
 650         MemCmd command = req->isSwap() ? MemCmd::SwapReq : MemCmd::WriteReq;
 651         PacketPtr data_pkt = new Packet(req, command,
 652                                         Packet::Broadcast);
 653         data_pkt->dataStatic(inst->memData);
 654
 655         LSQSenderState *state = new LSQSenderState;
 656         state->isLoad = false;
 657         state->idx = storeWBIdx;
 658         state->inst = inst;
 659         data_pkt->senderState = state;
 660
 661         DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%#x "
 662                 "to Addr:%#x, data:%#x [sn:%lli]\n",
 663                 storeWBIdx, inst->readPC(),
 664                 req->getPaddr(), (int)*(inst->memData),
 665                 inst->seqNum);
 666
 667         // @todo: Remove this SC hack once the memory system handles it.
 668         if (inst->isStoreConditional()) {
 669             // Disable recording the result temporarily.  Writing to
 670             // misc regs normally updates the result, but this is not
 671             // the desired behavior when handling store conditionals.
 672             inst->recordResult = false;
 673             bool success = TheISA::handleLockedWrite(inst.get(), req);
 674             inst->recordResult = true;
 675
 676             if (!success) {
 677                 // Instantly complete this store.
 678                 DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed.  "
 679                         "Instantly completing it.\n",
 680                         inst->seqNum);
 681                 WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
 682                 wb->schedule(curTick + 1);
 683                 delete state;
 684                 completeStore(storeWBIdx);
 685                 incrStIdx(storeWBIdx);
 686                 continue;
 687             }
 688         } else {
 689             // Non-store conditionals do not need a writeback.
 690             state->noWB = true;
 691         }
 692
 693         if (!dcachePort->sendTiming(data_pkt)) {
 694             if (data_pkt->result == Packet::BadAddress) {
 695                 panic("LSQ sent out a bad address for a completed store!");
 696             }
 697             // Need to handle becoming blocked on a store.
 698             DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will"
 699                     "retry later\n",
 700                     inst->seqNum);
 701             isStoreBlocked = true;
 702             ++lsqCacheBlocked;
 703             assert(retryPkt == NULL);
 704             retryPkt = data_pkt;
 705             lsq->setRetryTid(lsqID);
 706         } else {
 707             storePostSend(data_pkt);
 708         }
 709     }
 710
 711     // Not sure this should set it to 0.
 712     usedPorts = 0;
 713
 714     assert(stores >= 0 && storesToWB >= 0);
 715 }
 716
 717 /*template <class Impl>
 718 void
 719 LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum)
 720 {
 721     list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
 722                                               mshrSeqNums.end(),
 723                                               seqNum);
 724
 725     if (mshr_it != mshrSeqNums.end()) {
 726         mshrSeqNums.erase(mshr_it);
 727         DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size());
 728     }
 729 }*/
 730
 731 template <class Impl>
 732 void
 733 LSQUnit<Impl>::squash(const InstSeqNum &squashed_num)
 734 {
 735     DPRINTF(LSQUnit, "Squashing until [sn:%lli]!"
 736             "(Loads:%i Stores:%i)\n", squashed_num, loads, stores);
 737
 738     int load_idx = loadTail;
 739     decrLdIdx(load_idx);
 740
 741     while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
 742         DPRINTF(LSQUnit,"Load Instruction PC %#x squashed, "
 743                 "[sn:%lli]\n",
 744                 loadQueue[load_idx]->readPC(),
 745                 loadQueue[load_idx]->seqNum);
 746
 747         if (isStalled() && load_idx == stallingLoadIdx) {
 748             stalled = false;
 749             stallingStoreIsn = 0;
 750             stallingLoadIdx = 0;
 751         }
 752
 753         // Clear the smart pointer to make sure it is decremented.
 754         loadQueue[load_idx]->setSquashed();
 755         loadQueue[load_idx] = NULL;
 756         --loads;
 757
 758         // Inefficient!
 759         loadTail = load_idx;
 760
 761         decrLdIdx(load_idx);
 762         ++lsqSquashedLoads;
 763     }
 764
 765     if (isLoadBlocked) {
 766         if (squashed_num < blockedLoadSeqNum) {
 767             isLoadBlocked = false;
 768             loadBlockedHandled = false;
 769             blockedLoadSeqNum = 0;
 770         }
 771     }
 772
 773     if (memDepViolator && squashed_num < memDepViolator->seqNum) {
 774         memDepViolator = NULL;
 775     }
 776
 777     int store_idx = storeTail;
 778     decrStIdx(store_idx);
 779
 780     while (stores != 0 &&
 781            storeQueue[store_idx].inst->seqNum > squashed_num) {
 782         // Instructions marked as can WB are already committed.
 783         if (storeQueue[store_idx].canWB) {
 784             break;
 785         }
 786
 787         DPRINTF(LSQUnit,"Store Instruction PC %#x squashed, "
 788                 "idx:%i [sn:%lli]\n",
 789                 storeQueue[store_idx].inst->readPC(),
 790                 store_idx, storeQueue[store_idx].inst->seqNum);
 791
 792         // I don't think this can happen.  It should have been cleared
 793         // by the stalling load.
 794         if (isStalled() &&
 795             storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
 796             panic("Is stalled should have been cleared by stalling load!\n");
 797             stalled = false;
 798             stallingStoreIsn = 0;
 799         }
 800
 801         // Clear the smart pointer to make sure it is decremented.
 802         storeQueue[store_idx].inst->setSquashed();
 803         storeQueue[store_idx].inst = NULL;
 804         storeQueue[store_idx].canWB = 0;
 805
 806         // Must delete request now that it wasn't handed off to
 807         // memory.  This is quite ugly.  @todo: Figure out the proper
 808         // place to really handle request deletes.
 809         delete storeQueue[store_idx].req;
 810
 811         storeQueue[store_idx].req = NULL;
 812         --stores;
 813
 814         // Inefficient!
 815         storeTail = store_idx;
 816
 817         decrStIdx(store_idx);
 818         ++lsqSquashedStores;
 819     }
 820 }
 821
 822 template <class Impl>
 823 void
 824 LSQUnit<Impl>::storePostSend(PacketPtr pkt)
 825 {
 826     if (isStalled() &&
 827         storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
 828         DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
 829                 "load idx:%i\n",
 830                 stallingStoreIsn, stallingLoadIdx);
 831         stalled = false;
 832         stallingStoreIsn = 0;
 833         iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
 834     }
 835
 836     if (!storeQueue[storeWBIdx].inst->isStoreConditional()) {
 837         // The store is basically completed at this time. This
 838         // only works so long as the checker doesn't try to
 839         // verify the value in memory for stores.
 840         storeQueue[storeWBIdx].inst->setCompleted();
 841 #if USE_CHECKER
 842         if (cpu->checker) {
 843             cpu->checker->verify(storeQueue[storeWBIdx].inst);
 844         }
 845 #endif
 846     }
 847
 848     if (pkt->result != Packet::Success) {
 849         DPRINTF(LSQUnit,"D-Cache Write Miss on idx:%i!\n",
 850                 storeWBIdx);
 851
 852         DPRINTF(Activity, "Active st accessing mem miss [sn:%lli]\n",
 853                 storeQueue[storeWBIdx].inst->seqNum);
 854
 855         //mshrSeqNums.push_back(storeQueue[storeWBIdx].inst->seqNum);
 856
 857         //DPRINTF(LSQUnit, "Added MSHR. count = %i\n",mshrSeqNums.size());
 858
 859         // @todo: Increment stat here.
 860     } else {
 861         DPRINTF(LSQUnit,"D-Cache: Write Hit on idx:%i !\n",
 862                 storeWBIdx);
 863
 864         DPRINTF(Activity, "Active st accessing mem hit [sn:%lli]\n",
 865                 storeQueue[storeWBIdx].inst->seqNum);
 866     }
 867
 868     incrStIdx(storeWBIdx);
 869 }
 870
 871 template <class Impl>
 872 void
 873 LSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt)
 874 {
 875     iewStage->wakeCPU();
 876
 877     // Squashed instructions do not need to complete their access.
 878     if (inst->isSquashed()) {
 879         iewStage->decrWb(inst->seqNum);
 880         assert(!inst->isStore());
 881         ++lsqIgnoredResponses;
 882         return;
 883     }
 884
 885     if (!inst->isExecuted()) {
 886         inst->setExecuted();
 887
 888         // Complete access to copy data to proper place.
 889         inst->completeAcc(pkt);
 890     }
 891
 892     // Need to insert instruction into queue to commit
 893     iewStage->instToCommit(inst);
 894
 895     iewStage->activityThisCycle();
 896 }
 897
 898 template <class Impl>
 899 void
 900 LSQUnit<Impl>::completeStore(int store_idx)
 901 {
 902     assert(storeQueue[store_idx].inst);
 903     storeQueue[store_idx].completed = true;
 904     --storesToWB;
 905     // A bit conservative because a store completion may not free up entries,
 906     // but hopefully avoids two store completions in one cycle from making
 907     // the CPU tick twice.
 908     cpu->wakeCPU();
 909     cpu->activityThisCycle();
 910
 911     if (store_idx == storeHead) {
 912         do {
 913             incrStIdx(storeHead);
 914
 915             --stores;
 916         } while (storeQueue[storeHead].completed &&
 917                  storeHead != storeTail);
 918
 919         iewStage->updateLSQNextCycle = true;
 920     }
 921
 922     DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head "
 923             "idx:%i\n",
 924             storeQueue[store_idx].inst->seqNum, store_idx, storeHead);
 925
 926     if (isStalled() &&
 927         storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
 928         DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
 929                 "load idx:%i\n",
 930                 stallingStoreIsn, stallingLoadIdx);
 931         stalled = false;
 932         stallingStoreIsn = 0;
 933         iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
 934     }
 935
 936     storeQueue[store_idx].inst->setCompleted();
 937
 938     // Tell the checker we've completed this instruction.  Some stores
 939     // may get reported twice to the checker, but the checker can
 940     // handle that case.
 941 #if USE_CHECKER
 942     if (cpu->checker) {
 943         cpu->checker->verify(storeQueue[store_idx].inst);
 944     }
 945 #endif
 946 }
 947
 948 template <class Impl>
 949 void
 950 LSQUnit<Impl>::recvRetry()
 951 {
 952     if (isStoreBlocked) {
 953         assert(retryPkt != NULL);
 954
 955         if (dcachePort->sendTiming(retryPkt)) {
 956             if (retryPkt->result == Packet::BadAddress) {
 957                 panic("LSQ sent out a bad address for a completed store!");
 958             }
 959             storePostSend(retryPkt);
 960             retryPkt = NULL;
 961             isStoreBlocked = false;
 962             lsq->setRetryTid(-1);
 963         } else {
 964             // Still blocked!
 965             ++lsqCacheBlocked;
 966             lsq->setRetryTid(lsqID);
 967         }
 968     } else if (isLoadBlocked) {
 969         DPRINTF(LSQUnit, "Loads squash themselves and all younger insts, "
 970                 "no need to resend packet.\n");
 971     } else {
 972         DPRINTF(LSQUnit, "Retry received but LSQ is no longer blocked.\n");
 973     }
 974 }
 975
 976 template <class Impl>
 977 inline void
 978 LSQUnit<Impl>::incrStIdx(int &store_idx)
 979 {
 980     if (++store_idx >= SQEntries)
 981         store_idx = 0;
 982 }
 983
 984 template <class Impl>
 985 inline void
 986 LSQUnit<Impl>::decrStIdx(int &store_idx)
 987 {
 988     if (--store_idx < 0)
 989         store_idx += SQEntries;
 990 }
 991
 992 template <class Impl>
 993 inline void
 994 LSQUnit<Impl>::incrLdIdx(int &load_idx)
 995 {
 996     if (++load_idx >= LQEntries)
 997         load_idx = 0;
 998 }
 999
1000 template <class Impl>
1001 inline void
1002 LSQUnit<Impl>::decrLdIdx(int &load_idx)
1003 {
1004     if (--load_idx < 0)
1005         load_idx += LQEntries;
1006 }
1007
1008 template <class Impl>
1009 void
1010 LSQUnit<Impl>::dumpInsts()
1011 {
1012     cprintf("Load store queue: Dumping instructions.\n");
1013     cprintf("Load queue size: %i\n", loads);
1014     cprintf("Load queue: ");
1015
1016     int load_idx = loadHead;
1017
1018     while (load_idx != loadTail && loadQueue[load_idx]) {
1019         cprintf("%#x ", loadQueue[load_idx]->readPC());
1020
1021         incrLdIdx(load_idx);
1022     }
1023
1024     cprintf("Store queue size: %i\n", stores);
1025     cprintf("Store queue: ");
1026
1027     int store_idx = storeHead;
1028
1029     while (store_idx != storeTail && storeQueue[store_idx].inst) {
1030         cprintf("%#x ", storeQueue[store_idx].inst->readPC());
1031
1032         incrStIdx(store_idx);
1033     }
1034
1035     cprintf("\n");
1036 }