src/cpu/o3/lsq_unit_impl.hh

   1 /*
   2  * Copyright (c) 2010 ARM Limited
   3  * All rights reserved
   4  *
   5  * The license below extends only to copyright in the software and shall
   6  * not be construed as granting a license to any other intellectual
   7  * property including but not limited to intellectual property relating
   8  * to a hardware implementation of the functionality of the software
   9  * licensed hereunder.  You may use the software subject to the license
  10  * terms below provided that you ensure that this notice is replicated
  11  * unmodified and in its entirety in all distributions of the software,
  12  * modified or unmodified, in source code or in binary form.
  13  *
  14  * Copyright (c) 2004-2005 The Regents of The University of Michigan
  15  * All rights reserved.
  16  *
  17  * Redistribution and use in source and binary forms, with or without
  18  * modification, are permitted provided that the following conditions are
  19  * met: redistributions of source code must retain the above copyright
  20  * notice, this list of conditions and the following disclaimer;
  21  * redistributions in binary form must reproduce the above copyright
  22  * notice, this list of conditions and the following disclaimer in the
  23  * documentation and/or other materials provided with the distribution;
  24  * neither the name of the copyright holders nor the names of its
  25  * contributors may be used to endorse or promote products derived from
  26  * this software without specific prior written permission.
  27  *
  28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  33  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  34  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  35  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  36  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  37  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  38  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  39  *
  40  * Authors: Kevin Lim
  41  *          Korey Sewell
  42  */
  43
  44 #include "arch/generic/debugfaults.hh"
  45 #include "arch/locked_mem.hh"
  46 #include "base/str.hh"
  47 #include "config/the_isa.hh"
  48 #include "config/use_checker.hh"
  49 #include "cpu/o3/lsq.hh"
  50 #include "cpu/o3/lsq_unit.hh"
  51 #include "debug/Activity.hh"
  52 #include "debug/IEW.hh"
  53 #include "debug/LSQUnit.hh"
  54 #include "mem/packet.hh"
  55 #include "mem/request.hh"
  56
  57 #if USE_CHECKER
  58 #include "cpu/checker/cpu.hh"
  59 #endif
  60
  61 template<class Impl>
  62 LSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt,
  63                                               LSQUnit *lsq_ptr)
  64     : Event(Default_Pri, AutoDelete),
  65       inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
  66 {
  67 }
  68
  69 template<class Impl>
  70 void
  71 LSQUnit<Impl>::WritebackEvent::process()
  72 {
  73     if (!lsqPtr->isSwitchedOut()) {
  74         lsqPtr->writeback(inst, pkt);
  75     }
  76
  77     if (pkt->senderState)
  78         delete pkt->senderState;
  79
  80     delete pkt->req;
  81     delete pkt;
  82 }
  83
  84 template<class Impl>
  85 const char *
  86 LSQUnit<Impl>::WritebackEvent::description() const
  87 {
  88     return "Store writeback";
  89 }
  90
  91 template<class Impl>
  92 void
  93 LSQUnit<Impl>::completeDataAccess(PacketPtr pkt)
  94 {
  95     LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState);
  96     DynInstPtr inst = state->inst;
  97     DPRINTF(IEW, "Writeback event [sn:%lli].\n", inst->seqNum);
  98     DPRINTF(Activity, "Activity: Writeback event [sn:%lli].\n", inst->seqNum);
  99
 100     //iewStage->ldstQueue.removeMSHR(inst->threadNumber,inst->seqNum);
 101
 102     assert(!pkt->wasNacked());
 103
 104     // If this is a split access, wait until all packets are received.
 105     if (TheISA::HasUnalignedMemAcc && !state->complete()) {
 106         delete pkt->req;
 107         delete pkt;
 108         return;
 109     }
 110
 111     if (isSwitchedOut() || inst->isSquashed()) {
 112         iewStage->decrWb(inst->seqNum);
 113     } else {
 114         if (!state->noWB) {
 115             if (!TheISA::HasUnalignedMemAcc || !state->isSplit ||
 116                 !state->isLoad) {
 117                 writeback(inst, pkt);
 118             } else {
 119                 writeback(inst, state->mainPkt);
 120             }
 121         }
 122
 123         if (inst->isStore()) {
 124             completeStore(state->idx);
 125         }
 126     }
 127
 128     if (TheISA::HasUnalignedMemAcc && state->isSplit && state->isLoad) {
 129         delete state->mainPkt->req;
 130         delete state->mainPkt;
 131     }
 132     delete state;
 133     delete pkt->req;
 134     delete pkt;
 135 }
 136
 137 template <class Impl>
 138 LSQUnit<Impl>::LSQUnit()
 139     : loads(0), stores(0), storesToWB(0), cacheBlockMask(0), stalled(false),
 140       isStoreBlocked(false), isLoadBlocked(false),
 141       loadBlockedHandled(false), hasPendingPkt(false)
 142 {
 143 }
 144
 145 template<class Impl>
 146 void
 147 LSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params,
 148         LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries,
 149         unsigned id)
 150 {
 151     cpu = cpu_ptr;
 152     iewStage = iew_ptr;
 153
 154     DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id);
 155
 156     switchedOut = false;
 157
 158     cacheBlockMask = 0;
 159
 160     lsq = lsq_ptr;
 161
 162     lsqID = id;
 163
 164     // Add 1 for the sentinel entry (they are circular queues).
 165     LQEntries = maxLQEntries + 1;
 166     SQEntries = maxSQEntries + 1;
 167
 168     loadQueue.resize(LQEntries);
 169     storeQueue.resize(SQEntries);
 170
 171     depCheckShift = params->LSQDepCheckShift;
 172     checkLoads = params->LSQCheckLoads;
 173
 174     loadHead = loadTail = 0;
 175
 176     storeHead = storeWBIdx = storeTail = 0;
 177
 178     usedPorts = 0;
 179     cachePorts = params->cachePorts;
 180
 181     retryPkt = NULL;
 182     memDepViolator = NULL;
 183
 184     blockedLoadSeqNum = 0;
 185 }
 186
 187 template<class Impl>
 188 std::string
 189 LSQUnit<Impl>::name() const
 190 {
 191     if (Impl::MaxThreads == 1) {
 192         return iewStage->name() + ".lsq";
 193     } else {
 194         return iewStage->name() + ".lsq.thread" + to_string(lsqID);
 195     }
 196 }
 197
 198 template<class Impl>
 199 void
 200 LSQUnit<Impl>::regStats()
 201 {
 202     lsqForwLoads
 203         .name(name() + ".forwLoads")
 204         .desc("Number of loads that had data forwarded from stores");
 205
 206     invAddrLoads
 207         .name(name() + ".invAddrLoads")
 208         .desc("Number of loads ignored due to an invalid address");
 209
 210     lsqSquashedLoads
 211         .name(name() + ".squashedLoads")
 212         .desc("Number of loads squashed");
 213
 214     lsqIgnoredResponses
 215         .name(name() + ".ignoredResponses")
 216         .desc("Number of memory responses ignored because the instruction is squashed");
 217
 218     lsqMemOrderViolation
 219         .name(name() + ".memOrderViolation")
 220         .desc("Number of memory ordering violations");
 221
 222     lsqSquashedStores
 223         .name(name() + ".squashedStores")
 224         .desc("Number of stores squashed");
 225
 226     invAddrSwpfs
 227         .name(name() + ".invAddrSwpfs")
 228         .desc("Number of software prefetches ignored due to an invalid address");
 229
 230     lsqBlockedLoads
 231         .name(name() + ".blockedLoads")
 232         .desc("Number of blocked loads due to partial load-store forwarding");
 233
 234     lsqRescheduledLoads
 235         .name(name() + ".rescheduledLoads")
 236         .desc("Number of loads that were rescheduled");
 237
 238     lsqCacheBlocked
 239         .name(name() + ".cacheBlocked")
 240         .desc("Number of times an access to memory failed due to the cache being blocked");
 241 }
 242
 243 template<class Impl>
 244 void
 245 LSQUnit<Impl>::setDcachePort(Port *dcache_port)
 246 {
 247     dcachePort = dcache_port;
 248
 249 #if USE_CHECKER
 250     if (cpu->checker) {
 251         cpu->checker->setDcachePort(dcachePort);
 252     }
 253 #endif
 254 }
 255
 256 template<class Impl>
 257 void
 258 LSQUnit<Impl>::clearLQ()
 259 {
 260     loadQueue.clear();
 261 }
 262
 263 template<class Impl>
 264 void
 265 LSQUnit<Impl>::clearSQ()
 266 {
 267     storeQueue.clear();
 268 }
 269
 270 template<class Impl>
 271 void
 272 LSQUnit<Impl>::switchOut()
 273 {
 274     switchedOut = true;
 275     for (int i = 0; i < loadQueue.size(); ++i) {
 276         assert(!loadQueue[i]);
 277         loadQueue[i] = NULL;
 278     }
 279
 280     assert(storesToWB == 0);
 281 }
 282
 283 template<class Impl>
 284 void
 285 LSQUnit<Impl>::takeOverFrom()
 286 {
 287     switchedOut = false;
 288     loads = stores = storesToWB = 0;
 289
 290     loadHead = loadTail = 0;
 291
 292     storeHead = storeWBIdx = storeTail = 0;
 293
 294     usedPorts = 0;
 295
 296     memDepViolator = NULL;
 297
 298     blockedLoadSeqNum = 0;
 299
 300     stalled = false;
 301     isLoadBlocked = false;
 302     loadBlockedHandled = false;
 303
 304     // Just incase the memory system changed out from under us
 305     cacheBlockMask = 0;
 306 }
 307
 308 template<class Impl>
 309 void
 310 LSQUnit<Impl>::resizeLQ(unsigned size)
 311 {
 312     unsigned size_plus_sentinel = size + 1;
 313     assert(size_plus_sentinel >= LQEntries);
 314
 315     if (size_plus_sentinel > LQEntries) {
 316         while (size_plus_sentinel > loadQueue.size()) {
 317             DynInstPtr dummy;
 318             loadQueue.push_back(dummy);
 319             LQEntries++;
 320         }
 321     } else {
 322         LQEntries = size_plus_sentinel;
 323     }
 324
 325 }
 326
 327 template<class Impl>
 328 void
 329 LSQUnit<Impl>::resizeSQ(unsigned size)
 330 {
 331     unsigned size_plus_sentinel = size + 1;
 332     if (size_plus_sentinel > SQEntries) {
 333         while (size_plus_sentinel > storeQueue.size()) {
 334             SQEntry dummy;
 335             storeQueue.push_back(dummy);
 336             SQEntries++;
 337         }
 338     } else {
 339         SQEntries = size_plus_sentinel;
 340     }
 341 }
 342
 343 template <class Impl>
 344 void
 345 LSQUnit<Impl>::insert(DynInstPtr &inst)
 346 {
 347     assert(inst->isMemRef());
 348
 349     assert(inst->isLoad() || inst->isStore());
 350
 351     if (inst->isLoad()) {
 352         insertLoad(inst);
 353     } else {
 354         insertStore(inst);
 355     }
 356
 357     inst->setInLSQ();
 358 }
 359
 360 template <class Impl>
 361 void
 362 LSQUnit<Impl>::insertLoad(DynInstPtr &load_inst)
 363 {
 364     assert((loadTail + 1) % LQEntries != loadHead);
 365     assert(loads < LQEntries);
 366
 367     DPRINTF(LSQUnit, "Inserting load PC %s, idx:%i [sn:%lli]\n",
 368             load_inst->pcState(), loadTail, load_inst->seqNum);
 369
 370     load_inst->lqIdx = loadTail;
 371
 372     if (stores == 0) {
 373         load_inst->sqIdx = -1;
 374     } else {
 375         load_inst->sqIdx = storeTail;
 376     }
 377
 378     loadQueue[loadTail] = load_inst;
 379
 380     incrLdIdx(loadTail);
 381
 382     ++loads;
 383 }
 384
 385 template <class Impl>
 386 void
 387 LSQUnit<Impl>::insertStore(DynInstPtr &store_inst)
 388 {
 389     // Make sure it is not full before inserting an instruction.
 390     assert((storeTail + 1) % SQEntries != storeHead);
 391     assert(stores < SQEntries);
 392
 393     DPRINTF(LSQUnit, "Inserting store PC %s, idx:%i [sn:%lli]\n",
 394             store_inst->pcState(), storeTail, store_inst->seqNum);
 395
 396     store_inst->sqIdx = storeTail;
 397     store_inst->lqIdx = loadTail;
 398
 399     storeQueue[storeTail] = SQEntry(store_inst);
 400
 401     incrStIdx(storeTail);
 402
 403     ++stores;
 404 }
 405
 406 template <class Impl>
 407 typename Impl::DynInstPtr
 408 LSQUnit<Impl>::getMemDepViolator()
 409 {
 410     DynInstPtr temp = memDepViolator;
 411
 412     memDepViolator = NULL;
 413
 414     return temp;
 415 }
 416
 417 template <class Impl>
 418 unsigned
 419 LSQUnit<Impl>::numFreeEntries()
 420 {
 421     unsigned free_lq_entries = LQEntries - loads;
 422     unsigned free_sq_entries = SQEntries - stores;
 423
 424     // Both the LQ and SQ entries have an extra dummy entry to differentiate
 425     // empty/full conditions.  Subtract 1 from the free entries.
 426     if (free_lq_entries < free_sq_entries) {
 427         return free_lq_entries - 1;
 428     } else {
 429         return free_sq_entries - 1;
 430     }
 431 }
 432
 433 template <class Impl>
 434 int
 435 LSQUnit<Impl>::numLoadsReady()
 436 {
 437     int load_idx = loadHead;
 438     int retval = 0;
 439
 440     while (load_idx != loadTail) {
 441         assert(loadQueue[load_idx]);
 442
 443         if (loadQueue[load_idx]->readyToIssue()) {
 444             ++retval;
 445         }
 446     }
 447
 448     return retval;
 449 }
 450
 451 template <class Impl>
 452 void
 453 LSQUnit<Impl>::checkSnoop(PacketPtr pkt)
 454 {
 455     int load_idx = loadHead;
 456
 457     if (!cacheBlockMask) {
 458         assert(dcachePort);
 459         Addr bs = dcachePort->peerBlockSize();
 460
 461         // Make sure we actually got a size
 462         assert(bs != 0);
 463
 464         cacheBlockMask = ~(bs - 1);
 465     }
 466
 467     // If this is the only load in the LSQ we don't care
 468     if (load_idx == loadTail)
 469         return;
 470     incrLdIdx(load_idx);
 471
 472     DPRINTF(LSQUnit, "Got snoop for address %#x\n", pkt->getAddr());
 473     Addr invalidate_addr = pkt->getAddr() & cacheBlockMask;
 474     while (load_idx != loadTail) {
 475         DynInstPtr ld_inst = loadQueue[load_idx];
 476
 477         if (!ld_inst->effAddrValid || ld_inst->uncacheable()) {
 478             incrLdIdx(load_idx);
 479             continue;
 480         }
 481
 482         Addr load_addr = ld_inst->physEffAddr & cacheBlockMask;
 483         DPRINTF(LSQUnit, "-- inst [sn:%lli] load_addr: %#x to pktAddr:%#x\n",
 484                     ld_inst->seqNum, load_addr, invalidate_addr);
 485
 486         if (load_addr == invalidate_addr) {
 487             if (ld_inst->possibleLoadViolation) {
 488                 DPRINTF(LSQUnit, "Conflicting load at addr %#x [sn:%lli]\n",
 489                         ld_inst->physEffAddr, pkt->getAddr(), ld_inst->seqNum);
 490
 491                 // Mark the load for re-execution
 492                 ld_inst->fault = new ReExec;
 493             } else {
 494                 // If a older load checks this and it's true
 495                 // then we might have missed the snoop
 496                 // in which case we need to invalidate to be sure
 497                 ld_inst->hitExternalSnoop = true;
 498             }
 499         }
 500         incrLdIdx(load_idx);
 501     }
 502     return;
 503 }
 504
 505 template <class Impl>
 506 Fault
 507 LSQUnit<Impl>::checkViolations(int load_idx, DynInstPtr &inst)
 508 {
 509     Addr inst_eff_addr1 = inst->effAddr >> depCheckShift;
 510     Addr inst_eff_addr2 = (inst->effAddr + inst->effSize - 1) >> depCheckShift;
 511
 512     /** @todo in theory you only need to check an instruction that has executed
 513      * however, there isn't a good way in the pipeline at the moment to check
 514      * all instructions that will execute before the store writes back. Thus,
 515      * like the implementation that came before it, we're overly conservative.
 516      */
 517     while (load_idx != loadTail) {
 518         DynInstPtr ld_inst = loadQueue[load_idx];
 519         if (!ld_inst->effAddrValid || ld_inst->uncacheable()) {
 520             incrLdIdx(load_idx);
 521             continue;
 522         }
 523
 524         Addr ld_eff_addr1 = ld_inst->effAddr >> depCheckShift;
 525         Addr ld_eff_addr2 =
 526             (ld_inst->effAddr + ld_inst->effSize - 1) >> depCheckShift;
 527
 528         if (inst_eff_addr2 >= ld_eff_addr1 && inst_eff_addr1 <= ld_eff_addr2) {
 529             if (inst->isLoad()) {
 530                 // If this load is to the same block as an external snoop
 531                 // invalidate that we've observed then the load needs to be
 532                 // squashed as it could have newer data
 533                 if (ld_inst->hitExternalSnoop) {
 534                     if (!memDepViolator ||
 535                             ld_inst->seqNum < memDepViolator->seqNum) {
 536                         DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] "
 537                                 "and [sn:%lli] at address %#x\n",
 538                                 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
 539                         memDepViolator = ld_inst;
 540
 541                         ++lsqMemOrderViolation;
 542
 543                         return new GenericISA::M5PanicFault(
 544                                 "Detected fault with inst [sn:%lli] and "
 545                                 "[sn:%lli] at address %#x\n",
 546                                 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
 547                     }
 548                 }
 549
 550                 // Otherwise, mark the load has a possible load violation
 551                 // and if we see a snoop before it's commited, we need to squash
 552                 ld_inst->possibleLoadViolation = true;
 553                 DPRINTF(LSQUnit, "Found possible load violaiton at addr: %#x"
 554                         " between instructions [sn:%lli] and [sn:%lli]\n",
 555                         inst_eff_addr1, inst->seqNum, ld_inst->seqNum);
 556             } else {
 557                 // A load/store incorrectly passed this store.
 558                 // Check if we already have a violator, or if it's newer
 559                 // squash and refetch.
 560                 if (memDepViolator && ld_inst->seqNum > memDepViolator->seqNum)
 561                     break;
 562
 563                 DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] and "
 564                         "[sn:%lli] at address %#x\n",
 565                         inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
 566                 memDepViolator = ld_inst;
 567
 568                 ++lsqMemOrderViolation;
 569
 570                 return new GenericISA::M5PanicFault("Detected fault with "
 571                         "inst [sn:%lli] and [sn:%lli] at address %#x\n",
 572                         inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
 573             }
 574         }
 575
 576         incrLdIdx(load_idx);
 577     }
 578     return NoFault;
 579 }
 580
 581
 582
 583
 584 template <class Impl>
 585 Fault
 586 LSQUnit<Impl>::executeLoad(DynInstPtr &inst)
 587 {
 588     using namespace TheISA;
 589     // Execute a specific load.
 590     Fault load_fault = NoFault;
 591
 592     DPRINTF(LSQUnit, "Executing load PC %s, [sn:%lli]\n",
 593             inst->pcState(), inst->seqNum);
 594
 595     assert(!inst->isSquashed());
 596
 597     load_fault = inst->initiateAcc();
 598
 599     if (inst->isTranslationDelayed() &&
 600         load_fault == NoFault)
 601         return load_fault;
 602
 603     // If the instruction faulted or predicated false, then we need to send it
 604     // along to commit without the instruction completing.
 605     if (load_fault != NoFault || inst->readPredicate() == false) {
 606         // Send this instruction to commit, also make sure iew stage
 607         // realizes there is activity.
 608         // Mark it as executed unless it is an uncached load that
 609         // needs to hit the head of commit.
 610         if (inst->readPredicate() == false)
 611             inst->forwardOldRegs();
 612         DPRINTF(LSQUnit, "Load [sn:%lli] not executed from %s\n",
 613                 inst->seqNum,
 614                 (load_fault != NoFault ? "fault" : "predication"));
 615         if (!(inst->hasRequest() && inst->uncacheable()) ||
 616             inst->isAtCommit()) {
 617             inst->setExecuted();
 618         }
 619         iewStage->instToCommit(inst);
 620         iewStage->activityThisCycle();
 621     } else if (!loadBlocked()) {
 622         assert(inst->effAddrValid);
 623         int load_idx = inst->lqIdx;
 624         incrLdIdx(load_idx);
 625
 626         if (checkLoads)
 627             return checkViolations(load_idx, inst);
 628     }
 629
 630     return load_fault;
 631 }
 632
 633 template <class Impl>
 634 Fault
 635 LSQUnit<Impl>::executeStore(DynInstPtr &store_inst)
 636 {
 637     using namespace TheISA;
 638     // Make sure that a store exists.
 639     assert(stores != 0);
 640
 641     int store_idx = store_inst->sqIdx;
 642
 643     DPRINTF(LSQUnit, "Executing store PC %s [sn:%lli]\n",
 644             store_inst->pcState(), store_inst->seqNum);
 645
 646     assert(!store_inst->isSquashed());
 647
 648     // Check the recently completed loads to see if any match this store's
 649     // address.  If so, then we have a memory ordering violation.
 650     int load_idx = store_inst->lqIdx;
 651
 652     Fault store_fault = store_inst->initiateAcc();
 653
 654     if (store_inst->isTranslationDelayed() &&
 655         store_fault == NoFault)
 656         return store_fault;
 657
 658     if (store_inst->readPredicate() == false)
 659         store_inst->forwardOldRegs();
 660
 661     if (storeQueue[store_idx].size == 0) {
 662         DPRINTF(LSQUnit,"Fault on Store PC %s, [sn:%lli], Size = 0\n",
 663                 store_inst->pcState(), store_inst->seqNum);
 664
 665         return store_fault;
 666     } else if (store_inst->readPredicate() == false) {
 667         DPRINTF(LSQUnit, "Store [sn:%lli] not executed from predication\n",
 668                 store_inst->seqNum);
 669         return store_fault;
 670     }
 671
 672     assert(store_fault == NoFault);
 673
 674     if (store_inst->isStoreConditional()) {
 675         // Store conditionals need to set themselves as able to
 676         // writeback if we haven't had a fault by here.
 677         storeQueue[store_idx].canWB = true;
 678
 679         ++storesToWB;
 680     }
 681
 682     return checkViolations(load_idx, store_inst);
 683
 684 }
 685
 686 template <class Impl>
 687 void
 688 LSQUnit<Impl>::commitLoad()
 689 {
 690     assert(loadQueue[loadHead]);
 691
 692     DPRINTF(LSQUnit, "Committing head load instruction, PC %s\n",
 693             loadQueue[loadHead]->pcState());
 694
 695     loadQueue[loadHead] = NULL;
 696
 697     incrLdIdx(loadHead);
 698
 699     --loads;
 700 }
 701
 702 template <class Impl>
 703 void
 704 LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst)
 705 {
 706     assert(loads == 0 || loadQueue[loadHead]);
 707
 708     while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
 709         commitLoad();
 710     }
 711 }
 712
 713 template <class Impl>
 714 void
 715 LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst)
 716 {
 717     assert(stores == 0 || storeQueue[storeHead].inst);
 718
 719     int store_idx = storeHead;
 720
 721     while (store_idx != storeTail) {
 722         assert(storeQueue[store_idx].inst);
 723         // Mark any stores that are now committed and have not yet
 724         // been marked as able to write back.
 725         if (!storeQueue[store_idx].canWB) {
 726             if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
 727                 break;
 728             }
 729             DPRINTF(LSQUnit, "Marking store as able to write back, PC "
 730                     "%s [sn:%lli]\n",
 731                     storeQueue[store_idx].inst->pcState(),
 732                     storeQueue[store_idx].inst->seqNum);
 733
 734             storeQueue[store_idx].canWB = true;
 735
 736             ++storesToWB;
 737         }
 738
 739         incrStIdx(store_idx);
 740     }
 741 }
 742
 743 template <class Impl>
 744 void
 745 LSQUnit<Impl>::writebackPendingStore()
 746 {
 747     if (hasPendingPkt) {
 748         assert(pendingPkt != NULL);
 749
 750         // If the cache is blocked, this will store the packet for retry.
 751         if (sendStore(pendingPkt)) {
 752             storePostSend(pendingPkt);
 753         }
 754         pendingPkt = NULL;
 755         hasPendingPkt = false;
 756     }
 757 }
 758
 759 template <class Impl>
 760 void
 761 LSQUnit<Impl>::writebackStores()
 762 {
 763     // First writeback the second packet from any split store that didn't
 764     // complete last cycle because there weren't enough cache ports available.
 765     if (TheISA::HasUnalignedMemAcc) {
 766         writebackPendingStore();
 767     }
 768
 769     while (storesToWB > 0 &&
 770            storeWBIdx != storeTail &&
 771            storeQueue[storeWBIdx].inst &&
 772            storeQueue[storeWBIdx].canWB &&
 773            usedPorts < cachePorts) {
 774
 775         if (isStoreBlocked || lsq->cacheBlocked()) {
 776             DPRINTF(LSQUnit, "Unable to write back any more stores, cache"
 777                     " is blocked!\n");
 778             break;
 779         }
 780
 781         // Store didn't write any data so no need to write it back to
 782         // memory.
 783         if (storeQueue[storeWBIdx].size == 0) {
 784             completeStore(storeWBIdx);
 785
 786             incrStIdx(storeWBIdx);
 787
 788             continue;
 789         }
 790
 791         ++usedPorts;
 792
 793         if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
 794             incrStIdx(storeWBIdx);
 795
 796             continue;
 797         }
 798
 799         assert(storeQueue[storeWBIdx].req);
 800         assert(!storeQueue[storeWBIdx].committed);
 801
 802         if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) {
 803             assert(storeQueue[storeWBIdx].sreqLow);
 804             assert(storeQueue[storeWBIdx].sreqHigh);
 805         }
 806
 807         DynInstPtr inst = storeQueue[storeWBIdx].inst;
 808
 809         Request *req = storeQueue[storeWBIdx].req;
 810         RequestPtr sreqLow = storeQueue[storeWBIdx].sreqLow;
 811         RequestPtr sreqHigh = storeQueue[storeWBIdx].sreqHigh;
 812
 813         storeQueue[storeWBIdx].committed = true;
 814
 815         assert(!inst->memData);
 816         inst->memData = new uint8_t[64];
 817
 818         memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize());
 819
 820         MemCmd command =
 821             req->isSwap() ? MemCmd::SwapReq :
 822             (req->isLLSC() ? MemCmd::StoreCondReq : MemCmd::WriteReq);
 823         PacketPtr data_pkt;
 824         PacketPtr snd_data_pkt = NULL;
 825
 826         LSQSenderState *state = new LSQSenderState;
 827         state->isLoad = false;
 828         state->idx = storeWBIdx;
 829         state->inst = inst;
 830
 831         if (!TheISA::HasUnalignedMemAcc || !storeQueue[storeWBIdx].isSplit) {
 832
 833             // Build a single data packet if the store isn't split.
 834             data_pkt = new Packet(req, command, Packet::Broadcast);
 835             data_pkt->dataStatic(inst->memData);
 836             data_pkt->senderState = state;
 837         } else {
 838             // Create two packets if the store is split in two.
 839             data_pkt = new Packet(sreqLow, command, Packet::Broadcast);
 840             snd_data_pkt = new Packet(sreqHigh, command, Packet::Broadcast);
 841
 842             data_pkt->dataStatic(inst->memData);
 843             snd_data_pkt->dataStatic(inst->memData + sreqLow->getSize());
 844
 845             data_pkt->senderState = state;
 846             snd_data_pkt->senderState = state;
 847
 848             state->isSplit = true;
 849             state->outstanding = 2;
 850
 851             // Can delete the main request now.
 852             delete req;
 853             req = sreqLow;
 854         }
 855
 856         DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%s "
 857                 "to Addr:%#x, data:%#x [sn:%lli]\n",
 858                 storeWBIdx, inst->pcState(),
 859                 req->getPaddr(), (int)*(inst->memData),
 860                 inst->seqNum);
 861
 862         // @todo: Remove this SC hack once the memory system handles it.
 863         if (inst->isStoreConditional()) {
 864             assert(!storeQueue[storeWBIdx].isSplit);
 865             // Disable recording the result temporarily.  Writing to
 866             // misc regs normally updates the result, but this is not
 867             // the desired behavior when handling store conditionals.
 868             inst->recordResult = false;
 869             bool success = TheISA::handleLockedWrite(inst.get(), req);
 870             inst->recordResult = true;
 871
 872             if (!success) {
 873                 // Instantly complete this store.
 874                 DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed.  "
 875                         "Instantly completing it.\n",
 876                         inst->seqNum);
 877                 WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
 878                 cpu->schedule(wb, curTick() + 1);
 879                 completeStore(storeWBIdx);
 880                 incrStIdx(storeWBIdx);
 881                 continue;
 882             }
 883         } else {
 884             // Non-store conditionals do not need a writeback.
 885             state->noWB = true;
 886         }
 887
 888         bool split =
 889             TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit;
 890
 891         ThreadContext *thread = cpu->tcBase(lsqID);
 892
 893         if (req->isMmappedIpr()) {
 894             assert(!inst->isStoreConditional());
 895             TheISA::handleIprWrite(thread, data_pkt);
 896             delete data_pkt;
 897             if (split) {
 898                 assert(snd_data_pkt->req->isMmappedIpr());
 899                 TheISA::handleIprWrite(thread, snd_data_pkt);
 900                 delete snd_data_pkt;
 901                 delete sreqLow;
 902                 delete sreqHigh;
 903             }
 904             delete state;
 905             delete req;
 906             completeStore(storeWBIdx);
 907             incrStIdx(storeWBIdx);
 908         } else if (!sendStore(data_pkt)) {
 909             DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will"
 910                     "retry later\n",
 911                     inst->seqNum);
 912
 913             // Need to store the second packet, if split.
 914             if (split) {
 915                 state->pktToSend = true;
 916                 state->pendingPacket = snd_data_pkt;
 917             }
 918         } else {
 919
 920             // If split, try to send the second packet too
 921             if (split) {
 922                 assert(snd_data_pkt);
 923
 924                 // Ensure there are enough ports to use.
 925                 if (usedPorts < cachePorts) {
 926                     ++usedPorts;
 927                     if (sendStore(snd_data_pkt)) {
 928                         storePostSend(snd_data_pkt);
 929                     } else {
 930                         DPRINTF(IEW, "D-Cache became blocked when writing"
 931                                 " [sn:%lli] second packet, will retry later\n",
 932                                 inst->seqNum);
 933                     }
 934                 } else {
 935
 936                     // Store the packet for when there's free ports.
 937                     assert(pendingPkt == NULL);
 938                     pendingPkt = snd_data_pkt;
 939                     hasPendingPkt = true;
 940                 }
 941             } else {
 942
 943                 // Not a split store.
 944                 storePostSend(data_pkt);
 945             }
 946         }
 947     }
 948
 949     // Not sure this should set it to 0.
 950     usedPorts = 0;
 951
 952     assert(stores >= 0 && storesToWB >= 0);
 953 }
 954
 955 /*template <class Impl>
 956 void
 957 LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum)
 958 {
 959     list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
 960                                               mshrSeqNums.end(),
 961                                               seqNum);
 962
 963     if (mshr_it != mshrSeqNums.end()) {
 964         mshrSeqNums.erase(mshr_it);
 965         DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size());
 966     }
 967 }*/
 968
 969 template <class Impl>
 970 void
 971 LSQUnit<Impl>::squash(const InstSeqNum &squashed_num)
 972 {
 973     DPRINTF(LSQUnit, "Squashing until [sn:%lli]!"
 974             "(Loads:%i Stores:%i)\n", squashed_num, loads, stores);
 975
 976     int load_idx = loadTail;
 977     decrLdIdx(load_idx);
 978
 979     while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
 980         DPRINTF(LSQUnit,"Load Instruction PC %s squashed, "
 981                 "[sn:%lli]\n",
 982                 loadQueue[load_idx]->pcState(),
 983                 loadQueue[load_idx]->seqNum);
 984
 985         if (isStalled() && load_idx == stallingLoadIdx) {
 986             stalled = false;
 987             stallingStoreIsn = 0;
 988             stallingLoadIdx = 0;
 989         }
 990
 991         // Clear the smart pointer to make sure it is decremented.
 992         loadQueue[load_idx]->setSquashed();
 993         loadQueue[load_idx] = NULL;
 994         --loads;
 995
 996         // Inefficient!
 997         loadTail = load_idx;
 998
 999         decrLdIdx(load_idx);
1000         ++lsqSquashedLoads;
1001     }
1002
1003     if (isLoadBlocked) {
1004         if (squashed_num < blockedLoadSeqNum) {
1005             isLoadBlocked = false;
1006             loadBlockedHandled = false;
1007             blockedLoadSeqNum = 0;
1008         }
1009     }
1010
1011     if (memDepViolator && squashed_num < memDepViolator->seqNum) {
1012         memDepViolator = NULL;
1013     }
1014
1015     int store_idx = storeTail;
1016     decrStIdx(store_idx);
1017
1018     while (stores != 0 &&
1019            storeQueue[store_idx].inst->seqNum > squashed_num) {
1020         // Instructions marked as can WB are already committed.
1021         if (storeQueue[store_idx].canWB) {
1022             break;
1023         }
1024
1025         DPRINTF(LSQUnit,"Store Instruction PC %s squashed, "
1026                 "idx:%i [sn:%lli]\n",
1027                 storeQueue[store_idx].inst->pcState(),
1028                 store_idx, storeQueue[store_idx].inst->seqNum);
1029
1030         // I don't think this can happen.  It should have been cleared
1031         // by the stalling load.
1032         if (isStalled() &&
1033             storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
1034             panic("Is stalled should have been cleared by stalling load!\n");
1035             stalled = false;
1036             stallingStoreIsn = 0;
1037         }
1038
1039         // Clear the smart pointer to make sure it is decremented.
1040         storeQueue[store_idx].inst->setSquashed();
1041         storeQueue[store_idx].inst = NULL;
1042         storeQueue[store_idx].canWB = 0;
1043
1044         // Must delete request now that it wasn't handed off to
1045         // memory.  This is quite ugly.  @todo: Figure out the proper
1046         // place to really handle request deletes.
1047         delete storeQueue[store_idx].req;
1048         if (TheISA::HasUnalignedMemAcc && storeQueue[store_idx].isSplit) {
1049             delete storeQueue[store_idx].sreqLow;
1050             delete storeQueue[store_idx].sreqHigh;
1051
1052             storeQueue[store_idx].sreqLow = NULL;
1053             storeQueue[store_idx].sreqHigh = NULL;
1054         }
1055
1056         storeQueue[store_idx].req = NULL;
1057         --stores;
1058
1059         // Inefficient!
1060         storeTail = store_idx;
1061
1062         decrStIdx(store_idx);
1063         ++lsqSquashedStores;
1064     }
1065 }
1066
1067 template <class Impl>
1068 void
1069 LSQUnit<Impl>::storePostSend(PacketPtr pkt)
1070 {
1071     if (isStalled() &&
1072         storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
1073         DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1074                 "load idx:%i\n",
1075                 stallingStoreIsn, stallingLoadIdx);
1076         stalled = false;
1077         stallingStoreIsn = 0;
1078         iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
1079     }
1080
1081     if (!storeQueue[storeWBIdx].inst->isStoreConditional()) {
1082         // The store is basically completed at this time. This
1083         // only works so long as the checker doesn't try to
1084         // verify the value in memory for stores.
1085         storeQueue[storeWBIdx].inst->setCompleted();
1086 #if USE_CHECKER
1087         if (cpu->checker) {
1088             cpu->checker->verify(storeQueue[storeWBIdx].inst);
1089         }
1090 #endif
1091     }
1092
1093     incrStIdx(storeWBIdx);
1094 }
1095
1096 template <class Impl>
1097 void
1098 LSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt)
1099 {
1100     iewStage->wakeCPU();
1101
1102     // Squashed instructions do not need to complete their access.
1103     if (inst->isSquashed()) {
1104         iewStage->decrWb(inst->seqNum);
1105         assert(!inst->isStore());
1106         ++lsqIgnoredResponses;
1107         return;
1108     }
1109
1110     if (!inst->isExecuted()) {
1111         inst->setExecuted();
1112
1113         // Complete access to copy data to proper place.
1114         inst->completeAcc(pkt);
1115     }
1116
1117     // Need to insert instruction into queue to commit
1118     iewStage->instToCommit(inst);
1119
1120     iewStage->activityThisCycle();
1121
1122     // see if this load changed the PC
1123     iewStage->checkMisprediction(inst);
1124 }
1125
1126 template <class Impl>
1127 void
1128 LSQUnit<Impl>::completeStore(int store_idx)
1129 {
1130     assert(storeQueue[store_idx].inst);
1131     storeQueue[store_idx].completed = true;
1132     --storesToWB;
1133     // A bit conservative because a store completion may not free up entries,
1134     // but hopefully avoids two store completions in one cycle from making
1135     // the CPU tick twice.
1136     cpu->wakeCPU();
1137     cpu->activityThisCycle();
1138
1139     if (store_idx == storeHead) {
1140         do {
1141             incrStIdx(storeHead);
1142
1143             --stores;
1144         } while (storeQueue[storeHead].completed &&
1145                  storeHead != storeTail);
1146
1147         iewStage->updateLSQNextCycle = true;
1148     }
1149
1150     DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head "
1151             "idx:%i\n",
1152             storeQueue[store_idx].inst->seqNum, store_idx, storeHead);
1153
1154     if (isStalled() &&
1155         storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
1156         DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1157                 "load idx:%i\n",
1158                 stallingStoreIsn, stallingLoadIdx);
1159         stalled = false;
1160         stallingStoreIsn = 0;
1161         iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
1162     }
1163
1164     storeQueue[store_idx].inst->setCompleted();
1165
1166     // Tell the checker we've completed this instruction.  Some stores
1167     // may get reported twice to the checker, but the checker can
1168     // handle that case.
1169 #if USE_CHECKER
1170     if (cpu->checker) {
1171         cpu->checker->verify(storeQueue[store_idx].inst);
1172     }
1173 #endif
1174 }
1175
1176 template <class Impl>
1177 bool
1178 LSQUnit<Impl>::sendStore(PacketPtr data_pkt)
1179 {
1180     if (!dcachePort->sendTiming(data_pkt)) {
1181         // Need to handle becoming blocked on a store.
1182         isStoreBlocked = true;
1183         ++lsqCacheBlocked;
1184         assert(retryPkt == NULL);
1185         retryPkt = data_pkt;
1186         lsq->setRetryTid(lsqID);
1187         return false;
1188     }
1189     return true;
1190 }
1191
1192 template <class Impl>
1193 void
1194 LSQUnit<Impl>::recvRetry()
1195 {
1196     if (isStoreBlocked) {
1197         DPRINTF(LSQUnit, "Receiving retry: store blocked\n");
1198         assert(retryPkt != NULL);
1199
1200         LSQSenderState *state =
1201             dynamic_cast<LSQSenderState *>(retryPkt->senderState);
1202
1203         if (dcachePort->sendTiming(retryPkt)) {
1204             // Don't finish the store unless this is the last packet.
1205             if (!TheISA::HasUnalignedMemAcc || !state->pktToSend ||
1206                     state->pendingPacket == retryPkt) {
1207                 state->pktToSend = false;
1208                 storePostSend(retryPkt);
1209             }
1210             retryPkt = NULL;
1211             isStoreBlocked = false;
1212             lsq->setRetryTid(InvalidThreadID);
1213
1214             // Send any outstanding packet.
1215             if (TheISA::HasUnalignedMemAcc && state->pktToSend) {
1216                 assert(state->pendingPacket);
1217                 if (sendStore(state->pendingPacket)) {
1218                     storePostSend(state->pendingPacket);
1219                 }
1220             }
1221         } else {
1222             // Still blocked!
1223             ++lsqCacheBlocked;
1224             lsq->setRetryTid(lsqID);
1225         }
1226     } else if (isLoadBlocked) {
1227         DPRINTF(LSQUnit, "Loads squash themselves and all younger insts, "
1228                 "no need to resend packet.\n");
1229     } else {
1230         DPRINTF(LSQUnit, "Retry received but LSQ is no longer blocked.\n");
1231     }
1232 }
1233
1234 template <class Impl>
1235 inline void
1236 LSQUnit<Impl>::incrStIdx(int &store_idx)
1237 {
1238     if (++store_idx >= SQEntries)
1239         store_idx = 0;
1240 }
1241
1242 template <class Impl>
1243 inline void
1244 LSQUnit<Impl>::decrStIdx(int &store_idx)
1245 {
1246     if (--store_idx < 0)
1247         store_idx += SQEntries;
1248 }
1249
1250 template <class Impl>
1251 inline void
1252 LSQUnit<Impl>::incrLdIdx(int &load_idx)
1253 {
1254     if (++load_idx >= LQEntries)
1255         load_idx = 0;
1256 }
1257
1258 template <class Impl>
1259 inline void
1260 LSQUnit<Impl>::decrLdIdx(int &load_idx)
1261 {
1262     if (--load_idx < 0)
1263         load_idx += LQEntries;
1264 }
1265
1266 template <class Impl>
1267 void
1268 LSQUnit<Impl>::dumpInsts()
1269 {
1270     cprintf("Load store queue: Dumping instructions.\n");
1271     cprintf("Load queue size: %i\n", loads);
1272     cprintf("Load queue: ");
1273
1274     int load_idx = loadHead;
1275
1276     while (load_idx != loadTail && loadQueue[load_idx]) {
1277         cprintf("%s ", loadQueue[load_idx]->pcState());
1278
1279         incrLdIdx(load_idx);
1280     }
1281
1282     cprintf("Store queue size: %i\n", stores);
1283     cprintf("Store queue: ");
1284
1285     int store_idx = storeHead;
1286
1287     while (store_idx != storeTail && storeQueue[store_idx].inst) {
1288         cprintf("%s ", storeQueue[store_idx].inst->pcState());
1289
1290         incrStIdx(store_idx);
1291     }
1292
1293     cprintf("\n");
1294 }