src/cpu/o3/lsq_unit.hh

   1 /*
   2  * Copyright (c) 2004-2006 The Regents of The University of Michigan
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are
   7  * met: redistributions of source code must retain the above copyright
   8  * notice, this list of conditions and the following disclaimer;
   9  * redistributions in binary form must reproduce the above copyright
  10  * notice, this list of conditions and the following disclaimer in the
  11  * documentation and/or other materials provided with the distribution;
  12  * neither the name of the copyright holders nor the names of its
  13  * contributors may be used to endorse or promote products derived from
  14  * this software without specific prior written permission.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  17  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  18  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  19  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  20  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  26  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * Authors: Kevin Lim
  29  *          Korey Sewell
  30  */
  31
  32 #ifndef __CPU_O3_LSQ_UNIT_HH__
  33 #define __CPU_O3_LSQ_UNIT_HH__
  34
  35 #include <algorithm>
  36 #include <cstring>
  37 #include <map>
  38 #include <queue>
  39
  40 #include "arch/faults.hh"
  41 #include "arch/generic/debugfaults.hh"
  42 #include "arch/isa_traits.hh"
  43 #include "arch/locked_mem.hh"
  44 #include "arch/mmapped_ipr.hh"
  45 #include "base/fast_alloc.hh"
  46 #include "base/hashmap.hh"
  47 #include "config/the_isa.hh"
  48 #include "cpu/inst_seq.hh"
  49 #include "cpu/timebuf.hh"
  50 #include "debug/LSQUnit.hh"
  51 #include "mem/packet.hh"
  52 #include "mem/port.hh"
  53
  54 struct DerivO3CPUParams;
  55
  56 /**
  57  * Class that implements the actual LQ and SQ for each specific
  58  * thread.  Both are circular queues; load entries are freed upon
  59  * committing, while store entries are freed once they writeback. The
  60  * LSQUnit tracks if there are memory ordering violations, and also
  61  * detects partial load to store forwarding cases (a store only has
  62  * part of a load's data) that requires the load to wait until the
  63  * store writes back. In the former case it holds onto the instruction
  64  * until the dependence unit looks at it, and in the latter it stalls
  65  * the LSQ until the store writes back. At that point the load is
  66  * replayed.
  67  */
  68 template <class Impl>
  69 class LSQUnit {
  70   public:
  71     typedef typename Impl::O3CPU O3CPU;
  72     typedef typename Impl::DynInstPtr DynInstPtr;
  73     typedef typename Impl::CPUPol::IEW IEW;
  74     typedef typename Impl::CPUPol::LSQ LSQ;
  75     typedef typename Impl::CPUPol::IssueStruct IssueStruct;
  76
  77   public:
  78     /** Constructs an LSQ unit. init() must be called prior to use. */
  79     LSQUnit();
  80
  81     /** Initializes the LSQ unit with the specified number of entries. */
  82     void init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params,
  83             LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries,
  84             unsigned id);
  85
  86     /** Returns the name of the LSQ unit. */
  87     std::string name() const;
  88
  89     /** Registers statistics. */
  90     void regStats();
  91
  92     /** Sets the pointer to the dcache port. */
  93     void setDcachePort(Port *dcache_port);
  94
  95     /** Switches out LSQ unit. */
  96     void switchOut();
  97
  98     /** Takes over from another CPU's thread. */
  99     void takeOverFrom();
 100
 101     /** Returns if the LSQ is switched out. */
 102     bool isSwitchedOut() { return switchedOut; }
 103
 104     /** Ticks the LSQ unit, which in this case only resets the number of
 105      * used cache ports.
 106      * @todo: Move the number of used ports up to the LSQ level so it can
 107      * be shared by all LSQ units.
 108      */
 109     void tick() { usedPorts = 0; }
 110
 111     /** Inserts an instruction. */
 112     void insert(DynInstPtr &inst);
 113     /** Inserts a load instruction. */
 114     void insertLoad(DynInstPtr &load_inst);
 115     /** Inserts a store instruction. */
 116     void insertStore(DynInstPtr &store_inst);
 117
 118     /** Check for ordering violations in the LSQ. For a store squash if we
 119      * ever find a conflicting load. For a load, only squash if we
 120      * an external snoop invalidate has been seen for that load address
 121      * @param load_idx index to start checking at
 122      * @param inst the instruction to check
 123      */
 124     Fault checkViolations(int load_idx, DynInstPtr &inst);
 125
 126     /** Check if an incoming invalidate hits in the lsq on a load
 127      * that might have issued out of order wrt another load beacuse
 128      * of the intermediate invalidate.
 129      */
 130     void checkSnoop(PacketPtr pkt);
 131
 132     /** Executes a load instruction. */
 133     Fault executeLoad(DynInstPtr &inst);
 134
 135     Fault executeLoad(int lq_idx) { panic("Not implemented"); return NoFault; }
 136     /** Executes a store instruction. */
 137     Fault executeStore(DynInstPtr &inst);
 138
 139     /** Commits the head load. */
 140     void commitLoad();
 141     /** Commits loads older than a specific sequence number. */
 142     void commitLoads(InstSeqNum &youngest_inst);
 143
 144     /** Commits stores older than a specific sequence number. */
 145     void commitStores(InstSeqNum &youngest_inst);
 146
 147     /** Writes back stores. */
 148     void writebackStores();
 149
 150     /** Completes the data access that has been returned from the
 151      * memory system. */
 152     void completeDataAccess(PacketPtr pkt);
 153
 154     /** Clears all the entries in the LQ. */
 155     void clearLQ();
 156
 157     /** Clears all the entries in the SQ. */
 158     void clearSQ();
 159
 160     /** Resizes the LQ to a given size. */
 161     void resizeLQ(unsigned size);
 162
 163     /** Resizes the SQ to a given size. */
 164     void resizeSQ(unsigned size);
 165
 166     /** Squashes all instructions younger than a specific sequence number. */
 167     void squash(const InstSeqNum &squashed_num);
 168
 169     /** Returns if there is a memory ordering violation. Value is reset upon
 170      * call to getMemDepViolator().
 171      */
 172     bool violation() { return memDepViolator; }
 173
 174     /** Returns the memory ordering violator. */
 175     DynInstPtr getMemDepViolator();
 176
 177     /** Returns if a load became blocked due to the memory system. */
 178     bool loadBlocked()
 179     { return isLoadBlocked; }
 180
 181     /** Clears the signal that a load became blocked. */
 182     void clearLoadBlocked()
 183     { isLoadBlocked = false; }
 184
 185     /** Returns if the blocked load was handled. */
 186     bool isLoadBlockedHandled()
 187     { return loadBlockedHandled; }
 188
 189     /** Records the blocked load as being handled. */
 190     void setLoadBlockedHandled()
 191     { loadBlockedHandled = true; }
 192
 193     /** Returns the number of free entries (min of free LQ and SQ entries). */
 194     unsigned numFreeEntries();
 195
 196     /** Returns the number of loads ready to execute. */
 197     int numLoadsReady();
 198
 199     /** Returns the number of loads in the LQ. */
 200     int numLoads() { return loads; }
 201
 202     /** Returns the number of stores in the SQ. */
 203     int numStores() { return stores; }
 204
 205     /** Returns if either the LQ or SQ is full. */
 206     bool isFull() { return lqFull() || sqFull(); }
 207
 208     /** Returns if the LQ is full. */
 209     bool lqFull() { return loads >= (LQEntries - 1); }
 210
 211     /** Returns if the SQ is full. */
 212     bool sqFull() { return stores >= (SQEntries - 1); }
 213
 214     /** Returns the number of instructions in the LSQ. */
 215     unsigned getCount() { return loads + stores; }
 216
 217     /** Returns if there are any stores to writeback. */
 218     bool hasStoresToWB() { return storesToWB; }
 219
 220     /** Returns the number of stores to writeback. */
 221     int numStoresToWB() { return storesToWB; }
 222
 223     /** Returns if the LSQ unit will writeback on this cycle. */
 224     bool willWB() { return storeQueue[storeWBIdx].canWB &&
 225                         !storeQueue[storeWBIdx].completed &&
 226                         !isStoreBlocked; }
 227
 228     /** Handles doing the retry. */
 229     void recvRetry();
 230
 231   private:
 232     /** Writes back the instruction, sending it to IEW. */
 233     void writeback(DynInstPtr &inst, PacketPtr pkt);
 234
 235     /** Writes back a store that couldn't be completed the previous cycle. */
 236     void writebackPendingStore();
 237
 238     /** Handles completing the send of a store to memory. */
 239     void storePostSend(PacketPtr pkt);
 240
 241     /** Completes the store at the specified index. */
 242     void completeStore(int store_idx);
 243
 244     /** Attempts to send a store to the cache. */
 245     bool sendStore(PacketPtr data_pkt);
 246
 247     /** Increments the given store index (circular queue). */
 248     inline void incrStIdx(int &store_idx);
 249     /** Decrements the given store index (circular queue). */
 250     inline void decrStIdx(int &store_idx);
 251     /** Increments the given load index (circular queue). */
 252     inline void incrLdIdx(int &load_idx);
 253     /** Decrements the given load index (circular queue). */
 254     inline void decrLdIdx(int &load_idx);
 255
 256   public:
 257     /** Debugging function to dump instructions in the LSQ. */
 258     void dumpInsts();
 259
 260   private:
 261     /** Pointer to the CPU. */
 262     O3CPU *cpu;
 263
 264     /** Pointer to the IEW stage. */
 265     IEW *iewStage;
 266
 267     /** Pointer to the LSQ. */
 268     LSQ *lsq;
 269
 270     /** Pointer to the dcache port.  Used only for sending. */
 271     Port *dcachePort;
 272
 273     /** Derived class to hold any sender state the LSQ needs. */
 274     class LSQSenderState : public Packet::SenderState, public FastAlloc
 275     {
 276       public:
 277         /** Default constructor. */
 278         LSQSenderState()
 279             : noWB(false), isSplit(false), pktToSend(false), outstanding(1),
 280               mainPkt(NULL), pendingPacket(NULL)
 281         { }
 282
 283         /** Instruction who initiated the access to memory. */
 284         DynInstPtr inst;
 285         /** Whether or not it is a load. */
 286         bool isLoad;
 287         /** The LQ/SQ index of the instruction. */
 288         int idx;
 289         /** Whether or not the instruction will need to writeback. */
 290         bool noWB;
 291         /** Whether or not this access is split in two. */
 292         bool isSplit;
 293         /** Whether or not there is a packet that needs sending. */
 294         bool pktToSend;
 295         /** Number of outstanding packets to complete. */
 296         int outstanding;
 297         /** The main packet from a split load, used during writeback. */
 298         PacketPtr mainPkt;
 299         /** A second packet from a split store that needs sending. */
 300         PacketPtr pendingPacket;
 301
 302         /** Completes a packet and returns whether the access is finished. */
 303         inline bool complete() { return --outstanding == 0; }
 304     };
 305
 306     /** Writeback event, specifically for when stores forward data to loads. */
 307     class WritebackEvent : public Event {
 308       public:
 309         /** Constructs a writeback event. */
 310         WritebackEvent(DynInstPtr &_inst, PacketPtr pkt, LSQUnit *lsq_ptr);
 311
 312         /** Processes the writeback event. */
 313         void process();
 314
 315         /** Returns the description of this event. */
 316         const char *description() const;
 317
 318       private:
 319         /** Instruction whose results are being written back. */
 320         DynInstPtr inst;
 321
 322         /** The packet that would have been sent to memory. */
 323         PacketPtr pkt;
 324
 325         /** The pointer to the LSQ unit that issued the store. */
 326         LSQUnit<Impl> *lsqPtr;
 327     };
 328
 329   public:
 330     struct SQEntry {
 331         /** Constructs an empty store queue entry. */
 332         SQEntry()
 333             : inst(NULL), req(NULL), size(0),
 334               canWB(0), committed(0), completed(0)
 335         {
 336             std::memset(data, 0, sizeof(data));
 337         }
 338
 339         /** Constructs a store queue entry for a given instruction. */
 340         SQEntry(DynInstPtr &_inst)
 341             : inst(_inst), req(NULL), sreqLow(NULL), sreqHigh(NULL), size(0),
 342               isSplit(0), canWB(0), committed(0), completed(0)
 343         {
 344             std::memset(data, 0, sizeof(data));
 345         }
 346
 347         /** The store instruction. */
 348         DynInstPtr inst;
 349         /** The request for the store. */
 350         RequestPtr req;
 351         /** The split requests for the store. */
 352         RequestPtr sreqLow;
 353         RequestPtr sreqHigh;
 354         /** The size of the store. */
 355         int size;
 356         /** The store data. */
 357         char data[16];
 358         /** Whether or not the store is split into two requests. */
 359         bool isSplit;
 360         /** Whether or not the store can writeback. */
 361         bool canWB;
 362         /** Whether or not the store is committed. */
 363         bool committed;
 364         /** Whether or not the store is completed. */
 365         bool completed;
 366     };
 367
 368   private:
 369     /** The LSQUnit thread id. */
 370     ThreadID lsqID;
 371
 372     /** The store queue. */
 373     std::vector<SQEntry> storeQueue;
 374
 375     /** The load queue. */
 376     std::vector<DynInstPtr> loadQueue;
 377
 378     /** The number of LQ entries, plus a sentinel entry (circular queue).
 379      *  @todo: Consider having var that records the true number of LQ entries.
 380      */
 381     unsigned LQEntries;
 382     /** The number of SQ entries, plus a sentinel entry (circular queue).
 383      *  @todo: Consider having var that records the true number of SQ entries.
 384      */
 385     unsigned SQEntries;
 386
 387     /** The number of places to shift addresses in the LSQ before checking
 388      * for dependency violations
 389      */
 390     unsigned depCheckShift;
 391
 392     /** Should loads be checked for dependency issues */
 393     bool checkLoads;
 394
 395     /** The number of load instructions in the LQ. */
 396     int loads;
 397     /** The number of store instructions in the SQ. */
 398     int stores;
 399     /** The number of store instructions in the SQ waiting to writeback. */
 400     int storesToWB;
 401
 402     /** The index of the head instruction in the LQ. */
 403     int loadHead;
 404     /** The index of the tail instruction in the LQ. */
 405     int loadTail;
 406
 407     /** The index of the head instruction in the SQ. */
 408     int storeHead;
 409     /** The index of the first instruction that may be ready to be
 410      * written back, and has not yet been written back.
 411      */
 412     int storeWBIdx;
 413     /** The index of the tail instruction in the SQ. */
 414     int storeTail;
 415
 416     /// @todo Consider moving to a more advanced model with write vs read ports
 417     /** The number of cache ports available each cycle. */
 418     int cachePorts;
 419
 420     /** The number of used cache ports in this cycle. */
 421     int usedPorts;
 422
 423     /** Is the LSQ switched out. */
 424     bool switchedOut;
 425
 426     //list<InstSeqNum> mshrSeqNums;
 427
 428     /** Address Mask for a cache block (e.g. ~(cache_block_size-1)) */
 429     Addr cacheBlockMask;
 430
 431     /** Wire to read information from the issue stage time queue. */
 432     typename TimeBuffer<IssueStruct>::wire fromIssue;
 433
 434     /** Whether or not the LSQ is stalled. */
 435     bool stalled;
 436     /** The store that causes the stall due to partial store to load
 437      * forwarding.
 438      */
 439     InstSeqNum stallingStoreIsn;
 440     /** The index of the above store. */
 441     int stallingLoadIdx;
 442
 443     /** The packet that needs to be retried. */
 444     PacketPtr retryPkt;
 445
 446     /** Whehter or not a store is blocked due to the memory system. */
 447     bool isStoreBlocked;
 448
 449     /** Whether or not a load is blocked due to the memory system. */
 450     bool isLoadBlocked;
 451
 452     /** Has the blocked load been handled. */
 453     bool loadBlockedHandled;
 454
 455     /** Whether or not a store is in flight. */
 456     bool storeInFlight;
 457
 458     /** The sequence number of the blocked load. */
 459     InstSeqNum blockedLoadSeqNum;
 460
 461     /** The oldest load that caused a memory ordering violation. */
 462     DynInstPtr memDepViolator;
 463
 464     /** Whether or not there is a packet that couldn't be sent because of
 465      * a lack of cache ports. */
 466     bool hasPendingPkt;
 467
 468     /** The packet that is pending free cache ports. */
 469     PacketPtr pendingPkt;
 470
 471     /** Flag for memory model. */
 472     bool needsTSO;
 473
 474     // Will also need how many read/write ports the Dcache has.  Or keep track
 475     // of that in stage that is one level up, and only call executeLoad/Store
 476     // the appropriate number of times.
 477     /** Total number of loads forwaded from LSQ stores. */
 478     Stats::Scalar lsqForwLoads;
 479
 480     /** Total number of loads ignored due to invalid addresses. */
 481     Stats::Scalar invAddrLoads;
 482
 483     /** Total number of squashed loads. */
 484     Stats::Scalar lsqSquashedLoads;
 485
 486     /** Total number of responses from the memory system that are
 487      * ignored due to the instruction already being squashed. */
 488     Stats::Scalar lsqIgnoredResponses;
 489
 490     /** Tota number of memory ordering violations. */
 491     Stats::Scalar lsqMemOrderViolation;
 492
 493     /** Total number of squashed stores. */
 494     Stats::Scalar lsqSquashedStores;
 495
 496     /** Total number of software prefetches ignored due to invalid addresses. */
 497     Stats::Scalar invAddrSwpfs;
 498
 499     /** Ready loads blocked due to partial store-forwarding. */
 500     Stats::Scalar lsqBlockedLoads;
 501
 502     /** Number of loads that were rescheduled. */
 503     Stats::Scalar lsqRescheduledLoads;
 504
 505     /** Number of times the LSQ is blocked due to the cache. */
 506     Stats::Scalar lsqCacheBlocked;
 507
 508   public:
 509     /** Executes the load at the given index. */
 510     Fault read(Request *req, Request *sreqLow, Request *sreqHigh,
 511                uint8_t *data, int load_idx);
 512
 513     /** Executes the store at the given index. */
 514     Fault write(Request *req, Request *sreqLow, Request *sreqHigh,
 515                 uint8_t *data, int store_idx);
 516
 517     /** Returns the index of the head load instruction. */
 518     int getLoadHead() { return loadHead; }
 519     /** Returns the sequence number of the head load instruction. */
 520     InstSeqNum getLoadHeadSeqNum()
 521     {
 522         if (loadQueue[loadHead]) {
 523             return loadQueue[loadHead]->seqNum;
 524         } else {
 525             return 0;
 526         }
 527
 528     }
 529
 530     /** Returns the index of the head store instruction. */
 531     int getStoreHead() { return storeHead; }
 532     /** Returns the sequence number of the head store instruction. */
 533     InstSeqNum getStoreHeadSeqNum()
 534     {
 535         if (storeQueue[storeHead].inst) {
 536             return storeQueue[storeHead].inst->seqNum;
 537         } else {
 538             return 0;
 539         }
 540
 541     }
 542
 543     /** Returns whether or not the LSQ unit is stalled. */
 544     bool isStalled()  { return stalled; }
 545 };
 546
 547 template <class Impl>
 548 Fault
 549 LSQUnit<Impl>::read(Request *req, Request *sreqLow, Request *sreqHigh,
 550                     uint8_t *data, int load_idx)
 551 {
 552     DynInstPtr load_inst = loadQueue[load_idx];
 553
 554     assert(load_inst);
 555
 556     assert(!load_inst->isExecuted());
 557
 558     // Make sure this isn't an uncacheable access
 559     // A bit of a hackish way to get uncached accesses to work only if they're
 560     // at the head of the LSQ and are ready to commit (at the head of the ROB
 561     // too).
 562     if (req->isUncacheable() &&
 563         (load_idx != loadHead || !load_inst->isAtCommit())) {
 564         iewStage->rescheduleMemInst(load_inst);
 565         ++lsqRescheduledLoads;
 566         DPRINTF(LSQUnit, "Uncachable load [sn:%lli] PC %s\n",
 567                 load_inst->seqNum, load_inst->pcState());
 568
 569         // Must delete request now that it wasn't handed off to
 570         // memory.  This is quite ugly.  @todo: Figure out the proper
 571         // place to really handle request deletes.
 572         delete req;
 573         if (TheISA::HasUnalignedMemAcc && sreqLow) {
 574             delete sreqLow;
 575             delete sreqHigh;
 576         }
 577         return new GenericISA::M5PanicFault(
 578                 "Uncachable load [sn:%llx] PC %s\n",
 579                 load_inst->seqNum, load_inst->pcState());
 580     }
 581
 582     // Check the SQ for any previous stores that might lead to forwarding
 583     int store_idx = load_inst->sqIdx;
 584
 585     int store_size = 0;
 586
 587     DPRINTF(LSQUnit, "Read called, load idx: %i, store idx: %i, "
 588             "storeHead: %i addr: %#x%s\n",
 589             load_idx, store_idx, storeHead, req->getPaddr(),
 590             sreqLow ? " split" : "");
 591
 592     if (req->isLLSC()) {
 593         assert(!sreqLow);
 594         // Disable recording the result temporarily.  Writing to misc
 595         // regs normally updates the result, but this is not the
 596         // desired behavior when handling store conditionals.
 597         load_inst->recordResult = false;
 598         TheISA::handleLockedRead(load_inst.get(), req);
 599         load_inst->recordResult = true;
 600     }
 601
 602     if (req->isMmappedIpr()) {
 603         assert(!load_inst->memData);
 604         load_inst->memData = new uint8_t[64];
 605
 606         ThreadContext *thread = cpu->tcBase(lsqID);
 607         Tick delay;
 608         PacketPtr data_pkt =
 609             new Packet(req, MemCmd::ReadReq, Packet::Broadcast);
 610
 611         if (!TheISA::HasUnalignedMemAcc || !sreqLow) {
 612             data_pkt->dataStatic(load_inst->memData);
 613             delay = TheISA::handleIprRead(thread, data_pkt);
 614         } else {
 615             assert(sreqLow->isMmappedIpr() && sreqHigh->isMmappedIpr());
 616             PacketPtr fst_data_pkt =
 617                 new Packet(sreqLow, MemCmd::ReadReq, Packet::Broadcast);
 618             PacketPtr snd_data_pkt =
 619                 new Packet(sreqHigh, MemCmd::ReadReq, Packet::Broadcast);
 620
 621             fst_data_pkt->dataStatic(load_inst->memData);
 622             snd_data_pkt->dataStatic(load_inst->memData + sreqLow->getSize());
 623
 624             delay = TheISA::handleIprRead(thread, fst_data_pkt);
 625             unsigned delay2 = TheISA::handleIprRead(thread, snd_data_pkt);
 626             if (delay2 > delay)
 627                 delay = delay2;
 628
 629             delete sreqLow;
 630             delete sreqHigh;
 631             delete fst_data_pkt;
 632             delete snd_data_pkt;
 633         }
 634         WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this);
 635         cpu->schedule(wb, curTick() + delay);
 636         return NoFault;
 637     }
 638
 639     while (store_idx != -1) {
 640         // End once we've reached the top of the LSQ
 641         if (store_idx == storeWBIdx) {
 642             break;
 643         }
 644
 645         // Move the index to one younger
 646         if (--store_idx < 0)
 647             store_idx += SQEntries;
 648
 649         assert(storeQueue[store_idx].inst);
 650
 651         store_size = storeQueue[store_idx].size;
 652
 653         if (store_size == 0)
 654             continue;
 655         else if (storeQueue[store_idx].inst->uncacheable())
 656             continue;
 657
 658         assert(storeQueue[store_idx].inst->effAddrValid);
 659
 660         // Check if the store data is within the lower and upper bounds of
 661         // addresses that the request needs.
 662         bool store_has_lower_limit =
 663             req->getVaddr() >= storeQueue[store_idx].inst->effAddr;
 664         bool store_has_upper_limit =
 665             (req->getVaddr() + req->getSize()) <=
 666             (storeQueue[store_idx].inst->effAddr + store_size);
 667         bool lower_load_has_store_part =
 668             req->getVaddr() < (storeQueue[store_idx].inst->effAddr +
 669                            store_size);
 670         bool upper_load_has_store_part =
 671             (req->getVaddr() + req->getSize()) >
 672             storeQueue[store_idx].inst->effAddr;
 673
 674         // If the store's data has all of the data needed, we can forward.
 675         if ((store_has_lower_limit && store_has_upper_limit)) {
 676             // Get shift amount for offset into the store's data.
 677             int shift_amt = req->getVaddr() - storeQueue[store_idx].inst->effAddr;
 678
 679             memcpy(data, storeQueue[store_idx].data + shift_amt,
 680                    req->getSize());
 681
 682             assert(!load_inst->memData);
 683             load_inst->memData = new uint8_t[64];
 684
 685             memcpy(load_inst->memData,
 686                     storeQueue[store_idx].data + shift_amt, req->getSize());
 687
 688             DPRINTF(LSQUnit, "Forwarding from store idx %i to load to "
 689                     "addr %#x, data %#x\n",
 690                     store_idx, req->getVaddr(), data);
 691
 692             PacketPtr data_pkt = new Packet(req, MemCmd::ReadReq,
 693                                             Packet::Broadcast);
 694             data_pkt->dataStatic(load_inst->memData);
 695
 696             WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this);
 697
 698             // We'll say this has a 1 cycle load-store forwarding latency
 699             // for now.
 700             // @todo: Need to make this a parameter.
 701             cpu->schedule(wb, curTick());
 702
 703             // Don't need to do anything special for split loads.
 704             if (TheISA::HasUnalignedMemAcc && sreqLow) {
 705                 delete sreqLow;
 706                 delete sreqHigh;
 707             }
 708
 709             ++lsqForwLoads;
 710             return NoFault;
 711         } else if ((store_has_lower_limit && lower_load_has_store_part) ||
 712                    (store_has_upper_limit && upper_load_has_store_part) ||
 713                    (lower_load_has_store_part && upper_load_has_store_part)) {
 714             // This is the partial store-load forwarding case where a store
 715             // has only part of the load's data.
 716
 717             // If it's already been written back, then don't worry about
 718             // stalling on it.
 719             if (storeQueue[store_idx].completed) {
 720                 panic("Should not check one of these");
 721                 continue;
 722             }
 723
 724             // Must stall load and force it to retry, so long as it's the oldest
 725             // load that needs to do so.
 726             if (!stalled ||
 727                 (stalled &&
 728                  load_inst->seqNum <
 729                  loadQueue[stallingLoadIdx]->seqNum)) {
 730                 stalled = true;
 731                 stallingStoreIsn = storeQueue[store_idx].inst->seqNum;
 732                 stallingLoadIdx = load_idx;
 733             }
 734
 735             // Tell IQ/mem dep unit that this instruction will need to be
 736             // rescheduled eventually
 737             iewStage->rescheduleMemInst(load_inst);
 738             iewStage->decrWb(load_inst->seqNum);
 739             load_inst->clearIssued();
 740             ++lsqRescheduledLoads;
 741
 742             // Do not generate a writeback event as this instruction is not
 743             // complete.
 744             DPRINTF(LSQUnit, "Load-store forwarding mis-match. "
 745                     "Store idx %i to load addr %#x\n",
 746                     store_idx, req->getVaddr());
 747
 748             // Must delete request now that it wasn't handed off to
 749             // memory.  This is quite ugly.  @todo: Figure out the
 750             // proper place to really handle request deletes.
 751             delete req;
 752             if (TheISA::HasUnalignedMemAcc && sreqLow) {
 753                 delete sreqLow;
 754                 delete sreqHigh;
 755             }
 756
 757             return NoFault;
 758         }
 759     }
 760
 761     // If there's no forwarding case, then go access memory
 762     DPRINTF(LSQUnit, "Doing memory access for inst [sn:%lli] PC %s\n",
 763             load_inst->seqNum, load_inst->pcState());
 764
 765     assert(!load_inst->memData);
 766     load_inst->memData = new uint8_t[64];
 767
 768     ++usedPorts;
 769
 770     // if we the cache is not blocked, do cache access
 771     bool completedFirst = false;
 772     if (!lsq->cacheBlocked()) {
 773         MemCmd command =
 774             req->isLLSC() ? MemCmd::LoadLockedReq : MemCmd::ReadReq;
 775         PacketPtr data_pkt = new Packet(req, command, Packet::Broadcast);
 776         PacketPtr fst_data_pkt = NULL;
 777         PacketPtr snd_data_pkt = NULL;
 778
 779         data_pkt->dataStatic(load_inst->memData);
 780
 781         LSQSenderState *state = new LSQSenderState;
 782         state->isLoad = true;
 783         state->idx = load_idx;
 784         state->inst = load_inst;
 785         data_pkt->senderState = state;
 786
 787         if (!TheISA::HasUnalignedMemAcc || !sreqLow) {
 788
 789             // Point the first packet at the main data packet.
 790             fst_data_pkt = data_pkt;
 791         } else {
 792
 793             // Create the split packets.
 794             fst_data_pkt = new Packet(sreqLow, command, Packet::Broadcast);
 795             snd_data_pkt = new Packet(sreqHigh, command, Packet::Broadcast);
 796
 797             fst_data_pkt->dataStatic(load_inst->memData);
 798             snd_data_pkt->dataStatic(load_inst->memData + sreqLow->getSize());
 799
 800             fst_data_pkt->senderState = state;
 801             snd_data_pkt->senderState = state;
 802
 803             state->isSplit = true;
 804             state->outstanding = 2;
 805             state->mainPkt = data_pkt;
 806         }
 807
 808         if (!dcachePort->sendTiming(fst_data_pkt)) {
 809             // Delete state and data packet because a load retry
 810             // initiates a pipeline restart; it does not retry.
 811             delete state;
 812             delete data_pkt->req;
 813             delete data_pkt;
 814             if (TheISA::HasUnalignedMemAcc && sreqLow) {
 815                 delete fst_data_pkt->req;
 816                 delete fst_data_pkt;
 817                 delete snd_data_pkt->req;
 818                 delete snd_data_pkt;
 819                 sreqLow = NULL;
 820                 sreqHigh = NULL;
 821             }
 822
 823             req = NULL;
 824
 825             // If the access didn't succeed, tell the LSQ by setting
 826             // the retry thread id.
 827             lsq->setRetryTid(lsqID);
 828         } else if (TheISA::HasUnalignedMemAcc && sreqLow) {
 829             completedFirst = true;
 830
 831             // The first packet was sent without problems, so send this one
 832             // too. If there is a problem with this packet then the whole
 833             // load will be squashed, so indicate this to the state object.
 834             // The first packet will return in completeDataAccess and be
 835             // handled there.
 836             ++usedPorts;
 837             if (!dcachePort->sendTiming(snd_data_pkt)) {
 838
 839                 // The main packet will be deleted in completeDataAccess.
 840                 delete snd_data_pkt->req;
 841                 delete snd_data_pkt;
 842
 843                 state->complete();
 844
 845                 req = NULL;
 846                 sreqHigh = NULL;
 847
 848                 lsq->setRetryTid(lsqID);
 849             }
 850         }
 851     }
 852
 853     // If the cache was blocked, or has become blocked due to the access,
 854     // handle it.
 855     if (lsq->cacheBlocked()) {
 856         if (req)
 857             delete req;
 858         if (TheISA::HasUnalignedMemAcc && sreqLow && !completedFirst) {
 859             delete sreqLow;
 860             delete sreqHigh;
 861         }
 862
 863         ++lsqCacheBlocked;
 864
 865         // If the first part of a split access succeeds, then let the LSQ
 866         // handle the decrWb when completeDataAccess is called upon return
 867         // of the requested first part of data
 868         if (!completedFirst)
 869             iewStage->decrWb(load_inst->seqNum);
 870
 871         // There's an older load that's already going to squash.
 872         if (isLoadBlocked && blockedLoadSeqNum < load_inst->seqNum)
 873             return NoFault;
 874
 875         // Record that the load was blocked due to memory.  This
 876         // load will squash all instructions after it, be
 877         // refetched, and re-executed.
 878         isLoadBlocked = true;
 879         loadBlockedHandled = false;
 880         blockedLoadSeqNum = load_inst->seqNum;
 881         // No fault occurred, even though the interface is blocked.
 882         return NoFault;
 883     }
 884
 885     return NoFault;
 886 }
 887
 888 template <class Impl>
 889 Fault
 890 LSQUnit<Impl>::write(Request *req, Request *sreqLow, Request *sreqHigh,
 891                      uint8_t *data, int store_idx)
 892 {
 893     assert(storeQueue[store_idx].inst);
 894
 895     DPRINTF(LSQUnit, "Doing write to store idx %i, addr %#x data %#x"
 896             " | storeHead:%i [sn:%i]\n",
 897             store_idx, req->getPaddr(), data, storeHead,
 898             storeQueue[store_idx].inst->seqNum);
 899
 900     storeQueue[store_idx].req = req;
 901     storeQueue[store_idx].sreqLow = sreqLow;
 902     storeQueue[store_idx].sreqHigh = sreqHigh;
 903     unsigned size = req->getSize();
 904     storeQueue[store_idx].size = size;
 905     assert(size <= sizeof(storeQueue[store_idx].data));
 906
 907     // Split stores can only occur in ISAs with unaligned memory accesses.  If
 908     // a store request has been split, sreqLow and sreqHigh will be non-null.
 909     if (TheISA::HasUnalignedMemAcc && sreqLow) {
 910         storeQueue[store_idx].isSplit = true;
 911     }
 912
 913     memcpy(storeQueue[store_idx].data, data, size);
 914
 915     // This function only writes the data to the store queue, so no fault
 916     // can happen here.
 917     return NoFault;
 918 }
 919
 920 #endif // __CPU_O3_LSQ_UNIT_HH__