src/cpu/minor/lsq.cc

   1 /*
   2  * Copyright (c) 2013-2014 ARM Limited
   3  * All rights reserved
   4  *
   5  * The license below extends only to copyright in the software and shall
   6  * not be construed as granting a license to any other intellectual
   7  * property including but not limited to intellectual property relating
   8  * to a hardware implementation of the functionality of the software
   9  * licensed hereunder.  You may use the software subject to the license
  10  * terms below provided that you ensure that this notice is replicated
  11  * unmodified and in its entirety in all distributions of the software,
  12  * modified or unmodified, in source code or in binary form.
  13  *
  14  * Redistribution and use in source and binary forms, with or without
  15  * modification, are permitted provided that the following conditions are
  16  * met: redistributions of source code must retain the above copyright
  17  * notice, this list of conditions and the following disclaimer;
  18  * redistributions in binary form must reproduce the above copyright
  19  * notice, this list of conditions and the following disclaimer in the
  20  * documentation and/or other materials provided with the distribution;
  21  * neither the name of the copyright holders nor the names of its
  22  * contributors may be used to endorse or promote products derived from
  23  * this software without specific prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  26  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  27  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  28  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  29  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  30  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  31  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  32  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  33  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  34  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  35  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  36  *
  37  * Authors: Andrew Bardsley
  38  */
  39
  40 #include <iomanip>
  41 #include <sstream>
  42
  43 #include "arch/locked_mem.hh"
  44 #include "arch/mmapped_ipr.hh"
  45 #include "cpu/minor/cpu.hh"
  46 #include "cpu/minor/exec_context.hh"
  47 #include "cpu/minor/execute.hh"
  48 #include "cpu/minor/lsq.hh"
  49 #include "cpu/minor/pipeline.hh"
  50 #include "debug/Activity.hh"
  51 #include "debug/MinorMem.hh"
  52
  53 namespace Minor
  54 {
  55
  56 /** Returns the offset of addr into an aligned a block of size block_size */
  57 static Addr
  58 addrBlockOffset(Addr addr, unsigned int block_size)
  59 {
  60     return addr & (block_size - 1);
  61 }
  62
  63 /** Returns true if the given [addr .. addr+size-1] transfer needs to be
  64  *  fragmented across a block size of block_size */
  65 static bool
  66 transferNeedsBurst(Addr addr, unsigned int size, unsigned int block_size)
  67 {
  68     return (addrBlockOffset(addr, block_size) + size) > block_size;
  69 }
  70
  71 LSQ::LSQRequest::LSQRequest(LSQ &port_, MinorDynInstPtr inst_, bool isLoad_,
  72     PacketDataPtr data_, uint64_t *res_) :
  73     SenderState(),
  74     port(port_),
  75     inst(inst_),
  76     isLoad(isLoad_),
  77     data(data_),
  78     packet(NULL),
  79     request(),
  80     fault(NoFault),
  81     res(res_),
  82     skipped(false),
  83     issuedToMemory(false),
  84     state(NotIssued)
  85 { }
  86
  87 LSQ::AddrRangeCoverage
  88 LSQ::LSQRequest::containsAddrRangeOf(
  89     Addr req1_addr, unsigned int req1_size,
  90     Addr req2_addr, unsigned int req2_size)
  91 {
  92     /* 'end' here means the address of the byte just past the request
  93      *  blocks */
  94     Addr req2_end_addr = req2_addr + req2_size;
  95     Addr req1_end_addr = req1_addr + req1_size;
  96
  97     AddrRangeCoverage ret;
  98
  99     if (req1_addr > req2_end_addr || req1_end_addr < req2_addr)
 100         ret = NoAddrRangeCoverage;
 101     else if (req1_addr <= req2_addr && req1_end_addr >= req2_end_addr)
 102         ret = FullAddrRangeCoverage;
 103     else
 104         ret = PartialAddrRangeCoverage;
 105
 106     return ret;
 107 }
 108
 109 LSQ::AddrRangeCoverage
 110 LSQ::LSQRequest::containsAddrRangeOf(LSQRequestPtr other_request)
 111 {
 112     return containsAddrRangeOf(request.getPaddr(), request.getSize(),
 113         other_request->request.getPaddr(), other_request->request.getSize());
 114 }
 115
 116 bool
 117 LSQ::LSQRequest::isBarrier()
 118 {
 119     return inst->isInst() && inst->staticInst->isMemBarrier();
 120 }
 121
 122 bool
 123 LSQ::LSQRequest::needsToBeSentToStoreBuffer()
 124 {
 125     return state == StoreToStoreBuffer;
 126 }
 127
 128 void
 129 LSQ::LSQRequest::setState(LSQRequestState new_state)
 130 {
 131     DPRINTFS(MinorMem, (&port), "Setting state from %d to %d for request:"
 132         " %s\n", state, new_state, *inst);
 133     state = new_state;
 134 }
 135
 136 bool
 137 LSQ::LSQRequest::isComplete() const
 138 {
 139     /* @todo, There is currently only one 'completed' state.  This
 140      *  may not be a good choice */
 141     return state == Complete;
 142 }
 143
 144 void
 145 LSQ::LSQRequest::reportData(std::ostream &os) const
 146 {
 147     os << (isLoad ? 'R' : 'W') << ';';
 148     inst->reportData(os);
 149     os << ';' << state;
 150 }
 151
 152 std::ostream &
 153 operator <<(std::ostream &os, LSQ::AddrRangeCoverage coverage)
 154 {
 155     switch (coverage) {
 156       case LSQ::PartialAddrRangeCoverage:
 157         os << "PartialAddrRangeCoverage";
 158         break;
 159       case LSQ::FullAddrRangeCoverage:
 160         os << "FullAddrRangeCoverage";
 161         break;
 162       case LSQ::NoAddrRangeCoverage:
 163         os << "NoAddrRangeCoverage";
 164         break;
 165       default:
 166         os << "AddrRangeCoverage-" << static_cast<int>(coverage);
 167         break;
 168     }
 169     return os;
 170 }
 171
 172 std::ostream &
 173 operator <<(std::ostream &os, LSQ::LSQRequest::LSQRequestState state)
 174 {
 175     switch (state) {
 176       case LSQ::LSQRequest::NotIssued:
 177         os << "NotIssued";
 178         break;
 179       case LSQ::LSQRequest::InTranslation:
 180         os << "InTranslation";
 181         break;
 182       case LSQ::LSQRequest::Translated:
 183         os << "Translated";
 184         break;
 185       case LSQ::LSQRequest::Failed:
 186         os << "Failed";
 187         break;
 188       case LSQ::LSQRequest::RequestIssuing:
 189         os << "RequestIssuing";
 190         break;
 191       case LSQ::LSQRequest::StoreToStoreBuffer:
 192         os << "StoreToStoreBuffer";
 193         break;
 194       case LSQ::LSQRequest::StoreInStoreBuffer:
 195         os << "StoreInStoreBuffer";
 196         break;
 197       case LSQ::LSQRequest::StoreBufferIssuing:
 198         os << "StoreBufferIssuing";
 199         break;
 200       case LSQ::LSQRequest::RequestNeedsRetry:
 201         os << "RequestNeedsRetry";
 202         break;
 203       case LSQ::LSQRequest::StoreBufferNeedsRetry:
 204         os << "StoreBufferNeedsRetry";
 205         break;
 206       case LSQ::LSQRequest::Complete:
 207         os << "Complete";
 208         break;
 209       default:
 210         os << "LSQRequestState-" << static_cast<int>(state);
 211         break;
 212     }
 213     return os;
 214 }
 215
 216 void
 217 LSQ::clearMemBarrier(MinorDynInstPtr inst)
 218 {
 219     bool is_last_barrier = inst->id.execSeqNum >= lastMemBarrier;
 220
 221     DPRINTF(MinorMem, "Moving %s barrier out of store buffer inst: %s\n",
 222         (is_last_barrier ? "last" : "a"), *inst);
 223
 224     if (is_last_barrier)
 225         lastMemBarrier = 0;
 226 }
 227
 228 void
 229 LSQ::SingleDataRequest::finish(Fault fault_, RequestPtr request_,
 230     ThreadContext *tc, BaseTLB::Mode mode)
 231 {
 232     fault = fault_;
 233
 234     port.numAccessesInDTLB--;
 235
 236     DPRINTFS(MinorMem, (&port), "Received translation response for"
 237         " request: %s\n", *inst);
 238
 239     makePacket();
 240
 241     setState(Translated);
 242     port.tryToSendToTransfers(this);
 243
 244     /* Let's try and wake up the processor for the next cycle */
 245     port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
 246 }
 247
 248 void
 249 LSQ::SingleDataRequest::startAddrTranslation()
 250 {
 251     ThreadContext *thread = port.cpu.getContext(
 252         inst->id.threadId);
 253
 254     port.numAccessesInDTLB++;
 255
 256     setState(LSQ::LSQRequest::InTranslation);
 257
 258     DPRINTFS(MinorMem, (&port), "Submitting DTLB request\n");
 259     /* Submit the translation request.  The response will come through
 260      *  finish/markDelayed on the LSQRequest as it bears the Translation
 261      *  interface */
 262     thread->getDTBPtr()->translateTiming(
 263         &request, thread, this, (isLoad ? BaseTLB::Read : BaseTLB::Write));
 264 }
 265
 266 void
 267 LSQ::SingleDataRequest::retireResponse(PacketPtr packet_)
 268 {
 269     DPRINTFS(MinorMem, (&port), "Retiring packet\n");
 270     packet = packet_;
 271     packetInFlight = false;
 272     setState(Complete);
 273 }
 274
 275 void
 276 LSQ::SplitDataRequest::finish(Fault fault_, RequestPtr request_,
 277     ThreadContext *tc, BaseTLB::Mode mode)
 278 {
 279     fault = fault_;
 280
 281     port.numAccessesInDTLB--;
 282
 283     unsigned int M5_VAR_USED expected_fragment_index =
 284         numTranslatedFragments;
 285
 286     numInTranslationFragments--;
 287     numTranslatedFragments++;
 288
 289     DPRINTFS(MinorMem, (&port), "Received translation response for fragment"
 290         " %d of request: %s\n", expected_fragment_index, *inst);
 291
 292     assert(request_ == fragmentRequests[expected_fragment_index]);
 293
 294     /* Wake up next cycle to get things going again in case the
 295      *  tryToSendToTransfers does take */
 296     port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
 297
 298     if (fault != NoFault) {
 299         /* tryToSendToTransfers will handle the fault */
 300
 301         DPRINTFS(MinorMem, (&port), "Faulting translation for fragment:"
 302             " %d of request: %s\n",
 303             expected_fragment_index, *inst);
 304
 305         setState(Translated);
 306         port.tryToSendToTransfers(this);
 307     } else if (numTranslatedFragments == numFragments) {
 308         makeFragmentPackets();
 309
 310         setState(Translated);
 311         port.tryToSendToTransfers(this);
 312     } else {
 313         /* Avoid calling translateTiming from within ::finish */
 314         assert(!translationEvent.scheduled());
 315         port.cpu.schedule(translationEvent, curTick());
 316     }
 317 }
 318
 319 LSQ::SplitDataRequest::SplitDataRequest(LSQ &port_, MinorDynInstPtr inst_,
 320     bool isLoad_, PacketDataPtr data_, uint64_t *res_) :
 321     LSQRequest(port_, inst_, isLoad_, data_, res_),
 322     translationEvent(*this),
 323     numFragments(0),
 324     numInTranslationFragments(0),
 325     numTranslatedFragments(0),
 326     numIssuedFragments(0),
 327     numRetiredFragments(0),
 328     fragmentRequests(),
 329     fragmentPackets()
 330 {
 331     /* Don't know how many elements are needed until the request is
 332      *  populated by the caller. */
 333 }
 334
 335 LSQ::SplitDataRequest::~SplitDataRequest()
 336 {
 337     for (auto i = fragmentRequests.begin();
 338         i != fragmentRequests.end(); i++)
 339     {
 340         delete *i;
 341     }
 342
 343     for (auto i = fragmentPackets.begin();
 344          i != fragmentPackets.end(); i++)
 345     {
 346         delete *i;
 347     }
 348 }
 349
 350 void
 351 LSQ::SplitDataRequest::makeFragmentRequests()
 352 {
 353     Addr base_addr = request.getVaddr();
 354     unsigned int whole_size = request.getSize();
 355     unsigned int line_width = port.lineWidth;
 356
 357     unsigned int fragment_size;
 358     Addr fragment_addr;
 359
 360     /* Assume that this transfer is across potentially many block snap
 361      * boundaries:
 362      *
 363      * |      _|________|________|________|___     |
 364      * |     |0| 1      | 2      | 3      | 4 |    |
 365      * |     |_|________|________|________|___|    |
 366      * |       |        |        |        |        |
 367      *
 368      *  The first transfer (0) can be up to lineWidth in size.
 369      *  All the middle transfers (1-3) are lineWidth in size
 370      *  The last transfer (4) can be from zero to lineWidth - 1 in size
 371      */
 372     unsigned int first_fragment_offset =
 373         addrBlockOffset(base_addr, line_width);
 374     unsigned int last_fragment_size =
 375         addrBlockOffset(base_addr + whole_size, line_width);
 376     unsigned int first_fragment_size =
 377         line_width - first_fragment_offset;
 378
 379     unsigned int middle_fragments_total_size =
 380         whole_size - (first_fragment_size + last_fragment_size);
 381
 382     assert(addrBlockOffset(middle_fragments_total_size, line_width) == 0);
 383
 384     unsigned int middle_fragment_count =
 385         middle_fragments_total_size / line_width;
 386
 387     numFragments = 1 /* first */ + middle_fragment_count +
 388         (last_fragment_size == 0 ? 0 : 1);
 389
 390     DPRINTFS(MinorMem, (&port), "Dividing transfer into %d fragmentRequests."
 391         " First fragment size: %d Last fragment size: %d\n",
 392         numFragments, first_fragment_size,
 393         (last_fragment_size == 0 ? line_width : last_fragment_size));
 394
 395     assert(((middle_fragment_count * line_width) +
 396         first_fragment_size + last_fragment_size) == whole_size);
 397
 398     fragment_addr = base_addr;
 399     fragment_size = first_fragment_size;
 400
 401     /* Just past the last address in the request */
 402     Addr end_addr = base_addr + whole_size;
 403
 404     for (unsigned int fragment_index = 0; fragment_index < numFragments;
 405          fragment_index++)
 406     {
 407         bool M5_VAR_USED is_last_fragment = false;
 408
 409         if (fragment_addr == base_addr) {
 410             /* First fragment */
 411             fragment_size = first_fragment_size;
 412         } else {
 413             if ((fragment_addr + line_width) > end_addr) {
 414                 /* Adjust size of last fragment */
 415                 fragment_size = end_addr - fragment_addr;
 416                 is_last_fragment = true;
 417             } else {
 418                 /* Middle fragments */
 419                 fragment_size = line_width;
 420             }
 421         }
 422
 423         Request *fragment = new Request();
 424
 425         fragment->setThreadContext(request.contextId(), /* thread id */ 0);
 426         fragment->setVirt(0 /* asid */,
 427             fragment_addr, fragment_size, request.getFlags(),
 428             request.masterId(),
 429             request.getPC());
 430
 431         DPRINTFS(MinorMem, (&port), "Generating fragment addr: 0x%x size: %d"
 432             " (whole request addr: 0x%x size: %d) %s\n",
 433             fragment_addr, fragment_size, base_addr, whole_size,
 434             (is_last_fragment ? "last fragment" : ""));
 435
 436         fragment_addr += fragment_size;
 437
 438         fragmentRequests.push_back(fragment);
 439     }
 440 }
 441
 442 void
 443 LSQ::SplitDataRequest::makeFragmentPackets()
 444 {
 445     Addr base_addr = request.getVaddr();
 446
 447     DPRINTFS(MinorMem, (&port), "Making packets for request: %s\n", *inst);
 448
 449     for (unsigned int fragment_index = 0; fragment_index < numFragments;
 450          fragment_index++)
 451     {
 452         Request *fragment = fragmentRequests[fragment_index];
 453
 454         DPRINTFS(MinorMem, (&port), "Making packet %d for request: %s"
 455             " (%d, 0x%x)\n",
 456             fragment_index, *inst,
 457             (fragment->hasPaddr() ? "has paddr" : "no paddr"),
 458             (fragment->hasPaddr() ? fragment->getPaddr() : 0));
 459
 460         Addr fragment_addr = fragment->getVaddr();
 461         unsigned int fragment_size = fragment->getSize();
 462
 463         uint8_t *request_data = NULL;
 464
 465         if (!isLoad) {
 466             /* Split data for Packets.  Will become the property of the
 467              *  outgoing Packets */
 468             request_data = new uint8_t[fragment_size];
 469             std::memcpy(request_data, data + (fragment_addr - base_addr),
 470                 fragment_size);
 471         }
 472
 473         assert(fragment->hasPaddr());
 474
 475         PacketPtr fragment_packet =
 476             makePacketForRequest(*fragment, isLoad, this, request_data);
 477
 478         fragmentPackets.push_back(fragment_packet);
 479     }
 480
 481     /* Might as well make the overall/response packet here */
 482     /* Get the physical address for the whole request/packet from the first
 483      *  fragment */
 484     request.setPaddr(fragmentRequests[0]->getPaddr());
 485     makePacket();
 486 }
 487
 488 void
 489 LSQ::SplitDataRequest::startAddrTranslation()
 490 {
 491     setState(LSQ::LSQRequest::InTranslation);
 492
 493     makeFragmentRequests();
 494
 495     numInTranslationFragments = 0;
 496     numTranslatedFragments = 0;
 497
 498     /* @todo, just do these in sequence for now with
 499      * a loop of:
 500      * do {
 501      *  sendNextFragmentToTranslation ; translateTiming ; finish
 502      * } while (numTranslatedFragments != numFragments);
 503      */
 504
 505     /* Do first translation */
 506     sendNextFragmentToTranslation();
 507 }
 508
 509 PacketPtr
 510 LSQ::SplitDataRequest::getHeadPacket()
 511 {
 512     assert(numIssuedFragments < numFragments);
 513
 514     return fragmentPackets[numIssuedFragments];
 515 }
 516
 517 void
 518 LSQ::SplitDataRequest::stepToNextPacket()
 519 {
 520     assert(numIssuedFragments < numFragments);
 521
 522     numIssuedFragments++;
 523 }
 524
 525 void
 526 LSQ::SplitDataRequest::retireResponse(PacketPtr response)
 527 {
 528     assert(numRetiredFragments < numFragments);
 529
 530     DPRINTFS(MinorMem, (&port), "Retiring fragment addr: 0x%x size: %d"
 531         " offset: 0x%x (retired fragment num: %d) %s\n",
 532         response->req->getVaddr(), response->req->getSize(),
 533         request.getVaddr() - response->req->getVaddr(),
 534         numRetiredFragments,
 535         (fault == NoFault ? "" : fault->name()));
 536
 537     numRetiredFragments++;
 538
 539     if (skipped) {
 540         /* Skip because we already knew the request had faulted or been
 541          *  skipped */
 542         DPRINTFS(MinorMem, (&port), "Skipping this fragment\n");
 543     } else if (response->isError()) {
 544         /* Mark up the error and leave to execute to handle it */
 545         DPRINTFS(MinorMem, (&port), "Fragment has an error, skipping\n");
 546         setSkipped();
 547         packet->copyError(response);
 548     } else {
 549         if (isLoad) {
 550             if (!data) {
 551                 /* For a split transfer, a Packet must be constructed
 552                  *  to contain all returning data.  This is that packet's
 553                  *  data */
 554                 data = new uint8_t[request.getSize()];
 555             }
 556
 557             /* Populate the portion of the overall response data represented
 558              *  by the response fragment */
 559             std::memcpy(
 560                 data + (response->req->getVaddr() - request.getVaddr()),
 561                 response->getPtr<uint8_t>(),
 562                 response->req->getSize());
 563         }
 564     }
 565
 566     /* Complete early if we're skipping are no more in-flight accesses */
 567     if (skipped && !hasPacketsInMemSystem()) {
 568         DPRINTFS(MinorMem, (&port), "Completed skipped burst\n");
 569         setState(Complete);
 570         if (packet->needsResponse())
 571             packet->makeResponse();
 572     }
 573
 574     if (numRetiredFragments == numFragments)
 575         setState(Complete);
 576
 577     if (!skipped && isComplete()) {
 578         DPRINTFS(MinorMem, (&port), "Completed burst %d\n", packet != NULL);
 579
 580         DPRINTFS(MinorMem, (&port), "Retired packet isRead: %d isWrite: %d"
 581              " needsResponse: %d packetSize: %s requestSize: %s responseSize:"
 582              " %s\n", packet->isRead(), packet->isWrite(),
 583              packet->needsResponse(), packet->getSize(), request.getSize(),
 584              response->getSize());
 585
 586         /* A request can become complete by several paths, this is a sanity
 587          *  check to make sure the packet's data is created */
 588         if (!data) {
 589             data = new uint8_t[request.getSize()];
 590         }
 591
 592         if (isLoad) {
 593             DPRINTFS(MinorMem, (&port), "Copying read data\n");
 594             std::memcpy(packet->getPtr<uint8_t>(), data, request.getSize());
 595         }
 596         packet->makeResponse();
 597     }
 598
 599     /* Packets are all deallocated together in ~SplitLSQRequest */
 600 }
 601
 602 void
 603 LSQ::SplitDataRequest::sendNextFragmentToTranslation()
 604 {
 605     unsigned int fragment_index = numTranslatedFragments;
 606
 607     ThreadContext *thread = port.cpu.getContext(
 608         inst->id.threadId);
 609
 610     DPRINTFS(MinorMem, (&port), "Submitting DTLB request for fragment: %d\n",
 611         fragment_index);
 612
 613     port.numAccessesInDTLB++;
 614     numInTranslationFragments++;
 615
 616     thread->getDTBPtr()->translateTiming(
 617         fragmentRequests[fragment_index], thread, this, (isLoad ?
 618         BaseTLB::Read : BaseTLB::Write));
 619 }
 620
 621 bool
 622 LSQ::StoreBuffer::canInsert() const
 623 {
 624     /* @todo, support store amalgamation */
 625     return slots.size() < numSlots;
 626 }
 627
 628 void
 629 LSQ::StoreBuffer::deleteRequest(LSQRequestPtr request)
 630 {
 631     auto found = std::find(slots.begin(), slots.end(), request);
 632
 633     if (found != slots.end()) {
 634         DPRINTF(MinorMem, "Deleting request: %s %s %s from StoreBuffer\n",
 635             request, *found, *(request->inst));
 636         slots.erase(found);
 637
 638         delete request;
 639     }
 640 }
 641
 642 void
 643 LSQ::StoreBuffer::insert(LSQRequestPtr request)
 644 {
 645     if (!canInsert()) {
 646         warn("%s: store buffer insertion without space to insert from"
 647             " inst: %s\n", name(), *(request->inst));
 648     }
 649
 650     DPRINTF(MinorMem, "Pushing store: %s into store buffer\n", request);
 651
 652     numUnissuedAccesses++;
 653
 654     if (request->state != LSQRequest::Complete)
 655         request->setState(LSQRequest::StoreInStoreBuffer);
 656
 657     slots.push_back(request);
 658
 659     /* Let's try and wake up the processor for the next cycle to step
 660      *  the store buffer */
 661     lsq.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
 662 }
 663
 664 LSQ::AddrRangeCoverage
 665 LSQ::StoreBuffer::canForwardDataToLoad(LSQRequestPtr request,
 666     unsigned int &found_slot)
 667 {
 668     unsigned int slot_index = slots.size() - 1;
 669     auto i = slots.rbegin();
 670     AddrRangeCoverage ret = NoAddrRangeCoverage;
 671
 672     /* Traverse the store buffer in reverse order (most to least recent)
 673      *  and try to find a slot whose address range overlaps this request */
 674     while (ret == NoAddrRangeCoverage && i != slots.rend()) {
 675         LSQRequestPtr slot = *i;
 676
 677         if (slot->packet) {
 678             AddrRangeCoverage coverage = slot->containsAddrRangeOf(request);
 679
 680             if (coverage != NoAddrRangeCoverage) {
 681                 DPRINTF(MinorMem, "Forwarding: slot: %d result: %s thisAddr:"
 682                     " 0x%x thisSize: %d slotAddr: 0x%x slotSize: %d\n",
 683                     slot_index, coverage,
 684                     request->request.getPaddr(), request->request.getSize(),
 685                     slot->request.getPaddr(), slot->request.getSize());
 686
 687                 found_slot = slot_index;
 688                 ret = coverage;
 689             }
 690         }
 691
 692         i++;
 693         slot_index--;
 694     }
 695
 696     return ret;
 697 }
 698
 699 /** Fill the given packet with appropriate date from slot slot_number */
 700 void
 701 LSQ::StoreBuffer::forwardStoreData(LSQRequestPtr load,
 702     unsigned int slot_number)
 703 {
 704     assert(slot_number < slots.size());
 705     assert(load->packet);
 706     assert(load->isLoad);
 707
 708     LSQRequestPtr store = slots[slot_number];
 709
 710     assert(store->packet);
 711     assert(store->containsAddrRangeOf(load) == FullAddrRangeCoverage);
 712
 713     Addr load_addr = load->request.getPaddr();
 714     Addr store_addr = store->request.getPaddr();
 715     Addr addr_offset = load_addr - store_addr;
 716
 717     unsigned int load_size = load->request.getSize();
 718
 719     DPRINTF(MinorMem, "Forwarding %d bytes for addr: 0x%x from store buffer"
 720         " slot: %d addr: 0x%x addressOffset: 0x%x\n",
 721         load_size, load_addr, slot_number,
 722         store_addr, addr_offset);
 723
 724     void *load_packet_data = load->packet->getPtr<void>();
 725     void *store_packet_data = store->packet->getPtr<uint8_t>() + addr_offset;
 726
 727     std::memcpy(load_packet_data, store_packet_data, load_size);
 728 }
 729
 730 void
 731 LSQ::StoreBuffer::step()
 732 {
 733     DPRINTF(MinorMem, "StoreBuffer step numUnissuedAccesses: %d\n",
 734         numUnissuedAccesses);
 735
 736     if (numUnissuedAccesses != 0 && lsq.state == LSQ::MemoryRunning) {
 737         /* Clear all the leading barriers */
 738         while (!slots.empty() &&
 739             slots.front()->isComplete() && slots.front()->isBarrier())
 740         {
 741             LSQRequestPtr barrier = slots.front();
 742
 743             DPRINTF(MinorMem, "Clearing barrier for inst: %s\n",
 744                 *(barrier->inst));
 745
 746             numUnissuedAccesses--;
 747             lsq.clearMemBarrier(barrier->inst);
 748             slots.pop_front();
 749
 750             delete barrier;
 751         }
 752
 753         auto i = slots.begin();
 754         bool issued = true;
 755         unsigned int issue_count = 0;
 756
 757         /* Skip trying if the memory system is busy */
 758         if (lsq.state == LSQ::MemoryNeedsRetry)
 759             issued = false;
 760
 761         /* Try to issue all stores in order starting from the head
 762          *  of the queue.  Responses are allowed to be retired
 763          *  out of order */
 764         while (issued &&
 765             issue_count < storeLimitPerCycle &&
 766             lsq.canSendToMemorySystem() &&
 767             i != slots.end())
 768         {
 769             LSQRequestPtr request = *i;
 770
 771             DPRINTF(MinorMem, "Considering request: %s, sentAllPackets: %d"
 772                 " state: %s\n",
 773                 *(request->inst), request->sentAllPackets(),
 774                 request->state);
 775
 776             if (request->isBarrier() && request->isComplete()) {
 777                 /* Give up at barriers */
 778                 issued = false;
 779             } else if (!(request->state == LSQRequest::StoreBufferIssuing &&
 780                 request->sentAllPackets()))
 781             {
 782                 DPRINTF(MinorMem, "Trying to send request: %s to memory"
 783                     " system\n", *(request->inst));
 784
 785                 if (lsq.tryToSend(request)) {
 786                     /* Barrier are accounted for as they are cleared from
 787                      *  the queue, not after their transfers are complete */
 788                     if (!request->isBarrier())
 789                         numUnissuedAccesses--;
 790                     issue_count++;
 791                 } else {
 792                     /* Don't step on to the next store buffer entry if this
 793                      *  one hasn't issued all its packets as the store
 794                      *  buffer must still enforce ordering */
 795                     issued = false;
 796                 }
 797             }
 798             i++;
 799         }
 800     }
 801 }
 802
 803 void
 804 LSQ::completeMemBarrierInst(MinorDynInstPtr inst,
 805     bool committed)
 806 {
 807     if (committed) {
 808         /* Not already sent to the store buffer as a store request? */
 809         if (!inst->inStoreBuffer) {
 810             /* Insert an entry into the store buffer to tick off barriers
 811              *  until there are none in flight */
 812             storeBuffer.insert(new BarrierDataRequest(*this, inst));
 813         }
 814     } else {
 815         /* Clear the barrier anyway if it wasn't actually committed */
 816         clearMemBarrier(inst);
 817     }
 818 }
 819
 820 void
 821 LSQ::StoreBuffer::minorTrace() const
 822 {
 823     unsigned int size = slots.size();
 824     unsigned int i = 0;
 825     std::ostringstream os;
 826
 827     while (i < size) {
 828         LSQRequestPtr request = slots[i];
 829
 830         request->reportData(os);
 831
 832         i++;
 833         if (i < numSlots)
 834             os << ',';
 835     }
 836
 837     while (i < numSlots) {
 838         os << '-';
 839
 840         i++;
 841         if (i < numSlots)
 842             os << ',';
 843     }
 844
 845     MINORTRACE("addr=%s num_unissued_stores=%d\n", os.str(),
 846         numUnissuedAccesses);
 847 }
 848
 849 void
 850 LSQ::tryToSendToTransfers(LSQRequestPtr request)
 851 {
 852     if (state == MemoryNeedsRetry) {
 853         DPRINTF(MinorMem, "Request needs retry, not issuing to"
 854             " memory until retry arrives\n");
 855         return;
 856     }
 857
 858     if (request->state == LSQRequest::InTranslation) {
 859         DPRINTF(MinorMem, "Request still in translation, not issuing to"
 860             " memory\n");
 861         return;
 862     }
 863
 864     assert(request->state == LSQRequest::Translated ||
 865         request->state == LSQRequest::RequestIssuing ||
 866         request->state == LSQRequest::Failed ||
 867         request->state == LSQRequest::Complete);
 868
 869     if (requests.empty() || requests.front() != request) {
 870         DPRINTF(MinorMem, "Request not at front of requests queue, can't"
 871             " issue to memory\n");
 872         return;
 873     }
 874
 875     if (transfers.unreservedRemainingSpace() == 0) {
 876         DPRINTF(MinorMem, "No space to insert request into transfers"
 877             " queue\n");
 878         return;
 879     }
 880
 881     if (request->isComplete() || request->state == LSQRequest::Failed) {
 882         DPRINTF(MinorMem, "Passing a %s transfer on to transfers"
 883             " queue\n", (request->isComplete() ? "completed" : "failed"));
 884         request->setState(LSQRequest::Complete);
 885         request->setSkipped();
 886         moveFromRequestsToTransfers(request);
 887         return;
 888     }
 889
 890     if (!execute.instIsRightStream(request->inst)) {
 891         /* Wrong stream, try to abort the transfer but only do so if
 892          *  there are no packets in flight */
 893         if (request->hasPacketsInMemSystem()) {
 894             DPRINTF(MinorMem, "Request's inst. is from the wrong stream,"
 895                 " waiting for responses before aborting request\n");
 896         } else {
 897             DPRINTF(MinorMem, "Request's inst. is from the wrong stream,"
 898                 " aborting request\n");
 899             request->setState(LSQRequest::Complete);
 900             request->setSkipped();
 901             moveFromRequestsToTransfers(request);
 902         }
 903         return;
 904     }
 905
 906     if (request->fault != NoFault) {
 907         if (request->inst->staticInst->isPrefetch()) {
 908             DPRINTF(MinorMem, "Not signalling fault for faulting prefetch\n");
 909         }
 910         DPRINTF(MinorMem, "Moving faulting request into the transfers"
 911             " queue\n");
 912         request->setState(LSQRequest::Complete);
 913         request->setSkipped();
 914         moveFromRequestsToTransfers(request);
 915         return;
 916     }
 917
 918     bool is_load = request->isLoad;
 919     bool is_llsc = request->request.isLLSC();
 920     bool is_swap = request->request.isSwap();
 921     bool bufferable = !(request->request.isUncacheable() ||
 922         is_llsc || is_swap);
 923
 924     if (is_load) {
 925         if (numStoresInTransfers != 0) {
 926             DPRINTF(MinorMem, "Load request with stores still in transfers"
 927                 " queue, stalling\n");
 928             return;
 929         }
 930     } else {
 931         /* Store.  Can it be sent to the store buffer? */
 932         if (bufferable && !request->request.isMmappedIpr()) {
 933             request->setState(LSQRequest::StoreToStoreBuffer);
 934             moveFromRequestsToTransfers(request);
 935             DPRINTF(MinorMem, "Moving store into transfers queue\n");
 936             return;
 937         }
 938     }
 939
 940     /* Check if this is the head instruction (and so must be executable as
 941      *  its stream sequence number was checked above) for loads which must
 942      *  not be speculatively issued and stores which must be issued here */
 943     if (!bufferable) {
 944         if (!execute.instIsHeadInst(request->inst)) {
 945             DPRINTF(MinorMem, "Memory access not the head inst., can't be"
 946                 " sure it can be performed, not issuing\n");
 947             return;
 948         }
 949
 950         unsigned int forwarding_slot = 0;
 951
 952         if (storeBuffer.canForwardDataToLoad(request, forwarding_slot) !=
 953             NoAddrRangeCoverage)
 954         {
 955             DPRINTF(MinorMem, "Memory access can receive forwarded data"
 956                 " from the store buffer, need to wait for store buffer to"
 957                 " drain\n");
 958             return;
 959         }
 960     }
 961
 962     /* True: submit this packet to the transfers queue to be sent to the
 963      * memory system.
 964      * False: skip the memory and push a packet for this request onto
 965      * requests */
 966     bool do_access = true;
 967
 968     if (!is_llsc) {
 969         /* Check for match in the store buffer */
 970         if (is_load) {
 971             unsigned int forwarding_slot = 0;
 972             AddrRangeCoverage forwarding_result =
 973                 storeBuffer.canForwardDataToLoad(request,
 974                 forwarding_slot);
 975
 976             switch (forwarding_result) {
 977               case FullAddrRangeCoverage:
 978                 /* Forward data from the store buffer into this request and
 979                  *  repurpose this request's packet into a response packet */
 980                 storeBuffer.forwardStoreData(request, forwarding_slot);
 981                 request->packet->makeResponse();
 982
 983                 /* Just move between queues, no access */
 984                 do_access = false;
 985                 break;
 986               case PartialAddrRangeCoverage:
 987                 DPRINTF(MinorMem, "Load partly satisfied by store buffer"
 988                     " data. Must wait for the store to complete\n");
 989                 return;
 990                 break;
 991               case NoAddrRangeCoverage:
 992                 DPRINTF(MinorMem, "No forwardable data from store buffer\n");
 993                 /* Fall through to try access */
 994                 break;
 995             }
 996         }
 997     } else {
 998         if (!canSendToMemorySystem()) {
 999             DPRINTF(MinorMem, "Can't send request to memory system yet\n");
1000             return;
1001         }
1002
1003         SimpleThread &thread = *cpu.threads[request->inst->id.threadId];
1004
1005         TheISA::PCState old_pc = thread.pcState();
1006         ExecContext context(cpu, thread, execute, request->inst);
1007
1008         /* Handle LLSC requests and tests */
1009         if (is_load) {
1010             TheISA::handleLockedRead(&context, &request->request);
1011         } else {
1012             do_access = TheISA::handleLockedWrite(&context,
1013                 &request->request, cacheBlockMask);
1014
1015             if (!do_access) {
1016                 DPRINTF(MinorMem, "Not perfoming a memory "
1017                     "access for store conditional\n");
1018             }
1019         }
1020         thread.pcState(old_pc);
1021     }
1022
1023     /* See the do_access comment above */
1024     if (do_access) {
1025         if (!canSendToMemorySystem()) {
1026             DPRINTF(MinorMem, "Can't send request to memory system yet\n");
1027             return;
1028         }
1029
1030         /* Remember if this is an access which can't be idly
1031          *  discarded by an interrupt */
1032         if (!bufferable) {
1033             numAccessesIssuedToMemory++;
1034             request->issuedToMemory = true;
1035         }
1036
1037         if (tryToSend(request))
1038             moveFromRequestsToTransfers(request);
1039     } else {
1040         request->setState(LSQRequest::Complete);
1041         moveFromRequestsToTransfers(request);
1042     }
1043 }
1044
1045 bool
1046 LSQ::tryToSend(LSQRequestPtr request)
1047 {
1048     bool ret = false;
1049
1050     if (!canSendToMemorySystem()) {
1051         DPRINTF(MinorMem, "Can't send request: %s yet, no space in memory\n",
1052             *(request->inst));
1053     } else {
1054         PacketPtr packet = request->getHeadPacket();
1055
1056         DPRINTF(MinorMem, "Trying to send request: %s addr: 0x%x\n",
1057             *(request->inst), packet->req->getVaddr());
1058
1059         /* The sender state of the packet *must* be an LSQRequest
1060          *  so the response can be correctly handled */
1061         assert(packet->findNextSenderState<LSQRequest>());
1062
1063         if (request->request.isMmappedIpr()) {
1064             ThreadContext *thread =
1065                 cpu.getContext(request->request.threadId());
1066
1067             if (request->isLoad) {
1068                 DPRINTF(MinorMem, "IPR read inst: %s\n", *(request->inst));
1069                 TheISA::handleIprRead(thread, packet);
1070             } else {
1071                 DPRINTF(MinorMem, "IPR write inst: %s\n", *(request->inst));
1072                 TheISA::handleIprWrite(thread, packet);
1073             }
1074
1075             request->stepToNextPacket();
1076             ret = request->sentAllPackets();
1077
1078             if (!ret) {
1079                 DPRINTF(MinorMem, "IPR access has another packet: %s\n",
1080                     *(request->inst));
1081             }
1082
1083             if (ret)
1084                 request->setState(LSQRequest::Complete);
1085             else
1086                 request->setState(LSQRequest::RequestIssuing);
1087         } else if (dcachePort.sendTimingReq(packet)) {
1088             DPRINTF(MinorMem, "Sent data memory request\n");
1089
1090             numAccessesInMemorySystem++;
1091
1092             request->stepToNextPacket();
1093
1094             ret = request->sentAllPackets();
1095
1096             switch (request->state) {
1097               case LSQRequest::Translated:
1098               case LSQRequest::RequestIssuing:
1099                 /* Fully or partially issued a request in the transfers
1100                  *  queue */
1101                 request->setState(LSQRequest::RequestIssuing);
1102                 break;
1103               case LSQRequest::StoreInStoreBuffer:
1104               case LSQRequest::StoreBufferIssuing:
1105                 /* Fully or partially issued a request in the store
1106                  *  buffer */
1107                 request->setState(LSQRequest::StoreBufferIssuing);
1108                 break;
1109               default:
1110                 assert(false);
1111                 break;
1112             }
1113
1114             state = MemoryRunning;
1115         } else {
1116             DPRINTF(MinorMem,
1117                 "Sending data memory request - needs retry\n");
1118
1119             /* Needs to be resent, wait for that */
1120             state = MemoryNeedsRetry;
1121             retryRequest = request;
1122
1123             switch (request->state) {
1124               case LSQRequest::Translated:
1125               case LSQRequest::RequestIssuing:
1126                 request->setState(LSQRequest::RequestNeedsRetry);
1127                 break;
1128               case LSQRequest::StoreInStoreBuffer:
1129               case LSQRequest::StoreBufferIssuing:
1130                 request->setState(LSQRequest::StoreBufferNeedsRetry);
1131                 break;
1132               default:
1133                 assert(false);
1134                 break;
1135             }
1136         }
1137     }
1138
1139     return ret;
1140 }
1141
1142 void
1143 LSQ::moveFromRequestsToTransfers(LSQRequestPtr request)
1144 {
1145     assert(!requests.empty() && requests.front() == request);
1146     assert(transfers.unreservedRemainingSpace() != 0);
1147
1148     /* Need to count the number of stores in the transfers
1149      *  queue so that loads know when their store buffer forwarding
1150      *  results will be correct (only when all those stores
1151      *  have reached the store buffer) */
1152     if (!request->isLoad)
1153         numStoresInTransfers++;
1154
1155     requests.pop();
1156     transfers.push(request);
1157 }
1158
1159 bool
1160 LSQ::canSendToMemorySystem()
1161 {
1162     return state == MemoryRunning &&
1163         numAccessesInMemorySystem < inMemorySystemLimit;
1164 }
1165
1166 bool
1167 LSQ::recvTimingResp(PacketPtr response)
1168 {
1169     LSQRequestPtr request =
1170         safe_cast<LSQRequestPtr>(response->popSenderState());
1171
1172     DPRINTF(MinorMem, "Received response packet inst: %s"
1173         " addr: 0x%x cmd: %s\n",
1174         *(request->inst), response->getAddr(),
1175         response->cmd.toString());
1176
1177     numAccessesInMemorySystem--;
1178
1179     if (response->isError()) {
1180         DPRINTF(MinorMem, "Received error response packet: %s\n",
1181             *request->inst);
1182     }
1183
1184     switch (request->state) {
1185       case LSQRequest::RequestIssuing:
1186       case LSQRequest::RequestNeedsRetry:
1187         /* Response to a request from the transfers queue */
1188         request->retireResponse(response);
1189
1190         DPRINTF(MinorMem, "Has outstanding packets?: %d %d\n",
1191             request->hasPacketsInMemSystem(), request->isComplete());
1192
1193         break;
1194       case LSQRequest::StoreBufferIssuing:
1195       case LSQRequest::StoreBufferNeedsRetry:
1196         /* Response to a request from the store buffer */
1197         request->retireResponse(response);
1198
1199         /* Remove completed requests unless they are barrier (which will
1200          *  need to be removed in order */
1201         if (request->isComplete()) {
1202             if (!request->isBarrier()) {
1203                 storeBuffer.deleteRequest(request);
1204             } else {
1205                 DPRINTF(MinorMem, "Completed transfer for barrier: %s"
1206                     " leaving the request as it is also a barrier\n",
1207                     *(request->inst));
1208             }
1209         }
1210         break;
1211       default:
1212         /* Shouldn't be allowed to receive a response from another
1213          *  state */
1214         assert(false);
1215         break;
1216     }
1217
1218     /* We go to idle even if there are more things in the requests queue
1219      * as it's the job of step to actually step us on to the next
1220      * transaction */
1221
1222     /* Let's try and wake up the processor for the next cycle */
1223     cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
1224
1225     /* Never busy */
1226     return true;
1227 }
1228
1229 void
1230 LSQ::recvRetry()
1231 {
1232     DPRINTF(MinorMem, "Received retry request\n");
1233
1234     assert(state == MemoryNeedsRetry);
1235
1236     switch (retryRequest->state) {
1237       case LSQRequest::RequestNeedsRetry:
1238         /* Retry in the requests queue */
1239         retryRequest->setState(LSQRequest::Translated);
1240         break;
1241       case LSQRequest::StoreBufferNeedsRetry:
1242         /* Retry in the store buffer */
1243         retryRequest->setState(LSQRequest::StoreInStoreBuffer);
1244         break;
1245       default:
1246         assert(false);
1247     }
1248
1249     /* Set state back to MemoryRunning so that the following
1250      *  tryToSend can actually send.  Note that this won't
1251      *  allow another transfer in as tryToSend should
1252      *  issue a memory request and either succeed for this
1253      *  request or return the LSQ back to MemoryNeedsRetry */
1254     state = MemoryRunning;
1255
1256     /* Try to resend the request */
1257     if (tryToSend(retryRequest)) {
1258         /* Successfully sent, need to move the request */
1259         switch (retryRequest->state) {
1260           case LSQRequest::RequestIssuing:
1261             /* In the requests queue */
1262             moveFromRequestsToTransfers(retryRequest);
1263             break;
1264           case LSQRequest::StoreBufferIssuing:
1265             /* In the store buffer */
1266             storeBuffer.numUnissuedAccesses--;
1267             break;
1268           default:
1269             assert(false);
1270             break;
1271         }
1272     }
1273
1274     retryRequest = NULL;
1275 }
1276
1277 LSQ::LSQ(std::string name_, std::string dcache_port_name_,
1278     MinorCPU &cpu_, Execute &execute_,
1279     unsigned int in_memory_system_limit, unsigned int line_width,
1280     unsigned int requests_queue_size, unsigned int transfers_queue_size,
1281     unsigned int store_buffer_size,
1282     unsigned int store_buffer_cycle_store_limit) :
1283     Named(name_),
1284     cpu(cpu_),
1285     execute(execute_),
1286     dcachePort(dcache_port_name_, *this, cpu_),
1287     lastMemBarrier(0),
1288     state(MemoryRunning),
1289     inMemorySystemLimit(in_memory_system_limit),
1290     lineWidth((line_width == 0 ? cpu.cacheLineSize() : line_width)),
1291     requests(name_ + ".requests", "addr", requests_queue_size),
1292     transfers(name_ + ".transfers", "addr", transfers_queue_size),
1293     storeBuffer(name_ + ".storeBuffer",
1294         *this, store_buffer_size, store_buffer_cycle_store_limit),
1295     numAccessesInMemorySystem(0),
1296     numAccessesInDTLB(0),
1297     numStoresInTransfers(0),
1298     numAccessesIssuedToMemory(0),
1299     retryRequest(NULL),
1300     cacheBlockMask(~(cpu_.cacheLineSize() - 1))
1301 {
1302     if (in_memory_system_limit < 1) {
1303         fatal("%s: executeMaxAccessesInMemory must be >= 1 (%d)\n", name_,
1304             in_memory_system_limit);
1305     }
1306
1307     if (store_buffer_cycle_store_limit < 1) {
1308         fatal("%s: executeLSQMaxStoreBufferStoresPerCycle must be"
1309             " >= 1 (%d)\n", name_, store_buffer_cycle_store_limit);
1310     }
1311
1312     if (requests_queue_size < 1) {
1313         fatal("%s: executeLSQRequestsQueueSize must be"
1314             " >= 1 (%d)\n", name_, requests_queue_size);
1315     }
1316
1317     if (transfers_queue_size < 1) {
1318         fatal("%s: executeLSQTransfersQueueSize must be"
1319             " >= 1 (%d)\n", name_, transfers_queue_size);
1320     }
1321
1322     if (store_buffer_size < 1) {
1323         fatal("%s: executeLSQStoreBufferSize must be"
1324             " >= 1 (%d)\n", name_, store_buffer_size);
1325     }
1326
1327     if ((lineWidth & (lineWidth - 1)) != 0) {
1328         fatal("%s: lineWidth: %d must be a power of 2\n", name(), lineWidth);
1329     }
1330 }
1331
1332 LSQ::~LSQ()
1333 { }
1334
1335 LSQ::LSQRequest::~LSQRequest()
1336 {
1337     if (packet)
1338         delete packet;
1339     if (data)
1340         delete [] data;
1341 }
1342
1343 /**
1344  *  Step the memory access mechanism on to its next state.  In reality, most
1345  *  of the stepping is done by the callbacks on the LSQ but this
1346  *  function is responsible for issuing memory requests lodged in the
1347  *  requests queue.
1348  */
1349 void
1350 LSQ::step()
1351 {
1352     /* Try to move address-translated requests between queues and issue
1353      *  them */
1354     if (!requests.empty())
1355         tryToSendToTransfers(requests.front());
1356
1357     storeBuffer.step();
1358 }
1359
1360 LSQ::LSQRequestPtr
1361 LSQ::findResponse(MinorDynInstPtr inst)
1362 {
1363     LSQ::LSQRequestPtr ret = NULL;
1364
1365     if (!transfers.empty()) {
1366         LSQRequestPtr request = transfers.front();
1367
1368         /* Same instruction and complete access or a store that's
1369          *  capable of being moved to the store buffer */
1370         if (request->inst->id == inst->id) {
1371             if (request->isComplete() ||
1372                 (request->state == LSQRequest::StoreToStoreBuffer &&
1373                 storeBuffer.canInsert()))
1374             {
1375                 ret = request;
1376             }
1377         }
1378     }
1379
1380     if (ret) {
1381         DPRINTF(MinorMem, "Found matching memory response for inst: %s\n",
1382             *inst);
1383     } else {
1384         DPRINTF(MinorMem, "No matching memory response for inst: %s\n",
1385             *inst);
1386     }
1387
1388     return ret;
1389 }
1390
1391 void
1392 LSQ::popResponse(LSQ::LSQRequestPtr response)
1393 {
1394     assert(!transfers.empty() && transfers.front() == response);
1395
1396     transfers.pop();
1397
1398     if (!response->isLoad)
1399         numStoresInTransfers--;
1400
1401     if (response->issuedToMemory)
1402         numAccessesIssuedToMemory--;
1403
1404     if (response->state != LSQRequest::StoreInStoreBuffer) {
1405         DPRINTF(MinorMem, "Deleting %s request: %s\n",
1406             (response->isLoad ? "load" : "store"),
1407             *(response->inst));
1408
1409         delete response;
1410     }
1411 }
1412
1413 void
1414 LSQ::sendStoreToStoreBuffer(LSQRequestPtr request)
1415 {
1416     assert(request->state == LSQRequest::StoreToStoreBuffer);
1417
1418     DPRINTF(MinorMem, "Sending store: %s to store buffer\n",
1419         *(request->inst));
1420
1421     request->inst->inStoreBuffer = true;
1422
1423     storeBuffer.insert(request);
1424 }
1425
1426 bool
1427 LSQ::isDrained()
1428 {
1429     return requests.empty() && transfers.empty() &&
1430         storeBuffer.isDrained();
1431 }
1432
1433 bool
1434 LSQ::needsToTick()
1435 {
1436     bool ret = false;
1437
1438     if (canSendToMemorySystem()) {
1439         bool have_translated_requests = !requests.empty() &&
1440             requests.front()->state != LSQRequest::InTranslation &&
1441             transfers.unreservedRemainingSpace() != 0;
1442
1443         ret = have_translated_requests ||
1444             storeBuffer.numUnissuedStores() != 0;
1445     }
1446
1447     if (ret)
1448         DPRINTF(Activity, "Need to tick\n");
1449
1450     return ret;
1451 }
1452
1453 void
1454 LSQ::pushRequest(MinorDynInstPtr inst, bool isLoad, uint8_t *data,
1455     unsigned int size, Addr addr, unsigned int flags, uint64_t *res)
1456 {
1457     bool needs_burst = transferNeedsBurst(addr, size, lineWidth);
1458     LSQRequestPtr request;
1459
1460     /* Copy given data into the request.  The request will pass this to the
1461      *  packet and then it will own the data */
1462     uint8_t *request_data = NULL;
1463
1464     DPRINTF(MinorMem, "Pushing request (%s) addr: 0x%x size: %d flags:"
1465         " 0x%x%s lineWidth : 0x%x\n",
1466         (isLoad ? "load" : "store"), addr, size, flags,
1467             (needs_burst ? " (needs burst)" : ""), lineWidth);
1468
1469     if (!isLoad) {
1470         /* request_data becomes the property of a ...DataRequest (see below)
1471          *  and destroyed by its destructor */
1472         request_data = new uint8_t[size];
1473         if (flags & Request::CACHE_BLOCK_ZERO) {
1474             /* For cache zeroing, just use zeroed data */
1475             std::memset(request_data, 0, size);
1476         } else {
1477             std::memcpy(request_data, data, size);
1478         }
1479     }
1480
1481     if (needs_burst) {
1482         request = new SplitDataRequest(
1483             *this, inst, isLoad, request_data, res);
1484     } else {
1485         request = new SingleDataRequest(
1486             *this, inst, isLoad, request_data, res);
1487     }
1488
1489     if (inst->traceData)
1490         inst->traceData->setAddr(addr);
1491
1492     request->request.setThreadContext(cpu.cpuId(), /* thread id */ 0);
1493     request->request.setVirt(0 /* asid */,
1494         addr, size, flags, cpu.instMasterId(),
1495         /* I've no idea why we need the PC, but give it */
1496         inst->pc.instAddr());
1497
1498     requests.push(request);
1499     request->startAddrTranslation();
1500 }
1501
1502 void
1503 LSQ::pushFailedRequest(MinorDynInstPtr inst)
1504 {
1505     LSQRequestPtr request = new FailedDataRequest(*this, inst);
1506     requests.push(request);
1507 }
1508
1509 void
1510 LSQ::minorTrace() const
1511 {
1512     MINORTRACE("state=%s in_tlb_mem=%d/%d stores_in_transfers=%d"
1513         " lastMemBarrier=%d\n",
1514         state, numAccessesInDTLB, numAccessesInMemorySystem,
1515         numStoresInTransfers, lastMemBarrier);
1516     requests.minorTrace();
1517     transfers.minorTrace();
1518     storeBuffer.minorTrace();
1519 }
1520
1521 LSQ::StoreBuffer::StoreBuffer(std::string name_, LSQ &lsq_,
1522     unsigned int store_buffer_size,
1523     unsigned int store_limit_per_cycle) :
1524     Named(name_), lsq(lsq_),
1525     numSlots(store_buffer_size),
1526     storeLimitPerCycle(store_limit_per_cycle),
1527     slots(),
1528     numUnissuedAccesses(0)
1529 {
1530 }
1531
1532 PacketPtr
1533 makePacketForRequest(Request &request, bool isLoad,
1534     Packet::SenderState *sender_state, PacketDataPtr data)
1535 {
1536     MemCmd command;
1537
1538     /* Make a ret with the right command type to match the request */
1539     if (request.isLLSC()) {
1540         command = (isLoad ? MemCmd::LoadLockedReq : MemCmd::StoreCondReq);
1541     } else if (request.isSwap()) {
1542         command = MemCmd::SwapReq;
1543     } else {
1544         command = (isLoad ? MemCmd::ReadReq : MemCmd::WriteReq);
1545     }
1546
1547     PacketPtr ret = new Packet(&request, command);
1548
1549     if (sender_state)
1550         ret->pushSenderState(sender_state);
1551
1552     if (isLoad)
1553         ret->allocate();
1554     else
1555         ret->dataDynamicArray(data);
1556
1557     return ret;
1558 }
1559
1560 void
1561 LSQ::issuedMemBarrierInst(MinorDynInstPtr inst)
1562 {
1563     assert(inst->isInst() && inst->staticInst->isMemBarrier());
1564     assert(inst->id.execSeqNum > lastMemBarrier);
1565
1566     /* Remember the barrier.  We only have a notion of one
1567      *  barrier so this may result in some mem refs being
1568      *  delayed if they are between barriers */
1569     lastMemBarrier = inst->id.execSeqNum;
1570 }
1571
1572 void
1573 LSQ::LSQRequest::makePacket()
1574 {
1575     /* Make the function idempotent */
1576     if (packet)
1577         return;
1578
1579     packet = makePacketForRequest(request, isLoad, this, data);
1580     /* Null the ret data so we know not to deallocate it when the
1581      * ret is destroyed.  The data now belongs to the ret and
1582      * the ret is responsible for its destruction */
1583     data = NULL;
1584 }
1585
1586 std::ostream &
1587 operator <<(std::ostream &os, LSQ::MemoryState state)
1588 {
1589     switch (state) {
1590       case LSQ::MemoryRunning:
1591         os << "MemoryRunning";
1592         break;
1593       case LSQ::MemoryNeedsRetry:
1594         os << "MemoryNeedsRetry";
1595         break;
1596       default:
1597         os << "MemoryState-" << static_cast<int>(state);
1598         break;
1599     }
1600     return os;
1601 }
1602
1603 void
1604 LSQ::recvTimingSnoopReq(PacketPtr pkt)
1605 {
1606     /* LLSC operations in Minor can't be speculative and are executed from
1607      * the head of the requests queue.  We shouldn't need to do more than
1608      * this action on snoops. */
1609
1610     /* THREAD */
1611     TheISA::handleLockedSnoop(cpu.getContext(0), pkt, cacheBlockMask);
1612 }
1613
1614 }