src/mem/cache/cache_impl.hh

   1 /*
   2  * Copyright (c) 2002-2005 The Regents of The University of Michigan
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are
   7  * met: redistributions of source code must retain the above copyright
   8  * notice, this list of conditions and the following disclaimer;
   9  * redistributions in binary form must reproduce the above copyright
  10  * notice, this list of conditions and the following disclaimer in the
  11  * documentation and/or other materials provided with the distribution;
  12  * neither the name of the copyright holders nor the names of its
  13  * contributors may be used to endorse or promote products derived from
  14  * this software without specific prior written permission.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  17  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  18  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  19  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  20  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  26  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * Authors: Erik Hallnor
  29  *          Dave Greene
  30  *          Nathan Binkert
  31  *          Steve Reinhardt
  32  *          Ron Dreslinski
  33  */
  34
  35 /**
  36  * @file
  37  * Cache definitions.
  38  */
  39
  40 #include "sim/host.hh"
  41 #include "base/fast_alloc.hh"
  42 #include "base/misc.hh"
  43 #include "base/range.hh"
  44
  45 #include "mem/cache/cache.hh"
  46 #include "mem/cache/blk.hh"
  47 #include "mem/cache/mshr.hh"
  48 #include "mem/cache/prefetch/base.hh"
  49
  50 #include "sim/sim_exit.hh" // for SimExitEvent
  51
  52
  53 template<class TagStore>
  54 Cache<TagStore>::Cache(const Params *p, TagStore *tags, BasePrefetcher *pf)
  55     : BaseCache(p),
  56       tags(tags),
  57       prefetcher(pf),
  58       doFastWrites(true),
  59       prefetchOnAccess(p->prefetch_on_access)
  60 {
  61     tempBlock = new BlkType();
  62     tempBlock->data = new uint8_t[blkSize];
  63
  64     cpuSidePort = new CpuSidePort(p->name + "-cpu_side_port", this,
  65                                   "CpuSidePort");
  66     memSidePort = new MemSidePort(p->name + "-mem_side_port", this,
  67                                   "MemSidePort");
  68     cpuSidePort->setOtherPort(memSidePort);
  69     memSidePort->setOtherPort(cpuSidePort);
  70
  71     tags->setCache(this);
  72     if (prefetcher)
  73         prefetcher->setCache(this);
  74 }
  75
  76 template<class TagStore>
  77 void
  78 Cache<TagStore>::regStats()
  79 {
  80     BaseCache::regStats();
  81     tags->regStats(name());
  82     if (prefetcher)
  83         prefetcher->regStats(name());
  84 }
  85
  86 template<class TagStore>
  87 Port *
  88 Cache<TagStore>::getPort(const std::string &if_name, int idx)
  89 {
  90     if (if_name == "" || if_name == "cpu_side") {
  91         return cpuSidePort;
  92     } else if (if_name == "mem_side") {
  93         return memSidePort;
  94     } else if (if_name == "functional") {
  95         CpuSidePort *funcPort =
  96             new CpuSidePort(name() + "-cpu_side_funcport", this,
  97                             "CpuSideFuncPort");
  98         funcPort->setOtherPort(memSidePort);
  99         return funcPort;
 100     } else {
 101         panic("Port name %s unrecognized\n", if_name);
 102     }
 103 }
 104
 105 template<class TagStore>
 106 void
 107 Cache<TagStore>::deletePortRefs(Port *p)
 108 {
 109     if (cpuSidePort == p || memSidePort == p)
 110         panic("Can only delete functional ports\n");
 111
 112     delete p;
 113 }
 114
 115
 116 template<class TagStore>
 117 void
 118 Cache<TagStore>::cmpAndSwap(BlkType *blk, PacketPtr pkt)
 119 {
 120     uint64_t overwrite_val;
 121     bool overwrite_mem;
 122     uint64_t condition_val64;
 123     uint32_t condition_val32;
 124
 125     int offset = tags->extractBlkOffset(pkt->getAddr());
 126     uint8_t *blk_data = blk->data + offset;
 127
 128     assert(sizeof(uint64_t) >= pkt->getSize());
 129
 130     overwrite_mem = true;
 131     // keep a copy of our possible write value, and copy what is at the
 132     // memory address into the packet
 133     pkt->writeData((uint8_t *)&overwrite_val);
 134     pkt->setData(blk_data);
 135
 136     if (pkt->req->isCondSwap()) {
 137         if (pkt->getSize() == sizeof(uint64_t)) {
 138             condition_val64 = pkt->req->getExtraData();
 139             overwrite_mem = !std::memcmp(&condition_val64, blk_data,
 140                                          sizeof(uint64_t));
 141         } else if (pkt->getSize() == sizeof(uint32_t)) {
 142             condition_val32 = (uint32_t)pkt->req->getExtraData();
 143             overwrite_mem = !std::memcmp(&condition_val32, blk_data,
 144                                          sizeof(uint32_t));
 145         } else
 146             panic("Invalid size for conditional read/write\n");
 147     }
 148
 149     if (overwrite_mem) {
 150         std::memcpy(blk_data, &overwrite_val, pkt->getSize());
 151         blk->status |= BlkDirty;
 152     }
 153 }
 154
 155
 156 template<class TagStore>
 157 void
 158 Cache<TagStore>::satisfyCpuSideRequest(PacketPtr pkt, BlkType *blk)
 159 {
 160     assert(blk);
 161     // Occasionally this is not true... if we are a lower-level cache
 162     // satisfying a string of Read and ReadEx requests from
 163     // upper-level caches, a Read will mark the block as shared but we
 164     // can satisfy a following ReadEx anyway since we can rely on the
 165     // Read requester(s) to have buffered the ReadEx snoop and to
 166     // invalidate their blocks after receiving them.
 167     // assert(pkt->needsExclusive() ? blk->isWritable() : blk->isValid());
 168     assert(pkt->getOffset(blkSize) + pkt->getSize() <= blkSize);
 169
 170     // Check RMW operations first since both isRead() and
 171     // isWrite() will be true for them
 172     if (pkt->cmd == MemCmd::SwapReq) {
 173         cmpAndSwap(blk, pkt);
 174     } else if (pkt->isWrite()) {
 175         blk->status |= BlkDirty;
 176         if (blk->checkWrite(pkt)) {
 177             pkt->writeDataToBlock(blk->data, blkSize);
 178         }
 179     } else if (pkt->isRead()) {
 180         if (pkt->isLLSC()) {
 181             blk->trackLoadLocked(pkt);
 182         }
 183         pkt->setDataFromBlock(blk->data, blkSize);
 184         if (pkt->getSize() == blkSize) {
 185             // special handling for coherent block requests from
 186             // upper-level caches
 187             if (pkt->needsExclusive()) {
 188                 // on ReadExReq we give up our copy
 189                 tags->invalidateBlk(blk);
 190             } else {
 191                 // on ReadReq we create shareable copies here and in
 192                 // the requester
 193                 pkt->assertShared();
 194                 blk->status &= ~BlkWritable;
 195             }
 196         }
 197     } else {
 198         // Not a read or write... must be an upgrade.  it's OK
 199         // to just ack those as long as we have an exclusive
 200         // copy at this level.
 201         assert(pkt->cmd == MemCmd::UpgradeReq);
 202         tags->invalidateBlk(blk);
 203     }
 204 }
 205
 206
 207 /////////////////////////////////////////////////////
 208 //
 209 // MSHR helper functions
 210 //
 211 /////////////////////////////////////////////////////
 212
 213
 214 template<class TagStore>
 215 void
 216 Cache<TagStore>::markInService(MSHR *mshr)
 217 {
 218     markInServiceInternal(mshr);
 219 #if 0
 220         if (mshr->originalCmd == MemCmd::HardPFReq) {
 221             DPRINTF(HWPrefetch, "%s:Marking a HW_PF in service\n",
 222                     name());
 223             //Also clear pending if need be
 224             if (!prefetcher->havePending())
 225             {
 226                 deassertMemSideBusRequest(Request_PF);
 227             }
 228         }
 229 #endif
 230 }
 231
 232
 233 template<class TagStore>
 234 void
 235 Cache<TagStore>::squash(int threadNum)
 236 {
 237     bool unblock = false;
 238     BlockedCause cause = NUM_BLOCKED_CAUSES;
 239
 240     if (noTargetMSHR && noTargetMSHR->threadNum == threadNum) {
 241         noTargetMSHR = NULL;
 242         unblock = true;
 243         cause = Blocked_NoTargets;
 244     }
 245     if (mshrQueue.isFull()) {
 246         unblock = true;
 247         cause = Blocked_NoMSHRs;
 248     }
 249     mshrQueue.squash(threadNum);
 250     if (unblock && !mshrQueue.isFull()) {
 251         clearBlocked(cause);
 252     }
 253 }
 254
 255 /////////////////////////////////////////////////////
 256 //
 257 // Access path: requests coming in from the CPU side
 258 //
 259 /////////////////////////////////////////////////////
 260
 261 template<class TagStore>
 262 bool
 263 Cache<TagStore>::access(PacketPtr pkt, BlkType *&blk,
 264                         int &lat, PacketList &writebacks)
 265 {
 266     if (pkt->req->isUncacheable())  {
 267         blk = NULL;
 268         lat = hitLatency;
 269         return false;
 270     }
 271
 272     blk = tags->accessBlock(pkt->getAddr(), lat);
 273
 274     DPRINTF(Cache, "%s %x %s\n", pkt->cmdString(), pkt->getAddr(),
 275             (blk) ? "hit" : "miss");
 276
 277     if (blk != NULL) {
 278
 279         if (pkt->needsExclusive() ? blk->isWritable() : blk->isReadable()) {
 280             // OK to satisfy access
 281             hits[pkt->cmdToIndex()][0/*pkt->req->threadId()*/]++;
 282             satisfyCpuSideRequest(pkt, blk);
 283             return true;
 284         }
 285     }
 286
 287     // Can't satisfy access normally... either no block (blk == NULL)
 288     // or have block but need exclusive & only have shared.
 289
 290     // Writeback handling is special case.  We can write the block
 291     // into the cache without having a writeable copy (or any copy at
 292     // all).
 293     if (pkt->cmd == MemCmd::Writeback) {
 294         assert(blkSize == pkt->getSize());
 295         if (blk == NULL) {
 296             // need to do a replacement
 297             blk = allocateBlock(pkt->getAddr(), writebacks);
 298             if (blk == NULL) {
 299                 // no replaceable block available, give up.
 300                 // writeback will be forwarded to next level.
 301                 incMissCount(pkt);
 302                 return false;
 303             }
 304             tags->insertBlock(pkt->getAddr(), blk);
 305             blk->status = BlkValid | BlkReadable;
 306         }
 307         std::memcpy(blk->data, pkt->getPtr<uint8_t>(), blkSize);
 308         blk->status |= BlkDirty;
 309         // nothing else to do; writeback doesn't expect response
 310         assert(!pkt->needsResponse());
 311         hits[pkt->cmdToIndex()][0/*pkt->req->threadId()*/]++;
 312         return true;
 313     }
 314
 315     incMissCount(pkt);
 316
 317     if (blk == NULL && pkt->isLLSC() && pkt->isWrite()) {
 318         // complete miss on store conditional... just give up now
 319         pkt->req->setExtraData(0);
 320         return true;
 321     }
 322
 323     return false;
 324 }
 325
 326
 327 class ForwardResponseRecord : public Packet::SenderState, public FastAlloc
 328 {
 329     Packet::SenderState *prevSenderState;
 330     int prevSrc;
 331 #ifndef NDEBUG
 332     BaseCache *cache;
 333 #endif
 334   public:
 335     ForwardResponseRecord(Packet *pkt, BaseCache *_cache)
 336         : prevSenderState(pkt->senderState), prevSrc(pkt->getSrc())
 337 #ifndef NDEBUG
 338           , cache(_cache)
 339 #endif
 340     {}
 341     void restore(Packet *pkt, BaseCache *_cache)
 342     {
 343         assert(_cache == cache);
 344         pkt->senderState = prevSenderState;
 345         pkt->setDest(prevSrc);
 346     }
 347 };
 348
 349
 350 template<class TagStore>
 351 bool
 352 Cache<TagStore>::timingAccess(PacketPtr pkt)
 353 {
 354 //@todo Add back in MemDebug Calls
 355 //    MemDebug::cacheAccess(pkt);
 356
 357     // we charge hitLatency for doing just about anything here
 358     Tick time =  curTick + hitLatency;
 359
 360     if (pkt->isResponse()) {
 361         // must be cache-to-cache response from upper to lower level
 362         ForwardResponseRecord *rec =
 363             dynamic_cast<ForwardResponseRecord *>(pkt->senderState);
 364         assert(rec != NULL);
 365         rec->restore(pkt, this);
 366         delete rec;
 367         memSidePort->respond(pkt, time);
 368         return true;
 369     }
 370
 371     assert(pkt->isRequest());
 372
 373     if (pkt->memInhibitAsserted()) {
 374         DPRINTF(Cache, "mem inhibited on 0x%x: not responding\n",
 375                 pkt->getAddr());
 376         assert(!pkt->req->isUncacheable());
 377         // Special tweak for multilevel coherence: snoop downward here
 378         // on invalidates since there may be other caches below here
 379         // that have shared copies.  Not necessary if we know that
 380         // supplier had exclusive copy to begin with.
 381         if (pkt->needsExclusive() && !pkt->isSupplyExclusive()) {
 382             Packet *snoopPkt = new Packet(pkt, true);  // clear flags
 383             snoopPkt->setExpressSnoop();
 384             snoopPkt->assertMemInhibit();
 385             memSidePort->sendTiming(snoopPkt);
 386             // main memory will delete snoopPkt
 387         }
 388         // since we're the official target but we aren't responding,
 389         // delete the packet now.
 390         delete pkt;
 391         return true;
 392     }
 393
 394     if (pkt->req->isUncacheable()) {
 395         // writes go in write buffer, reads use MSHR
 396         if (pkt->isWrite() && !pkt->isRead()) {
 397             allocateWriteBuffer(pkt, time, true);
 398         } else {
 399             allocateUncachedReadBuffer(pkt, time, true);
 400         }
 401         assert(pkt->needsResponse()); // else we should delete it here??
 402         return true;
 403     }
 404
 405     int lat = hitLatency;
 406     BlkType *blk = NULL;
 407     PacketList writebacks;
 408
 409     bool satisfied = access(pkt, blk, lat, writebacks);
 410
 411 #if 0
 412     /** @todo make the fast write alloc (wh64) work with coherence. */
 413
 414     // If this is a block size write/hint (WH64) allocate the block here
 415     // if the coherence protocol allows it.
 416     if (!blk && pkt->getSize() >= blkSize && coherence->allowFastWrites() &&
 417         (pkt->cmd == MemCmd::WriteReq
 418          || pkt->cmd == MemCmd::WriteInvalidateReq) ) {
 419         // not outstanding misses, can do this
 420         MSHR *outstanding_miss = mshrQueue.findMatch(pkt->getAddr());
 421         if (pkt->cmd == MemCmd::WriteInvalidateReq || !outstanding_miss) {
 422             if (outstanding_miss) {
 423                 warn("WriteInv doing a fastallocate"
 424                      "with an outstanding miss to the same address\n");
 425             }
 426             blk = handleFill(NULL, pkt, BlkValid | BlkWritable,
 427                                    writebacks);
 428             ++fastWrites;
 429         }
 430     }
 431 #endif
 432
 433     // track time of availability of next prefetch, if any
 434     Tick next_pf_time = 0;
 435
 436     bool needsResponse = pkt->needsResponse();
 437
 438     if (satisfied) {
 439         if (needsResponse) {
 440             pkt->makeTimingResponse();
 441             cpuSidePort->respond(pkt, curTick+lat);
 442         } else {
 443             delete pkt;
 444         }
 445
 446         if (prefetcher && (prefetchOnAccess || (blk && blk->wasPrefetched()))) {
 447             if (blk)
 448                 blk->status &= ~BlkHWPrefetched;
 449             next_pf_time = prefetcher->notify(pkt, time);
 450         }
 451     } else {
 452         // miss
 453
 454         Addr blk_addr = pkt->getAddr() & ~(Addr(blkSize-1));
 455         MSHR *mshr = mshrQueue.findMatch(blk_addr);
 456
 457         if (mshr) {
 458             // MSHR hit
 459             //@todo remove hw_pf here
 460             mshr_hits[pkt->cmdToIndex()][0/*pkt->req->threadId()*/]++;
 461             if (mshr->threadNum != 0/*pkt->req->threadId()*/) {
 462                 mshr->threadNum = -1;
 463             }
 464             mshr->allocateTarget(pkt, time, order++);
 465             if (mshr->getNumTargets() == numTarget) {
 466                 noTargetMSHR = mshr;
 467                 setBlocked(Blocked_NoTargets);
 468                 // need to be careful with this... if this mshr isn't
 469                 // ready yet (i.e. time > curTick_, we don't want to
 470                 // move it ahead of mshrs that are ready
 471                 // mshrQueue.moveToFront(mshr);
 472             }
 473         } else {
 474             // no MSHR
 475             mshr_misses[pkt->cmdToIndex()][0/*pkt->req->threadId()*/]++;
 476             // always mark as cache fill for now... if we implement
 477             // no-write-allocate or bypass accesses this will have to
 478             // be changed.
 479             if (pkt->cmd == MemCmd::Writeback) {
 480                 allocateWriteBuffer(pkt, time, true);
 481             } else {
 482                 if (blk && blk->isValid()) {
 483                     // If we have a write miss to a valid block, we
 484                     // need to mark the block non-readable.  Otherwise
 485                     // if we allow reads while there's an outstanding
 486                     // write miss, the read could return stale data
 487                     // out of the cache block... a more aggressive
 488                     // system could detect the overlap (if any) and
 489                     // forward data out of the MSHRs, but we don't do
 490                     // that yet.  Note that we do need to leave the
 491                     // block valid so that it stays in the cache, in
 492                     // case we get an upgrade response (and hence no
 493                     // new data) when the write miss completes.
 494                     // As long as CPUs do proper store/load forwarding
 495                     // internally, and have a sufficiently weak memory
 496                     // model, this is probably unnecessary, but at some
 497                     // point it must have seemed like we needed it...
 498                     assert(pkt->needsExclusive() && !blk->isWritable());
 499                     blk->status &= ~BlkReadable;
 500                 }
 501
 502                 allocateMissBuffer(pkt, time, true);
 503             }
 504
 505             if (prefetcher) {
 506                 next_pf_time = prefetcher->notify(pkt, time);
 507             }
 508         }
 509     }
 510
 511     if (next_pf_time != 0)
 512         requestMemSideBus(Request_PF, std::max(time, next_pf_time));
 513
 514     // copy writebacks to write buffer
 515     while (!writebacks.empty()) {
 516         PacketPtr wbPkt = writebacks.front();
 517         allocateWriteBuffer(wbPkt, time, true);
 518         writebacks.pop_front();
 519     }
 520
 521     return true;
 522 }
 523
 524
 525 // See comment in cache.hh.
 526 template<class TagStore>
 527 PacketPtr
 528 Cache<TagStore>::getBusPacket(PacketPtr cpu_pkt, BlkType *blk,
 529                               bool needsExclusive)
 530 {
 531     bool blkValid = blk && blk->isValid();
 532
 533     if (cpu_pkt->req->isUncacheable()) {
 534         assert(blk == NULL);
 535         return NULL;
 536     }
 537
 538     if (!blkValid && (cpu_pkt->cmd == MemCmd::Writeback ||
 539                       cpu_pkt->cmd == MemCmd::UpgradeReq)) {
 540         // Writebacks that weren't allocated in access() and upgrades
 541         // from upper-level caches that missed completely just go
 542         // through.
 543         return NULL;
 544     }
 545
 546     assert(cpu_pkt->needsResponse());
 547
 548     MemCmd cmd;
 549     // @TODO make useUpgrades a parameter.
 550     // Note that ownership protocols require upgrade, otherwise a
 551     // write miss on a shared owned block will generate a ReadExcl,
 552     // which will clobber the owned copy.
 553     const bool useUpgrades = true;
 554     if (blkValid && useUpgrades) {
 555         // only reason to be here is that blk is shared
 556         // (read-only) and we need exclusive
 557         assert(needsExclusive && !blk->isWritable());
 558         cmd = MemCmd::UpgradeReq;
 559     } else {
 560         // block is invalid
 561         cmd = needsExclusive ? MemCmd::ReadExReq : MemCmd::ReadReq;
 562     }
 563     PacketPtr pkt = new Packet(cpu_pkt->req, cmd, Packet::Broadcast, blkSize);
 564
 565     pkt->allocate();
 566     return pkt;
 567 }
 568
 569
 570 template<class TagStore>
 571 Tick
 572 Cache<TagStore>::atomicAccess(PacketPtr pkt)
 573 {
 574     int lat = hitLatency;
 575
 576     // @TODO: make this a parameter
 577     bool last_level_cache = false;
 578
 579     if (pkt->memInhibitAsserted()) {
 580         assert(!pkt->req->isUncacheable());
 581         // have to invalidate ourselves and any lower caches even if
 582         // upper cache will be responding
 583         if (pkt->isInvalidate()) {
 584             BlkType *blk = tags->findBlock(pkt->getAddr());
 585             if (blk && blk->isValid()) {
 586                 tags->invalidateBlk(blk);
 587                 DPRINTF(Cache, "rcvd mem-inhibited %s on 0x%x: invalidating\n",
 588                         pkt->cmdString(), pkt->getAddr());
 589             }
 590             if (!last_level_cache) {
 591                 DPRINTF(Cache, "forwarding mem-inhibited %s on 0x%x\n",
 592                         pkt->cmdString(), pkt->getAddr());
 593                 lat += memSidePort->sendAtomic(pkt);
 594             }
 595         } else {
 596             DPRINTF(Cache, "rcvd mem-inhibited %s on 0x%x: not responding\n",
 597                     pkt->cmdString(), pkt->getAddr());
 598         }
 599
 600         return lat;
 601     }
 602
 603     // should assert here that there are no outstanding MSHRs or
 604     // writebacks... that would mean that someone used an atomic
 605     // access in timing mode
 606
 607     BlkType *blk = NULL;
 608     PacketList writebacks;
 609
 610     if (!access(pkt, blk, lat, writebacks)) {
 611         // MISS
 612         PacketPtr bus_pkt = getBusPacket(pkt, blk, pkt->needsExclusive());
 613
 614         bool is_forward = (bus_pkt == NULL);
 615
 616         if (is_forward) {
 617             // just forwarding the same request to the next level
 618             // no local cache operation involved
 619             bus_pkt = pkt;
 620         }
 621
 622         DPRINTF(Cache, "Sending an atomic %s for %x\n",
 623                 bus_pkt->cmdString(), bus_pkt->getAddr());
 624
 625 #if TRACING_ON
 626         CacheBlk::State old_state = blk ? blk->status : 0;
 627 #endif
 628
 629         lat += memSidePort->sendAtomic(bus_pkt);
 630
 631         DPRINTF(Cache, "Receive response: %s for addr %x in state %i\n",
 632                 bus_pkt->cmdString(), bus_pkt->getAddr(), old_state);
 633
 634         assert(!bus_pkt->wasNacked());
 635
 636         // If packet was a forward, the response (if any) is already
 637         // in place in the bus_pkt == pkt structure, so we don't need
 638         // to do anything.  Otherwise, use the separate bus_pkt to
 639         // generate response to pkt and then delete it.
 640         if (!is_forward) {
 641             if (pkt->needsResponse()) {
 642                 assert(bus_pkt->isResponse());
 643                 if (bus_pkt->isError()) {
 644                     pkt->makeAtomicResponse();
 645                     pkt->copyError(bus_pkt);
 646                 } else if (bus_pkt->isRead() ||
 647                            bus_pkt->cmd == MemCmd::UpgradeResp) {
 648                     // we're updating cache state to allow us to
 649                     // satisfy the upstream request from the cache
 650                     blk = handleFill(bus_pkt, blk, writebacks);
 651                     satisfyCpuSideRequest(pkt, blk);
 652                 } else {
 653                     // we're satisfying the upstream request without
 654                     // modifying cache state, e.g., a write-through
 655                     pkt->makeAtomicResponse();
 656                 }
 657             }
 658             delete bus_pkt;
 659         }
 660     }
 661
 662     // Note that we don't invoke the prefetcher at all in atomic mode.
 663     // It's not clear how to do it properly, particularly for
 664     // prefetchers that aggressively generate prefetch candidates and
 665     // rely on bandwidth contention to throttle them; these will tend
 666     // to pollute the cache in atomic mode since there is no bandwidth
 667     // contention.  If we ever do want to enable prefetching in atomic
 668     // mode, though, this is the place to do it... see timingAccess()
 669     // for an example (though we'd want to issue the prefetch(es)
 670     // immediately rather than calling requestMemSideBus() as we do
 671     // there).
 672
 673     // Handle writebacks if needed
 674     while (!writebacks.empty()){
 675         PacketPtr wbPkt = writebacks.front();
 676         memSidePort->sendAtomic(wbPkt);
 677         writebacks.pop_front();
 678         delete wbPkt;
 679     }
 680
 681     // We now have the block one way or another (hit or completed miss)
 682
 683     if (pkt->needsResponse()) {
 684         pkt->makeAtomicResponse();
 685     }
 686
 687     return lat;
 688 }
 689
 690
 691 template<class TagStore>
 692 void
 693 Cache<TagStore>::functionalAccess(PacketPtr pkt,
 694                                   CachePort *incomingPort,
 695                                   CachePort *otherSidePort)
 696 {
 697     Addr blk_addr = pkt->getAddr() & ~(blkSize - 1);
 698     BlkType *blk = tags->findBlock(pkt->getAddr());
 699
 700     pkt->pushLabel(name());
 701
 702     CacheBlkPrintWrapper cbpw(blk);
 703     bool done =
 704         (blk && pkt->checkFunctional(&cbpw, blk_addr, blkSize, blk->data))
 705         || incomingPort->checkFunctional(pkt)
 706         || mshrQueue.checkFunctional(pkt, blk_addr)
 707         || writeBuffer.checkFunctional(pkt, blk_addr)
 708         || otherSidePort->checkFunctional(pkt);
 709
 710     // We're leaving the cache, so pop cache->name() label
 711     pkt->popLabel();
 712
 713     if (!done) {
 714         otherSidePort->sendFunctional(pkt);
 715     }
 716 }
 717
 718
 719 /////////////////////////////////////////////////////
 720 //
 721 // Response handling: responses from the memory side
 722 //
 723 /////////////////////////////////////////////////////
 724
 725
 726 template<class TagStore>
 727 void
 728 Cache<TagStore>::handleResponse(PacketPtr pkt)
 729 {
 730     Tick time = curTick + hitLatency;
 731     MSHR *mshr = dynamic_cast<MSHR*>(pkt->senderState);
 732     bool is_error = pkt->isError();
 733
 734     assert(mshr);
 735
 736     if (pkt->wasNacked()) {
 737         //pkt->reinitFromRequest();
 738         warn("NACKs from devices not connected to the same bus "
 739              "not implemented\n");
 740         return;
 741     }
 742     if (is_error) {
 743         DPRINTF(Cache, "Cache received packet with error for address %x, "
 744                 "cmd: %s\n", pkt->getAddr(), pkt->cmdString());
 745     }
 746
 747     DPRINTF(Cache, "Handling response to %x\n", pkt->getAddr());
 748
 749     MSHRQueue *mq = mshr->queue;
 750     bool wasFull = mq->isFull();
 751
 752     if (mshr == noTargetMSHR) {
 753         // we always clear at least one target
 754         clearBlocked(Blocked_NoTargets);
 755         noTargetMSHR = NULL;
 756     }
 757
 758     // Initial target is used just for stats
 759     MSHR::Target *initial_tgt = mshr->getTarget();
 760     BlkType *blk = tags->findBlock(pkt->getAddr());
 761     int stats_cmd_idx = initial_tgt->pkt->cmdToIndex();
 762     Tick miss_latency = curTick - initial_tgt->recvTime;
 763     PacketList writebacks;
 764
 765     if (pkt->req->isUncacheable()) {
 766         mshr_uncacheable_lat[stats_cmd_idx][0/*pkt->req->threadId()*/] +=
 767             miss_latency;
 768     } else {
 769         mshr_miss_latency[stats_cmd_idx][0/*pkt->req->threadId()*/] +=
 770             miss_latency;
 771     }
 772
 773     bool is_fill = !mshr->isForward &&
 774         (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);
 775
 776     if (is_fill && !is_error) {
 777         DPRINTF(Cache, "Block for addr %x being updated in Cache\n",
 778                 pkt->getAddr());
 779
 780         // give mshr a chance to do some dirty work
 781         mshr->handleFill(pkt, blk);
 782
 783         blk = handleFill(pkt, blk, writebacks);
 784         assert(blk != NULL);
 785     }
 786
 787     // First offset for critical word first calculations
 788     int initial_offset = 0;
 789
 790     if (mshr->hasTargets()) {
 791         initial_offset = mshr->getTarget()->pkt->getOffset(blkSize);
 792     }
 793
 794     while (mshr->hasTargets()) {
 795         MSHR::Target *target = mshr->getTarget();
 796
 797         switch (target->source) {
 798           case MSHR::Target::FromCPU:
 799             Tick completion_time;
 800             if (is_fill) {
 801                 satisfyCpuSideRequest(target->pkt, blk);
 802                 // How many bytes past the first request is this one
 803                 int transfer_offset =
 804                     target->pkt->getOffset(blkSize) - initial_offset;
 805                 if (transfer_offset < 0) {
 806                     transfer_offset += blkSize;
 807                 }
 808
 809                 // If critical word (no offset) return first word time
 810                 completion_time = tags->getHitLatency() +
 811                     (transfer_offset ? pkt->finishTime : pkt->firstWordTime);
 812
 813                 assert(!target->pkt->req->isUncacheable());
 814                 missLatency[target->pkt->cmdToIndex()][0/*pkt->req->threadId()*/] +=
 815                     completion_time - target->recvTime;
 816             } else {
 817                 // not a cache fill, just forwarding response
 818                 completion_time = tags->getHitLatency() + pkt->finishTime;
 819                 if (pkt->isRead() && !is_error) {
 820                     target->pkt->setData(pkt->getPtr<uint8_t>());
 821                 }
 822             }
 823             target->pkt->makeTimingResponse();
 824             // if this packet is an error copy that to the new packet
 825             if (is_error)
 826                 target->pkt->copyError(pkt);
 827             if (pkt->isInvalidate()) {
 828                 // If intermediate cache got ReadRespWithInvalidate,
 829                 // propagate that.  Response should not have
 830                 // isInvalidate() set otherwise.
 831                 assert(target->pkt->cmd == MemCmd::ReadResp);
 832                 assert(pkt->cmd == MemCmd::ReadRespWithInvalidate);
 833                 target->pkt->cmd = MemCmd::ReadRespWithInvalidate;
 834             }
 835             cpuSidePort->respond(target->pkt, completion_time);
 836             break;
 837
 838           case MSHR::Target::FromPrefetcher:
 839             assert(target->pkt->cmd == MemCmd::HardPFReq);
 840             if (blk)
 841                 blk->status |= BlkHWPrefetched;
 842             delete target->pkt->req;
 843             delete target->pkt;
 844             break;
 845
 846           case MSHR::Target::FromSnoop:
 847             // I don't believe that a snoop can be in an error state
 848             assert(!is_error);
 849             // response to snoop request
 850             DPRINTF(Cache, "processing deferred snoop...\n");
 851             handleSnoop(target->pkt, blk, true, true,
 852                         mshr->pendingInvalidate || pkt->isInvalidate());
 853             break;
 854
 855           default:
 856             panic("Illegal target->source enum %d\n", target->source);
 857         }
 858
 859         mshr->popTarget();
 860     }
 861
 862     if (pkt->isInvalidate()) {
 863         tags->invalidateBlk(blk);
 864     }
 865
 866     if (mshr->promoteDeferredTargets()) {
 867         // avoid later read getting stale data while write miss is
 868         // outstanding.. see comment in timingAccess()
 869         blk->status &= ~BlkReadable;
 870         MSHRQueue *mq = mshr->queue;
 871         mq->markPending(mshr);
 872         requestMemSideBus((RequestCause)mq->index, pkt->finishTime);
 873     } else {
 874         mq->deallocate(mshr);
 875         if (wasFull && !mq->isFull()) {
 876             clearBlocked((BlockedCause)mq->index);
 877         }
 878     }
 879
 880     // copy writebacks to write buffer
 881     while (!writebacks.empty()) {
 882         PacketPtr wbPkt = writebacks.front();
 883         allocateWriteBuffer(wbPkt, time, true);
 884         writebacks.pop_front();
 885     }
 886     // if we used temp block, clear it out
 887     if (blk == tempBlock) {
 888         if (blk->isDirty()) {
 889             allocateWriteBuffer(writebackBlk(blk), time, true);
 890         }
 891         tags->invalidateBlk(blk);
 892     }
 893
 894     delete pkt;
 895 }
 896
 897
 898
 899
 900 template<class TagStore>
 901 PacketPtr
 902 Cache<TagStore>::writebackBlk(BlkType *blk)
 903 {
 904     assert(blk && blk->isValid() && blk->isDirty());
 905
 906     writebacks[0/*pkt->req->threadId()*/]++;
 907
 908     Request *writebackReq =
 909         new Request(tags->regenerateBlkAddr(blk->tag, blk->set), blkSize, 0);
 910     PacketPtr writeback = new Packet(writebackReq, MemCmd::Writeback, -1);
 911     writeback->allocate();
 912     std::memcpy(writeback->getPtr<uint8_t>(), blk->data, blkSize);
 913
 914     blk->status &= ~BlkDirty;
 915     return writeback;
 916 }
 917
 918
 919 template<class TagStore>
 920 typename Cache<TagStore>::BlkType*
 921 Cache<TagStore>::allocateBlock(Addr addr, PacketList &writebacks)
 922 {
 923     BlkType *blk = tags->findVictim(addr, writebacks);
 924
 925     if (blk->isValid()) {
 926         Addr repl_addr = tags->regenerateBlkAddr(blk->tag, blk->set);
 927         MSHR *repl_mshr = mshrQueue.findMatch(repl_addr);
 928         if (repl_mshr) {
 929             // must be an outstanding upgrade request on block
 930             // we're about to replace...
 931             assert(!blk->isWritable());
 932             assert(repl_mshr->needsExclusive());
 933             // too hard to replace block with transient state
 934             // allocation failed, block not inserted
 935             return NULL;
 936         } else {
 937             DPRINTF(Cache, "replacement: replacing %x with %x: %s\n",
 938                     repl_addr, addr,
 939                     blk->isDirty() ? "writeback" : "clean");
 940
 941             if (blk->isDirty()) {
 942                 // Save writeback packet for handling by caller
 943                 writebacks.push_back(writebackBlk(blk));
 944             }
 945         }
 946     }
 947
 948     return blk;
 949 }
 950
 951
 952 // Note that the reason we return a list of writebacks rather than
 953 // inserting them directly in the write buffer is that this function
 954 // is called by both atomic and timing-mode accesses, and in atomic
 955 // mode we don't mess with the write buffer (we just perform the
 956 // writebacks atomically once the original request is complete).
 957 template<class TagStore>
 958 typename Cache<TagStore>::BlkType*
 959 Cache<TagStore>::handleFill(PacketPtr pkt, BlkType *blk,
 960                             PacketList &writebacks)
 961 {
 962     Addr addr = pkt->getAddr();
 963 #if TRACING_ON
 964     CacheBlk::State old_state = blk ? blk->status : 0;
 965 #endif
 966
 967     if (blk == NULL) {
 968         // better have read new data...
 969         assert(pkt->hasData());
 970         // need to do a replacement
 971         blk = allocateBlock(addr, writebacks);
 972         if (blk == NULL) {
 973             // No replaceable block... just use temporary storage to
 974             // complete the current request and then get rid of it
 975             assert(!tempBlock->isValid());
 976             blk = tempBlock;
 977             tempBlock->set = tags->extractSet(addr);
 978             tempBlock->tag = tags->extractTag(addr);
 979             DPRINTF(Cache, "using temp block for %x\n", addr);
 980         } else {
 981             tags->insertBlock(addr, blk);
 982         }
 983     } else {
 984         // existing block... probably an upgrade
 985         assert(blk->tag == tags->extractTag(addr));
 986         // either we're getting new data or the block should already be valid
 987         assert(pkt->hasData() || blk->isValid());
 988     }
 989
 990     if (!pkt->sharedAsserted()) {
 991         blk->status = BlkValid | BlkReadable | BlkWritable;
 992     } else {
 993         assert(!pkt->needsExclusive());
 994         blk->status = BlkValid | BlkReadable;
 995     }
 996
 997     DPRINTF(Cache, "Block addr %x moving from state %i to %i\n",
 998             addr, old_state, blk->status);
 999
1000     // if we got new data, copy it in
1001     if (pkt->isRead()) {
1002         std::memcpy(blk->data, pkt->getPtr<uint8_t>(), blkSize);
1003     }
1004
1005     blk->whenReady = pkt->finishTime;
1006
1007     return blk;
1008 }
1009
1010
1011 /////////////////////////////////////////////////////
1012 //
1013 // Snoop path: requests coming in from the memory side
1014 //
1015 /////////////////////////////////////////////////////
1016
1017 template<class TagStore>
1018 void
1019 Cache<TagStore>::
1020 doTimingSupplyResponse(PacketPtr req_pkt, uint8_t *blk_data,
1021                        bool already_copied, bool pending_inval)
1022 {
1023     // timing-mode snoop responses require a new packet, unless we
1024     // already made a copy...
1025     PacketPtr pkt = already_copied ? req_pkt : new Packet(req_pkt, true);
1026     if (!req_pkt->isInvalidate()) {
1027         // note that we're ignoring the shared flag on req_pkt... it's
1028         // basically irrelevant, as we'll always assert shared unless
1029         // it's an exclusive request, in which case the shared line
1030         // should never be asserted1
1031         pkt->assertShared();
1032     }
1033     pkt->allocate();
1034     pkt->makeTimingResponse();
1035     if (pkt->isRead()) {
1036         pkt->setDataFromBlock(blk_data, blkSize);
1037     }
1038     if (pkt->cmd == MemCmd::ReadResp && pending_inval) {
1039         // Assume we defer a response to a read from a far-away cache
1040         // A, then later defer a ReadExcl from a cache B on the same
1041         // bus as us.  We'll assert MemInhibit in both cases, but in
1042         // the latter case MemInhibit will keep the invalidation from
1043         // reaching cache A.  This special response tells cache A that
1044         // it gets the block to satisfy its read, but must immediately
1045         // invalidate it.
1046         pkt->cmd = MemCmd::ReadRespWithInvalidate;
1047     }
1048     memSidePort->respond(pkt, curTick + hitLatency);
1049 }
1050
1051 template<class TagStore>
1052 void
1053 Cache<TagStore>::handleSnoop(PacketPtr pkt, BlkType *blk,
1054                              bool is_timing, bool is_deferred,
1055                              bool pending_inval)
1056 {
1057     // deferred snoops can only happen in timing mode
1058     assert(!(is_deferred && !is_timing));
1059     // pending_inval only makes sense on deferred snoops
1060     assert(!(pending_inval && !is_deferred));
1061     assert(pkt->isRequest());
1062
1063     if (forwardSnoops) {
1064         // first propagate snoop upward to see if anyone above us wants to
1065         // handle it.  save & restore packet src since it will get
1066         // rewritten to be relative to cpu-side bus (if any)
1067         bool alreadyResponded = pkt->memInhibitAsserted();
1068         if (is_timing) {
1069             Packet *snoopPkt = new Packet(pkt, true);  // clear flags
1070             snoopPkt->setExpressSnoop();
1071             snoopPkt->senderState = new ForwardResponseRecord(pkt, this);
1072             cpuSidePort->sendTiming(snoopPkt);
1073             if (snoopPkt->memInhibitAsserted()) {
1074                 // cache-to-cache response from some upper cache
1075                 assert(!alreadyResponded);
1076                 pkt->assertMemInhibit();
1077             } else {
1078                 delete snoopPkt->senderState;
1079             }
1080             if (snoopPkt->sharedAsserted()) {
1081                 pkt->assertShared();
1082             }
1083             delete snoopPkt;
1084         } else {
1085             int origSrc = pkt->getSrc();
1086             cpuSidePort->sendAtomic(pkt);
1087             if (!alreadyResponded && pkt->memInhibitAsserted()) {
1088                 // cache-to-cache response from some upper cache:
1089                 // forward response to original requester
1090                 assert(pkt->isResponse());
1091             }
1092             pkt->setSrc(origSrc);
1093         }
1094     }
1095
1096     if (!blk || !blk->isValid()) {
1097         return;
1098     }
1099
1100     // we may end up modifying both the block state and the packet (if
1101     // we respond in atomic mode), so just figure out what to do now
1102     // and then do it later
1103     bool respond = blk->isDirty() && pkt->needsResponse();
1104     bool have_exclusive = blk->isWritable();
1105     bool invalidate = pkt->isInvalidate();
1106
1107     if (pkt->isRead() && !pkt->isInvalidate()) {
1108         assert(!pkt->needsExclusive());
1109         pkt->assertShared();
1110         int bits_to_clear = BlkWritable;
1111         const bool haveOwnershipState = true; // for now
1112         if (!haveOwnershipState) {
1113             // if we don't support pure ownership (dirty && !writable),
1114             // have to clear dirty bit here, assume memory snarfs data
1115             // on cache-to-cache xfer
1116             bits_to_clear |= BlkDirty;
1117         }
1118         blk->status &= ~bits_to_clear;
1119     }
1120
1121     DPRINTF(Cache, "snooped a %s request for addr %x, %snew state is %i\n",
1122             pkt->cmdString(), blockAlign(pkt->getAddr()),
1123             respond ? "responding, " : "", invalidate ? 0 : blk->status);
1124
1125     if (respond) {
1126         assert(!pkt->memInhibitAsserted());
1127         pkt->assertMemInhibit();
1128         if (have_exclusive) {
1129             pkt->setSupplyExclusive();
1130         }
1131         if (is_timing) {
1132             doTimingSupplyResponse(pkt, blk->data, is_deferred, pending_inval);
1133         } else {
1134             pkt->makeAtomicResponse();
1135             pkt->setDataFromBlock(blk->data, blkSize);
1136         }
1137     } else if (is_timing && is_deferred) {
1138         // if it's a deferred timing snoop then we've made a copy of
1139         // the packet, and so if we're not using that copy to respond
1140         // then we need to delete it here.
1141         delete pkt;
1142     }
1143
1144     // Do this last in case it deallocates block data or something
1145     // like that
1146     if (invalidate) {
1147         tags->invalidateBlk(blk);
1148     }
1149 }
1150
1151
1152 template<class TagStore>
1153 void
1154 Cache<TagStore>::snoopTiming(PacketPtr pkt)
1155 {
1156     // Note that some deferred snoops don't have requests, since the
1157     // original access may have already completed
1158     if ((pkt->req && pkt->req->isUncacheable()) ||
1159         pkt->cmd == MemCmd::Writeback) {
1160         //Can't get a hit on an uncacheable address
1161         //Revisit this for multi level coherence
1162         return;
1163     }
1164
1165     BlkType *blk = tags->findBlock(pkt->getAddr());
1166
1167     Addr blk_addr = pkt->getAddr() & ~(Addr(blkSize-1));
1168     MSHR *mshr = mshrQueue.findMatch(blk_addr);
1169
1170     // Let the MSHR itself track the snoop and decide whether we want
1171     // to go ahead and do the regular cache snoop
1172     if (mshr && mshr->handleSnoop(pkt, order++)) {
1173         DPRINTF(Cache, "Deferring snoop on in-service MSHR to blk %x\n",
1174                 blk_addr);
1175         if (mshr->getNumTargets() > numTarget)
1176             warn("allocating bonus target for snoop"); //handle later
1177         return;
1178     }
1179
1180     //We also need to check the writeback buffers and handle those
1181     std::vector<MSHR *> writebacks;
1182     if (writeBuffer.findMatches(blk_addr, writebacks)) {
1183         DPRINTF(Cache, "Snoop hit in writeback to addr: %x\n",
1184                 pkt->getAddr());
1185
1186         //Look through writebacks for any non-uncachable writes, use that
1187         for (int i = 0; i < writebacks.size(); i++) {
1188             mshr = writebacks[i];
1189             assert(!mshr->isUncacheable());
1190             assert(mshr->getNumTargets() == 1);
1191             PacketPtr wb_pkt = mshr->getTarget()->pkt;
1192             assert(wb_pkt->cmd == MemCmd::Writeback);
1193
1194             assert(!pkt->memInhibitAsserted());
1195             pkt->assertMemInhibit();
1196             if (!pkt->needsExclusive()) {
1197                 pkt->assertShared();
1198             } else {
1199                 // if we're not asserting the shared line, we need to
1200                 // invalidate our copy.  we'll do that below as long as
1201                 // the packet's invalidate flag is set...
1202                 assert(pkt->isInvalidate());
1203             }
1204             doTimingSupplyResponse(pkt, wb_pkt->getPtr<uint8_t>(),
1205                                    false, false);
1206
1207             if (pkt->isInvalidate()) {
1208                 // Invalidation trumps our writeback... discard here
1209                 markInService(mshr);
1210                 delete wb_pkt;
1211             }
1212
1213             // If this was a shared writeback, there may still be
1214             // other shared copies above that require invalidation.
1215             // We could be more selective and return here if the
1216             // request is non-exclusive or if the writeback is
1217             // exclusive.
1218             break;
1219         }
1220     }
1221
1222     handleSnoop(pkt, blk, true, false, false);
1223 }
1224
1225
1226 template<class TagStore>
1227 Tick
1228 Cache<TagStore>::snoopAtomic(PacketPtr pkt)
1229 {
1230     if (pkt->req->isUncacheable() || pkt->cmd == MemCmd::Writeback) {
1231         // Can't get a hit on an uncacheable address
1232         // Revisit this for multi level coherence
1233         return hitLatency;
1234     }
1235
1236     BlkType *blk = tags->findBlock(pkt->getAddr());
1237     handleSnoop(pkt, blk, false, false, false);
1238     return hitLatency;
1239 }
1240
1241
1242 template<class TagStore>
1243 MSHR *
1244 Cache<TagStore>::getNextMSHR()
1245 {
1246     // Check both MSHR queue and write buffer for potential requests
1247     MSHR *miss_mshr  = mshrQueue.getNextMSHR();
1248     MSHR *write_mshr = writeBuffer.getNextMSHR();
1249
1250     // Now figure out which one to send... some cases are easy
1251     if (miss_mshr && !write_mshr) {
1252         return miss_mshr;
1253     }
1254     if (write_mshr && !miss_mshr) {
1255         return write_mshr;
1256     }
1257
1258     if (miss_mshr && write_mshr) {
1259         // We have one of each... normally we favor the miss request
1260         // unless the write buffer is full
1261         if (writeBuffer.isFull() && writeBuffer.inServiceEntries == 0) {
1262             // Write buffer is full, so we'd like to issue a write;
1263             // need to search MSHR queue for conflicting earlier miss.
1264             MSHR *conflict_mshr =
1265                 mshrQueue.findPending(write_mshr->addr, write_mshr->size);
1266
1267             if (conflict_mshr && conflict_mshr->order < write_mshr->order) {
1268                 // Service misses in order until conflict is cleared.
1269                 return conflict_mshr;
1270             }
1271
1272             // No conflicts; issue write
1273             return write_mshr;
1274         }
1275
1276         // Write buffer isn't full, but need to check it for
1277         // conflicting earlier writeback
1278         MSHR *conflict_mshr =
1279             writeBuffer.findPending(miss_mshr->addr, miss_mshr->size);
1280         if (conflict_mshr) {
1281             // not sure why we don't check order here... it was in the
1282             // original code but commented out.
1283
1284             // The only way this happens is if we are
1285             // doing a write and we didn't have permissions
1286             // then subsequently saw a writeback (owned got evicted)
1287             // We need to make sure to perform the writeback first
1288             // To preserve the dirty data, then we can issue the write
1289
1290             // should we return write_mshr here instead?  I.e. do we
1291             // have to flush writes in order?  I don't think so... not
1292             // for Alpha anyway.  Maybe for x86?
1293             return conflict_mshr;
1294         }
1295
1296         // No conflicts; issue read
1297         return miss_mshr;
1298     }
1299
1300     // fall through... no pending requests.  Try a prefetch.
1301     assert(!miss_mshr && !write_mshr);
1302     if (!mshrQueue.isFull()) {
1303         // If we have a miss queue slot, we can try a prefetch
1304         PacketPtr pkt = prefetcher->getPacket();
1305         if (pkt) {
1306             // Update statistic on number of prefetches issued
1307             // (hwpf_mshr_misses)
1308             mshr_misses[pkt->cmdToIndex()][0/*pkt->req->threadId()*/]++;
1309             // Don't request bus, since we already have it
1310             return allocateMissBuffer(pkt, curTick, false);
1311         }
1312     }
1313
1314     return NULL;
1315 }
1316
1317
1318 template<class TagStore>
1319 PacketPtr
1320 Cache<TagStore>::getTimingPacket()
1321 {
1322     MSHR *mshr = getNextMSHR();
1323
1324     if (mshr == NULL) {
1325         return NULL;
1326     }
1327
1328     // use request from 1st target
1329     PacketPtr tgt_pkt = mshr->getTarget()->pkt;
1330     PacketPtr pkt = NULL;
1331
1332     if (mshr->isForwardNoResponse()) {
1333         // no response expected, just forward packet as it is
1334         assert(tags->findBlock(mshr->addr) == NULL);
1335         pkt = tgt_pkt;
1336     } else {
1337         BlkType *blk = tags->findBlock(mshr->addr);
1338         pkt = getBusPacket(tgt_pkt, blk, mshr->needsExclusive());
1339
1340         mshr->isForward = (pkt == NULL);
1341
1342         if (mshr->isForward) {
1343             // not a cache block request, but a response is expected
1344             // make copy of current packet to forward, keep current
1345             // copy for response handling
1346             pkt = new Packet(tgt_pkt);
1347             pkt->allocate();
1348             if (pkt->isWrite()) {
1349                 pkt->setData(tgt_pkt->getPtr<uint8_t>());
1350             }
1351         }
1352     }
1353
1354     assert(pkt != NULL);
1355     pkt->senderState = mshr;
1356     return pkt;
1357 }
1358
1359
1360 template<class TagStore>
1361 Tick
1362 Cache<TagStore>::nextMSHRReadyTime()
1363 {
1364     Tick nextReady = std::min(mshrQueue.nextMSHRReadyTime(),
1365                               writeBuffer.nextMSHRReadyTime());
1366
1367     if (prefetcher) {
1368         nextReady = std::min(nextReady,
1369                              prefetcher->nextPrefetchReadyTime());
1370     }
1371
1372     return nextReady;
1373 }
1374
1375
1376 ///////////////
1377 //
1378 // CpuSidePort
1379 //
1380 ///////////////
1381
1382 template<class TagStore>
1383 void
1384 Cache<TagStore>::CpuSidePort::
1385 getDeviceAddressRanges(AddrRangeList &resp, bool &snoop)
1386 {
1387     // CPU side port doesn't snoop; it's a target only.  It can
1388     // potentially respond to any address.
1389     snoop = false;
1390     resp.push_back(myCache()->getAddrRange());
1391 }
1392
1393
1394 template<class TagStore>
1395 bool
1396 Cache<TagStore>::CpuSidePort::recvTiming(PacketPtr pkt)
1397 {
1398     // illegal to block responses... can lead to deadlock
1399     if (pkt->isRequest() && !pkt->memInhibitAsserted() && blocked) {
1400         DPRINTF(Cache,"Scheduling a retry while blocked\n");
1401         mustSendRetry = true;
1402         return false;
1403     }
1404
1405     myCache()->timingAccess(pkt);
1406     return true;
1407 }
1408
1409
1410 template<class TagStore>
1411 Tick
1412 Cache<TagStore>::CpuSidePort::recvAtomic(PacketPtr pkt)
1413 {
1414     return myCache()->atomicAccess(pkt);
1415 }
1416
1417
1418 template<class TagStore>
1419 void
1420 Cache<TagStore>::CpuSidePort::recvFunctional(PacketPtr pkt)
1421 {
1422     myCache()->functionalAccess(pkt, this, otherPort);
1423 }
1424
1425
1426 template<class TagStore>
1427 Cache<TagStore>::
1428 CpuSidePort::CpuSidePort(const std::string &_name, Cache<TagStore> *_cache,
1429                          const std::string &_label)
1430     : BaseCache::CachePort(_name, _cache, _label)
1431 {
1432 }
1433
1434 ///////////////
1435 //
1436 // MemSidePort
1437 //
1438 ///////////////
1439
1440 template<class TagStore>
1441 void
1442 Cache<TagStore>::MemSidePort::
1443 getDeviceAddressRanges(AddrRangeList &resp, bool &snoop)
1444 {
1445     // Memory-side port always snoops, but never passes requests
1446     // through to targets on the cpu side (so we don't add anything to
1447     // the address range list).
1448     snoop = true;
1449 }
1450
1451
1452 template<class TagStore>
1453 bool
1454 Cache<TagStore>::MemSidePort::recvTiming(PacketPtr pkt)
1455 {
1456     // this needs to be fixed so that the cache updates the mshr and sends the
1457     // packet back out on the link, but it probably won't happen so until this
1458     // gets fixed, just panic when it does
1459     if (pkt->wasNacked())
1460         panic("Need to implement cache resending nacked packets!\n");
1461
1462     if (pkt->isRequest() && blocked) {
1463         DPRINTF(Cache,"Scheduling a retry while blocked\n");
1464         mustSendRetry = true;
1465         return false;
1466     }
1467
1468     if (pkt->isResponse()) {
1469         myCache()->handleResponse(pkt);
1470     } else {
1471         myCache()->snoopTiming(pkt);
1472     }
1473     return true;
1474 }
1475
1476
1477 template<class TagStore>
1478 Tick
1479 Cache<TagStore>::MemSidePort::recvAtomic(PacketPtr pkt)
1480 {
1481     // in atomic mode, responses go back to the sender via the
1482     // function return from sendAtomic(), not via a separate
1483     // sendAtomic() from the responder.  Thus we should never see a
1484     // response packet in recvAtomic() (anywhere, not just here).
1485     assert(!pkt->isResponse());
1486     return myCache()->snoopAtomic(pkt);
1487 }
1488
1489
1490 template<class TagStore>
1491 void
1492 Cache<TagStore>::MemSidePort::recvFunctional(PacketPtr pkt)
1493 {
1494     myCache()->functionalAccess(pkt, this, otherPort);
1495 }
1496
1497
1498
1499 template<class TagStore>
1500 void
1501 Cache<TagStore>::MemSidePort::sendPacket()
1502 {
1503     // if we have responses that are ready, they take precedence
1504     if (deferredPacketReady()) {
1505         bool success = sendTiming(transmitList.front().pkt);
1506
1507         if (success) {
1508             //send successful, remove packet
1509             transmitList.pop_front();
1510         }
1511
1512         waitingOnRetry = !success;
1513     } else {
1514         // check for non-response packets (requests & writebacks)
1515         PacketPtr pkt = myCache()->getTimingPacket();
1516         if (pkt == NULL) {
1517             // can happen if e.g. we attempt a writeback and fail, but
1518             // before the retry, the writeback is eliminated because
1519             // we snoop another cache's ReadEx.
1520             waitingOnRetry = false;
1521         } else {
1522             MSHR *mshr = dynamic_cast<MSHR*>(pkt->senderState);
1523
1524             bool success = sendTiming(pkt);
1525
1526             waitingOnRetry = !success;
1527             if (waitingOnRetry) {
1528                 DPRINTF(CachePort, "now waiting on a retry\n");
1529                 if (!mshr->isForwardNoResponse()) {
1530                     delete pkt;
1531                 }
1532             } else {
1533                 myCache()->markInService(mshr);
1534             }
1535         }
1536     }
1537
1538
1539     // tried to send packet... if it was successful (no retry), see if
1540     // we need to rerequest bus or not
1541     if (!waitingOnRetry) {
1542         Tick nextReady = std::min(deferredPacketReadyTime(),
1543                                   myCache()->nextMSHRReadyTime());
1544         // @TODO: need to facotr in prefetch requests here somehow
1545         if (nextReady != MaxTick) {
1546             DPRINTF(CachePort, "more packets to send @ %d\n", nextReady);
1547             schedule(sendEvent, std::max(nextReady, curTick + 1));
1548         } else {
1549             // no more to send right now: if we're draining, we may be done
1550             if (drainEvent) {
1551                 drainEvent->process();
1552                 drainEvent = NULL;
1553             }
1554         }
1555     }
1556 }
1557
1558 template<class TagStore>
1559 void
1560 Cache<TagStore>::MemSidePort::recvRetry()
1561 {
1562     assert(waitingOnRetry);
1563     sendPacket();
1564 }
1565
1566
1567 template<class TagStore>
1568 void
1569 Cache<TagStore>::MemSidePort::processSendEvent()
1570 {
1571     assert(!waitingOnRetry);
1572     sendPacket();
1573 }
1574
1575
1576 template<class TagStore>
1577 Cache<TagStore>::
1578 MemSidePort::MemSidePort(const std::string &_name, Cache<TagStore> *_cache,
1579                          const std::string &_label)
1580     : BaseCache::CachePort(_name, _cache, _label)
1581 {
1582     // override default send event from SimpleTimingPort
1583     delete sendEvent;
1584     sendEvent = new SendEvent(this);
1585 }