Merge zizzer:/bk/newmem
[gem5.git] / src / mem / cache / cache_impl.hh
1 /*
2 * Copyright (c) 2002-2005 The Regents of The University of Michigan
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * Authors: Erik Hallnor
29 * Dave Greene
30 * Nathan Binkert
31 */
32
33 /**
34 * @file
35 * Cache definitions.
36 */
37
38 #include <assert.h>
39 #include <math.h>
40
41 #include <cassert>
42 #include <iostream>
43 #include <cstring>
44 #include <string>
45
46 #include "sim/host.hh"
47 #include "base/misc.hh"
48 #include "cpu/smt.hh"
49
50 #include "mem/cache/cache.hh"
51 #include "mem/cache/cache_blk.hh"
52 #include "mem/cache/miss/mshr.hh"
53 #include "mem/cache/prefetch/base_prefetcher.hh"
54
55 #include "sim/sim_exit.hh" // for SimExitEvent
56
57 bool SIGNAL_NACK_HACK;
58
59 template<class TagStore, class Coherence>
60 void
61 Cache<TagStore,Coherence>::
62 recvStatusChange(Port::Status status, bool isCpuSide)
63 {
64
65 }
66
67
68 template<class TagStore, class Coherence>
69 Cache<TagStore,Coherence>::
70 Cache(const std::string &_name,
71 Cache<TagStore,Coherence>::Params &params)
72 : BaseCache(_name, params.baseParams),
73 prefetchAccess(params.prefetchAccess),
74 tags(params.tags), missQueue(params.missQueue),
75 coherence(params.coherence), prefetcher(params.prefetcher),
76 hitLatency(params.hitLatency),
77 compressionAlg(params.compressionAlg),
78 blkSize(params.blkSize),
79 doFastWrites(params.doFastWrites),
80 prefetchMiss(params.prefetchMiss),
81 storeCompressed(params.storeCompressed),
82 compressOnWriteback(params.compressOnWriteback),
83 compLatency(params.compLatency),
84 adaptiveCompression(params.adaptiveCompression),
85 writebackCompressed(params.writebackCompressed)
86 {
87 tags->setCache(this);
88 missQueue->setCache(this);
89 missQueue->setPrefetcher(prefetcher);
90 coherence->setCache(this);
91 prefetcher->setCache(this);
92 invalidateReq = new Request((Addr) NULL, blkSize, 0);
93 invalidatePkt = new Packet(invalidateReq, MemCmd::InvalidateReq, 0);
94 }
95
96 template<class TagStore, class Coherence>
97 void
98 Cache<TagStore,Coherence>::regStats()
99 {
100 BaseCache::regStats();
101 tags->regStats(name());
102 missQueue->regStats(name());
103 coherence->regStats(name());
104 prefetcher->regStats(name());
105 }
106
107 template<class TagStore, class Coherence>
108 typename Cache<TagStore,Coherence>::BlkType*
109 Cache<TagStore,Coherence>::handleAccess(PacketPtr &pkt, int & lat,
110 PacketList & writebacks, bool update)
111 {
112 // Set the block offset here
113 int offset = tags->extractBlkOffset(pkt->getAddr());
114
115 BlkType *blk = NULL;
116 if (update) {
117 blk = tags->findBlock(pkt->getAddr(), lat);
118 } else {
119 blk = tags->findBlock(pkt->getAddr());
120 lat = 0;
121 }
122 if (blk != NULL) {
123
124 if (!update) {
125
126 if (pkt->isWrite()){
127 assert(offset < blkSize);
128 assert(pkt->getSize() <= blkSize);
129 assert(offset+pkt->getSize() <= blkSize);
130 std::memcpy(blk->data + offset, pkt->getPtr<uint8_t>(),
131 pkt->getSize());
132 } else if (pkt->isReadWrite()) {
133 cmpAndSwap(blk, pkt);
134 } else if (!(pkt->flags & SATISFIED)) {
135 pkt->flags |= SATISFIED;
136 pkt->result = Packet::Success;
137 assert(offset < blkSize);
138 assert(pkt->getSize() <= blkSize);
139 assert(offset + pkt->getSize() <=blkSize);
140 std::memcpy(pkt->getPtr<uint8_t>(), blk->data + offset,
141 pkt->getSize());
142 }
143 return blk;
144 }
145
146 // Hit
147 if (blk->isPrefetch()) {
148 //Signal that this was a hit under prefetch (no need for
149 //use prefetch (only can get here if true)
150 DPRINTF(HWPrefetch, "Hit a block that was prefetched\n");
151 blk->status &= ~BlkHWPrefetched;
152 if (prefetchMiss) {
153 //If we are using the miss stream, signal the
154 //prefetcher otherwise the access stream would have
155 //already signaled this hit
156 prefetcher->handleMiss(pkt, curTick);
157 }
158 }
159
160 if ((pkt->isReadWrite() && blk->isWritable()) ||
161 (pkt->isWrite() && blk->isWritable()) ||
162 (pkt->isRead() && blk->isValid())) {
163
164 // We are satisfying the request
165 pkt->flags |= SATISFIED;
166
167 if (blk->isCompressed()) {
168 // If the data is compressed, need to increase the latency
169 lat += (compLatency/4);
170 }
171
172 bool write_data = false;
173
174 assert(verifyData(blk));
175
176 assert(offset < blkSize);
177 assert(pkt->getSize() <= blkSize);
178 assert(offset+pkt->getSize() <= blkSize);
179
180 if (pkt->isWrite()) {
181 if (blk->checkWrite(pkt->req)) {
182 write_data = true;
183 blk->status |= BlkDirty;
184 std::memcpy(blk->data + offset, pkt->getPtr<uint8_t>(),
185 pkt->getSize());
186 }
187 } else if (pkt->isReadWrite()) {
188 cmpAndSwap(blk, pkt);
189 } else {
190 assert(pkt->isRead());
191 if (pkt->req->isLocked()) {
192 blk->trackLoadLocked(pkt->req);
193 }
194 std::memcpy(pkt->getPtr<uint8_t>(), blk->data + offset,
195 pkt->getSize());
196 }
197
198 if (write_data ||
199 (adaptiveCompression && blk->isCompressed()))
200 {
201 // If we wrote data, need to update the internal block
202 // data.
203 updateData(blk, writebacks,
204 !(adaptiveCompression &&
205 blk->isReferenced()));
206 }
207 } else {
208 // permission violation, treat it as a miss
209 blk = NULL;
210 }
211 } else {
212 // complete miss (no matching block)
213 if (pkt->req->isLocked() && pkt->isWrite()) {
214 // miss on store conditional... just give up now
215 pkt->req->setExtraData(0);
216 pkt->flags |= SATISFIED;
217 }
218 }
219
220 return blk;
221 }
222
223 template<class TagStore, class Coherence>
224 void
225 Cache<TagStore,Coherence>::cmpAndSwap(BlkType *blk, PacketPtr &pkt){
226 uint64_t overwrite_val;
227 bool overwrite_mem;
228 uint64_t condition_val64;
229 uint32_t condition_val32;
230
231 int offset = tags->extractBlkOffset(pkt->getAddr());
232
233 assert(sizeof(uint64_t) >= pkt->getSize());
234
235 overwrite_mem = true;
236 // keep a copy of our possible write value, and copy what is at the
237 // memory address into the packet
238 std::memcpy(&overwrite_val, pkt->getPtr<uint8_t>(), pkt->getSize());
239 std::memcpy(pkt->getPtr<uint8_t>(), blk->data + offset,
240 pkt->getSize());
241
242 if (pkt->req->isCondSwap()) {
243 if (pkt->getSize() == sizeof(uint64_t)) {
244 condition_val64 = pkt->req->getExtraData();
245 overwrite_mem = !std::memcmp(&condition_val64, blk->data + offset,
246 sizeof(uint64_t));
247 } else if (pkt->getSize() == sizeof(uint32_t)) {
248 condition_val32 = (uint32_t)pkt->req->getExtraData();
249 overwrite_mem = !std::memcmp(&condition_val32, blk->data + offset,
250 sizeof(uint32_t));
251 } else
252 panic("Invalid size for conditional read/write\n");
253 }
254
255 if (overwrite_mem)
256 std::memcpy(blk->data + offset,
257 &overwrite_val, pkt->getSize());
258
259 }
260
261 template<class TagStore, class Coherence>
262 typename Cache<TagStore,Coherence>::BlkType*
263 Cache<TagStore,Coherence>::handleFill(BlkType *blk, PacketPtr &pkt,
264 CacheBlk::State new_state,
265 PacketList & writebacks,
266 PacketPtr target)
267 {
268 #ifndef NDEBUG
269 BlkType *tmp_blk = tags->findBlock(pkt->getAddr());
270 assert(tmp_blk == blk);
271 #endif
272 blk = doReplacement(blk, pkt, new_state, writebacks);
273
274
275 if (pkt->isRead()) {
276 std::memcpy(blk->data, pkt->getPtr<uint8_t>(), blkSize);
277 }
278
279 blk->whenReady = pkt->finishTime;
280
281 // Respond to target, if any
282 if (target) {
283
284 target->flags |= SATISFIED;
285
286 if (target->cmd == MemCmd::InvalidateReq) {
287 tags->invalidateBlk(blk);
288 blk = NULL;
289 }
290
291 if (blk && ((target->isWrite() || target->isReadWrite()) ?
292 blk->isWritable() : blk->isValid())) {
293 assert(target->isWrite() || target->isReadWrite() || target->isRead());
294 assert(target->getOffset(blkSize) + target->getSize() <= blkSize);
295 if (target->isWrite()) {
296 if (blk->checkWrite(pkt->req)) {
297 blk->status |= BlkDirty;
298 std::memcpy(blk->data + target->getOffset(blkSize),
299 target->getPtr<uint8_t>(), target->getSize());
300 }
301 } else if (target->isReadWrite()) {
302 cmpAndSwap(blk, pkt);
303 } else {
304 if (pkt->req->isLocked()) {
305 blk->trackLoadLocked(pkt->req);
306 }
307 std::memcpy(target->getPtr<uint8_t>(),
308 blk->data + target->getOffset(blkSize),
309 target->getSize());
310 }
311 }
312 }
313
314 if (blk) {
315 // Need to write the data into the block
316 updateData(blk, writebacks, !adaptiveCompression || true);
317 }
318 return blk;
319 }
320
321 template<class TagStore, class Coherence>
322 typename Cache<TagStore,Coherence>::BlkType*
323 Cache<TagStore,Coherence>::handleFill(BlkType *blk, MSHR * mshr,
324 CacheBlk::State new_state,
325 PacketList & writebacks, PacketPtr pkt)
326 {
327 /*
328 #ifndef NDEBUG
329 BlkType *tmp_blk = findBlock(mshr->pkt->getAddr());
330 assert(tmp_blk == blk);
331 #endif
332 PacketPtr pkt = mshr->pkt;*/
333 blk = doReplacement(blk, pkt, new_state, writebacks);
334
335 if (pkt->isRead()) {
336 std::memcpy(blk->data, pkt->getPtr<uint8_t>(), blkSize);
337 }
338
339 blk->whenReady = pkt->finishTime;
340
341
342 // respond to MSHR targets, if any
343
344 // First offset for critical word first calculations
345 int initial_offset = 0;
346
347 if (mshr->hasTargets()) {
348 initial_offset = mshr->getTarget()->getOffset(blkSize);
349 }
350
351 while (mshr->hasTargets()) {
352 PacketPtr target = mshr->getTarget();
353
354 target->flags |= SATISFIED;
355
356 // How many bytes pass the first request is this one
357 int transfer_offset = target->getOffset(blkSize) - initial_offset;
358 if (transfer_offset < 0) {
359 transfer_offset += blkSize;
360 }
361
362 // If critical word (no offset) return first word time
363 Tick completion_time = tags->getHitLatency() +
364 transfer_offset ? pkt->finishTime : pkt->firstWordTime;
365
366 if (target->cmd == MemCmd::InvalidateReq) {
367 //Mark the blk as invalid now, if it hasn't been already
368 if (blk) {
369 tags->invalidateBlk(blk);
370 blk = NULL;
371 }
372
373 //Also get rid of the invalidate
374 mshr->popTarget();
375
376 DPRINTF(Cache, "Popping off a Invalidate for addr %x\n",
377 pkt->getAddr());
378
379 continue;
380 }
381
382 if (blk && ((target->isWrite() || target->isReadWrite()) ?
383 blk->isWritable() : blk->isValid())) {
384 assert(target->isWrite() || target->isRead() || target->isReadWrite() );
385 assert(target->getOffset(blkSize) + target->getSize() <= blkSize);
386 if (target->isWrite()) {
387 if (blk->checkWrite(pkt->req)) {
388 blk->status |= BlkDirty;
389 std::memcpy(blk->data + target->getOffset(blkSize),
390 target->getPtr<uint8_t>(), target->getSize());
391 }
392 } else if (target->isReadWrite()) {
393 cmpAndSwap(blk, pkt);
394 } else {
395 if (target->req->isLocked()) {
396 blk->trackLoadLocked(target->req);
397 }
398 std::memcpy(target->getPtr<uint8_t>(),
399 blk->data + target->getOffset(blkSize),
400 target->getSize());
401 }
402 } else {
403 // Invalid access, need to do another request
404 // can occur if block is invalidated, or not correct
405 // permissions
406 // mshr->pkt = pkt;
407 break;
408 }
409 respondToMiss(target, completion_time);
410 mshr->popTarget();
411 }
412
413 if (blk) {
414 // Need to write the data into the block
415 updateData(blk, writebacks, !adaptiveCompression || true);
416 }
417
418 return blk;
419 }
420
421
422 template<class TagStore, class Coherence>
423 void
424 Cache<TagStore,Coherence>::handleSnoop(BlkType *blk,
425 CacheBlk::State new_state,
426 PacketPtr &pkt)
427 {
428 //Must have the block to supply
429 assert(blk);
430 // Can only supply data, and if it hasn't already been supllied
431 assert(pkt->isRead());
432 assert(!(pkt->flags & SATISFIED));
433 pkt->flags |= SATISFIED;
434 Addr offset = pkt->getOffset(blkSize);
435 assert(offset < blkSize);
436 assert(pkt->getSize() <= blkSize);
437 assert(offset + pkt->getSize() <=blkSize);
438 std::memcpy(pkt->getPtr<uint8_t>(), blk->data + offset, pkt->getSize());
439
440 handleSnoop(blk, new_state);
441 }
442
443 template<class TagStore, class Coherence>
444 void
445 Cache<TagStore,Coherence>::handleSnoop(BlkType *blk,
446 CacheBlk::State new_state)
447 {
448 if (blk && blk->status != new_state) {
449 if ((new_state && BlkValid) == 0) {
450 tags->invalidateBlk(blk);
451 } else {
452 assert(new_state >= 0 && new_state < 128);
453 blk->status = new_state;
454 }
455 }
456 }
457
458 template<class TagStore, class Coherence>
459 PacketPtr
460 Cache<TagStore,Coherence>::writebackBlk(BlkType *blk)
461 {
462 assert(blk && blk->isValid() && blk->isModified());
463 int data_size = blkSize;
464 data_size = blk->size;
465 if (compressOnWriteback) {
466 // not already compressed
467 // need to compress to ship it
468 assert(data_size == blkSize);
469 uint8_t *tmp_data = new uint8_t[blkSize];
470 data_size = compressionAlg->compress(tmp_data,blk->data,
471 data_size);
472 delete [] tmp_data;
473 }
474
475 /* PacketPtr writeback =
476 buildWritebackReq(tags->regenerateBlkAddr(blk->tag, blk->set),
477 blk->asid, blkSize,
478 blk->data, data_size);
479 */
480
481 Request *writebackReq =
482 new Request(tags->regenerateBlkAddr(blk->tag, blk->set), blkSize, 0);
483 PacketPtr writeback = new Packet(writebackReq, MemCmd::Writeback, -1);
484 writeback->allocate();
485 std::memcpy(writeback->getPtr<uint8_t>(),blk->data,blkSize);
486
487 blk->status &= ~BlkDirty;
488 return writeback;
489 }
490
491
492 template<class TagStore, class Coherence>
493 bool
494 Cache<TagStore,Coherence>::verifyData(BlkType *blk)
495 {
496 bool retval;
497 // The data stored in the blk
498 uint8_t *blk_data = new uint8_t[blkSize];
499 tags->readData(blk, blk_data);
500 // Pointer for uncompressed data, assumed uncompressed
501 uint8_t *tmp_data = blk_data;
502 // The size of the data being stored, assumed uncompressed
503 int data_size = blkSize;
504
505 // If the block is compressed need to uncompress to access
506 if (blk->isCompressed()){
507 // Allocate new storage for the data
508 tmp_data = new uint8_t[blkSize];
509 data_size = compressionAlg->uncompress(tmp_data,blk_data, blk->size);
510 assert(data_size == blkSize);
511 // Don't need to keep blk_data around
512 delete [] blk_data;
513 } else {
514 assert(blkSize == blk->size);
515 }
516
517 retval = std::memcmp(tmp_data, blk->data, blkSize) == 0;
518 delete [] tmp_data;
519 return retval;
520 }
521
522 template<class TagStore, class Coherence>
523 void
524 Cache<TagStore,Coherence>::updateData(BlkType *blk, PacketList &writebacks,
525 bool compress_block)
526 {
527 if (storeCompressed && compress_block) {
528 uint8_t *comp_data = new uint8_t[blkSize];
529 int new_size = compressionAlg->compress(comp_data, blk->data, blkSize);
530 if (new_size > (blkSize - tags->getSubBlockSize())){
531 // no benefit to storing it compressed
532 blk->status &= ~BlkCompressed;
533 tags->writeData(blk, blk->data, blkSize,
534 writebacks);
535 } else {
536 // Store the data compressed
537 blk->status |= BlkCompressed;
538 tags->writeData(blk, comp_data, new_size,
539 writebacks);
540 }
541 delete [] comp_data;
542 } else {
543 blk->status &= ~BlkCompressed;
544 tags->writeData(blk, blk->data, blkSize, writebacks);
545 }
546 }
547
548 template<class TagStore, class Coherence>
549 typename Cache<TagStore,Coherence>::BlkType*
550 Cache<TagStore,Coherence>::doReplacement(BlkType *blk, PacketPtr &pkt,
551 CacheBlk::State new_state,
552 PacketList &writebacks)
553 {
554 if (blk == NULL) {
555 // need to do a replacement
556 BlkList compress_list;
557 blk = tags->findReplacement(pkt, writebacks, compress_list);
558 while (adaptiveCompression && !compress_list.empty()) {
559 updateData(compress_list.front(), writebacks, true);
560 compress_list.pop_front();
561 }
562 if (blk->isValid()) {
563 DPRINTF(Cache, "replacement: replacing %x with %x: %s\n",
564 tags->regenerateBlkAddr(blk->tag,blk->set), pkt->getAddr(),
565 (blk->isModified()) ? "writeback" : "clean");
566
567 if (blk->isModified()) {
568 // Need to write the data back
569 writebacks.push_back(writebackBlk(blk));
570 }
571 }
572 blk->tag = tags->extractTag(pkt->getAddr(), blk);
573 } else {
574 // must be a status change
575 // assert(blk->status != new_state);
576 if (blk->status == new_state) warn("Changing state to same value\n");
577 }
578
579 blk->status = new_state;
580 return blk;
581 }
582
583
584 template<class TagStore, class Coherence>
585 bool
586 Cache<TagStore,Coherence>::access(PacketPtr &pkt)
587 {
588 //@todo Add back in MemDebug Calls
589 // MemDebug::cacheAccess(pkt);
590 BlkType *blk = NULL;
591 PacketList writebacks;
592 int size = blkSize;
593 int lat = hitLatency;
594 if (prefetchAccess) {
595 //We are determining prefetches on access stream, call prefetcher
596 prefetcher->handleMiss(pkt, curTick);
597 }
598
599 Addr blk_addr = pkt->getAddr() & ~(Addr(blkSize-1));
600
601 if (!pkt->req->isUncacheable()) {
602 if (!missQueue->findMSHR(blk_addr)) {
603 blk = handleAccess(pkt, lat, writebacks);
604 }
605 } else {
606 size = pkt->getSize();
607 }
608 // If this is a block size write/hint (WH64) allocate the block here
609 // if the coherence protocol allows it.
610 /** @todo make the fast write alloc (wh64) work with coherence. */
611 /** @todo Do we want to do fast writes for writebacks as well? */
612 if (!blk && pkt->getSize() >= blkSize && coherence->allowFastWrites() &&
613 (pkt->cmd == MemCmd::WriteReq
614 || pkt->cmd == MemCmd::WriteInvalidateReq) ) {
615 // not outstanding misses, can do this
616 MSHR* outstanding_miss = missQueue->findMSHR(pkt->getAddr());
617 if (pkt->cmd == MemCmd::WriteInvalidateReq || !outstanding_miss) {
618 if (outstanding_miss) {
619 warn("WriteInv doing a fastallocate"
620 "with an outstanding miss to the same address\n");
621 }
622 blk = handleFill(NULL, pkt, BlkValid | BlkWritable,
623 writebacks);
624 ++fastWrites;
625 }
626 }
627 while (!writebacks.empty()) {
628 PacketPtr wbPkt = writebacks.front();
629 missQueue->doWriteback(wbPkt);
630 writebacks.pop_front();
631 delete wbPkt;
632 }
633
634 DPRINTF(Cache, "%s %x %s\n", pkt->cmdString(), pkt->getAddr(),
635 (blk) ? "hit" : "miss");
636
637 if (blk) {
638 // Hit
639 hits[pkt->cmdToIndex()][0/*pkt->req->getThreadNum()*/]++;
640 // clear dirty bit if write through
641 respond(pkt, curTick+lat);
642 return true;
643 }
644
645 // Miss
646 if (!pkt->req->isUncacheable()) {
647 misses[pkt->cmdToIndex()][0/*pkt->req->getThreadNum()*/]++;
648 /** @todo Move miss count code into BaseCache */
649 if (missCount) {
650 --missCount;
651 if (missCount == 0)
652 exitSimLoop("A cache reached the maximum miss count");
653 }
654 }
655
656 if (pkt->flags & SATISFIED) {
657 // happens when a store conditional fails because it missed
658 // the cache completely
659 respond(pkt, curTick+lat);
660 } else {
661 missQueue->handleMiss(pkt, size, curTick + hitLatency);
662 }
663
664 if (!pkt->needsResponse()) {
665 //Need to clean up the packet on a writeback miss, but leave the request
666 //for the next level.
667 delete pkt;
668 }
669
670 return true;
671 }
672
673
674 template<class TagStore, class Coherence>
675 PacketPtr
676 Cache<TagStore,Coherence>::getPacket()
677 {
678 assert(missQueue->havePending());
679 PacketPtr pkt = missQueue->getPacket();
680 if (pkt) {
681 if (!pkt->req->isUncacheable()) {
682 if (pkt->cmd == MemCmd::HardPFReq)
683 misses[MemCmd::HardPFReq][0/*pkt->req->getThreadNum()*/]++;
684 BlkType *blk = tags->findBlock(pkt->getAddr());
685 MemCmd cmd =
686 coherence->getBusCmd(pkt->cmd, (blk) ? blk->status : 0);
687 missQueue->setBusCmd(pkt, cmd);
688 }
689 }
690
691 assert(!doMasterRequest() || missQueue->havePending());
692 assert(!pkt || pkt->time <= curTick);
693 SIGNAL_NACK_HACK = false;
694 return pkt;
695 }
696
697 template<class TagStore, class Coherence>
698 void
699 Cache<TagStore,Coherence>::sendResult(PacketPtr &pkt, MSHR* mshr,
700 bool success)
701 {
702 if (success && !(SIGNAL_NACK_HACK)) {
703 //Remember if it was an upgrade because writeback MSHR's are removed
704 //in Mark in Service
705 bool upgrade = (mshr->pkt && mshr->pkt->cmd == MemCmd::UpgradeReq);
706
707 missQueue->markInService(mshr->pkt, mshr);
708
709 //Temp Hack for UPGRADES
710 if (upgrade) {
711 assert(pkt); //Upgrades need to be fixed
712 pkt->flags &= ~CACHE_LINE_FILL;
713 BlkType *blk = tags->findBlock(pkt->getAddr());
714 CacheBlk::State old_state = (blk) ? blk->status : 0;
715 CacheBlk::State new_state = coherence->getNewState(pkt,old_state);
716 if (old_state != new_state)
717 DPRINTF(Cache, "Block for blk addr %x moving from state "
718 "%i to %i\n", pkt->getAddr(), old_state, new_state);
719 //Set the state on the upgrade
720 std::memcpy(pkt->getPtr<uint8_t>(), blk->data, blkSize);
721 PacketList writebacks;
722 handleFill(blk, mshr, new_state, writebacks, pkt);
723 assert(writebacks.empty());
724 missQueue->handleResponse(pkt, curTick + hitLatency);
725 }
726 } else if (pkt && !pkt->req->isUncacheable()) {
727 pkt->flags &= ~NACKED_LINE;
728 SIGNAL_NACK_HACK = false;
729 pkt->flags &= ~SATISFIED;
730 pkt->flags &= ~SNOOP_COMMIT;
731
732 //Rmove copy from mshr
733 delete mshr->pkt;
734 mshr->pkt = pkt;
735
736 missQueue->restoreOrigCmd(pkt);
737 }
738 }
739
740 template<class TagStore, class Coherence>
741 void
742 Cache<TagStore,Coherence>::handleResponse(PacketPtr &pkt)
743 {
744 BlkType *blk = NULL;
745 if (pkt->senderState) {
746 //Delete temp copy in MSHR, restore it.
747 delete ((MSHR*)pkt->senderState)->pkt;
748 ((MSHR*)pkt->senderState)->pkt = pkt;
749 if (pkt->result == Packet::Nacked) {
750 //pkt->reinitFromRequest();
751 warn("NACKs from devices not connected to the same bus "
752 "not implemented\n");
753 return;
754 }
755 if (pkt->result == Packet::BadAddress) {
756 //Make the response a Bad address and send it
757 }
758 // MemDebug::cacheResponse(pkt);
759 DPRINTF(Cache, "Handling reponse to %x\n", pkt->getAddr());
760
761 if (pkt->isCacheFill() && !pkt->isNoAllocate()) {
762 DPRINTF(Cache, "Block for addr %x being updated in Cache\n",
763 pkt->getAddr());
764 blk = tags->findBlock(pkt->getAddr());
765 CacheBlk::State old_state = (blk) ? blk->status : 0;
766 PacketList writebacks;
767 CacheBlk::State new_state = coherence->getNewState(pkt,old_state);
768 if (old_state != new_state)
769 DPRINTF(Cache, "Block for blk addr %x moving from "
770 "state %i to %i\n",
771 pkt->getAddr(),
772 old_state, new_state);
773 blk = handleFill(blk, (MSHR*)pkt->senderState,
774 new_state, writebacks, pkt);
775 while (!writebacks.empty()) {
776 PacketPtr wbPkt = writebacks.front();
777 missQueue->doWriteback(wbPkt);
778 writebacks.pop_front();
779 delete wbPkt;
780 }
781 }
782 missQueue->handleResponse(pkt, curTick + hitLatency);
783 }
784 }
785
786 template<class TagStore, class Coherence>
787 PacketPtr
788 Cache<TagStore,Coherence>::getCoherencePacket()
789 {
790 return coherence->getPacket();
791 }
792
793 template<class TagStore, class Coherence>
794 void
795 Cache<TagStore,Coherence>::sendCoherenceResult(PacketPtr &pkt,
796 MSHR *cshr,
797 bool success)
798 {
799 coherence->sendResult(pkt, cshr, success);
800 }
801
802
803 template<class TagStore, class Coherence>
804 void
805 Cache<TagStore,Coherence>::snoop(PacketPtr &pkt)
806 {
807 if (pkt->req->isUncacheable()) {
808 //Can't get a hit on an uncacheable address
809 //Revisit this for multi level coherence
810 return;
811 }
812
813 //Send a timing (true) invalidate up if the protocol calls for it
814 if (coherence->propogateInvalidate(pkt, true)) {
815 //Temp hack, we had a functional read hit in the L1, mark as success
816 pkt->flags |= SATISFIED;
817 pkt->result = Packet::Success;
818 respondToSnoop(pkt, curTick + hitLatency);
819 return;
820 }
821
822 Addr blk_addr = pkt->getAddr() & ~(Addr(blkSize-1));
823 BlkType *blk = tags->findBlock(pkt->getAddr());
824 MSHR *mshr = missQueue->findMSHR(blk_addr);
825 if (coherence->hasProtocol() || pkt->isInvalidate()) {
826 //@todo Move this into handle bus req
827 //If we find an mshr, and it is in service, we need to NACK or
828 //invalidate
829 if (mshr) {
830 if (mshr->inService) {
831 if ((mshr->pkt->isInvalidate() || !mshr->pkt->isCacheFill())
832 && (pkt->cmd != MemCmd::InvalidateReq
833 && pkt->cmd != MemCmd::WriteInvalidateReq)) {
834 //If the outstanding request was an invalidate
835 //(upgrade,readex,..) Then we need to ACK the request
836 //until we get the data Also NACK if the outstanding
837 //request is not a cachefill (writeback)
838 assert(!(pkt->flags & SATISFIED));
839 pkt->flags |= SATISFIED;
840 pkt->flags |= NACKED_LINE;
841 SIGNAL_NACK_HACK = true;
842 ///@todo NACK's from other levels
843 //warn("NACKs from devices not connected to the same bus "
844 //"not implemented\n");
845 //respondToSnoop(pkt, curTick + hitLatency);
846 return;
847 }
848 else {
849 //The supplier will be someone else, because we are
850 //waiting for the data. This should cause this cache to
851 //be forced to go to the shared state, not the exclusive
852 //even though the shared line won't be asserted. But for
853 //now we will just invlidate ourselves and allow the other
854 //cache to go into the exclusive state. @todo Make it so
855 //a read to a pending read doesn't invalidate. @todo Make
856 //it so that a read to a pending read can't be exclusive
857 //now.
858
859 //Set the address so find match works
860 //panic("Don't have invalidates yet\n");
861 invalidatePkt->addrOverride(pkt->getAddr());
862
863 //Append the invalidate on
864 missQueue->addTarget(mshr,invalidatePkt);
865 DPRINTF(Cache, "Appending Invalidate to addr: %x\n",
866 pkt->getAddr());
867 return;
868 }
869 }
870 }
871 //We also need to check the writeback buffers and handle those
872 std::vector<MSHR *> writebacks;
873 if (missQueue->findWrites(blk_addr, writebacks)) {
874 DPRINTF(Cache, "Snoop hit in writeback to addr: %x\n",
875 pkt->getAddr());
876
877 //Look through writebacks for any non-uncachable writes, use that
878 for (int i=0; i<writebacks.size(); i++) {
879 mshr = writebacks[i];
880
881 if (!mshr->pkt->req->isUncacheable()) {
882 if (pkt->isRead()) {
883 //Only Upgrades don't get here
884 //Supply the data
885 assert(!(pkt->flags & SATISFIED));
886 pkt->flags |= SATISFIED;
887
888 //If we are in an exclusive protocol, make it ask again
889 //to get write permissions (upgrade), signal shared
890 pkt->flags |= SHARED_LINE;
891
892 assert(pkt->isRead());
893 Addr offset = pkt->getAddr() & (blkSize - 1);
894 assert(offset < blkSize);
895 assert(pkt->getSize() <= blkSize);
896 assert(offset + pkt->getSize() <=blkSize);
897 std::memcpy(pkt->getPtr<uint8_t>(), mshr->pkt->getPtr<uint8_t>() + offset, pkt->getSize());
898
899 respondToSnoop(pkt, curTick + hitLatency);
900 }
901
902 if (pkt->isInvalidate()) {
903 //This must be an upgrade or other cache will take
904 //ownership
905 missQueue->markInService(mshr->pkt, mshr);
906 }
907 return;
908 }
909 }
910 }
911 }
912 CacheBlk::State new_state;
913 bool satisfy = coherence->handleBusRequest(pkt,blk,mshr, new_state);
914
915 if (blk && mshr && !mshr->inService && new_state == 0) {
916 //There was a outstanding write to a shared block, not need ReadEx
917 //not update, so change No Allocate param in MSHR
918 mshr->pkt->flags &= ~NO_ALLOCATE;
919 }
920
921 if (satisfy) {
922 DPRINTF(Cache, "Cache snooped a %s request for addr %x and "
923 "now supplying data, new state is %i\n",
924 pkt->cmdString(), blk_addr, new_state);
925
926 handleSnoop(blk, new_state, pkt);
927 respondToSnoop(pkt, curTick + hitLatency);
928 return;
929 }
930 if (blk)
931 DPRINTF(Cache, "Cache snooped a %s request for addr %x, "
932 "new state is %i\n", pkt->cmdString(), blk_addr, new_state);
933
934 handleSnoop(blk, new_state);
935 }
936
937 template<class TagStore, class Coherence>
938 void
939 Cache<TagStore,Coherence>::snoopResponse(PacketPtr &pkt)
940 {
941 //Need to handle the response, if NACKED
942 if (pkt->flags & NACKED_LINE) {
943 //Need to mark it as not in service, and retry for bus
944 assert(0); //Yeah, we saw a NACK come through
945
946 //For now this should never get called, we return false when we see a
947 //NACK instead, by doing this we allow the bus_blocked mechanism to
948 //handle the retry For now it retrys in just 2 cycles, need to figure
949 //out how to change that Eventually we will want to also have success
950 //come in as a parameter Need to make sure that we handle the
951 //functionality that happens on successufl return of the sendAddr
952 //function
953 }
954 }
955
956
957 /**
958 * @todo Fix to not assume write allocate
959 */
960 template<class TagStore, class Coherence>
961 Tick
962 Cache<TagStore,Coherence>::probe(PacketPtr &pkt, bool update,
963 CachePort* otherSidePort)
964 {
965 // MemDebug::cacheProbe(pkt);
966 if (!pkt->req->isUncacheable()) {
967 if (pkt->isInvalidate() && !pkt->isRead() && !pkt->isWrite()) {
968 //Upgrade or Invalidate, satisfy it, don't forward
969 DPRINTF(Cache, "%s %x ?\n", pkt->cmdString(), pkt->getAddr());
970 pkt->flags |= SATISFIED;
971 return 0;
972 }
973 }
974
975 if (!update && (otherSidePort == cpuSidePort)) {
976 // Still need to change data in all locations.
977 otherSidePort->checkAndSendFunctional(pkt);
978 if (pkt->isRead() && pkt->result == Packet::Success)
979 return 0;
980 }
981
982 PacketList writebacks;
983 int lat;
984
985 BlkType *blk = handleAccess(pkt, lat, writebacks, update);
986
987 DPRINTF(Cache, "%s %x %s\n", pkt->cmdString(),
988 pkt->getAddr(), (blk) ? "hit" : "miss");
989
990
991 // Need to check for outstanding misses and writes
992 Addr blk_addr = pkt->getAddr() & ~(blkSize - 1);
993
994 // There can only be one matching outstanding miss.
995 MSHR* mshr = missQueue->findMSHR(blk_addr);
996
997 // There can be many matching outstanding writes.
998 std::vector<MSHR*> writes;
999 missQueue->findWrites(blk_addr, writes);
1000
1001 if (!update) {
1002 bool notDone = !(pkt->flags & SATISFIED); //Hit in cache (was a block)
1003 // Check for data in MSHR and writebuffer.
1004 if (mshr) {
1005 MSHR::TargetList *targets = mshr->getTargetList();
1006 MSHR::TargetList::iterator i = targets->begin();
1007 MSHR::TargetList::iterator end = targets->end();
1008 for (; i != end && notDone; ++i) {
1009 PacketPtr target = *i;
1010 // If the target contains data, and it overlaps the
1011 // probed request, need to update data
1012 if (target->intersect(pkt)) {
1013 DPRINTF(Cache, "Functional %s access to blk_addr %x intersects a MSHR\n",
1014 pkt->cmdString(), blk_addr);
1015 notDone = fixPacket(pkt, target);
1016 }
1017 }
1018 }
1019 for (int i = 0; i < writes.size() && notDone; ++i) {
1020 PacketPtr write = writes[i]->pkt;
1021 if (write->intersect(pkt)) {
1022 DPRINTF(Cache, "Functional %s access to blk_addr %x intersects a writeback\n",
1023 pkt->cmdString(), blk_addr);
1024 notDone = fixPacket(pkt, write);
1025 }
1026 }
1027 if (notDone && otherSidePort == memSidePort) {
1028 otherSidePort->checkAndSendFunctional(pkt);
1029 assert(pkt->result == Packet::Success);
1030 }
1031 return 0;
1032 } else if (!blk && !(pkt->flags & SATISFIED)) {
1033 // update the cache state and statistics
1034 if (mshr || !writes.empty()){
1035 // Can't handle it, return request unsatisfied.
1036 panic("Atomic access ran into outstanding MSHR's or WB's!");
1037 }
1038 if (!pkt->req->isUncacheable() /*Uncacheables just go through*/
1039 && (pkt->cmd != MemCmd::Writeback)/*Writebacks on miss fall through*/) {
1040 // Fetch the cache block to fill
1041 BlkType *blk = tags->findBlock(pkt->getAddr());
1042 MemCmd temp_cmd =
1043 coherence->getBusCmd(pkt->cmd, (blk) ? blk->status : 0);
1044
1045 PacketPtr busPkt = new Packet(pkt->req,temp_cmd, -1, blkSize);
1046
1047 busPkt->allocate();
1048
1049 busPkt->time = curTick;
1050
1051 DPRINTF(Cache, "Sending a atomic %s for %x\n",
1052 busPkt->cmdString(), busPkt->getAddr());
1053
1054 lat = memSidePort->sendAtomic(busPkt);
1055
1056 //Be sure to flip the response to a request for coherence
1057 if (busPkt->needsResponse()) {
1058 busPkt->makeAtomicResponse();
1059 }
1060
1061 /* if (!(busPkt->flags & SATISFIED)) {
1062 // blocked at a higher level, just return
1063 return 0;
1064 }
1065
1066 */ misses[pkt->cmdToIndex()][0/*pkt->req->getThreadNum()*/]++;
1067
1068 CacheBlk::State old_state = (blk) ? blk->status : 0;
1069 CacheBlk::State new_state =
1070 coherence->getNewState(busPkt, old_state);
1071 DPRINTF(Cache, "Receive response: %s for addr %x in state %i\n",
1072 busPkt->cmdString(), busPkt->getAddr(), old_state);
1073 if (old_state != new_state)
1074 DPRINTF(Cache, "Block for blk addr %x moving from state "
1075 "%i to %i\n", busPkt->getAddr(), old_state, new_state);
1076
1077 handleFill(blk, busPkt, new_state, writebacks, pkt);
1078 //Free the packet
1079 delete busPkt;
1080
1081 // Handle writebacks if needed
1082 while (!writebacks.empty()){
1083 PacketPtr wbPkt = writebacks.front();
1084 memSidePort->sendAtomic(wbPkt);
1085 writebacks.pop_front();
1086 delete wbPkt;
1087 }
1088 return lat + hitLatency;
1089 } else {
1090 return memSidePort->sendAtomic(pkt);
1091 }
1092 } else {
1093 if (blk) {
1094 // There was a cache hit.
1095 // Handle writebacks if needed
1096 while (!writebacks.empty()){
1097 PacketPtr wbPkt = writebacks.front();
1098 memSidePort->sendAtomic(wbPkt);
1099 writebacks.pop_front();
1100 delete wbPkt;
1101 }
1102
1103 hits[pkt->cmdToIndex()][0/*pkt->req->getThreadNum()*/]++;
1104 }
1105
1106 return hitLatency;
1107 }
1108
1109 return 0;
1110 }
1111
1112 template<class TagStore, class Coherence>
1113 Tick
1114 Cache<TagStore,Coherence>::snoopProbe(PacketPtr &pkt)
1115 {
1116 //Send a atomic (false) invalidate up if the protocol calls for it
1117 if (coherence->propogateInvalidate(pkt, false)) {
1118 //Temp hack, we had a functional read hit in the L1, mark as success
1119 pkt->flags |= SATISFIED;
1120 pkt->result = Packet::Success;
1121 return hitLatency;
1122 }
1123
1124 Addr blk_addr = pkt->getAddr() & ~(Addr(blkSize-1));
1125 BlkType *blk = tags->findBlock(pkt->getAddr());
1126 MSHR *mshr = missQueue->findMSHR(blk_addr);
1127 CacheBlk::State new_state = 0;
1128 bool satisfy = coherence->handleBusRequest(pkt,blk,mshr, new_state);
1129 if (satisfy) {
1130 DPRINTF(Cache, "Cache snooped a %s request for addr %x and "
1131 "now supplying data, new state is %i\n",
1132 pkt->cmdString(), blk_addr, new_state);
1133
1134 handleSnoop(blk, new_state, pkt);
1135 return hitLatency;
1136 }
1137 if (blk)
1138 DPRINTF(Cache, "Cache snooped a %s request for addr %x, "
1139 "new state is %i\n",
1140 pkt->cmdString(), blk_addr, new_state);
1141 handleSnoop(blk, new_state);
1142 return 0;
1143 }
1144
1145 template<class TagStore, class Coherence>
1146 Port *
1147 Cache<TagStore,Coherence>::getPort(const std::string &if_name, int idx)
1148 {
1149 if (if_name == "")
1150 {
1151 if (cpuSidePort == NULL) {
1152 cpuSidePort = new CpuSidePort(name() + "-cpu_side_port", this);
1153 sendEvent = new CacheEvent(cpuSidePort, true);
1154 }
1155 return cpuSidePort;
1156 }
1157 else if (if_name == "functional")
1158 {
1159 return new CpuSidePort(name() + "-cpu_side_funcport", this);
1160 }
1161 else if (if_name == "cpu_side")
1162 {
1163 if (cpuSidePort == NULL) {
1164 cpuSidePort = new CpuSidePort(name() + "-cpu_side_port", this);
1165 sendEvent = new CacheEvent(cpuSidePort, true);
1166 }
1167 return cpuSidePort;
1168 }
1169 else if (if_name == "mem_side")
1170 {
1171 if (memSidePort != NULL)
1172 panic("Already have a mem side for this cache\n");
1173 memSidePort = new MemSidePort(name() + "-mem_side_port", this);
1174 memSendEvent = new CacheEvent(memSidePort, true);
1175 return memSidePort;
1176 }
1177 else panic("Port name %s unrecognized\n", if_name);
1178 }
1179
1180 template<class TagStore, class Coherence>
1181 void
1182 Cache<TagStore,Coherence>::deletePortRefs(Port *p)
1183 {
1184 if (cpuSidePort == p || memSidePort == p)
1185 panic("Can only delete functional ports\n");
1186 // nothing else to do
1187 }
1188
1189
1190 template<class TagStore, class Coherence>
1191 bool
1192 Cache<TagStore,Coherence>::CpuSidePort::recvTiming(PacketPtr pkt)
1193 {
1194 if (!pkt->req->isUncacheable()
1195 && pkt->isInvalidate()
1196 && !pkt->isRead() && !pkt->isWrite()) {
1197 //Upgrade or Invalidate
1198 //Look into what happens if two slave caches on bus
1199 DPRINTF(Cache, "%s %x ?\n", pkt->cmdString(), pkt->getAddr());
1200
1201 assert(!(pkt->flags & SATISFIED));
1202 pkt->flags |= SATISFIED;
1203 //Invalidates/Upgrades need no response if they get the bus
1204 return true;
1205 }
1206
1207 if (pkt->isRequest() && blocked)
1208 {
1209 DPRINTF(Cache,"Scheduling a retry while blocked\n");
1210 mustSendRetry = true;
1211 return false;
1212 }
1213
1214 if (pkt->isWrite() && (pkt->req->isLocked())) {
1215 pkt->req->setExtraData(1);
1216 }
1217 myCache()->access(pkt);
1218 return true;
1219 }
1220
1221 template<class TagStore, class Coherence>
1222 Tick
1223 Cache<TagStore,Coherence>::CpuSidePort::recvAtomic(PacketPtr pkt)
1224 {
1225 myCache()->probe(pkt, true, NULL);
1226 //TEMP ALWAYS SUCCES FOR NOW
1227 pkt->result = Packet::Success;
1228 //Fix this timing info
1229 return myCache()->hitLatency;
1230 }
1231
1232 template<class TagStore, class Coherence>
1233 void
1234 Cache<TagStore,Coherence>::CpuSidePort::recvFunctional(PacketPtr pkt)
1235 {
1236 if (checkFunctional(pkt)) {
1237 //TEMP USE CPU?THREAD 0 0
1238 pkt->req->setThreadContext(0,0);
1239
1240 myCache()->probe(pkt, false, cache->memSidePort);
1241 //TEMP ALWAYS SUCCESFUL FOR NOW
1242 pkt->result = Packet::Success;
1243 }
1244 }
1245
1246
1247 template<class TagStore, class Coherence>
1248 bool
1249 Cache<TagStore,Coherence>::MemSidePort::recvTiming(PacketPtr pkt)
1250 {
1251 if (pkt->isRequest() && blocked)
1252 {
1253 DPRINTF(Cache,"Scheduling a retry while blocked\n");
1254 mustSendRetry = true;
1255 return false;
1256 }
1257
1258 if (pkt->isResponse())
1259 myCache()->handleResponse(pkt);
1260 else {
1261 //Check if we should do the snoop
1262 if (pkt->flags & SNOOP_COMMIT)
1263 myCache()->snoop(pkt);
1264 }
1265 return true;
1266 }
1267
1268 template<class TagStore, class Coherence>
1269 Tick
1270 Cache<TagStore,Coherence>::MemSidePort::recvAtomic(PacketPtr pkt)
1271 {
1272 if (pkt->isResponse())
1273 myCache()->handleResponse(pkt);
1274 else
1275 return myCache()->snoopProbe(pkt);
1276 //Fix this timing info
1277 return myCache()->hitLatency;
1278 }
1279
1280 template<class TagStore, class Coherence>
1281 void
1282 Cache<TagStore,Coherence>::MemSidePort::recvFunctional(PacketPtr pkt)
1283 {
1284 if (checkFunctional(pkt)) {
1285 myCache()->probe(pkt, false, cache->cpuSidePort);
1286 }
1287 }
1288
1289
1290 template<class TagStore, class Coherence>
1291 Cache<TagStore,Coherence>::
1292 CpuSidePort::CpuSidePort(const std::string &_name,
1293 Cache<TagStore,Coherence> *_cache)
1294 : BaseCache::CachePort(_name, _cache, true)
1295 {
1296 }
1297
1298 template<class TagStore, class Coherence>
1299 Cache<TagStore,Coherence>::
1300 MemSidePort::MemSidePort(const std::string &_name,
1301 Cache<TagStore,Coherence> *_cache)
1302 : BaseCache::CachePort(_name, _cache, false)
1303 {
1304 }
1305