src/mem/packet.cc

   1 /*
   2  * Copyright (c) 2011-2019 ARM Limited
   3  * All rights reserved
   4  *
   5  * The license below extends only to copyright in the software and shall
   6  * not be construed as granting a license to any other intellectual
   7  * property including but not limited to intellectual property relating
   8  * to a hardware implementation of the functionality of the software
   9  * licensed hereunder.  You may use the software subject to the license
  10  * terms below provided that you ensure that this notice is replicated
  11  * unmodified and in its entirety in all distributions of the software,
  12  * modified or unmodified, in source code or in binary form.
  13  *
  14  * Copyright (c) 2006 The Regents of The University of Michigan
  15  * Copyright (c) 2010,2015 Advanced Micro Devices, Inc.
  16  * All rights reserved.
  17  *
  18  * Redistribution and use in source and binary forms, with or without
  19  * modification, are permitted provided that the following conditions are
  20  * met: redistributions of source code must retain the above copyright
  21  * notice, this list of conditions and the following disclaimer;
  22  * redistributions in binary form must reproduce the above copyright
  23  * notice, this list of conditions and the following disclaimer in the
  24  * documentation and/or other materials provided with the distribution;
  25  * neither the name of the copyright holders nor the names of its
  26  * contributors may be used to endorse or promote products derived from
  27  * this software without specific prior written permission.
  28  *
  29  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  30  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  31  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  32  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  33  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  34  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  35  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  36  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  37  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  38  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  39  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  40  */
  41
  42 /**
  43  * @file
  44  * Definition of the Packet Class, a packet is a transaction occuring
  45  * between a single level of the memory heirarchy (ie L1->L2).
  46  */
  47
  48 #include "mem/packet.hh"
  49
  50 #include <algorithm>
  51 #include <cstring>
  52 #include <iostream>
  53 #include <sstream>
  54 #include <string>
  55
  56 #include "base/cprintf.hh"
  57 #include "base/logging.hh"
  58 #include "base/trace.hh"
  59 #include "mem/packet_access.hh"
  60
  61 // The one downside to bitsets is that static initializers can get ugly.
  62 #define SET1(a1)                     (1 << (a1))
  63 #define SET2(a1, a2)                 (SET1(a1) | SET1(a2))
  64 #define SET3(a1, a2, a3)             (SET2(a1, a2) | SET1(a3))
  65 #define SET4(a1, a2, a3, a4)         (SET3(a1, a2, a3) | SET1(a4))
  66 #define SET5(a1, a2, a3, a4, a5)     (SET4(a1, a2, a3, a4) | SET1(a5))
  67 #define SET6(a1, a2, a3, a4, a5, a6) (SET5(a1, a2, a3, a4, a5) | SET1(a6))
  68 #define SET7(a1, a2, a3, a4, a5, a6, a7) (SET6(a1, a2, a3, a4, a5, a6) | \
  69                                           SET1(a7))
  70
  71 const MemCmd::CommandInfo
  72 MemCmd::commandInfo[] =
  73 {
  74     /* InvalidCmd */
  75     { 0, InvalidCmd, "InvalidCmd" },
  76     /* ReadReq - Read issued by a non-caching agent such as a CPU or
  77      * device, with no restrictions on alignment. */
  78     { SET3(IsRead, IsRequest, NeedsResponse), ReadResp, "ReadReq" },
  79     /* ReadResp */
  80     { SET3(IsRead, IsResponse, HasData), InvalidCmd, "ReadResp" },
  81     /* ReadRespWithInvalidate */
  82     { SET4(IsRead, IsResponse, HasData, IsInvalidate),
  83             InvalidCmd, "ReadRespWithInvalidate" },
  84     /* WriteReq */
  85     { SET5(IsWrite, NeedsWritable, IsRequest, NeedsResponse, HasData),
  86             WriteResp, "WriteReq" },
  87     /* WriteResp */
  88     { SET2(IsWrite, IsResponse), InvalidCmd, "WriteResp" },
  89     /* WriteCompleteResp - The WriteCompleteResp command is needed
  90      * because in the GPU memory model we use a WriteResp to indicate
  91      * that a write has reached the cache controller so we can free
  92      * resources at the coalescer. Later, when the write succesfully
  93      * completes we send a WriteCompleteResp to the CU so its wait
  94      * counters can be updated. Wait counters in the CU is how memory
  95      * dependences are handled in the GPU ISA. */
  96     { SET2(IsWrite, IsResponse), InvalidCmd, "WriteCompleteResp" },
  97     /* WritebackDirty */
  98     { SET5(IsWrite, IsRequest, IsEviction, HasData, FromCache),
  99             InvalidCmd, "WritebackDirty" },
 100     /* WritebackClean - This allows the upstream cache to writeback a
 101      * line to the downstream cache without it being considered
 102      * dirty. */
 103     { SET5(IsWrite, IsRequest, IsEviction, HasData, FromCache),
 104             InvalidCmd, "WritebackClean" },
 105     /* WriteClean - This allows a cache to write a dirty block to a memory
 106        below without evicting its copy. */
 107     { SET4(IsWrite, IsRequest, HasData, FromCache), InvalidCmd, "WriteClean" },
 108     /* CleanEvict */
 109     { SET3(IsRequest, IsEviction, FromCache), InvalidCmd, "CleanEvict" },
 110     /* SoftPFReq */
 111     { SET4(IsRead, IsRequest, IsSWPrefetch, NeedsResponse),
 112             SoftPFResp, "SoftPFReq" },
 113     /* SoftPFExReq */
 114     { SET6(IsRead, NeedsWritable, IsInvalidate, IsRequest,
 115            IsSWPrefetch, NeedsResponse), SoftPFResp, "SoftPFExReq" },
 116     /* HardPFReq */
 117     { SET5(IsRead, IsRequest, IsHWPrefetch, NeedsResponse, FromCache),
 118             HardPFResp, "HardPFReq" },
 119     /* SoftPFResp */
 120     { SET4(IsRead, IsResponse, IsSWPrefetch, HasData),
 121             InvalidCmd, "SoftPFResp" },
 122     /* HardPFResp */
 123     { SET4(IsRead, IsResponse, IsHWPrefetch, HasData),
 124             InvalidCmd, "HardPFResp" },
 125     /* WriteLineReq */
 126     { SET5(IsWrite, NeedsWritable, IsRequest, NeedsResponse, HasData),
 127             WriteResp, "WriteLineReq" },
 128     /* UpgradeReq */
 129     { SET6(IsInvalidate, NeedsWritable, IsUpgrade, IsRequest, NeedsResponse,
 130             FromCache),
 131             UpgradeResp, "UpgradeReq" },
 132     /* SCUpgradeReq: response could be UpgradeResp or UpgradeFailResp */
 133     { SET7(IsInvalidate, NeedsWritable, IsUpgrade, IsLlsc,
 134            IsRequest, NeedsResponse, FromCache),
 135             UpgradeResp, "SCUpgradeReq" },
 136     /* UpgradeResp */
 137     { SET2(IsUpgrade, IsResponse),
 138             InvalidCmd, "UpgradeResp" },
 139     /* SCUpgradeFailReq: generates UpgradeFailResp but still gets the data */
 140     { SET7(IsRead, NeedsWritable, IsInvalidate,
 141            IsLlsc, IsRequest, NeedsResponse, FromCache),
 142             UpgradeFailResp, "SCUpgradeFailReq" },
 143     /* UpgradeFailResp - Behaves like a ReadExReq, but notifies an SC
 144      * that it has failed, acquires line as Dirty*/
 145     { SET3(IsRead, IsResponse, HasData),
 146             InvalidCmd, "UpgradeFailResp" },
 147     /* ReadExReq - Read issues by a cache, always cache-line aligned,
 148      * and the response is guaranteed to be writeable (exclusive or
 149      * even modified) */
 150     { SET6(IsRead, NeedsWritable, IsInvalidate, IsRequest, NeedsResponse,
 151             FromCache),
 152             ReadExResp, "ReadExReq" },
 153     /* ReadExResp - Response matching a read exclusive, as we check
 154      * the need for exclusive also on responses */
 155     { SET3(IsRead, IsResponse, HasData),
 156             InvalidCmd, "ReadExResp" },
 157     /* ReadCleanReq - Read issued by a cache, always cache-line
 158      * aligned, and the response is guaranteed to not contain dirty data
 159      * (exclusive or shared).*/
 160     { SET4(IsRead, IsRequest, NeedsResponse, FromCache),
 161             ReadResp, "ReadCleanReq" },
 162     /* ReadSharedReq - Read issued by a cache, always cache-line
 163      * aligned, response is shared, possibly exclusive, owned or even
 164      * modified. */
 165     { SET4(IsRead, IsRequest, NeedsResponse, FromCache),
 166             ReadResp, "ReadSharedReq" },
 167     /* LoadLockedReq: note that we use plain ReadResp as response, so that
 168      *                we can also use ReadRespWithInvalidate when needed */
 169     { SET4(IsRead, IsLlsc, IsRequest, NeedsResponse),
 170             ReadResp, "LoadLockedReq" },
 171     /* StoreCondReq */
 172     { SET6(IsWrite, NeedsWritable, IsLlsc,
 173            IsRequest, NeedsResponse, HasData),
 174             StoreCondResp, "StoreCondReq" },
 175     /* StoreCondFailReq: generates failing StoreCondResp */
 176     { SET6(IsWrite, NeedsWritable, IsLlsc,
 177            IsRequest, NeedsResponse, HasData),
 178             StoreCondResp, "StoreCondFailReq" },
 179     /* StoreCondResp */
 180     { SET3(IsWrite, IsLlsc, IsResponse),
 181             InvalidCmd, "StoreCondResp" },
 182     /* SwapReq -- for Swap ldstub type operations */
 183     { SET6(IsRead, IsWrite, NeedsWritable, IsRequest, HasData, NeedsResponse),
 184         SwapResp, "SwapReq" },
 185     /* SwapResp -- for Swap ldstub type operations */
 186     { SET4(IsRead, IsWrite, IsResponse, HasData),
 187             InvalidCmd, "SwapResp" },
 188     { 0, InvalidCmd, "Deprecated_MessageReq" },
 189     { 0, InvalidCmd, "Deprecated_MessageResp" },
 190     /* MemFenceReq -- for synchronization requests */
 191     {SET2(IsRequest, NeedsResponse), MemFenceResp, "MemFenceReq"},
 192     /* MemSyncReq */
 193     {SET2(IsRequest, NeedsResponse), MemSyncResp, "MemSyncReq"},
 194     /* MemSyncResp */
 195     {SET1(IsResponse), InvalidCmd, "MemSyncResp"},
 196     /* MemFenceResp -- for synchronization responses */
 197     {SET1(IsResponse), InvalidCmd, "MemFenceResp"},
 198     /* Cache Clean Request -- Update with the latest data all existing
 199        copies of the block down to the point indicated by the
 200        request */
 201     { SET4(IsRequest, IsClean, NeedsResponse, FromCache),
 202       CleanSharedResp, "CleanSharedReq" },
 203     /* Cache Clean Response - Indicates that all caches up to the
 204        specified point of reference have a up-to-date copy of the
 205        cache block or no copy at all */
 206     { SET2(IsResponse, IsClean), InvalidCmd, "CleanSharedResp" },
 207     /* Cache Clean and Invalidate Request -- Invalidate all existing
 208        copies down to the point indicated by the request */
 209     { SET5(IsRequest, IsInvalidate, IsClean, NeedsResponse, FromCache),
 210       CleanInvalidResp, "CleanInvalidReq" },
 211      /* Cache Clean and Invalidate Respose -- Indicates that no cache
 212         above the specified point holds the block and that the block
 213         was written to a memory below the specified point. */
 214     { SET3(IsResponse, IsInvalidate, IsClean),
 215       InvalidCmd, "CleanInvalidResp" },
 216     /* InvalidDestError  -- packet dest field invalid */
 217     { SET2(IsResponse, IsError), InvalidCmd, "InvalidDestError" },
 218     /* BadAddressError   -- memory address invalid */
 219     { SET2(IsResponse, IsError), InvalidCmd, "BadAddressError" },
 220     /* FunctionalReadError */
 221     { SET3(IsRead, IsResponse, IsError), InvalidCmd, "FunctionalReadError" },
 222     /* FunctionalWriteError */
 223     { SET3(IsWrite, IsResponse, IsError), InvalidCmd, "FunctionalWriteError" },
 224     /* PrintReq */
 225     { SET2(IsRequest, IsPrint), InvalidCmd, "PrintReq" },
 226     /* Flush Request */
 227     { SET3(IsRequest, IsFlush, NeedsWritable), InvalidCmd, "FlushReq" },
 228     /* Invalidation Request */
 229     { SET5(IsInvalidate, IsRequest, NeedsWritable, NeedsResponse, FromCache),
 230       InvalidateResp, "InvalidateReq" },
 231     /* Invalidation Response */
 232     { SET2(IsInvalidate, IsResponse),
 233       InvalidCmd, "InvalidateResp" },
 234       // hardware transactional memory
 235     { SET3(IsRead, IsRequest, NeedsResponse), HTMReqResp, "HTMReq" },
 236     { SET2(IsRead, IsResponse), InvalidCmd, "HTMReqResp" },
 237     { SET2(IsRead, IsRequest), InvalidCmd, "HTMAbort" },
 238 };
 239
 240 AddrRange
 241 Packet::getAddrRange() const
 242 {
 243     return RangeSize(getAddr(), getSize());
 244 }
 245
 246 bool
 247 Packet::trySatisfyFunctional(Printable *obj, Addr addr, bool is_secure, int size,
 248                         uint8_t *_data)
 249 {
 250     const Addr func_start = getAddr();
 251     const Addr func_end   = getAddr() + getSize() - 1;
 252     const Addr val_start  = addr;
 253     const Addr val_end    = val_start + size - 1;
 254
 255     if (is_secure != _isSecure || func_start > val_end ||
 256         val_start > func_end) {
 257         // no intersection
 258         return false;
 259     }
 260
 261     // check print first since it doesn't require data
 262     if (isPrint()) {
 263         assert(!_data);
 264         safe_cast<PrintReqState*>(senderState)->printObj(obj);
 265         return false;
 266     }
 267
 268     // we allow the caller to pass NULL to signify the other packet
 269     // has no data
 270     if (!_data) {
 271         return false;
 272     }
 273
 274     const Addr val_offset = func_start > val_start ?
 275         func_start - val_start : 0;
 276     const Addr func_offset = func_start < val_start ?
 277         val_start - func_start : 0;
 278     const Addr overlap_size = std::min(val_end, func_end)+1 -
 279         std::max(val_start, func_start);
 280
 281     if (isRead()) {
 282         std::memcpy(getPtr<uint8_t>() + func_offset,
 283                _data + val_offset,
 284                overlap_size);
 285
 286         // initialise the tracking of valid bytes if we have not
 287         // used it already
 288         if (bytesValid.empty())
 289             bytesValid.resize(getSize(), false);
 290
 291         // track if we are done filling the functional access
 292         bool all_bytes_valid = true;
 293
 294         int i = 0;
 295
 296         // check up to func_offset
 297         for (; all_bytes_valid && i < func_offset; ++i)
 298             all_bytes_valid &= bytesValid[i];
 299
 300         // update the valid bytes
 301         for (i = func_offset; i < func_offset + overlap_size; ++i)
 302             bytesValid[i] = true;
 303
 304         // check the bit after the update we just made
 305         for (; all_bytes_valid && i < getSize(); ++i)
 306             all_bytes_valid &= bytesValid[i];
 307
 308         return all_bytes_valid;
 309     } else if (isWrite()) {
 310         std::memcpy(_data + val_offset,
 311                getConstPtr<uint8_t>() + func_offset,
 312                overlap_size);
 313     } else {
 314         panic("Don't know how to handle command %s\n", cmdString());
 315     }
 316
 317     // keep going with request by default
 318     return false;
 319 }
 320
 321 void
 322 Packet::copyResponderFlags(const PacketPtr pkt)
 323 {
 324     assert(isRequest());
 325     // If we have already found a responder, no other cache should
 326     // commit to responding
 327     assert(!pkt->cacheResponding() || !cacheResponding());
 328     flags.set(pkt->flags & RESPONDER_FLAGS);
 329 }
 330
 331 void
 332 Packet::pushSenderState(Packet::SenderState *sender_state)
 333 {
 334     assert(sender_state != NULL);
 335     sender_state->predecessor = senderState;
 336     senderState = sender_state;
 337 }
 338
 339 Packet::SenderState *
 340 Packet::popSenderState()
 341 {
 342     assert(senderState != NULL);
 343     SenderState *sender_state = senderState;
 344     senderState = sender_state->predecessor;
 345     sender_state->predecessor = NULL;
 346     return sender_state;
 347 }
 348
 349 uint64_t
 350 Packet::getUintX(ByteOrder endian) const
 351 {
 352     switch(getSize()) {
 353       case 1:
 354         return (uint64_t)get<uint8_t>(endian);
 355       case 2:
 356         return (uint64_t)get<uint16_t>(endian);
 357       case 4:
 358         return (uint64_t)get<uint32_t>(endian);
 359       case 8:
 360         return (uint64_t)get<uint64_t>(endian);
 361       default:
 362         panic("%i isn't a supported word size.\n", getSize());
 363     }
 364 }
 365
 366 void
 367 Packet::setUintX(uint64_t w, ByteOrder endian)
 368 {
 369     switch(getSize()) {
 370       case 1:
 371         set<uint8_t>((uint8_t)w, endian);
 372         break;
 373       case 2:
 374         set<uint16_t>((uint16_t)w, endian);
 375         break;
 376       case 4:
 377         set<uint32_t>((uint32_t)w, endian);
 378         break;
 379       case 8:
 380         set<uint64_t>((uint64_t)w, endian);
 381         break;
 382       default:
 383         panic("%i isn't a supported word size.\n", getSize());
 384     }
 385
 386 }
 387
 388 void
 389 Packet::print(std::ostream &o, const int verbosity,
 390               const std::string &prefix) const
 391 {
 392     ccprintf(o, "%s%s [%x:%x]%s%s%s%s%s%s", prefix, cmdString(),
 393              getAddr(), getAddr() + getSize() - 1,
 394              req->isSecure() ? " (s)" : "",
 395              req->isInstFetch() ? " IF" : "",
 396              req->isUncacheable() ? " UC" : "",
 397              isExpressSnoop() ? " ES" : "",
 398              req->isToPOC() ? " PoC" : "",
 399              req->isToPOU() ? " PoU" : "");
 400 }
 401
 402 std::string
 403 Packet::print() const {
 404     std::ostringstream str;
 405     print(str);
 406     return str.str();
 407 }
 408
 409 bool
 410 Packet::matchBlockAddr(const Addr addr, const bool is_secure,
 411                        const int blk_size) const
 412 {
 413     return (getBlockAddr(blk_size) == addr) && (isSecure() == is_secure);
 414 }
 415
 416 bool
 417 Packet::matchBlockAddr(const PacketPtr pkt, const int blk_size) const
 418 {
 419     return matchBlockAddr(pkt->getBlockAddr(blk_size), pkt->isSecure(),
 420                           blk_size);
 421 }
 422
 423 bool
 424 Packet::matchAddr(const Addr addr, const bool is_secure) const
 425 {
 426     return (getAddr() == addr) && (isSecure() == is_secure);
 427 }
 428
 429 bool
 430 Packet::matchAddr(const PacketPtr pkt) const
 431 {
 432     return matchAddr(pkt->getAddr(), pkt->isSecure());
 433 }
 434
 435 Packet::PrintReqState::PrintReqState(std::ostream &_os, int _verbosity)
 436     : curPrefixPtr(new std::string("")), os(_os), verbosity(_verbosity)
 437 {
 438     labelStack.push_back(LabelStackEntry("", curPrefixPtr));
 439 }
 440
 441 Packet::PrintReqState::~PrintReqState()
 442 {
 443     labelStack.pop_back();
 444     assert(labelStack.empty());
 445     delete curPrefixPtr;
 446 }
 447
 448 Packet::PrintReqState::
 449 LabelStackEntry::LabelStackEntry(const std::string &_label,
 450                                  std::string *_prefix)
 451     : label(_label), prefix(_prefix), labelPrinted(false)
 452 {
 453 }
 454
 455 void
 456 Packet::PrintReqState::pushLabel(const std::string &lbl,
 457                                  const std::string &prefix)
 458 {
 459     labelStack.push_back(LabelStackEntry(lbl, curPrefixPtr));
 460     curPrefixPtr = new std::string(*curPrefixPtr);
 461     *curPrefixPtr += prefix;
 462 }
 463
 464 void
 465 Packet::PrintReqState::popLabel()
 466 {
 467     delete curPrefixPtr;
 468     curPrefixPtr = labelStack.back().prefix;
 469     labelStack.pop_back();
 470     assert(!labelStack.empty());
 471 }
 472
 473 void
 474 Packet::PrintReqState::printLabels()
 475 {
 476     if (!labelStack.back().labelPrinted) {
 477         LabelStack::iterator i = labelStack.begin();
 478         LabelStack::iterator end = labelStack.end();
 479         while (i != end) {
 480             if (!i->labelPrinted) {
 481                 ccprintf(os, "%s%s\n", *(i->prefix), i->label);
 482                 i->labelPrinted = true;
 483             }
 484             i++;
 485         }
 486     }
 487 }
 488
 489
 490 void
 491 Packet::PrintReqState::printObj(Printable *obj)
 492 {
 493     printLabels();
 494     obj->print(os, verbosity, curPrefix());
 495 }
 496
 497 void
 498 Packet::makeHtmTransactionalReqResponse(
 499     const HtmCacheFailure htm_return_code)
 500 {
 501     assert(needsResponse());
 502     assert(isRequest());
 503
 504     cmd = cmd.responseCommand();
 505
 506     setHtmTransactionFailedInCache(htm_return_code);
 507
 508     // responses are never express, even if the snoop that
 509     // triggered them was
 510     flags.clear(EXPRESS_SNOOP);
 511 }
 512
 513 void
 514 Packet::setHtmTransactionFailedInCache(
 515     const HtmCacheFailure htm_return_code)
 516 {
 517     if (htm_return_code != HtmCacheFailure::NO_FAIL)
 518         flags.set(FAILS_TRANSACTION);
 519
 520     htmReturnReason = htm_return_code;
 521 }
 522
 523 bool
 524 Packet::htmTransactionFailedInCache() const
 525 {
 526     return flags.isSet(FAILS_TRANSACTION);
 527 }
 528
 529 HtmCacheFailure
 530 Packet::getHtmTransactionFailedInCacheRC() const
 531 {
 532     assert(htmTransactionFailedInCache());
 533     return htmReturnReason;
 534 }
 535
 536 void
 537 Packet::setHtmTransactional(uint64_t htm_uid)
 538 {
 539     flags.set(FROM_TRANSACTION);
 540     htmTransactionUid = htm_uid;
 541 }
 542
 543 bool
 544 Packet::isHtmTransactional() const
 545 {
 546     return flags.isSet(FROM_TRANSACTION);
 547 }
 548
 549 uint64_t
 550 Packet::getHtmTransactionUid() const
 551 {
 552     assert(flags.isSet(FROM_TRANSACTION));
 553     return htmTransactionUid;
 554 }