/*
- * Copyright (c) 2012-2014 ARM Limited
+ * Copyright (c) 2012-2019 ARM Limited
* All rights reserved
*
* The license below extends only to copyright in the software and shall
* modified or unmodified, in source code or in binary form.
*
* Copyright (c) 2006 The Regents of The University of Michigan
- * Copyright (c) 2010 Advanced Micro Devices, Inc.
+ * Copyright (c) 2010,2015 Advanced Micro Devices, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * Authors: Ron Dreslinski
- * Steve Reinhardt
- * Ali Saidi
- * Andreas Hansson
*/
/**
#include <cassert>
#include <list>
+#include "base/addr_range.hh"
#include "base/cast.hh"
#include "base/compiler.hh"
#include "base/flags.hh"
-#include "base/misc.hh"
+#include "base/logging.hh"
#include "base/printable.hh"
#include "base/types.hh"
+#include "mem/htm.hh"
#include "mem/request.hh"
#include "sim/core.hh"
typedef Packet *PacketPtr;
typedef uint8_t* PacketDataPtr;
typedef std::list<PacketPtr> PacketList;
+typedef uint64_t PacketId;
class MemCmd
{
ReadRespWithInvalidate,
WriteReq,
WriteResp,
- Writeback,
+ WriteCompleteResp,
+ WritebackDirty,
+ WritebackClean,
+ WriteClean, // writes dirty data below without evicting
+ CleanEvict,
SoftPFReq,
+ SoftPFExReq,
HardPFReq,
SoftPFResp,
HardPFResp,
- WriteInvalidateReq,
- WriteInvalidateResp,
+ WriteLineReq,
UpgradeReq,
SCUpgradeReq, // Special "weak" upgrade for StoreCond
UpgradeResp,
UpgradeFailResp, // Valid for SCUpgradeReq only
ReadExReq,
ReadExResp,
+ ReadCleanReq,
+ ReadSharedReq,
LoadLockedReq,
StoreCondReq,
StoreCondFailReq, // Failed StoreCondReq in MSHR (never sent)
StoreCondResp,
SwapReq,
SwapResp,
- MessageReq,
- MessageResp,
+ // MessageReq and MessageResp are deprecated.
+ MemFenceReq = SwapResp + 3,
+ MemSyncReq, // memory synchronization request (e.g., cache invalidate)
+ MemSyncResp, // memory synchronization response
+ MemFenceResp,
+ CleanSharedReq,
+ CleanSharedResp,
+ CleanInvalidReq,
+ CleanInvalidResp,
// Error responses
// @TODO these should be classified as responses rather than
// requests; coding them as requests initially for backwards
// Fake simulator-only commands
PrintReq, // Print state matching address
FlushReq, //request for a cache flush
- InvalidationReq, // request for address to be invalidated from lsq
+ InvalidateReq, // request for address to be invalidated
+ InvalidateResp,
+ // hardware transactional memory
+ HTMReq,
+ HTMReqResp,
+ HTMAbort,
NUM_MEM_CMDS
};
IsWrite, //!< Data flows from requester to responder
IsUpgrade,
IsInvalidate,
- NeedsExclusive, //!< Requires exclusive copy to complete in-cache
+ IsClean, //!< Cleans any existing dirty blocks
+ NeedsWritable, //!< Requires writable copy to complete in-cache
IsRequest, //!< Issued by requester
IsResponse, //!< Issue by responder
NeedsResponse, //!< Requester needs response from target
+ IsEviction,
IsSWPrefetch,
IsHWPrefetch,
IsLlsc, //!< Alpha/MIPS LL or SC access
IsError, //!< Error response
IsPrint, //!< Print state matching address (for debugging)
IsFlush, //!< Flush the address from caches
+ FromCache, //!< Request originated from a caching agent
NUM_COMMAND_ATTRIBUTES
};
public:
- bool isRead() const { return testCmdAttrib(IsRead); }
- bool isWrite() const { return testCmdAttrib(IsWrite); }
- bool isUpgrade() const { return testCmdAttrib(IsUpgrade); }
- bool isRequest() const { return testCmdAttrib(IsRequest); }
- bool isResponse() const { return testCmdAttrib(IsResponse); }
- bool needsExclusive() const { return testCmdAttrib(NeedsExclusive); }
- bool needsResponse() const { return testCmdAttrib(NeedsResponse); }
- bool isInvalidate() const { return testCmdAttrib(IsInvalidate); }
+ bool isRead() const { return testCmdAttrib(IsRead); }
+ bool isWrite() const { return testCmdAttrib(IsWrite); }
+ bool isUpgrade() const { return testCmdAttrib(IsUpgrade); }
+ bool isRequest() const { return testCmdAttrib(IsRequest); }
+ bool isResponse() const { return testCmdAttrib(IsResponse); }
+ bool needsWritable() const { return testCmdAttrib(NeedsWritable); }
+ bool needsResponse() const { return testCmdAttrib(NeedsResponse); }
+ bool isInvalidate() const { return testCmdAttrib(IsInvalidate); }
+ bool isEviction() const { return testCmdAttrib(IsEviction); }
+ bool isClean() const { return testCmdAttrib(IsClean); }
+ bool fromCache() const { return testCmdAttrib(FromCache); }
+
+ /**
+ * A writeback is an eviction that carries data.
+ */
+ bool isWriteback() const { return testCmdAttrib(IsEviction) &&
+ testCmdAttrib(HasData); }
/**
* Check if this particular packet type carries payload data. Note
bool isPrint() const { return testCmdAttrib(IsPrint); }
bool isFlush() const { return testCmdAttrib(IsFlush); }
- const Command
+ Command
responseCommand() const
{
return commandInfo[cmd].response;
/**
* A Packet is used to encapsulate a transfer between two objects in
* the memory system (e.g., the L1 and L2 cache). (In contrast, a
- * single Request travels all the way from the requester to the
+ * single Request travels all the way from the requestor to the
* ultimate destination and back, possibly being conveyed by several
* different Packets along the way.)
*/
typedef ::Flags<FlagsType> Flags;
private:
- static const FlagsType PUBLIC_FLAGS = 0x00000000;
- static const FlagsType PRIVATE_FLAGS = 0x00007F0F;
- static const FlagsType COPY_FLAGS = 0x0000000F;
-
- static const FlagsType SHARED = 0x00000001;
- // Special control flags
- /// Special timing-mode atomic snoop for multi-level coherence.
- static const FlagsType EXPRESS_SNOOP = 0x00000002;
- /// Does supplier have exclusive copy?
- /// Useful for multi-level coherence.
- static const FlagsType SUPPLY_EXCLUSIVE = 0x00000004;
- // Snoop response flags
- static const FlagsType MEM_INHIBIT = 0x00000008;
- /// Are the 'addr' and 'size' fields valid?
- static const FlagsType VALID_ADDR = 0x00000100;
- static const FlagsType VALID_SIZE = 0x00000200;
- /// Is the data pointer set to a value that shouldn't be freed
- /// when the packet is destroyed?
- static const FlagsType STATIC_DATA = 0x00001000;
- /// The data pointer points to a value that should be freed when
- /// the packet is destroyed. The pointer is assumed to be pointing
- /// to an array, and delete [] is consequently called
- static const FlagsType DYNAMIC_DATA = 0x00002000;
- /// suppress the error if this packet encounters a functional
- /// access failure.
- static const FlagsType SUPPRESS_FUNC_ERROR = 0x00008000;
- // Signal prefetch squash through express snoop flag
- static const FlagsType PREFETCH_SNOOP_SQUASH = 0x00010000;
+
+ enum : FlagsType {
+ // Flags to transfer across when copying a packet
+ COPY_FLAGS = 0x000000FF,
+
+ // Flags that are used to create reponse packets
+ RESPONDER_FLAGS = 0x00000009,
+
+ // Does this packet have sharers (which means it should not be
+ // considered writable) or not. See setHasSharers below.
+ HAS_SHARERS = 0x00000001,
+
+ // Special control flags
+ /// Special timing-mode atomic snoop for multi-level coherence.
+ EXPRESS_SNOOP = 0x00000002,
+
+ /// Allow a responding cache to inform the cache hierarchy
+ /// that it had a writable copy before responding. See
+ /// setResponderHadWritable below.
+ RESPONDER_HAD_WRITABLE = 0x00000004,
+
+ // Snoop co-ordination flag to indicate that a cache is
+ // responding to a snoop. See setCacheResponding below.
+ CACHE_RESPONDING = 0x00000008,
+
+ // The writeback/writeclean should be propagated further
+ // downstream by the receiver
+ WRITE_THROUGH = 0x00000010,
+
+ // Response co-ordination flag for cache maintenance
+ // operations
+ SATISFIED = 0x00000020,
+
+ // hardware transactional memory
+
+ // Indicates that this packet/request has returned from the
+ // cache hierarchy in a failed transaction. The core is
+ // notified like this.
+ FAILS_TRANSACTION = 0x00000040,
+
+ // Indicates that this packet/request originates in the CPU executing
+ // in transactional mode, i.e. in a transaction.
+ FROM_TRANSACTION = 0x00000080,
+
+ /// Are the 'addr' and 'size' fields valid?
+ VALID_ADDR = 0x00000100,
+ VALID_SIZE = 0x00000200,
+
+ /// Is the data pointer set to a value that shouldn't be freed
+ /// when the packet is destroyed?
+ STATIC_DATA = 0x00001000,
+ /// The data pointer points to a value that should be freed when
+ /// the packet is destroyed. The pointer is assumed to be pointing
+ /// to an array, and delete [] is consequently called
+ DYNAMIC_DATA = 0x00002000,
+
+ /// suppress the error if this packet encounters a functional
+ /// access failure.
+ SUPPRESS_FUNC_ERROR = 0x00008000,
+
+ // Signal block present to squash prefetch and cache evict packets
+ // through express snoop flag
+ BLOCK_CACHED = 0x00010000
+ };
Flags flags;
/// The command field of the packet.
MemCmd cmd;
+ const PacketId id;
+
/// A pointer to the original request.
- const RequestPtr req;
+ RequestPtr req;
private:
/**
- * A pointer to the data being transfered. It can be differnt
- * sizes at each level of the heirarchy so it belongs in the
+ * A pointer to the data being transferred. It can be different
+ * sizes at each level of the hierarchy so it belongs to the
* packet, not request. This may or may not be populated when a
- * responder recieves the packet. If not populated it memory should
+ * responder receives the packet. If not populated memory should
* be allocated.
*/
PacketDataPtr data;
unsigned size;
/**
- * Source port identifier set on a request packet to enable
- * appropriate routing of the responses. The source port
- * identifier is set by any multiplexing component, e.g. a
- * crossbar, as the timing responses need this information to be
- * routed back to the appropriate port at a later point in
- * time. The field can be updated (over-written) as the request
- * packet passes through additional multiplexing components, and
- * it is their responsibility to remember the original source port
- * identifier, for example by using an appropriate sender
- * state. The latter is done in the cache and bridge.
+ * Track the bytes found that satisfy a functional read.
*/
- PortID src;
+ std::vector<bool> bytesValid;
- /**
- * Destination port identifier that is present on all response
- * packets that passed through a multiplexing component as a
- * request packet. The source port identifier is turned into a
- * destination port identifier when the packet is turned into a
- * response, and the destination is used, e.g. by the crossbar, to
- * select the appropriate path through the interconnect.
- */
- PortID dest;
+ // Quality of Service priority value
+ uint8_t _qosValue;
+
+ // hardware transactional memory
/**
- * The original value of the command field. Only valid when the
- * current command field is an error condition; in that case, the
- * previous contents of the command field are copied here. This
- * field is *not* set on non-error responses.
+ * Holds the return status of the transaction.
+ * The default case will be NO_FAIL, otherwise this will specify the
+ * reason for the transaction's failure in the memory subsystem.
*/
- MemCmd origCmd;
+ HtmCacheFailure htmReturnReason;
/**
- * These values specify the range of bytes found that satisfy a
- * functional read.
+ * A global unique identifier of the transaction.
+ * This is used for correctness/debugging only.
*/
- uint16_t bytesValidStart;
- uint16_t bytesValidEnd;
+ uint64_t htmTransactionUid;
public:
/**
- * The extra delay from seeing the packet until the first word is
+ * The extra delay from seeing the packet until the header is
* transmitted. This delay is used to communicate the crossbar
* forwarding latency to the neighbouring object (e.g. a cache)
* that actually makes the packet wait. As the delay is relative,
* a 32-bit unsigned should be sufficient.
*/
- uint32_t firstWordDelay;
+ uint32_t headerDelay;
+
+ /**
+ * Keep track of the extra delay incurred by snooping upwards
+ * before sending a request down the memory system. This is used
+ * by the coherent crossbar to account for the additional request
+ * delay.
+ */
+ uint32_t snoopDelay;
/**
- * The extra pipelining delay from seeing the packet until the
- * last word is transmitted by the component that provided it (if
- * any). This includes the first word delay. Similar to the first
- * word delay, this is used to make up for the fact that the
+ * The extra pipelining delay from seeing the packet until the end of
+ * payload is transmitted by the component that provided it (if
+ * any). This includes the header delay. Similar to the header
+ * delay, this is used to make up for the fact that the
* crossbar does not make the packet wait. As the delay is
* relative, a 32-bit unsigned should be sufficient.
*/
- uint32_t lastWordDelay;
+ uint32_t payloadDelay;
/**
* A virtual base opaque structure used to hold state associated
- * with the packet (e.g., an MSHR), specific to a MemObject that
+ * with the packet (e.g., an MSHR), specific to a SimObject that
* sees the packet. A pointer to this state is returned in the
- * packet's response so that the MemObject in question can quickly
+ * packet's response so that the SimObject in question can quickly
* look up the state needed to process it. A specific subclass
* would be derived from this to carry state specific to a
* particular sending device.
*
- * As multiple MemObjects may add their SenderState throughout the
+ * As multiple SimObjects may add their SenderState throughout the
* memory system, the SenderStates create a stack, where a
- * MemObject can add a new Senderstate, as long as the
+ * SimObject can add a new Senderstate, as long as the
* predecessing SenderState is restored when the response comes
* back. For this reason, the predecessor should always be
* populated with the current SenderState of a packet before
/// Return the index of this command.
inline int cmdToIndex() const { return cmd.toInt(); }
- bool isRead() const { return cmd.isRead(); }
- bool isWrite() const { return cmd.isWrite(); }
- bool isUpgrade() const { return cmd.isUpgrade(); }
- bool isRequest() const { return cmd.isRequest(); }
- bool isResponse() const { return cmd.isResponse(); }
- bool needsExclusive() const { return cmd.needsExclusive(); }
- bool needsResponse() const { return cmd.needsResponse(); }
- bool isInvalidate() const { return cmd.isInvalidate(); }
- bool hasData() const { return cmd.hasData(); }
- bool isLLSC() const { return cmd.isLLSC(); }
- bool isError() const { return cmd.isError(); }
- bool isPrint() const { return cmd.isPrint(); }
- bool isFlush() const { return cmd.isFlush(); }
-
- // Snoop flags
- void assertMemInhibit()
+ bool isRead() const { return cmd.isRead(); }
+ bool isWrite() const { return cmd.isWrite(); }
+ bool isUpgrade() const { return cmd.isUpgrade(); }
+ bool isRequest() const { return cmd.isRequest(); }
+ bool isResponse() const { return cmd.isResponse(); }
+ bool needsWritable() const
+ {
+ // we should never check if a response needsWritable, the
+ // request has this flag, and for a response we should rather
+ // look at the hasSharers flag (if not set, the response is to
+ // be considered writable)
+ assert(isRequest());
+ return cmd.needsWritable();
+ }
+ bool needsResponse() const { return cmd.needsResponse(); }
+ bool isInvalidate() const { return cmd.isInvalidate(); }
+ bool isEviction() const { return cmd.isEviction(); }
+ bool isClean() const { return cmd.isClean(); }
+ bool fromCache() const { return cmd.fromCache(); }
+ bool isWriteback() const { return cmd.isWriteback(); }
+ bool hasData() const { return cmd.hasData(); }
+ bool hasRespData() const
+ {
+ MemCmd resp_cmd = cmd.responseCommand();
+ return resp_cmd.hasData();
+ }
+ bool isLLSC() const { return cmd.isLLSC(); }
+ bool isError() const { return cmd.isError(); }
+ bool isPrint() const { return cmd.isPrint(); }
+ bool isFlush() const { return cmd.isFlush(); }
+
+ bool isWholeLineWrite(unsigned blk_size)
+ {
+ return (cmd == MemCmd::WriteReq || cmd == MemCmd::WriteLineReq) &&
+ getOffset(blk_size) == 0 && getSize() == blk_size;
+ }
+
+ //@{
+ /// Snoop flags
+ /**
+ * Set the cacheResponding flag. This is used by the caches to
+ * signal another cache that they are responding to a request. A
+ * cache will only respond to snoops if it has the line in either
+ * Modified or Owned state. Note that on snoop hits we always pass
+ * the line as Modified and never Owned. In the case of an Owned
+ * line we proceed to invalidate all other copies.
+ *
+ * On a cache fill (see Cache::handleFill), we check hasSharers
+ * first, ignoring the cacheResponding flag if hasSharers is set.
+ * A line is consequently allocated as:
+ *
+ * hasSharers cacheResponding state
+ * true false Shared
+ * true true Shared
+ * false false Exclusive
+ * false true Modified
+ */
+ void setCacheResponding()
{
assert(isRequest());
- assert(!flags.isSet(MEM_INHIBIT));
- flags.set(MEM_INHIBIT);
- }
- bool memInhibitAsserted() const { return flags.isSet(MEM_INHIBIT); }
- void assertShared() { flags.set(SHARED); }
- bool sharedAsserted() const { return flags.isSet(SHARED); }
-
- // Special control flags
- void setExpressSnoop() { flags.set(EXPRESS_SNOOP); }
- bool isExpressSnoop() const { return flags.isSet(EXPRESS_SNOOP); }
- void setSupplyExclusive() { flags.set(SUPPLY_EXCLUSIVE); }
- void clearSupplyExclusive() { flags.clear(SUPPLY_EXCLUSIVE); }
- bool isSupplyExclusive() const { return flags.isSet(SUPPLY_EXCLUSIVE); }
+ assert(!flags.isSet(CACHE_RESPONDING));
+ flags.set(CACHE_RESPONDING);
+ }
+ bool cacheResponding() const { return flags.isSet(CACHE_RESPONDING); }
+ /**
+ * On fills, the hasSharers flag is used by the caches in
+ * combination with the cacheResponding flag, as clarified
+ * above. If the hasSharers flag is not set, the packet is passing
+ * writable. Thus, a response from a memory passes the line as
+ * writable by default.
+ *
+ * The hasSharers flag is also used by upstream caches to inform a
+ * downstream cache that they have the block (by calling
+ * setHasSharers on snoop request packets that hit in upstream
+ * cachs tags or MSHRs). If the snoop packet has sharers, a
+ * downstream cache is prevented from passing a dirty line upwards
+ * if it was not explicitly asked for a writable copy. See
+ * Cache::satisfyCpuSideRequest.
+ *
+ * The hasSharers flag is also used on writebacks, in
+ * combination with the WritbackClean or WritebackDirty commands,
+ * to allocate the block downstream either as:
+ *
+ * command hasSharers state
+ * WritebackDirty false Modified
+ * WritebackDirty true Owned
+ * WritebackClean false Exclusive
+ * WritebackClean true Shared
+ */
+ void setHasSharers() { flags.set(HAS_SHARERS); }
+ bool hasSharers() const { return flags.isSet(HAS_SHARERS); }
+ //@}
+
+ /**
+ * The express snoop flag is used for two purposes. Firstly, it is
+ * used to bypass flow control for normal (non-snoop) requests
+ * going downstream in the memory system. In cases where a cache
+ * is responding to a snoop from another cache (it had a dirty
+ * line), but the line is not writable (and there are possibly
+ * other copies), the express snoop flag is set by the downstream
+ * cache to invalidate all other copies in zero time. Secondly,
+ * the express snoop flag is also set to be able to distinguish
+ * snoop packets that came from a downstream cache, rather than
+ * snoop packets from neighbouring caches.
+ */
+ void setExpressSnoop() { flags.set(EXPRESS_SNOOP); }
+ bool isExpressSnoop() const { return flags.isSet(EXPRESS_SNOOP); }
+
+ /**
+ * On responding to a snoop request (which only happens for
+ * Modified or Owned lines), make sure that we can transform an
+ * Owned response to a Modified one. If this flag is not set, the
+ * responding cache had the line in the Owned state, and there are
+ * possibly other Shared copies in the memory system. A downstream
+ * cache helps in orchestrating the invalidation of these copies
+ * by sending out the appropriate express snoops.
+ */
+ void setResponderHadWritable()
+ {
+ assert(cacheResponding());
+ assert(!responderHadWritable());
+ flags.set(RESPONDER_HAD_WRITABLE);
+ }
+ bool responderHadWritable() const
+ { return flags.isSet(RESPONDER_HAD_WRITABLE); }
+
+ /**
+ * Copy the reponse flags from an input packet to this packet. The
+ * reponse flags determine whether a responder has been found and
+ * the state at which the block will be at the destination.
+ *
+ * @pkt The packet that we will copy flags from
+ */
+ void copyResponderFlags(const PacketPtr pkt);
+
+ /**
+ * A writeback/writeclean cmd gets propagated further downstream
+ * by the receiver when the flag is set.
+ */
+ void setWriteThrough()
+ {
+ assert(cmd.isWrite() &&
+ (cmd.isEviction() || cmd == MemCmd::WriteClean));
+ flags.set(WRITE_THROUGH);
+ }
+ void clearWriteThrough() { flags.clear(WRITE_THROUGH); }
+ bool writeThrough() const { return flags.isSet(WRITE_THROUGH); }
+
+ /**
+ * Set when a request hits in a cache and the cache is not going
+ * to respond. This is used by the crossbar to coordinate
+ * responses for cache maintenance operations.
+ */
+ void setSatisfied()
+ {
+ assert(cmd.isClean());
+ assert(!flags.isSet(SATISFIED));
+ flags.set(SATISFIED);
+ }
+ bool satisfied() const { return flags.isSet(SATISFIED); }
+
void setSuppressFuncError() { flags.set(SUPPRESS_FUNC_ERROR); }
bool suppressFuncError() const { return flags.isSet(SUPPRESS_FUNC_ERROR); }
- void setPrefetchSquashed() { flags.set(PREFETCH_SNOOP_SQUASH); }
- bool prefetchSquashed() const { return flags.isSet(PREFETCH_SNOOP_SQUASH); }
+ void setBlockCached() { flags.set(BLOCK_CACHED); }
+ bool isBlockCached() const { return flags.isSet(BLOCK_CACHED); }
+ void clearBlockCached() { flags.clear(BLOCK_CACHED); }
+
+ /**
+ * QoS Value getter
+ * Returns 0 if QoS value was never set (constructor default).
+ *
+ * @return QoS priority value of the packet
+ */
+ inline uint8_t qosValue() const { return _qosValue; }
+
+ /**
+ * QoS Value setter
+ * Interface for setting QoS priority value of the packet.
+ *
+ * @param qos_value QoS priority value
+ */
+ inline void qosValue(const uint8_t qos_value)
+ { _qosValue = qos_value; }
+
+ inline RequestorID requestorId() const { return req->requestorId(); }
// Network error conditions... encapsulate them as methods since
// their encoding keeps changing (from result field to command
cmd = MemCmd::BadAddressError;
}
- bool hadBadAddress() const { return cmd == MemCmd::BadAddressError; }
void copyError(Packet *pkt) { assert(pkt->isError()); cmd = pkt->cmd; }
- bool isSrcValid() const { return src != InvalidPortID; }
- /// Accessor function to get the source index of the packet.
- PortID getSrc() const { assert(isSrcValid()); return src; }
- /// Accessor function to set the source index of the packet.
- void setSrc(PortID _src) { src = _src; }
-
- bool isDestValid() const { return dest != InvalidPortID; }
- /// Accessor function for the destination index of the packet.
- PortID getDest() const { assert(isDestValid()); return dest; }
- /// Accessor function to set the destination index of the packet.
- void setDest(PortID _dest) { dest = _dest; }
- /// Reset destination field, e.g. to turn a response into a request again.
- void clearDest() { dest = InvalidPortID; }
-
Addr getAddr() const { assert(flags.isSet(VALID_ADDR)); return addr; }
/**
* Update the address of this packet mid-transaction. This is used
void setAddr(Addr _addr) { assert(flags.isSet(VALID_ADDR)); addr = _addr; }
unsigned getSize() const { assert(flags.isSet(VALID_SIZE)); return size; }
- Addr getOffset(int blkSize) const { return getAddr() & (Addr)(blkSize - 1); }
+
+ /**
+ * Get address range to which this packet belongs.
+ *
+ * @return Address range of this packet.
+ */
+ AddrRange getAddrRange() const;
+
+ Addr getOffset(unsigned int blk_size) const
+ {
+ return getAddr() & Addr(blk_size - 1);
+ }
+
+ Addr getBlockAddr(unsigned int blk_size) const
+ {
+ return getAddr() & ~(Addr(blk_size - 1));
+ }
bool isSecure() const
{
return _isSecure;
}
+ /**
+ * Accessor function to atomic op.
+ */
+ AtomicOpFunctor *getAtomicOp() const { return req->getAtomicOpFunctor(); }
+ bool isAtomicOp() const { return req->isAtomic(); }
+
/**
* It has been determined that the SC packet should successfully update
- * memory. Therefore, convert this SC packet to a normal write.
+ * memory. Therefore, convert this SC packet to a normal write.
*/
void
convertScToWrite()
}
/**
- * When ruby is in use, Ruby will monitor the cache line and thus M5
- * phys memory should treat LL ops as normal reads.
+ * When ruby is in use, Ruby will monitor the cache line and the
+ * phys memory should treat LL ops as normal reads.
*/
void
convertLlToRead()
}
/**
- * Constructor. Note that a Request object must be constructed
+ * Constructor. Note that a Request object must be constructed
* first, but the Requests's physical address and size fields need
* not be valid. The command must be supplied.
*/
- Packet(const RequestPtr _req, MemCmd _cmd)
- : cmd(_cmd), req(_req), data(nullptr), addr(0), _isSecure(false),
- size(0), src(InvalidPortID), dest(InvalidPortID),
- bytesValidStart(0), bytesValidEnd(0),
- firstWordDelay(0), lastWordDelay(0),
- senderState(NULL)
+ Packet(const RequestPtr &_req, MemCmd _cmd)
+ : cmd(_cmd), id((PacketId)_req.get()), req(_req),
+ data(nullptr), addr(0), _isSecure(false), size(0),
+ _qosValue(0),
+ htmReturnReason(HtmCacheFailure::NO_FAIL),
+ htmTransactionUid(0),
+ headerDelay(0), snoopDelay(0),
+ payloadDelay(0), senderState(NULL)
{
+ flags.clear();
if (req->hasPaddr()) {
addr = req->getPaddr();
flags.set(VALID_ADDR);
_isSecure = req->isSecure();
}
+
+ /**
+ * hardware transactional memory
+ *
+ * This is a bit of a hack!
+ * Technically the address of a HTM command is set to zero
+ * but is not valid. The reason that we pretend it's valid is
+ * to void the getAddr() function from failing. It would be
+ * cumbersome to add control flow in many places to check if the
+ * packet represents a HTM command before calling getAddr().
+ */
+ if (req->isHTMCmd()) {
+ flags.set(VALID_ADDR);
+ assert(addr == 0x0);
+ }
if (req->hasSize()) {
size = req->getSize();
flags.set(VALID_SIZE);
* a request that is for a whole block, not the address from the
* req. this allows for overriding the size/addr of the req.
*/
- Packet(const RequestPtr _req, MemCmd _cmd, int _blkSize)
- : cmd(_cmd), req(_req), data(nullptr), addr(0), _isSecure(false),
- src(InvalidPortID), dest(InvalidPortID),
- bytesValidStart(0), bytesValidEnd(0),
- firstWordDelay(0), lastWordDelay(0),
- senderState(NULL)
+ Packet(const RequestPtr &_req, MemCmd _cmd, int _blkSize, PacketId _id = 0)
+ : cmd(_cmd), id(_id ? _id : (PacketId)_req.get()), req(_req),
+ data(nullptr), addr(0), _isSecure(false),
+ _qosValue(0),
+ htmReturnReason(HtmCacheFailure::NO_FAIL),
+ htmTransactionUid(0),
+ headerDelay(0),
+ snoopDelay(0), payloadDelay(0), senderState(NULL)
{
+ flags.clear();
if (req->hasPaddr()) {
addr = req->getPaddr() & ~(_blkSize - 1);
flags.set(VALID_ADDR);
* less than that of the original packet. In this case the new
* packet should allocate its own data.
*/
- Packet(PacketPtr pkt, bool clear_flags, bool alloc_data)
- : cmd(pkt->cmd), req(pkt->req),
+ Packet(const PacketPtr pkt, bool clear_flags, bool alloc_data)
+ : cmd(pkt->cmd), id(pkt->id), req(pkt->req),
data(nullptr),
addr(pkt->addr), _isSecure(pkt->_isSecure), size(pkt->size),
- src(pkt->src), dest(pkt->dest),
- bytesValidStart(pkt->bytesValidStart),
- bytesValidEnd(pkt->bytesValidEnd),
- firstWordDelay(pkt->firstWordDelay),
- lastWordDelay(pkt->lastWordDelay),
+ bytesValid(pkt->bytesValid),
+ _qosValue(pkt->qosValue()),
+ htmReturnReason(HtmCacheFailure::NO_FAIL),
+ htmTransactionUid(0),
+ headerDelay(pkt->headerDelay),
+ snoopDelay(0),
+ payloadDelay(pkt->payloadDelay),
senderState(pkt->senderState)
{
if (!clear_flags)
flags.set(pkt->flags & (VALID_ADDR|VALID_SIZE));
+ if (pkt->isHtmTransactional())
+ setHtmTransactional(pkt->getHtmTransactionUid());
+
+ if (pkt->htmTransactionFailedInCache()) {
+ setHtmTransactionFailedInCache(
+ pkt->getHtmTransactionFailedInCacheRC()
+ );
+ }
+
// should we allocate space for data, or not, the express
// snoops do not need to carry any data as they only serve to
// co-ordinate state changes
}
/**
- * Change the packet type based on request type.
+ * Generate the appropriate read MemCmd based on the Request flags.
*/
- void
- refineCommand()
+ static MemCmd
+ makeReadCmd(const RequestPtr &req)
{
- if (cmd == MemCmd::ReadReq) {
- if (req->isLLSC()) {
- cmd = MemCmd::LoadLockedReq;
- } else if (req->isPrefetch()) {
- cmd = MemCmd::SoftPFReq;
- }
- } else if (cmd == MemCmd::WriteReq) {
- if (req->isLLSC()) {
- cmd = MemCmd::StoreCondReq;
- } else if (req->isSwap()) {
- cmd = MemCmd::SwapReq;
- }
- }
+ if (req->isHTMCmd()) {
+ if (req->isHTMAbort())
+ return MemCmd::HTMAbort;
+ else
+ return MemCmd::HTMReq;
+ } else if (req->isLLSC())
+ return MemCmd::LoadLockedReq;
+ else if (req->isPrefetchEx())
+ return MemCmd::SoftPFExReq;
+ else if (req->isPrefetch())
+ return MemCmd::SoftPFReq;
+ else
+ return MemCmd::ReadReq;
+ }
+
+ /**
+ * Generate the appropriate write MemCmd based on the Request flags.
+ */
+ static MemCmd
+ makeWriteCmd(const RequestPtr &req)
+ {
+ if (req->isLLSC())
+ return MemCmd::StoreCondReq;
+ else if (req->isSwap() || req->isAtomic())
+ return MemCmd::SwapReq;
+ else if (req->isCacheInvalidate()) {
+ return req->isCacheClean() ? MemCmd::CleanInvalidReq :
+ MemCmd::InvalidateReq;
+ } else if (req->isCacheClean()) {
+ return MemCmd::CleanSharedReq;
+ } else
+ return MemCmd::WriteReq;
}
/**
* Constructor-like methods that return Packets based on Request objects.
- * Will call refineCommand() to fine-tune the Packet type if it's not a
- * vanilla read or write.
+ * Fine-tune the MemCmd type if it's not a vanilla read or write.
*/
static PacketPtr
- createRead(const RequestPtr req)
+ createRead(const RequestPtr &req)
{
- PacketPtr pkt = new Packet(req, MemCmd::ReadReq);
- pkt->refineCommand();
- return pkt;
+ return new Packet(req, makeReadCmd(req));
}
static PacketPtr
- createWrite(const RequestPtr req)
+ createWrite(const RequestPtr &req)
{
- PacketPtr pkt = new Packet(req, MemCmd::WriteReq);
- pkt->refineCommand();
- return pkt;
+ return new Packet(req, makeWriteCmd(req));
}
/**
*/
~Packet()
{
- // If this is a request packet for which there's no response,
- // delete the request object here, since the requester will
- // never get the chance.
- if (req && isRequest() && !needsResponse())
- delete req;
deleteData();
}
/**
* Take a request packet and modify it in place to be suitable for
- * returning as a response to that request. The source field is
- * turned into the destination, and subsequently cleared. Note
- * that the latter is not necessary for atomic requests, but
- * causes no harm as neither field is valid.
+ * returning as a response to that request.
*/
void
makeResponse()
{
assert(needsResponse());
assert(isRequest());
- origCmd = cmd;
cmd = cmd.responseCommand();
// responses are never express, even if the snoop that
// triggered them was
flags.clear(EXPRESS_SNOOP);
-
- dest = src;
- src = InvalidPortID;
}
void
flags.set(VALID_SIZE);
}
+ /**
+ * Check if packet corresponds to a given block-aligned address and
+ * address space.
+ *
+ * @param addr The address to compare against.
+ * @param is_secure Whether addr belongs to the secure address space.
+ * @param blk_size Block size in bytes.
+ * @return Whether packet matches description.
+ */
+ bool matchBlockAddr(const Addr addr, const bool is_secure,
+ const int blk_size) const;
+
+ /**
+ * Check if this packet refers to the same block-aligned address and
+ * address space as another packet.
+ *
+ * @param pkt The packet to compare against.
+ * @param blk_size Block size in bytes.
+ * @return Whether packet matches description.
+ */
+ bool matchBlockAddr(const PacketPtr pkt, const int blk_size) const;
+
+ /**
+ * Check if packet corresponds to a given address and address space.
+ *
+ * @param addr The address to compare against.
+ * @param is_secure Whether addr belongs to the secure address space.
+ * @return Whether packet matches description.
+ */
+ bool matchAddr(const Addr addr, const bool is_secure) const;
+
+ /**
+ * Check if this packet refers to the same address and address space as
+ * another packet.
+ *
+ * @param pkt The packet to compare against.
+ * @return Whether packet matches description.
+ */
+ bool matchAddr(const PacketPtr pkt) const;
+
+ public:
+ /**
+ * @{
+ * @name Data accessor mehtods
+ */
/**
* Set the data pointer to the following value that should not be
getPtr()
{
assert(flags.isSet(STATIC_DATA|DYNAMIC_DATA));
+ assert(!isMaskedWrite());
return (T*)data;
}
}
/**
- * return the value of what is pointed to in the packet.
+ * Get the data in the packet byte swapped from big endian to
+ * host endian.
*/
template <typename T>
- T get() const;
+ T getBE() const;
+
+ /**
+ * Get the data in the packet byte swapped from little endian to
+ * host endian.
+ */
+ template <typename T>
+ T getLE() const;
+
+ /**
+ * Get the data in the packet byte swapped from the specified
+ * endianness.
+ */
+ template <typename T>
+ T get(ByteOrder endian) const;
+
+ /** Set the value in the data pointer to v as big endian. */
+ template <typename T>
+ void setBE(T v);
+
+ /** Set the value in the data pointer to v as little endian. */
+ template <typename T>
+ void setLE(T v);
/**
- * set the value in the data pointer to v.
+ * Set the value in the data pointer to v using the specified
+ * endianness.
*/
template <typename T>
- void set(T v);
+ void set(T v, ByteOrder endian);
+
+ /**
+ * Get the data in the packet byte swapped from the specified
+ * endianness and zero-extended to 64 bits.
+ */
+ uint64_t getUintX(ByteOrder endian) const;
+
+ /**
+ * Set the value in the word w after truncating it to the length
+ * of the packet and then byteswapping it to the desired
+ * endianness.
+ */
+ void setUintX(uint64_t w, ByteOrder endian);
/**
* Copy data into the packet from the provided pointer.
// same pointer from source to destination and back
assert(p != getPtr<uint8_t>() || flags.isSet(STATIC_DATA));
- if (p != getPtr<uint8_t>())
+ if (p != getPtr<uint8_t>()) {
// for packet with allocated dynamic data, we copy data from
// one to the other, e.g. a forwarded response to a response
std::memcpy(getPtr<uint8_t>(), p, getSize());
+ }
}
/**
}
/**
- * Copy data from the packet to the provided block pointer, which
- * is aligned to the given block size.
+ * Copy data from the packet to the memory at the provided pointer.
+ * @param p Pointer to which data will be copied.
*/
void
writeData(uint8_t *p) const
{
- std::memcpy(p, getConstPtr<uint8_t>(), getSize());
+ if (!isMaskedWrite()) {
+ std::memcpy(p, getConstPtr<uint8_t>(), getSize());
+ } else {
+ assert(req->getByteEnable().size() == getSize());
+ // Write only the enabled bytes
+ const uint8_t *base = getConstPtr<uint8_t>();
+ for (int i = 0; i < getSize(); i++) {
+ if (req->getByteEnable()[i]) {
+ p[i] = *(base + i);
+ }
+ // Disabled bytes stay untouched
+ }
+ }
}
/**
- * Copy data from the packet to the memory at the provided pointer.
+ * Copy data from the packet to the provided block pointer, which
+ * is aligned to the given block size.
+ * @param blk_data Pointer to block to which data will be copied.
+ * @param blkSize Block size in bytes.
*/
void
writeDataToBlock(uint8_t *blk_data, int blkSize) const
void
allocate()
{
- assert(flags.noneSet(STATIC_DATA|DYNAMIC_DATA));
- flags.set(DYNAMIC_DATA);
- data = new uint8_t[getSize()];
+ // if either this command or the response command has a data
+ // payload, actually allocate space
+ if (hasData() || hasRespData()) {
+ assert(flags.noneSet(STATIC_DATA|DYNAMIC_DATA));
+ flags.set(DYNAMIC_DATA);
+ data = new uint8_t[getSize()];
+ }
}
+ /** @} */
+
+ /** Get the data in the packet without byte swapping. */
+ template <typename T>
+ T getRaw() const;
+
+ /** Set the value in the data pointer to v without byte swapping. */
+ template <typename T>
+ void setRaw(T v);
+
+ public:
/**
* Check a functional request against a memory value stored in
* another packet (i.e. an in-transit request or
* accordingly.
*/
bool
- checkFunctional(PacketPtr other)
+ trySatisfyFunctional(PacketPtr other)
{
+ if (other->isMaskedWrite()) {
+ // Do not forward data if overlapping with a masked write
+ if (_isSecure == other->isSecure() &&
+ getAddr() <= (other->getAddr() + other->getSize() - 1) &&
+ other->getAddr() <= (getAddr() + getSize() - 1)) {
+ warn("Trying to check against a masked write, skipping."
+ " (addr: 0x%x, other addr: 0x%x)", getAddr(),
+ other->getAddr());
+ }
+ return false;
+ }
// all packets that are carrying a payload should have a valid
// data pointer
- return checkFunctional(other, other->getAddr(), other->isSecure(),
- other->getSize(),
- other->hasData() ?
- other->getPtr<uint8_t>() : NULL);
+ return trySatisfyFunctional(other, other->getAddr(), other->isSecure(),
+ other->getSize(),
+ other->hasData() ?
+ other->getPtr<uint8_t>() : NULL);
+ }
+
+ /**
+ * Does the request need to check for cached copies of the same block
+ * in the memory hierarchy above.
+ **/
+ bool
+ mustCheckAbove() const
+ {
+ return cmd == MemCmd::HardPFReq || isEviction();
+ }
+
+ /**
+ * Is this packet a clean eviction, including both actual clean
+ * evict packets, but also clean writebacks.
+ */
+ bool
+ isCleanEviction() const
+ {
+ return cmd == MemCmd::CleanEvict || cmd == MemCmd::WritebackClean;
+ }
+
+ bool
+ isMaskedWrite() const
+ {
+ return (cmd == MemCmd::WriteReq && req->isMasked());
}
/**
* memory value.
*/
bool
- checkFunctional(Printable *obj, Addr base, bool is_secure, int size,
- uint8_t *_data);
+ trySatisfyFunctional(Printable *obj, Addr base, bool is_secure, int size,
+ uint8_t *_data);
/**
* Push label for PrintReq (safe to call unconditionally).
* @return string with the request's type and start<->end addresses
*/
std::string print() const;
+
+ // hardware transactional memory
+
+ /**
+ * Communicates to the core that a packet was processed by the memory
+ * subsystem while running in transactional mode.
+ * It may happen that the transaction has failed at the memory subsystem
+ * and this needs to be communicated to the core somehow.
+ * This function decorates the response packet with flags to indicate
+ * such a situation has occurred.
+ */
+ void makeHtmTransactionalReqResponse(const HtmCacheFailure ret_code);
+
+ /**
+ * Stipulates that this packet/request originates in the CPU executing
+ * in transactional mode, i.e. within a transaction.
+ */
+ void setHtmTransactional(uint64_t val);
+
+ /**
+ * Returns whether or not this packet/request originates in the CPU
+ * executing in transactional mode, i.e. within a transaction.
+ */
+ bool isHtmTransactional() const;
+
+ /**
+ * If a packet/request originates in a CPU executing in transactional
+ * mode, i.e. within a transaction, this function returns the unique ID
+ * of the transaction. This is used for verifying correctness
+ * and debugging.
+ */
+ uint64_t getHtmTransactionUid() const;
+
+ /**
+ * Stipulates that this packet/request has returned from the
+ * cache hierarchy in a failed transaction. The core is
+ * notified like this.
+ */
+ void setHtmTransactionFailedInCache(const HtmCacheFailure ret_code);
+
+ /**
+ * Returns whether or not this packet/request has returned from the
+ * cache hierarchy in a failed transaction. The core is
+ * notified liked this.
+ */
+ bool htmTransactionFailedInCache() const;
+
+ /**
+ * If a packet/request has returned from the cache hierarchy in a
+ * failed transaction, this function returns the failure reason.
+ */
+ HtmCacheFailure getHtmTransactionFailedInCacheRC() const;
};
#endif //__MEM_PACKET_HH
projects / gem5.git / blobdiff