X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fmem%2Fpacket.hh;h=130cc41adc5df6ee23de8bfdbae8d22ed2a32312;hb=e33b3aa6692b172f6db5957774a9e0289e81fa5b;hp=390d9672f81755f44af2c82ddb356f925882a0eb;hpb=8e5c441a54b481085d6311f14af66e41b5766f91;p=gem5.git diff --git a/src/mem/packet.hh b/src/mem/packet.hh index 390d9672f..130cc41ad 100644 --- a/src/mem/packet.hh +++ b/src/mem/packet.hh @@ -1,6 +1,18 @@ /* + * Copyright (c) 2012-2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * * Copyright (c) 2006 The Regents of The University of Michigan - * Copyright (c) 2010 Advancec Micro Devices, Inc. + * Copyright (c) 2010,2015 Advanced Micro Devices, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -29,6 +41,8 @@ * Authors: Ron Dreslinski * Steve Reinhardt * Ali Saidi + * Andreas Hansson + * Nikos Nikoleris */ /** @@ -39,24 +53,26 @@ #ifndef __MEM_PACKET_HH__ #define __MEM_PACKET_HH__ +#include #include #include -#include +#include "base/addr_range.hh" #include "base/cast.hh" #include "base/compiler.hh" -#include "base/fast_alloc.hh" #include "base/flags.hh" -#include "base/misc.hh" +#include "base/logging.hh" #include "base/printable.hh" #include "base/types.hh" +#include "config/the_isa.hh" #include "mem/request.hh" #include "sim/core.hh" -struct Packet; +class Packet; typedef Packet *PacketPtr; typedef uint8_t* PacketDataPtr; typedef std::list PacketList; +typedef uint64_t PacketId; class MemCmd { @@ -74,13 +90,16 @@ class MemCmd ReadRespWithInvalidate, WriteReq, WriteResp, - Writeback, + WritebackDirty, + WritebackClean, + WriteClean, // writes dirty data below without evicting + CleanEvict, SoftPFReq, + SoftPFExReq, HardPFReq, SoftPFResp, HardPFResp, - WriteInvalidateReq, - WriteInvalidateResp, + WriteLineReq, UpgradeReq, SCUpgradeReq, // Special "weak" upgrade for StoreCond UpgradeResp, @@ -88,22 +107,35 @@ class MemCmd UpgradeFailResp, // Valid for SCUpgradeReq only ReadExReq, ReadExResp, + ReadCleanReq, + ReadSharedReq, LoadLockedReq, StoreCondReq, + StoreCondFailReq, // Failed StoreCondReq in MSHR (never sent) StoreCondResp, SwapReq, SwapResp, MessageReq, MessageResp, + MemFenceReq, + MemFenceResp, + CleanSharedReq, + CleanSharedResp, + CleanInvalidReq, + CleanInvalidResp, // Error responses // @TODO these should be classified as responses rather than // requests; coding them as requests initially for backwards // compatibility - NetworkNackError, // nacked at network layer (not by protocol) InvalidDestError, // packet dest field invalid BadAddressError, // memory address invalid + FunctionalReadError, // unable to fulfill functional read + FunctionalWriteError, // unable to fulfill functional write // Fake simulator-only commands PrintReq, // Print state matching address + FlushReq, //request for a cache flush + InvalidateReq, // request for address to be invalidated + InvalidateResp, NUM_MEM_CMDS }; @@ -116,18 +148,21 @@ class MemCmd IsRead, //!< Data flows from responder to requester IsWrite, //!< Data flows from requester to responder IsUpgrade, - IsPrefetch, //!< Not a demand access IsInvalidate, - NeedsExclusive, //!< Requires exclusive copy to complete in-cache + IsClean, //!< Cleans any existing dirty blocks + NeedsWritable, //!< Requires writable copy to complete in-cache IsRequest, //!< Issued by requester IsResponse, //!< Issue by responder NeedsResponse, //!< Requester needs response from target + IsEviction, IsSWPrefetch, IsHWPrefetch, IsLlsc, //!< Alpha/MIPS LL or SC access HasData, //!< There is an associated payload IsError, //!< Error response IsPrint, //!< Print state matching address (for debugging) + IsFlush, //!< Flush the address from caches + FromCache, //!< Request originated from a caching agent NUM_COMMAND_ATTRIBUTES }; @@ -161,21 +196,40 @@ class MemCmd public: - bool isRead() const { return testCmdAttrib(IsRead); } - bool isWrite() const { return testCmdAttrib(IsWrite); } - bool isUpgrade() const { return testCmdAttrib(IsUpgrade); } - bool isRequest() const { return testCmdAttrib(IsRequest); } - bool isResponse() const { return testCmdAttrib(IsResponse); } - bool needsExclusive() const { return testCmdAttrib(NeedsExclusive); } - bool needsResponse() const { return testCmdAttrib(NeedsResponse); } - bool isInvalidate() const { return testCmdAttrib(IsInvalidate); } + bool isRead() const { return testCmdAttrib(IsRead); } + bool isWrite() const { return testCmdAttrib(IsWrite); } + bool isUpgrade() const { return testCmdAttrib(IsUpgrade); } + bool isRequest() const { return testCmdAttrib(IsRequest); } + bool isResponse() const { return testCmdAttrib(IsResponse); } + bool needsWritable() const { return testCmdAttrib(NeedsWritable); } + bool needsResponse() const { return testCmdAttrib(NeedsResponse); } + bool isInvalidate() const { return testCmdAttrib(IsInvalidate); } + bool isEviction() const { return testCmdAttrib(IsEviction); } + bool isClean() const { return testCmdAttrib(IsClean); } + bool fromCache() const { return testCmdAttrib(FromCache); } + + /** + * A writeback is an eviction that carries data. + */ + bool isWriteback() const { return testCmdAttrib(IsEviction) && + testCmdAttrib(HasData); } + + /** + * Check if this particular packet type carries payload data. Note + * that this does not reflect if the data pointer of the packet is + * valid or not. + */ bool hasData() const { return testCmdAttrib(HasData); } - bool isReadWrite() const { return isRead() && isWrite(); } bool isLLSC() const { return testCmdAttrib(IsLlsc); } + bool isSWPrefetch() const { return testCmdAttrib(IsSWPrefetch); } + bool isHWPrefetch() const { return testCmdAttrib(IsHWPrefetch); } + bool isPrefetch() const { return testCmdAttrib(IsSWPrefetch) || + testCmdAttrib(IsHWPrefetch); } bool isError() const { return testCmdAttrib(IsError); } bool isPrint() const { return testCmdAttrib(IsPrint); } + bool isFlush() const { return testCmdAttrib(IsFlush); } - const Command + Command responseCommand() const { return commandInfo[cmd].response; @@ -200,42 +254,66 @@ class MemCmd * ultimate destination and back, possibly being conveyed by several * different Packets along the way.) */ -class Packet : public FastAlloc, public Printable +class Packet : public Printable { public: typedef uint32_t FlagsType; typedef ::Flags Flags; - typedef short NodeID; private: - static const FlagsType PUBLIC_FLAGS = 0x00000000; - static const FlagsType PRIVATE_FLAGS = 0x00007F0F; - static const FlagsType COPY_FLAGS = 0x0000000F; - - static const FlagsType SHARED = 0x00000001; - // Special control flags - /// Special timing-mode atomic snoop for multi-level coherence. - static const FlagsType EXPRESS_SNOOP = 0x00000002; - /// Does supplier have exclusive copy? - /// Useful for multi-level coherence. - static const FlagsType SUPPLY_EXCLUSIVE = 0x00000004; - // Snoop response flags - static const FlagsType MEM_INHIBIT = 0x00000008; - /// Are the 'addr' and 'size' fields valid? - static const FlagsType VALID_ADDR = 0x00000100; - static const FlagsType VALID_SIZE = 0x00000200; - /// Is the 'src' field valid? - static const FlagsType VALID_SRC = 0x00000400; - static const FlagsType VALID_DST = 0x00000800; - /// Is the data pointer set to a value that shouldn't be freed - /// when the packet is destroyed? - static const FlagsType STATIC_DATA = 0x00001000; - /// The data pointer points to a value that should be freed when - /// the packet is destroyed. - static const FlagsType DYNAMIC_DATA = 0x00002000; - /// the data pointer points to an array (thus delete []) needs to - /// be called on it rather than simply delete. - static const FlagsType ARRAY_DATA = 0x00004000; + + enum : FlagsType { + // Flags to transfer across when copying a packet + COPY_FLAGS = 0x0000003F, + + // Flags that are used to create reponse packets + RESPONDER_FLAGS = 0x00000009, + + // Does this packet have sharers (which means it should not be + // considered writable) or not. See setHasSharers below. + HAS_SHARERS = 0x00000001, + + // Special control flags + /// Special timing-mode atomic snoop for multi-level coherence. + EXPRESS_SNOOP = 0x00000002, + + /// Allow a responding cache to inform the cache hierarchy + /// that it had a writable copy before responding. See + /// setResponderHadWritable below. + RESPONDER_HAD_WRITABLE = 0x00000004, + + // Snoop co-ordination flag to indicate that a cache is + // responding to a snoop. See setCacheResponding below. + CACHE_RESPONDING = 0x00000008, + + // The writeback/writeclean should be propagated further + // downstream by the receiver + WRITE_THROUGH = 0x00000010, + + // Response co-ordination flag for cache maintenance + // operations + SATISFIED = 0x00000020, + + /// Are the 'addr' and 'size' fields valid? + VALID_ADDR = 0x00000100, + VALID_SIZE = 0x00000200, + + /// Is the data pointer set to a value that shouldn't be freed + /// when the packet is destroyed? + STATIC_DATA = 0x00001000, + /// The data pointer points to a value that should be freed when + /// the packet is destroyed. The pointer is assumed to be pointing + /// to an array, and delete [] is consequently called + DYNAMIC_DATA = 0x00002000, + + /// suppress the error if this packet encounters a functional + /// access failure. + SUPPRESS_FUNC_ERROR = 0x00008000, + + // Signal block present to squash prefetch and cache evict packets + // through express snoop flag + BLOCK_CACHED = 0x00010000 + }; Flags flags; @@ -245,15 +323,17 @@ class Packet : public FastAlloc, public Printable /// The command field of the packet. MemCmd cmd; + const PacketId id; + /// A pointer to the original request. RequestPtr req; private: /** - * A pointer to the data being transfered. It can be differnt - * sizes at each level of the heirarchy so it belongs in the + * A pointer to the data being transferred. It can be different + * sizes at each level of the hierarchy so it belongs to the * packet, not request. This may or may not be populated when a - * responder recieves the packet. If not populated it memory should + * responder receives the packet. If not populated memory should * be allocated. */ PacketDataPtr data; @@ -262,59 +342,70 @@ class Packet : public FastAlloc, public Printable /// physical, depending on the system configuration. Addr addr; + /// True if the request targets the secure memory space. + bool _isSecure; + /// The size of the request or transfer. unsigned size; /** - * Device address (e.g., bus ID) of the source of the - * transaction. The source is not responsible for setting this - * field; it is set implicitly by the interconnect when the packet - * is first sent. + * Track the bytes found that satisfy a functional read. */ - NodeID src; + std::vector bytesValid; + + // Quality of Service priority value + uint8_t _qosValue; + + public: /** - * Device address (e.g., bus ID) of the destination of the - * transaction. The special value Broadcast indicates that the - * packet should be routed based on its address. This field is - * initialized in the constructor and is thus always valid (unlike - * addr, size, and src). + * The extra delay from seeing the packet until the header is + * transmitted. This delay is used to communicate the crossbar + * forwarding latency to the neighbouring object (e.g. a cache) + * that actually makes the packet wait. As the delay is relative, + * a 32-bit unsigned should be sufficient. */ - NodeID dest; + uint32_t headerDelay; /** - * The original value of the command field. Only valid when the - * current command field is an error condition; in that case, the - * previous contents of the command field are copied here. This - * field is *not* set on non-error responses. + * Keep track of the extra delay incurred by snooping upwards + * before sending a request down the memory system. This is used + * by the coherent crossbar to account for the additional request + * delay. */ - MemCmd origCmd; - - public: - /// Used to calculate latencies for each packet. - Tick time; - - /// The time at which the packet will be fully transmitted - Tick finishTime; - - /// The time at which the first chunk of the packet will be transmitted - Tick firstWordTime; + uint32_t snoopDelay; - /// The special destination address indicating that the packet - /// should be routed based on its address. - static const NodeID Broadcast = -1; + /** + * The extra pipelining delay from seeing the packet until the end of + * payload is transmitted by the component that provided it (if + * any). This includes the header delay. Similar to the header + * delay, this is used to make up for the fact that the + * crossbar does not make the packet wait. As the delay is + * relative, a 32-bit unsigned should be sufficient. + */ + uint32_t payloadDelay; /** * A virtual base opaque structure used to hold state associated - * with the packet but specific to the sending device (e.g., an - * MSHR). A pointer to this state is returned in the packet's - * response so that the sender can quickly look up the state - * needed to process it. A specific subclass would be derived - * from this to carry state specific to a particular sending - * device. + * with the packet (e.g., an MSHR), specific to a SimObject that + * sees the packet. A pointer to this state is returned in the + * packet's response so that the SimObject in question can quickly + * look up the state needed to process it. A specific subclass + * would be derived from this to carry state specific to a + * particular sending device. + * + * As multiple SimObjects may add their SenderState throughout the + * memory system, the SenderStates create a stack, where a + * SimObject can add a new Senderstate, as long as the + * predecessing SenderState is restored when the response comes + * back. For this reason, the predecessor should always be + * populated with the current SenderState of a packet before + * modifying the senderState field in the request packet. */ struct SenderState { + SenderState* predecessor; + SenderState() : predecessor(NULL) {} virtual ~SenderState() {} }; @@ -322,7 +413,7 @@ class Packet : public FastAlloc, public Printable * Object used to maintain state of a PrintReq. The senderState * field of a PrintReq should always be of this type. */ - class PrintReqState : public SenderState, public FastAlloc + class PrintReqState : public SenderState { private: /** @@ -384,12 +475,51 @@ class Packet : public FastAlloc, public Printable * This packet's sender state. Devices should use dynamic_cast<> * to cast to the state appropriate to the sender. The intent of * this variable is to allow a device to attach extra information - * to a request. A response packet must return the sender state + * to a request. A response packet must return the sender state * that was attached to the original request (even if a new packet * is created). */ SenderState *senderState; + /** + * Push a new sender state to the packet and make the current + * sender state the predecessor of the new one. This should be + * prefered over direct manipulation of the senderState member + * variable. + * + * @param sender_state SenderState to push at the top of the stack + */ + void pushSenderState(SenderState *sender_state); + + /** + * Pop the top of the state stack and return a pointer to it. This + * assumes the current sender state is not NULL. This should be + * preferred over direct manipulation of the senderState member + * variable. + * + * @return The current top of the stack + */ + SenderState *popSenderState(); + + /** + * Go through the sender state stack and return the first instance + * that is of type T (as determined by a dynamic_cast). If there + * is no sender state of type T, NULL is returned. + * + * @return The topmost state of type T + */ + template + T * findNextSenderState() const + { + T *t = NULL; + SenderState* sender_state = senderState; + while (t == NULL && sender_state != NULL) { + t = dynamic_cast(sender_state); + sender_state = sender_state->predecessor; + } + return t; + } + /// Return the string name of the cmd field (for debugging and /// tracing). const std::string &cmdString() const { return cmd.toString(); } @@ -397,42 +527,195 @@ class Packet : public FastAlloc, public Printable /// Return the index of this command. inline int cmdToIndex() const { return cmd.toInt(); } - bool isRead() const { return cmd.isRead(); } - bool isWrite() const { return cmd.isWrite(); } - bool isUpgrade() const { return cmd.isUpgrade(); } - bool isRequest() const { return cmd.isRequest(); } - bool isResponse() const { return cmd.isResponse(); } - bool needsExclusive() const { return cmd.needsExclusive(); } - bool needsResponse() const { return cmd.needsResponse(); } - bool isInvalidate() const { return cmd.isInvalidate(); } - bool hasData() const { return cmd.hasData(); } - bool isReadWrite() const { return cmd.isReadWrite(); } - bool isLLSC() const { return cmd.isLLSC(); } - bool isError() const { return cmd.isError(); } - bool isPrint() const { return cmd.isPrint(); } - - // Snoop flags - void assertMemInhibit() { flags.set(MEM_INHIBIT); } - bool memInhibitAsserted() { return flags.isSet(MEM_INHIBIT); } - void assertShared() { flags.set(SHARED); } - bool sharedAsserted() { return flags.isSet(SHARED); } - - // Special control flags + bool isRead() const { return cmd.isRead(); } + bool isWrite() const { return cmd.isWrite(); } + bool isUpgrade() const { return cmd.isUpgrade(); } + bool isRequest() const { return cmd.isRequest(); } + bool isResponse() const { return cmd.isResponse(); } + bool needsWritable() const + { + // we should never check if a response needsWritable, the + // request has this flag, and for a response we should rather + // look at the hasSharers flag (if not set, the response is to + // be considered writable) + assert(isRequest()); + return cmd.needsWritable(); + } + bool needsResponse() const { return cmd.needsResponse(); } + bool isInvalidate() const { return cmd.isInvalidate(); } + bool isEviction() const { return cmd.isEviction(); } + bool isClean() const { return cmd.isClean(); } + bool fromCache() const { return cmd.fromCache(); } + bool isWriteback() const { return cmd.isWriteback(); } + bool hasData() const { return cmd.hasData(); } + bool hasRespData() const + { + MemCmd resp_cmd = cmd.responseCommand(); + return resp_cmd.hasData(); + } + bool isLLSC() const { return cmd.isLLSC(); } + bool isError() const { return cmd.isError(); } + bool isPrint() const { return cmd.isPrint(); } + bool isFlush() const { return cmd.isFlush(); } + + bool isWholeLineWrite(unsigned blk_size) + { + return (cmd == MemCmd::WriteReq || cmd == MemCmd::WriteLineReq) && + getOffset(blk_size) == 0 && getSize() == blk_size; + } + + //@{ + /// Snoop flags + /** + * Set the cacheResponding flag. This is used by the caches to + * signal another cache that they are responding to a request. A + * cache will only respond to snoops if it has the line in either + * Modified or Owned state. Note that on snoop hits we always pass + * the line as Modified and never Owned. In the case of an Owned + * line we proceed to invalidate all other copies. + * + * On a cache fill (see Cache::handleFill), we check hasSharers + * first, ignoring the cacheResponding flag if hasSharers is set. + * A line is consequently allocated as: + * + * hasSharers cacheResponding state + * true false Shared + * true true Shared + * false false Exclusive + * false true Modified + */ + void setCacheResponding() + { + assert(isRequest()); + assert(!flags.isSet(CACHE_RESPONDING)); + flags.set(CACHE_RESPONDING); + } + bool cacheResponding() const { return flags.isSet(CACHE_RESPONDING); } + /** + * On fills, the hasSharers flag is used by the caches in + * combination with the cacheResponding flag, as clarified + * above. If the hasSharers flag is not set, the packet is passing + * writable. Thus, a response from a memory passes the line as + * writable by default. + * + * The hasSharers flag is also used by upstream caches to inform a + * downstream cache that they have the block (by calling + * setHasSharers on snoop request packets that hit in upstream + * cachs tags or MSHRs). If the snoop packet has sharers, a + * downstream cache is prevented from passing a dirty line upwards + * if it was not explicitly asked for a writable copy. See + * Cache::satisfyCpuSideRequest. + * + * The hasSharers flag is also used on writebacks, in + * combination with the WritbackClean or WritebackDirty commands, + * to allocate the block downstream either as: + * + * command hasSharers state + * WritebackDirty false Modified + * WritebackDirty true Owned + * WritebackClean false Exclusive + * WritebackClean true Shared + */ + void setHasSharers() { flags.set(HAS_SHARERS); } + bool hasSharers() const { return flags.isSet(HAS_SHARERS); } + //@} + + /** + * The express snoop flag is used for two purposes. Firstly, it is + * used to bypass flow control for normal (non-snoop) requests + * going downstream in the memory system. In cases where a cache + * is responding to a snoop from another cache (it had a dirty + * line), but the line is not writable (and there are possibly + * other copies), the express snoop flag is set by the downstream + * cache to invalidate all other copies in zero time. Secondly, + * the express snoop flag is also set to be able to distinguish + * snoop packets that came from a downstream cache, rather than + * snoop packets from neighbouring caches. + */ void setExpressSnoop() { flags.set(EXPRESS_SNOOP); } - bool isExpressSnoop() { return flags.isSet(EXPRESS_SNOOP); } - void setSupplyExclusive() { flags.set(SUPPLY_EXCLUSIVE); } - bool isSupplyExclusive() { return flags.isSet(SUPPLY_EXCLUSIVE); } + bool isExpressSnoop() const { return flags.isSet(EXPRESS_SNOOP); } - // Network error conditions... encapsulate them as methods since - // their encoding keeps changing (from result field to command - // field, etc.) - void - setNacked() + /** + * On responding to a snoop request (which only happens for + * Modified or Owned lines), make sure that we can transform an + * Owned response to a Modified one. If this flag is not set, the + * responding cache had the line in the Owned state, and there are + * possibly other Shared copies in the memory system. A downstream + * cache helps in orchestrating the invalidation of these copies + * by sending out the appropriate express snoops. + */ + void setResponderHadWritable() { - assert(isResponse()); - cmd = MemCmd::NetworkNackError; + assert(cacheResponding()); + assert(!responderHadWritable()); + flags.set(RESPONDER_HAD_WRITABLE); + } + bool responderHadWritable() const + { return flags.isSet(RESPONDER_HAD_WRITABLE); } + + /** + * Copy the reponse flags from an input packet to this packet. The + * reponse flags determine whether a responder has been found and + * the state at which the block will be at the destination. + * + * @pkt The packet that we will copy flags from + */ + void copyResponderFlags(const PacketPtr pkt); + + /** + * A writeback/writeclean cmd gets propagated further downstream + * by the receiver when the flag is set. + */ + void setWriteThrough() + { + assert(cmd.isWrite() && + (cmd.isEviction() || cmd == MemCmd::WriteClean)); + flags.set(WRITE_THROUGH); } + void clearWriteThrough() { flags.clear(WRITE_THROUGH); } + bool writeThrough() const { return flags.isSet(WRITE_THROUGH); } + /** + * Set when a request hits in a cache and the cache is not going + * to respond. This is used by the crossbar to coordinate + * responses for cache maintenance operations. + */ + void setSatisfied() + { + assert(cmd.isClean()); + assert(!flags.isSet(SATISFIED)); + flags.set(SATISFIED); + } + bool satisfied() const { return flags.isSet(SATISFIED); } + + void setSuppressFuncError() { flags.set(SUPPRESS_FUNC_ERROR); } + bool suppressFuncError() const { return flags.isSet(SUPPRESS_FUNC_ERROR); } + void setBlockCached() { flags.set(BLOCK_CACHED); } + bool isBlockCached() const { return flags.isSet(BLOCK_CACHED); } + void clearBlockCached() { flags.clear(BLOCK_CACHED); } + + /** + * QoS Value getter + * Returns 0 if QoS value was never set (constructor default). + * + * @return QoS priority value of the packet + */ + inline uint8_t qosValue() const { return _qosValue; } + + /** + * QoS Value setter + * Interface for setting QoS priority value of the packet. + * + * @param qos_value QoS priority value + */ + inline void qosValue(const uint8_t qos_value) + { _qosValue = qos_value; } + + inline MasterID masterId() const { return req->masterId(); } + + // Network error conditions... encapsulate them as methods since + // their encoding keeps changing (from result field to command + // field, etc.) void setBadAddress() { @@ -440,31 +723,52 @@ class Packet : public FastAlloc, public Printable cmd = MemCmd::BadAddressError; } - bool wasNacked() const { return cmd == MemCmd::NetworkNackError; } - bool hadBadAddress() const { return cmd == MemCmd::BadAddressError; } void copyError(Packet *pkt) { assert(pkt->isError()); cmd = pkt->cmd; } - bool isSrcValid() { return flags.isSet(VALID_SRC); } - /// Accessor function to get the source index of the packet. - NodeID getSrc() const { assert(flags.isSet(VALID_SRC)); return src; } - /// Accessor function to set the source index of the packet. - void setSrc(NodeID _src) { src = _src; flags.set(VALID_SRC); } - /// Reset source field, e.g. to retransmit packet on different bus. - void clearSrc() { flags.clear(VALID_SRC); } - - bool isDestValid() { return flags.isSet(VALID_DST); } - /// Accessor function for the destination index of the packet. - NodeID getDest() const { assert(flags.isSet(VALID_DST)); return dest; } - /// Accessor function to set the destination index of the packet. - void setDest(NodeID _dest) { dest = _dest; flags.set(VALID_DST); } - Addr getAddr() const { assert(flags.isSet(VALID_ADDR)); return addr; } + /** + * Update the address of this packet mid-transaction. This is used + * by the address mapper to change an already set address to a new + * one based on the system configuration. It is intended to remap + * an existing address, so it asserts that the current address is + * valid. + */ + void setAddr(Addr _addr) { assert(flags.isSet(VALID_ADDR)); addr = _addr; } + unsigned getSize() const { assert(flags.isSet(VALID_SIZE)); return size; } - Addr getOffset(int blkSize) const { return getAddr() & (Addr)(blkSize - 1); } + + /** + * Get address range to which this packet belongs. + * + * @return Address range of this packet. + */ + AddrRange getAddrRange() const; + + Addr getOffset(unsigned int blk_size) const + { + return getAddr() & Addr(blk_size - 1); + } + + Addr getBlockAddr(unsigned int blk_size) const + { + return getAddr() & ~(Addr(blk_size - 1)); + } + + bool isSecure() const + { + assert(flags.isSet(VALID_ADDR)); + return _isSecure; + } + + /** + * Accessor function to atomic op. + */ + AtomicOpFunctor *getAtomicOp() const { return req->getAtomicOpFunctor(); } + bool isAtomicOp() const { return req->isAtomic(); } /** * It has been determined that the SC packet should successfully update - * memory. Therefore, convert this SC packet to a normal write. + * memory. Therefore, convert this SC packet to a normal write. */ void convertScToWrite() @@ -475,8 +779,8 @@ class Packet : public FastAlloc, public Printable } /** - * When ruby is in use, Ruby will monitor the cache line and thus M5 - * phys memory should treat LL ops as normal reads. + * When ruby is in use, Ruby will monitor the cache line and the + * phys memory should treat LL ops as normal reads. */ void convertLlToRead() @@ -487,18 +791,20 @@ class Packet : public FastAlloc, public Printable } /** - * Constructor. Note that a Request object must be constructed + * Constructor. Note that a Request object must be constructed * first, but the Requests's physical address and size fields need - * not be valid. The command and destination addresses must be - * supplied. + * not be valid. The command must be supplied. */ - Packet(Request *_req, MemCmd _cmd, NodeID _dest) - : flags(VALID_DST), cmd(_cmd), req(_req), data(NULL), - dest(_dest), time(curTick), senderState(NULL) + Packet(const RequestPtr &_req, MemCmd _cmd) + : cmd(_cmd), id((PacketId)_req.get()), req(_req), + data(nullptr), addr(0), _isSecure(false), size(0), + _qosValue(0), headerDelay(0), snoopDelay(0), + payloadDelay(0), senderState(NULL) { if (req->hasPaddr()) { addr = req->getPaddr(); flags.set(VALID_ADDR); + _isSecure = req->isSecure(); } if (req->hasSize()) { size = req->getSize(); @@ -511,13 +817,16 @@ class Packet : public FastAlloc, public Printable * a request that is for a whole block, not the address from the * req. this allows for overriding the size/addr of the req. */ - Packet(Request *_req, MemCmd _cmd, NodeID _dest, int _blkSize) - : flags(VALID_DST), cmd(_cmd), req(_req), data(NULL), - dest(_dest), time(curTick), senderState(NULL) + Packet(const RequestPtr &_req, MemCmd _cmd, int _blkSize, PacketId _id = 0) + : cmd(_cmd), id(_id ? _id : (PacketId)_req.get()), req(_req), + data(nullptr), addr(0), _isSecure(false), + _qosValue(0), headerDelay(0), + snoopDelay(0), payloadDelay(0), senderState(NULL) { if (req->hasPaddr()) { addr = req->getPaddr() & ~(_blkSize - 1); flags.set(VALID_ADDR); + _isSecure = req->isSecure(); } size = _blkSize; flags.set(VALID_SIZE); @@ -530,73 +839,112 @@ class Packet : public FastAlloc, public Printable * less than that of the original packet. In this case the new * packet should allocate its own data. */ - Packet(Packet *pkt, bool clearFlags = false) - : cmd(pkt->cmd), req(pkt->req), - data(pkt->flags.isSet(STATIC_DATA) ? pkt->data : NULL), - addr(pkt->addr), size(pkt->size), src(pkt->src), dest(pkt->dest), - time(curTick), senderState(pkt->senderState) + Packet(const PacketPtr pkt, bool clear_flags, bool alloc_data) + : cmd(pkt->cmd), id(pkt->id), req(pkt->req), + data(nullptr), + addr(pkt->addr), _isSecure(pkt->_isSecure), size(pkt->size), + bytesValid(pkt->bytesValid), + _qosValue(pkt->qosValue()), + headerDelay(pkt->headerDelay), + snoopDelay(0), + payloadDelay(pkt->payloadDelay), + senderState(pkt->senderState) { - if (!clearFlags) + if (!clear_flags) flags.set(pkt->flags & COPY_FLAGS); - flags.set(pkt->flags & (VALID_ADDR|VALID_SIZE|VALID_SRC|VALID_DST)); - flags.set(pkt->flags & STATIC_DATA); + flags.set(pkt->flags & (VALID_ADDR|VALID_SIZE)); + + // should we allocate space for data, or not, the express + // snoops do not need to carry any data as they only serve to + // co-ordinate state changes + if (alloc_data) { + // even if asked to allocate data, if the original packet + // holds static data, then the sender will not be doing + // any memcpy on receiving the response, thus we simply + // carry the pointer forward + if (pkt->flags.isSet(STATIC_DATA)) { + data = pkt->data; + flags.set(STATIC_DATA); + } else { + allocate(); + } + } } /** - * clean up packet variables + * Generate the appropriate read MemCmd based on the Request flags. */ - ~Packet() + static MemCmd + makeReadCmd(const RequestPtr &req) { - // If this is a request packet for which there's no response, - // delete the request object here, since the requester will - // never get the chance. - if (req && isRequest() && !needsResponse()) - delete req; - deleteData(); + if (req->isLLSC()) + return MemCmd::LoadLockedReq; + else if (req->isPrefetchEx()) + return MemCmd::SoftPFExReq; + else if (req->isPrefetch()) + return MemCmd::SoftPFReq; + else + return MemCmd::ReadReq; } /** - * Reinitialize packet address and size from the associated - * Request object, and reset other fields that may have been - * modified by a previous transaction. Typically called when a - * statically allocated Request/Packet pair is reused for multiple - * transactions. + * Generate the appropriate write MemCmd based on the Request flags. */ - void - reinitFromRequest() + static MemCmd + makeWriteCmd(const RequestPtr &req) { - assert(req->hasPaddr()); - flags = 0; - addr = req->getPaddr(); - size = req->getSize(); - time = req->time(); + if (req->isLLSC()) + return MemCmd::StoreCondReq; + else if (req->isSwap() || req->isAtomic()) + return MemCmd::SwapReq; + else if (req->isCacheInvalidate()) { + return req->isCacheClean() ? MemCmd::CleanInvalidReq : + MemCmd::InvalidateReq; + } else if (req->isCacheClean()) { + return MemCmd::CleanSharedReq; + } else + return MemCmd::WriteReq; + } - flags.set(VALID_ADDR|VALID_SIZE); + /** + * Constructor-like methods that return Packets based on Request objects. + * Fine-tune the MemCmd type if it's not a vanilla read or write. + */ + static PacketPtr + createRead(const RequestPtr &req) + { + return new Packet(req, makeReadCmd(req)); + } + + static PacketPtr + createWrite(const RequestPtr &req) + { + return new Packet(req, makeWriteCmd(req)); + } + + /** + * clean up packet variables + */ + ~Packet() + { deleteData(); } /** * Take a request packet and modify it in place to be suitable for - * returning as a response to that request. The source and - * destination fields are *not* modified, as is appropriate for - * atomic accesses. + * returning as a response to that request. */ void makeResponse() { assert(needsResponse()); assert(isRequest()); - origCmd = cmd; cmd = cmd.responseCommand(); // responses are never express, even if the snoop that // triggered them was flags.clear(EXPRESS_SNOOP); - - dest = src; - flags.set(VALID_DST, flags.isSet(VALID_SRC)); - flags.clear(VALID_SRC); } void @@ -611,18 +959,16 @@ class Packet : public FastAlloc, public Printable makeResponse(); } - /** - * Take a request packet that has been returned as NACKED and - * modify it so that it can be sent out again. Only packets that - * need a response can be NACKED, so verify that that is true. - */ void - reinitNacked() + setFunctionalResponseStatus(bool success) { - assert(wasNacked()); - cmd = origCmd; - assert(needsResponse()); - setDest(Broadcast); + if (!success) { + if (isWrite()) { + cmd = MemCmd::FunctionalWriteError; + } else { + cmd = MemCmd::FunctionalReadError; + } + } } void @@ -634,42 +980,106 @@ class Packet : public FastAlloc, public Printable flags.set(VALID_SIZE); } + /** + * Check if packet corresponds to a given block-aligned address and + * address space. + * + * @param addr The address to compare against. + * @param is_secure Whether addr belongs to the secure address space. + * @param blk_size Block size in bytes. + * @return Whether packet matches description. + */ + bool matchBlockAddr(const Addr addr, const bool is_secure, + const int blk_size) const; + + /** + * Check if this packet refers to the same block-aligned address and + * address space as another packet. + * + * @param pkt The packet to compare against. + * @param blk_size Block size in bytes. + * @return Whether packet matches description. + */ + bool matchBlockAddr(const PacketPtr pkt, const int blk_size) const; + + /** + * Check if packet corresponds to a given address and address space. + * + * @param addr The address to compare against. + * @param is_secure Whether addr belongs to the secure address space. + * @return Whether packet matches description. + */ + bool matchAddr(const Addr addr, const bool is_secure) const; + + /** + * Check if this packet refers to the same address and address space as + * another packet. + * + * @param pkt The packet to compare against. + * @return Whether packet matches description. + */ + bool matchAddr(const PacketPtr pkt) const; + + public: + /** + * @{ + * @name Data accessor mehtods + */ /** * Set the data pointer to the following value that should not be - * freed. + * freed. Static data allows us to do a single memcpy even if + * multiple packets are required to get from source to destination + * and back. In essence the pointer is set calling dataStatic on + * the original packet, and whenever this packet is copied and + * forwarded the same pointer is passed on. When a packet + * eventually reaches the destination holding the data, it is + * copied once into the location originally set. On the way back + * to the source, no copies are necessary. */ template void dataStatic(T *p) { - assert(flags.noneSet(STATIC_DATA|DYNAMIC_DATA|ARRAY_DATA)); + assert(flags.noneSet(STATIC_DATA|DYNAMIC_DATA)); data = (PacketDataPtr)p; flags.set(STATIC_DATA); } /** - * Set the data pointer to a value that should have delete [] - * called on it. + * Set the data pointer to the following value that should not be + * freed. This version of the function allows the pointer passed + * to us to be const. To avoid issues down the line we cast the + * constness away, the alternative would be to keep both a const + * and non-const data pointer and cleverly choose between + * them. Note that this is only allowed for static data. */ template void - dataDynamicArray(T *p) + dataStaticConst(const T *p) { - assert(flags.noneSet(STATIC_DATA|DYNAMIC_DATA|ARRAY_DATA)); - data = (PacketDataPtr)p; - flags.set(DYNAMIC_DATA|ARRAY_DATA); + assert(flags.noneSet(STATIC_DATA|DYNAMIC_DATA)); + data = const_cast(p); + flags.set(STATIC_DATA); } /** - * set the data pointer to a value that should have delete called - * on it. + * Set the data pointer to a value that should have delete [] + * called on it. Dynamic data is local to this packet, and as the + * packet travels from source to destination, forwarded packets + * will allocate their own data. When a packet reaches the final + * destination it will populate the dynamic data of that specific + * packet, and on the way back towards the source, memcpy will be + * invoked in every step where a new packet was created e.g. in + * the caches. Ultimately when the response reaches the source a + * final memcpy is needed to extract the data from the packet + * before it is deallocated. */ template void dataDynamic(T *p) { - assert(flags.noneSet(STATIC_DATA|DYNAMIC_DATA|ARRAY_DATA)); + assert(flags.noneSet(STATIC_DATA|DYNAMIC_DATA)); data = (PacketDataPtr)p; flags.set(DYNAMIC_DATA); } @@ -682,28 +1092,100 @@ class Packet : public FastAlloc, public Printable getPtr() { assert(flags.isSet(STATIC_DATA|DYNAMIC_DATA)); + assert(!isMaskedWrite()); return (T*)data; } + template + const T* + getConstPtr() const + { + assert(flags.isSet(STATIC_DATA|DYNAMIC_DATA)); + return (const T*)data; + } + + /** + * Get the data in the packet byte swapped from big endian to + * host endian. + */ + template + T getBE() const; + /** - * return the value of what is pointed to in the packet. + * Get the data in the packet byte swapped from little endian to + * host endian. */ template - T get(); + T getLE() const; /** - * set the value in the data pointer to v. + * Get the data in the packet byte swapped from the specified + * endianness. */ template - void set(T v); + T get(ByteOrder endian) const; + +#if THE_ISA != NULL_ISA + /** + * Get the data in the packet byte swapped from guest to host + * endian. + */ + template + T get() const + M5_DEPRECATED_MSG("The memory system should be ISA independent."); +#endif + + /** Set the value in the data pointer to v as big endian. */ + template + void setBE(T v); + + /** Set the value in the data pointer to v as little endian. */ + template + void setLE(T v); + + /** + * Set the value in the data pointer to v using the specified + * endianness. + */ + template + void set(T v, ByteOrder endian); + +#if THE_ISA != NULL_ISA + /** Set the value in the data pointer to v as guest endian. */ + template + void set(T v) + M5_DEPRECATED_MSG("The memory system should be ISA independent."); +#endif + + /** + * Get the data in the packet byte swapped from the specified + * endianness and zero-extended to 64 bits. + */ + uint64_t getUintX(ByteOrder endian) const; + + /** + * Set the value in the word w after truncating it to the length + * of the packet and then byteswapping it to the desired + * endianness. + */ + void setUintX(uint64_t w, ByteOrder endian); /** * Copy data into the packet from the provided pointer. */ void - setData(uint8_t *p) + setData(const uint8_t *p) { - std::memcpy(getPtr(), p, getSize()); + // we should never be copying data onto itself, which means we + // must idenfity packets with static data, as they carry the + // same pointer from source to destination and back + assert(p != getPtr() || flags.isSet(STATIC_DATA)); + + if (p != getPtr()) { + // for packet with allocated dynamic data, we copy data from + // one to the other, e.g. a forwarded response to a response + std::memcpy(getPtr(), p, getSize()); + } } /** @@ -711,26 +1193,41 @@ class Packet : public FastAlloc, public Printable * which is aligned to the given block size. */ void - setDataFromBlock(uint8_t *blk_data, int blkSize) + setDataFromBlock(const uint8_t *blk_data, int blkSize) { setData(blk_data + getOffset(blkSize)); } /** - * Copy data from the packet to the provided block pointer, which - * is aligned to the given block size. + * Copy data from the packet to the memory at the provided pointer. + * @param p Pointer to which data will be copied. */ void - writeData(uint8_t *p) + writeData(uint8_t *p) const { - std::memcpy(p, getPtr(), getSize()); + if (!isMaskedWrite()) { + std::memcpy(p, getConstPtr(), getSize()); + } else { + assert(req->getByteEnable().size() == getSize()); + // Write only the enabled bytes + const uint8_t *base = getConstPtr(); + for (int i = 0; i < getSize(); i++) { + if (req->getByteEnable()[i]) { + p[i] = *(base + i); + } + // Disabled bytes stay untouched + } + } } /** - * Copy data from the packet to the memory at the provided pointer. + * Copy data from the packet to the provided block pointer, which + * is aligned to the given block size. + * @param blk_data Pointer to block to which data will be copied. + * @param blkSize Block size in bytes. */ void - writeDataToBlock(uint8_t *blk_data, int blkSize) + writeDataToBlock(uint8_t *blk_data, int blkSize) const { writeData(blk_data + getOffset(blkSize)); } @@ -742,51 +1239,105 @@ class Packet : public FastAlloc, public Printable void deleteData() { - if (flags.isSet(ARRAY_DATA)) + if (flags.isSet(DYNAMIC_DATA)) delete [] data; - else if (flags.isSet(DYNAMIC_DATA)) - delete data; - flags.clear(STATIC_DATA|DYNAMIC_DATA|ARRAY_DATA); + flags.clear(STATIC_DATA|DYNAMIC_DATA); data = NULL; } - /** If there isn't data in the packet, allocate some. */ + /** Allocate memory for the packet. */ void allocate() { - if (data) { - assert(flags.isSet(STATIC_DATA|DYNAMIC_DATA)); - return; + // if either this command or the response command has a data + // payload, actually allocate space + if (hasData() || hasRespData()) { + assert(flags.noneSet(STATIC_DATA|DYNAMIC_DATA)); + flags.set(DYNAMIC_DATA); + data = new uint8_t[getSize()]; } - - assert(flags.noneSet(STATIC_DATA|DYNAMIC_DATA)); - flags.set(DYNAMIC_DATA|ARRAY_DATA); - data = new uint8_t[getSize()]; } + /** @} */ + + /** Get the data in the packet without byte swapping. */ + template + T getRaw() const; + + /** Set the value in the data pointer to v without byte swapping. */ + template + void setRaw(T v); + public: /** - * Check a functional request against a memory value represented - * by a base/size pair and an associated data array. If the - * functional request is a read, it may be satisfied by the memory - * value. If the functional request is a write, it may update the - * memory value. + * Check a functional request against a memory value stored in + * another packet (i.e. an in-transit request or + * response). Returns true if the current packet is a read, and + * the other packet provides the data, which is then copied to the + * current packet. If the current packet is a write, and the other + * packet intersects this one, then we update the data + * accordingly. */ - bool checkFunctional(Printable *obj, Addr base, int size, uint8_t *data); + bool + trySatisfyFunctional(PacketPtr other) + { + if (other->isMaskedWrite()) { + // Do not forward data if overlapping with a masked write + if (_isSecure == other->isSecure() && + getAddr() <= (other->getAddr() + other->getSize() - 1) && + other->getAddr() <= (getAddr() + getSize() - 1)) { + warn("Trying to check against a masked write, skipping." + " (addr: 0x%x, other addr: 0x%x)", getAddr(), + other->getAddr()); + } + return false; + } + // all packets that are carrying a payload should have a valid + // data pointer + return trySatisfyFunctional(other, other->getAddr(), other->isSecure(), + other->getSize(), + other->hasData() ? + other->getPtr() : NULL); + } /** - * Check a functional request against a memory value stored in - * another packet (i.e. an in-transit request or response). + * Does the request need to check for cached copies of the same block + * in the memory hierarchy above. + **/ + bool + mustCheckAbove() const + { + return cmd == MemCmd::HardPFReq || isEviction(); + } + + /** + * Is this packet a clean eviction, including both actual clean + * evict packets, but also clean writebacks. */ bool - checkFunctional(PacketPtr other) + isCleanEviction() const { - uint8_t *data = other->hasData() ? other->getPtr() : NULL; - return checkFunctional(other, other->getAddr(), other->getSize(), - data); + return cmd == MemCmd::CleanEvict || cmd == MemCmd::WritebackClean; + } + + bool + isMaskedWrite() const + { + return (cmd == MemCmd::WriteReq && !req->getByteEnable().empty()); } + /** + * Check a functional request against a memory value represented + * by a base/size pair and an associated data array. If the + * current packet is a read, it may be satisfied by the memory + * value. If the current packet is a write, it may update the + * memory value. + */ + bool + trySatisfyFunctional(Printable *obj, Addr base, bool is_secure, int size, + uint8_t *_data); + /** * Push label for PrintReq (safe to call unconditionally). */ @@ -809,6 +1360,14 @@ class Packet : public FastAlloc, public Printable void print(std::ostream &o, int verbosity = 0, const std::string &prefix = "") const; + + /** + * A no-args wrapper of print(std::ostream...) + * meant to be invoked from DPRINTFs + * avoiding string overheads in fast mode + * @return string with the request's type and start<->end addresses + */ + std::string print() const; }; #endif //__MEM_PACKET_HH