X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;ds=sidebyside;f=src%2Fmem%2Fpacket.hh;h=130cc41adc5df6ee23de8bfdbae8d22ed2a32312;hb=e33b3aa6692b172f6db5957774a9e0289e81fa5b;hp=155a7ff82cc82ba252d2ca8b7089a041975b64bc;hpb=f6f63ec0aa68f631691d9eccc18739722a0a9f17;p=gem5.git diff --git a/src/mem/packet.hh b/src/mem/packet.hh index 155a7ff82..130cc41ad 100644 --- a/src/mem/packet.hh +++ b/src/mem/packet.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2014 ARM Limited + * Copyright (c) 2012-2019 ARM Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -12,7 +12,7 @@ * modified or unmodified, in source code or in binary form. * * Copyright (c) 2006 The Regents of The University of Michigan - * Copyright (c) 2010 Advanced Micro Devices, Inc. + * Copyright (c) 2010,2015 Advanced Micro Devices, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -42,6 +42,7 @@ * Steve Reinhardt * Ali Saidi * Andreas Hansson + * Nikos Nikoleris */ /** @@ -56,12 +57,14 @@ #include #include +#include "base/addr_range.hh" #include "base/cast.hh" #include "base/compiler.hh" #include "base/flags.hh" -#include "base/misc.hh" +#include "base/logging.hh" #include "base/printable.hh" #include "base/types.hh" +#include "config/the_isa.hh" #include "mem/request.hh" #include "sim/core.hh" @@ -69,6 +72,7 @@ class Packet; typedef Packet *PacketPtr; typedef uint8_t* PacketDataPtr; typedef std::list PacketList; +typedef uint64_t PacketId; class MemCmd { @@ -86,13 +90,16 @@ class MemCmd ReadRespWithInvalidate, WriteReq, WriteResp, - Writeback, + WritebackDirty, + WritebackClean, + WriteClean, // writes dirty data below without evicting + CleanEvict, SoftPFReq, + SoftPFExReq, HardPFReq, SoftPFResp, HardPFResp, - WriteInvalidateReq, - WriteInvalidateResp, + WriteLineReq, UpgradeReq, SCUpgradeReq, // Special "weak" upgrade for StoreCond UpgradeResp, @@ -100,6 +107,8 @@ class MemCmd UpgradeFailResp, // Valid for SCUpgradeReq only ReadExReq, ReadExResp, + ReadCleanReq, + ReadSharedReq, LoadLockedReq, StoreCondReq, StoreCondFailReq, // Failed StoreCondReq in MSHR (never sent) @@ -108,6 +117,12 @@ class MemCmd SwapResp, MessageReq, MessageResp, + MemFenceReq, + MemFenceResp, + CleanSharedReq, + CleanSharedResp, + CleanInvalidReq, + CleanInvalidResp, // Error responses // @TODO these should be classified as responses rather than // requests; coding them as requests initially for backwards @@ -119,7 +134,8 @@ class MemCmd // Fake simulator-only commands PrintReq, // Print state matching address FlushReq, //request for a cache flush - InvalidationReq, // request for address to be invalidated from lsq + InvalidateReq, // request for address to be invalidated + InvalidateResp, NUM_MEM_CMDS }; @@ -133,10 +149,12 @@ class MemCmd IsWrite, //!< Data flows from requester to responder IsUpgrade, IsInvalidate, - NeedsExclusive, //!< Requires exclusive copy to complete in-cache + IsClean, //!< Cleans any existing dirty blocks + NeedsWritable, //!< Requires writable copy to complete in-cache IsRequest, //!< Issued by requester IsResponse, //!< Issue by responder NeedsResponse, //!< Requester needs response from target + IsEviction, IsSWPrefetch, IsHWPrefetch, IsLlsc, //!< Alpha/MIPS LL or SC access @@ -144,6 +162,7 @@ class MemCmd IsError, //!< Error response IsPrint, //!< Print state matching address (for debugging) IsFlush, //!< Flush the address from caches + FromCache, //!< Request originated from a caching agent NUM_COMMAND_ATTRIBUTES }; @@ -177,16 +196,30 @@ class MemCmd public: - bool isRead() const { return testCmdAttrib(IsRead); } - bool isWrite() const { return testCmdAttrib(IsWrite); } - bool isUpgrade() const { return testCmdAttrib(IsUpgrade); } - bool isRequest() const { return testCmdAttrib(IsRequest); } - bool isResponse() const { return testCmdAttrib(IsResponse); } - bool needsExclusive() const { return testCmdAttrib(NeedsExclusive); } - bool needsResponse() const { return testCmdAttrib(NeedsResponse); } - bool isInvalidate() const { return testCmdAttrib(IsInvalidate); } + bool isRead() const { return testCmdAttrib(IsRead); } + bool isWrite() const { return testCmdAttrib(IsWrite); } + bool isUpgrade() const { return testCmdAttrib(IsUpgrade); } + bool isRequest() const { return testCmdAttrib(IsRequest); } + bool isResponse() const { return testCmdAttrib(IsResponse); } + bool needsWritable() const { return testCmdAttrib(NeedsWritable); } + bool needsResponse() const { return testCmdAttrib(NeedsResponse); } + bool isInvalidate() const { return testCmdAttrib(IsInvalidate); } + bool isEviction() const { return testCmdAttrib(IsEviction); } + bool isClean() const { return testCmdAttrib(IsClean); } + bool fromCache() const { return testCmdAttrib(FromCache); } + + /** + * A writeback is an eviction that carries data. + */ + bool isWriteback() const { return testCmdAttrib(IsEviction) && + testCmdAttrib(HasData); } + + /** + * Check if this particular packet type carries payload data. Note + * that this does not reflect if the data pointer of the packet is + * valid or not. + */ bool hasData() const { return testCmdAttrib(HasData); } - bool isReadWrite() const { return isRead() && isWrite(); } bool isLLSC() const { return testCmdAttrib(IsLlsc); } bool isSWPrefetch() const { return testCmdAttrib(IsSWPrefetch); } bool isHWPrefetch() const { return testCmdAttrib(IsHWPrefetch); } @@ -196,7 +229,7 @@ class MemCmd bool isPrint() const { return testCmdAttrib(IsPrint); } bool isFlush() const { return testCmdAttrib(IsFlush); } - const Command + Command responseCommand() const { return commandInfo[cmd].response; @@ -228,36 +261,59 @@ class Packet : public Printable typedef ::Flags Flags; private: - static const FlagsType PUBLIC_FLAGS = 0x00000000; - static const FlagsType PRIVATE_FLAGS = 0x00007F0F; - static const FlagsType COPY_FLAGS = 0x0000000F; - - static const FlagsType SHARED = 0x00000001; - // Special control flags - /// Special timing-mode atomic snoop for multi-level coherence. - static const FlagsType EXPRESS_SNOOP = 0x00000002; - /// Does supplier have exclusive copy? - /// Useful for multi-level coherence. - static const FlagsType SUPPLY_EXCLUSIVE = 0x00000004; - // Snoop response flags - static const FlagsType MEM_INHIBIT = 0x00000008; - /// Are the 'addr' and 'size' fields valid? - static const FlagsType VALID_ADDR = 0x00000100; - static const FlagsType VALID_SIZE = 0x00000200; - /// Is the data pointer set to a value that shouldn't be freed - /// when the packet is destroyed? - static const FlagsType STATIC_DATA = 0x00001000; - /// The data pointer points to a value that should be freed when - /// the packet is destroyed. - static const FlagsType DYNAMIC_DATA = 0x00002000; - /// the data pointer points to an array (thus delete []) needs to - /// be called on it rather than simply delete. - static const FlagsType ARRAY_DATA = 0x00004000; - /// suppress the error if this packet encounters a functional - /// access failure. - static const FlagsType SUPPRESS_FUNC_ERROR = 0x00008000; - // Signal prefetch squash through express snoop flag - static const FlagsType PREFETCH_SNOOP_SQUASH = 0x00010000; + + enum : FlagsType { + // Flags to transfer across when copying a packet + COPY_FLAGS = 0x0000003F, + + // Flags that are used to create reponse packets + RESPONDER_FLAGS = 0x00000009, + + // Does this packet have sharers (which means it should not be + // considered writable) or not. See setHasSharers below. + HAS_SHARERS = 0x00000001, + + // Special control flags + /// Special timing-mode atomic snoop for multi-level coherence. + EXPRESS_SNOOP = 0x00000002, + + /// Allow a responding cache to inform the cache hierarchy + /// that it had a writable copy before responding. See + /// setResponderHadWritable below. + RESPONDER_HAD_WRITABLE = 0x00000004, + + // Snoop co-ordination flag to indicate that a cache is + // responding to a snoop. See setCacheResponding below. + CACHE_RESPONDING = 0x00000008, + + // The writeback/writeclean should be propagated further + // downstream by the receiver + WRITE_THROUGH = 0x00000010, + + // Response co-ordination flag for cache maintenance + // operations + SATISFIED = 0x00000020, + + /// Are the 'addr' and 'size' fields valid? + VALID_ADDR = 0x00000100, + VALID_SIZE = 0x00000200, + + /// Is the data pointer set to a value that shouldn't be freed + /// when the packet is destroyed? + STATIC_DATA = 0x00001000, + /// The data pointer points to a value that should be freed when + /// the packet is destroyed. The pointer is assumed to be pointing + /// to an array, and delete [] is consequently called + DYNAMIC_DATA = 0x00002000, + + /// suppress the error if this packet encounters a functional + /// access failure. + SUPPRESS_FUNC_ERROR = 0x00008000, + + // Signal block present to squash prefetch and cache evict packets + // through express snoop flag + BLOCK_CACHED = 0x00010000 + }; Flags flags; @@ -267,15 +323,17 @@ class Packet : public Printable /// The command field of the packet. MemCmd cmd; + const PacketId id; + /// A pointer to the original request. RequestPtr req; private: /** - * A pointer to the data being transfered. It can be differnt - * sizes at each level of the heirarchy so it belongs in the + * A pointer to the data being transferred. It can be different + * sizes at each level of the hierarchy so it belongs to the * packet, not request. This may or may not be populated when a - * responder recieves the packet. If not populated it memory should + * responder receives the packet. If not populated memory should * be allocated. */ PacketDataPtr data; @@ -291,76 +349,54 @@ class Packet : public Printable unsigned size; /** - * Source port identifier set on a request packet to enable - * appropriate routing of the responses. The source port - * identifier is set by any multiplexing component, e.g. a bus, as - * the timing responses need this information to be routed back to - * the appropriate port at a later point in time. The field can be - * updated (over-written) as the request packet passes through - * additional multiplexing components, and it is their - * responsibility to remember the original source port identifier, - * for example by using an appropriate sender state. The latter is - * done in the cache and bridge. + * Track the bytes found that satisfy a functional read. */ - PortID src; + std::vector bytesValid; - /** - * Destination port identifier that is present on all response - * packets that passed through a multiplexing component as a - * request packet. The source port identifier is turned into a - * destination port identifier when the packet is turned into a - * response, and the destination is used, e.g. by the bus, to - * select the appropriate path through the interconnect. - */ - PortID dest; + // Quality of Service priority value + uint8_t _qosValue; - /** - * The original value of the command field. Only valid when the - * current command field is an error condition; in that case, the - * previous contents of the command field are copied here. This - * field is *not* set on non-error responses. - */ - MemCmd origCmd; + public: /** - * These values specify the range of bytes found that satisfy a - * functional read. + * The extra delay from seeing the packet until the header is + * transmitted. This delay is used to communicate the crossbar + * forwarding latency to the neighbouring object (e.g. a cache) + * that actually makes the packet wait. As the delay is relative, + * a 32-bit unsigned should be sufficient. */ - uint16_t bytesValidStart; - uint16_t bytesValidEnd; - - public: + uint32_t headerDelay; /** - * The extra delay from seeing the packet until the first word is - * transmitted by the bus that provided it (if any). This delay is - * used to communicate the bus waiting time to the neighbouring - * object (e.g. a cache) that actually makes the packet wait. As - * the delay is relative, a 32-bit unsigned should be sufficient. + * Keep track of the extra delay incurred by snooping upwards + * before sending a request down the memory system. This is used + * by the coherent crossbar to account for the additional request + * delay. */ - uint32_t busFirstWordDelay; + uint32_t snoopDelay; /** - * The extra delay from seeing the packet until the last word is - * transmitted by the bus that provided it (if any). Similar to - * the first word time, this is used to make up for the fact that - * the bus does not make the packet wait. As the delay is relative, - * a 32-bit unsigned should be sufficient. + * The extra pipelining delay from seeing the packet until the end of + * payload is transmitted by the component that provided it (if + * any). This includes the header delay. Similar to the header + * delay, this is used to make up for the fact that the + * crossbar does not make the packet wait. As the delay is + * relative, a 32-bit unsigned should be sufficient. */ - uint32_t busLastWordDelay; + uint32_t payloadDelay; /** * A virtual base opaque structure used to hold state associated - * with the packet (e.g., an MSHR), specific to a MemObject that + * with the packet (e.g., an MSHR), specific to a SimObject that * sees the packet. A pointer to this state is returned in the - * packet's response so that the MemObject in question can quickly + * packet's response so that the SimObject in question can quickly * look up the state needed to process it. A specific subclass * would be derived from this to carry state specific to a * particular sending device. * - * As multiple MemObjects may add their SenderState throughout the + * As multiple SimObjects may add their SenderState throughout the * memory system, the SenderStates create a stack, where a - * MemObject can add a new Senderstate, as long as the + * SimObject can add a new Senderstate, as long as the * predecessing SenderState is restored when the response comes * back. For this reason, the predecessor should always be * populated with the current SenderState of a packet before @@ -491,37 +527,191 @@ class Packet : public Printable /// Return the index of this command. inline int cmdToIndex() const { return cmd.toInt(); } - bool isRead() const { return cmd.isRead(); } - bool isWrite() const { return cmd.isWrite(); } - bool isUpgrade() const { return cmd.isUpgrade(); } - bool isRequest() const { return cmd.isRequest(); } - bool isResponse() const { return cmd.isResponse(); } - bool needsExclusive() const { return cmd.needsExclusive(); } - bool needsResponse() const { return cmd.needsResponse(); } - bool isInvalidate() const { return cmd.isInvalidate(); } - bool hasData() const { return cmd.hasData(); } - bool isReadWrite() const { return cmd.isReadWrite(); } - bool isLLSC() const { return cmd.isLLSC(); } - bool isError() const { return cmd.isError(); } - bool isPrint() const { return cmd.isPrint(); } - bool isFlush() const { return cmd.isFlush(); } - - // Snoop flags - void assertMemInhibit() { flags.set(MEM_INHIBIT); } - bool memInhibitAsserted() const { return flags.isSet(MEM_INHIBIT); } - void assertShared() { flags.set(SHARED); } - bool sharedAsserted() const { return flags.isSet(SHARED); } - - // Special control flags - void setExpressSnoop() { flags.set(EXPRESS_SNOOP); } - bool isExpressSnoop() const { return flags.isSet(EXPRESS_SNOOP); } - void setSupplyExclusive() { flags.set(SUPPLY_EXCLUSIVE); } - void clearSupplyExclusive() { flags.clear(SUPPLY_EXCLUSIVE); } - bool isSupplyExclusive() const { return flags.isSet(SUPPLY_EXCLUSIVE); } + bool isRead() const { return cmd.isRead(); } + bool isWrite() const { return cmd.isWrite(); } + bool isUpgrade() const { return cmd.isUpgrade(); } + bool isRequest() const { return cmd.isRequest(); } + bool isResponse() const { return cmd.isResponse(); } + bool needsWritable() const + { + // we should never check if a response needsWritable, the + // request has this flag, and for a response we should rather + // look at the hasSharers flag (if not set, the response is to + // be considered writable) + assert(isRequest()); + return cmd.needsWritable(); + } + bool needsResponse() const { return cmd.needsResponse(); } + bool isInvalidate() const { return cmd.isInvalidate(); } + bool isEviction() const { return cmd.isEviction(); } + bool isClean() const { return cmd.isClean(); } + bool fromCache() const { return cmd.fromCache(); } + bool isWriteback() const { return cmd.isWriteback(); } + bool hasData() const { return cmd.hasData(); } + bool hasRespData() const + { + MemCmd resp_cmd = cmd.responseCommand(); + return resp_cmd.hasData(); + } + bool isLLSC() const { return cmd.isLLSC(); } + bool isError() const { return cmd.isError(); } + bool isPrint() const { return cmd.isPrint(); } + bool isFlush() const { return cmd.isFlush(); } + + bool isWholeLineWrite(unsigned blk_size) + { + return (cmd == MemCmd::WriteReq || cmd == MemCmd::WriteLineReq) && + getOffset(blk_size) == 0 && getSize() == blk_size; + } + + //@{ + /// Snoop flags + /** + * Set the cacheResponding flag. This is used by the caches to + * signal another cache that they are responding to a request. A + * cache will only respond to snoops if it has the line in either + * Modified or Owned state. Note that on snoop hits we always pass + * the line as Modified and never Owned. In the case of an Owned + * line we proceed to invalidate all other copies. + * + * On a cache fill (see Cache::handleFill), we check hasSharers + * first, ignoring the cacheResponding flag if hasSharers is set. + * A line is consequently allocated as: + * + * hasSharers cacheResponding state + * true false Shared + * true true Shared + * false false Exclusive + * false true Modified + */ + void setCacheResponding() + { + assert(isRequest()); + assert(!flags.isSet(CACHE_RESPONDING)); + flags.set(CACHE_RESPONDING); + } + bool cacheResponding() const { return flags.isSet(CACHE_RESPONDING); } + /** + * On fills, the hasSharers flag is used by the caches in + * combination with the cacheResponding flag, as clarified + * above. If the hasSharers flag is not set, the packet is passing + * writable. Thus, a response from a memory passes the line as + * writable by default. + * + * The hasSharers flag is also used by upstream caches to inform a + * downstream cache that they have the block (by calling + * setHasSharers on snoop request packets that hit in upstream + * cachs tags or MSHRs). If the snoop packet has sharers, a + * downstream cache is prevented from passing a dirty line upwards + * if it was not explicitly asked for a writable copy. See + * Cache::satisfyCpuSideRequest. + * + * The hasSharers flag is also used on writebacks, in + * combination with the WritbackClean or WritebackDirty commands, + * to allocate the block downstream either as: + * + * command hasSharers state + * WritebackDirty false Modified + * WritebackDirty true Owned + * WritebackClean false Exclusive + * WritebackClean true Shared + */ + void setHasSharers() { flags.set(HAS_SHARERS); } + bool hasSharers() const { return flags.isSet(HAS_SHARERS); } + //@} + + /** + * The express snoop flag is used for two purposes. Firstly, it is + * used to bypass flow control for normal (non-snoop) requests + * going downstream in the memory system. In cases where a cache + * is responding to a snoop from another cache (it had a dirty + * line), but the line is not writable (and there are possibly + * other copies), the express snoop flag is set by the downstream + * cache to invalidate all other copies in zero time. Secondly, + * the express snoop flag is also set to be able to distinguish + * snoop packets that came from a downstream cache, rather than + * snoop packets from neighbouring caches. + */ + void setExpressSnoop() { flags.set(EXPRESS_SNOOP); } + bool isExpressSnoop() const { return flags.isSet(EXPRESS_SNOOP); } + + /** + * On responding to a snoop request (which only happens for + * Modified or Owned lines), make sure that we can transform an + * Owned response to a Modified one. If this flag is not set, the + * responding cache had the line in the Owned state, and there are + * possibly other Shared copies in the memory system. A downstream + * cache helps in orchestrating the invalidation of these copies + * by sending out the appropriate express snoops. + */ + void setResponderHadWritable() + { + assert(cacheResponding()); + assert(!responderHadWritable()); + flags.set(RESPONDER_HAD_WRITABLE); + } + bool responderHadWritable() const + { return flags.isSet(RESPONDER_HAD_WRITABLE); } + + /** + * Copy the reponse flags from an input packet to this packet. The + * reponse flags determine whether a responder has been found and + * the state at which the block will be at the destination. + * + * @pkt The packet that we will copy flags from + */ + void copyResponderFlags(const PacketPtr pkt); + + /** + * A writeback/writeclean cmd gets propagated further downstream + * by the receiver when the flag is set. + */ + void setWriteThrough() + { + assert(cmd.isWrite() && + (cmd.isEviction() || cmd == MemCmd::WriteClean)); + flags.set(WRITE_THROUGH); + } + void clearWriteThrough() { flags.clear(WRITE_THROUGH); } + bool writeThrough() const { return flags.isSet(WRITE_THROUGH); } + + /** + * Set when a request hits in a cache and the cache is not going + * to respond. This is used by the crossbar to coordinate + * responses for cache maintenance operations. + */ + void setSatisfied() + { + assert(cmd.isClean()); + assert(!flags.isSet(SATISFIED)); + flags.set(SATISFIED); + } + bool satisfied() const { return flags.isSet(SATISFIED); } + void setSuppressFuncError() { flags.set(SUPPRESS_FUNC_ERROR); } bool suppressFuncError() const { return flags.isSet(SUPPRESS_FUNC_ERROR); } - void setPrefetchSquashed() { flags.set(PREFETCH_SNOOP_SQUASH); } - bool prefetchSquashed() const { return flags.isSet(PREFETCH_SNOOP_SQUASH); } + void setBlockCached() { flags.set(BLOCK_CACHED); } + bool isBlockCached() const { return flags.isSet(BLOCK_CACHED); } + void clearBlockCached() { flags.clear(BLOCK_CACHED); } + + /** + * QoS Value getter + * Returns 0 if QoS value was never set (constructor default). + * + * @return QoS priority value of the packet + */ + inline uint8_t qosValue() const { return _qosValue; } + + /** + * QoS Value setter + * Interface for setting QoS priority value of the packet. + * + * @param qos_value QoS priority value + */ + inline void qosValue(const uint8_t qos_value) + { _qosValue = qos_value; } + + inline MasterID masterId() const { return req->masterId(); } // Network error conditions... encapsulate them as methods since // their encoding keeps changing (from result field to command @@ -533,25 +723,8 @@ class Packet : public Printable cmd = MemCmd::BadAddressError; } - bool hadBadAddress() const { return cmd == MemCmd::BadAddressError; } void copyError(Packet *pkt) { assert(pkt->isError()); cmd = pkt->cmd; } - bool isSrcValid() const { return src != InvalidPortID; } - /// Accessor function to get the source index of the packet. - PortID getSrc() const { assert(isSrcValid()); return src; } - /// Accessor function to set the source index of the packet. - void setSrc(PortID _src) { src = _src; } - /// Reset source field, e.g. to retransmit packet on different bus. - void clearSrc() { src = InvalidPortID; } - - bool isDestValid() const { return dest != InvalidPortID; } - /// Accessor function for the destination index of the packet. - PortID getDest() const { assert(isDestValid()); return dest; } - /// Accessor function to set the destination index of the packet. - void setDest(PortID _dest) { dest = _dest; } - /// Reset destination field, e.g. to turn a response into a request again. - void clearDest() { dest = InvalidPortID; } - Addr getAddr() const { assert(flags.isSet(VALID_ADDR)); return addr; } /** * Update the address of this packet mid-transaction. This is used @@ -563,7 +736,23 @@ class Packet : public Printable void setAddr(Addr _addr) { assert(flags.isSet(VALID_ADDR)); addr = _addr; } unsigned getSize() const { assert(flags.isSet(VALID_SIZE)); return size; } - Addr getOffset(int blkSize) const { return getAddr() & (Addr)(blkSize - 1); } + + /** + * Get address range to which this packet belongs. + * + * @return Address range of this packet. + */ + AddrRange getAddrRange() const; + + Addr getOffset(unsigned int blk_size) const + { + return getAddr() & Addr(blk_size - 1); + } + + Addr getBlockAddr(unsigned int blk_size) const + { + return getAddr() & ~(Addr(blk_size - 1)); + } bool isSecure() const { @@ -571,9 +760,15 @@ class Packet : public Printable return _isSecure; } + /** + * Accessor function to atomic op. + */ + AtomicOpFunctor *getAtomicOp() const { return req->getAtomicOpFunctor(); } + bool isAtomicOp() const { return req->isAtomic(); } + /** * It has been determined that the SC packet should successfully update - * memory. Therefore, convert this SC packet to a normal write. + * memory. Therefore, convert this SC packet to a normal write. */ void convertScToWrite() @@ -584,8 +779,8 @@ class Packet : public Printable } /** - * When ruby is in use, Ruby will monitor the cache line and thus M5 - * phys memory should treat LL ops as normal reads. + * When ruby is in use, Ruby will monitor the cache line and the + * phys memory should treat LL ops as normal reads. */ void convertLlToRead() @@ -596,16 +791,15 @@ class Packet : public Printable } /** - * Constructor. Note that a Request object must be constructed + * Constructor. Note that a Request object must be constructed * first, but the Requests's physical address and size fields need * not be valid. The command must be supplied. */ - Packet(Request *_req, MemCmd _cmd) - : cmd(_cmd), req(_req), data(NULL), - src(InvalidPortID), dest(InvalidPortID), - bytesValidStart(0), bytesValidEnd(0), - busFirstWordDelay(0), busLastWordDelay(0), - senderState(NULL) + Packet(const RequestPtr &_req, MemCmd _cmd) + : cmd(_cmd), id((PacketId)_req.get()), req(_req), + data(nullptr), addr(0), _isSecure(false), size(0), + _qosValue(0), headerDelay(0), snoopDelay(0), + payloadDelay(0), senderState(NULL) { if (req->hasPaddr()) { addr = req->getPaddr(); @@ -623,12 +817,11 @@ class Packet : public Printable * a request that is for a whole block, not the address from the * req. this allows for overriding the size/addr of the req. */ - Packet(Request *_req, MemCmd _cmd, int _blkSize) - : cmd(_cmd), req(_req), data(NULL), - src(InvalidPortID), dest(InvalidPortID), - bytesValidStart(0), bytesValidEnd(0), - busFirstWordDelay(0), busLastWordDelay(0), - senderState(NULL) + Packet(const RequestPtr &_req, MemCmd _cmd, int _blkSize, PacketId _id = 0) + : cmd(_cmd), id(_id ? _id : (PacketId)_req.get()), req(_req), + data(nullptr), addr(0), _isSecure(false), + _qosValue(0), headerDelay(0), + snoopDelay(0), payloadDelay(0), senderState(NULL) { if (req->hasPaddr()) { addr = req->getPaddr() & ~(_blkSize - 1); @@ -646,64 +839,88 @@ class Packet : public Printable * less than that of the original packet. In this case the new * packet should allocate its own data. */ - Packet(Packet *pkt, bool clearFlags = false) - : cmd(pkt->cmd), req(pkt->req), - data(pkt->flags.isSet(STATIC_DATA) ? pkt->data : NULL), + Packet(const PacketPtr pkt, bool clear_flags, bool alloc_data) + : cmd(pkt->cmd), id(pkt->id), req(pkt->req), + data(nullptr), addr(pkt->addr), _isSecure(pkt->_isSecure), size(pkt->size), - src(pkt->src), dest(pkt->dest), - bytesValidStart(pkt->bytesValidStart), - bytesValidEnd(pkt->bytesValidEnd), - busFirstWordDelay(pkt->busFirstWordDelay), - busLastWordDelay(pkt->busLastWordDelay), + bytesValid(pkt->bytesValid), + _qosValue(pkt->qosValue()), + headerDelay(pkt->headerDelay), + snoopDelay(0), + payloadDelay(pkt->payloadDelay), senderState(pkt->senderState) { - if (!clearFlags) + if (!clear_flags) flags.set(pkt->flags & COPY_FLAGS); flags.set(pkt->flags & (VALID_ADDR|VALID_SIZE)); - flags.set(pkt->flags & STATIC_DATA); + + // should we allocate space for data, or not, the express + // snoops do not need to carry any data as they only serve to + // co-ordinate state changes + if (alloc_data) { + // even if asked to allocate data, if the original packet + // holds static data, then the sender will not be doing + // any memcpy on receiving the response, thus we simply + // carry the pointer forward + if (pkt->flags.isSet(STATIC_DATA)) { + data = pkt->data; + flags.set(STATIC_DATA); + } else { + allocate(); + } + } } /** - * Change the packet type based on request type. + * Generate the appropriate read MemCmd based on the Request flags. */ - void - refineCommand() + static MemCmd + makeReadCmd(const RequestPtr &req) { - if (cmd == MemCmd::ReadReq) { - if (req->isLLSC()) { - cmd = MemCmd::LoadLockedReq; - } else if (req->isPrefetch()) { - cmd = MemCmd::SoftPFReq; - } - } else if (cmd == MemCmd::WriteReq) { - if (req->isLLSC()) { - cmd = MemCmd::StoreCondReq; - } else if (req->isSwap()) { - cmd = MemCmd::SwapReq; - } - } + if (req->isLLSC()) + return MemCmd::LoadLockedReq; + else if (req->isPrefetchEx()) + return MemCmd::SoftPFExReq; + else if (req->isPrefetch()) + return MemCmd::SoftPFReq; + else + return MemCmd::ReadReq; + } + + /** + * Generate the appropriate write MemCmd based on the Request flags. + */ + static MemCmd + makeWriteCmd(const RequestPtr &req) + { + if (req->isLLSC()) + return MemCmd::StoreCondReq; + else if (req->isSwap() || req->isAtomic()) + return MemCmd::SwapReq; + else if (req->isCacheInvalidate()) { + return req->isCacheClean() ? MemCmd::CleanInvalidReq : + MemCmd::InvalidateReq; + } else if (req->isCacheClean()) { + return MemCmd::CleanSharedReq; + } else + return MemCmd::WriteReq; } /** * Constructor-like methods that return Packets based on Request objects. - * Will call refineCommand() to fine-tune the Packet type if it's not a - * vanilla read or write. + * Fine-tune the MemCmd type if it's not a vanilla read or write. */ static PacketPtr - createRead(Request *req) + createRead(const RequestPtr &req) { - PacketPtr pkt = new Packet(req, MemCmd::ReadReq); - pkt->refineCommand(); - return pkt; + return new Packet(req, makeReadCmd(req)); } static PacketPtr - createWrite(Request *req) + createWrite(const RequestPtr &req) { - PacketPtr pkt = new Packet(req, MemCmd::WriteReq); - pkt->refineCommand(); - return pkt; + return new Packet(req, makeWriteCmd(req)); } /** @@ -711,62 +928,23 @@ class Packet : public Printable */ ~Packet() { - // If this is a request packet for which there's no response, - // delete the request object here, since the requester will - // never get the chance. - if (req && isRequest() && !needsResponse()) - delete req; - deleteData(); - } - - /** - * Reinitialize packet address and size from the associated - * Request object, and reset other fields that may have been - * modified by a previous transaction. Typically called when a - * statically allocated Request/Packet pair is reused for multiple - * transactions. - */ - void - reinitFromRequest() - { - assert(req->hasPaddr()); - flags = 0; - addr = req->getPaddr(); - _isSecure = req->isSecure(); - size = req->getSize(); - - src = InvalidPortID; - dest = InvalidPortID; - bytesValidStart = 0; - bytesValidEnd = 0; - busFirstWordDelay = 0; - busLastWordDelay = 0; - - flags.set(VALID_ADDR|VALID_SIZE); deleteData(); } /** * Take a request packet and modify it in place to be suitable for - * returning as a response to that request. The source field is - * turned into the destination, and subsequently cleared. Note - * that the latter is not necessary for atomic requests, but - * causes no harm as neither field is valid. + * returning as a response to that request. */ void makeResponse() { assert(needsResponse()); assert(isRequest()); - origCmd = cmd; cmd = cmd.responseCommand(); // responses are never express, even if the snoop that // triggered them was flags.clear(EXPRESS_SNOOP); - - dest = src; - clearSrc(); } void @@ -802,42 +980,106 @@ class Packet : public Printable flags.set(VALID_SIZE); } + /** + * Check if packet corresponds to a given block-aligned address and + * address space. + * + * @param addr The address to compare against. + * @param is_secure Whether addr belongs to the secure address space. + * @param blk_size Block size in bytes. + * @return Whether packet matches description. + */ + bool matchBlockAddr(const Addr addr, const bool is_secure, + const int blk_size) const; + + /** + * Check if this packet refers to the same block-aligned address and + * address space as another packet. + * + * @param pkt The packet to compare against. + * @param blk_size Block size in bytes. + * @return Whether packet matches description. + */ + bool matchBlockAddr(const PacketPtr pkt, const int blk_size) const; + + /** + * Check if packet corresponds to a given address and address space. + * + * @param addr The address to compare against. + * @param is_secure Whether addr belongs to the secure address space. + * @return Whether packet matches description. + */ + bool matchAddr(const Addr addr, const bool is_secure) const; + + /** + * Check if this packet refers to the same address and address space as + * another packet. + * + * @param pkt The packet to compare against. + * @return Whether packet matches description. + */ + bool matchAddr(const PacketPtr pkt) const; + + public: + /** + * @{ + * @name Data accessor mehtods + */ /** * Set the data pointer to the following value that should not be - * freed. + * freed. Static data allows us to do a single memcpy even if + * multiple packets are required to get from source to destination + * and back. In essence the pointer is set calling dataStatic on + * the original packet, and whenever this packet is copied and + * forwarded the same pointer is passed on. When a packet + * eventually reaches the destination holding the data, it is + * copied once into the location originally set. On the way back + * to the source, no copies are necessary. */ template void dataStatic(T *p) { - assert(flags.noneSet(STATIC_DATA|DYNAMIC_DATA|ARRAY_DATA)); + assert(flags.noneSet(STATIC_DATA|DYNAMIC_DATA)); data = (PacketDataPtr)p; flags.set(STATIC_DATA); } /** - * Set the data pointer to a value that should have delete [] - * called on it. + * Set the data pointer to the following value that should not be + * freed. This version of the function allows the pointer passed + * to us to be const. To avoid issues down the line we cast the + * constness away, the alternative would be to keep both a const + * and non-const data pointer and cleverly choose between + * them. Note that this is only allowed for static data. */ template void - dataDynamicArray(T *p) + dataStaticConst(const T *p) { - assert(flags.noneSet(STATIC_DATA|DYNAMIC_DATA|ARRAY_DATA)); - data = (PacketDataPtr)p; - flags.set(DYNAMIC_DATA|ARRAY_DATA); + assert(flags.noneSet(STATIC_DATA|DYNAMIC_DATA)); + data = const_cast(p); + flags.set(STATIC_DATA); } /** - * set the data pointer to a value that should have delete called - * on it. + * Set the data pointer to a value that should have delete [] + * called on it. Dynamic data is local to this packet, and as the + * packet travels from source to destination, forwarded packets + * will allocate their own data. When a packet reaches the final + * destination it will populate the dynamic data of that specific + * packet, and on the way back towards the source, memcpy will be + * invoked in every step where a new packet was created e.g. in + * the caches. Ultimately when the response reaches the source a + * final memcpy is needed to extract the data from the packet + * before it is deallocated. */ template void dataDynamic(T *p) { - assert(flags.noneSet(STATIC_DATA|DYNAMIC_DATA|ARRAY_DATA)); + assert(flags.noneSet(STATIC_DATA|DYNAMIC_DATA)); data = (PacketDataPtr)p; flags.set(DYNAMIC_DATA); } @@ -847,32 +1089,103 @@ class Packet : public Printable */ template T* - getPtr(bool null_ok = false) + getPtr() { - assert(null_ok || flags.isSet(STATIC_DATA|DYNAMIC_DATA)); + assert(flags.isSet(STATIC_DATA|DYNAMIC_DATA)); + assert(!isMaskedWrite()); return (T*)data; } + template + const T* + getConstPtr() const + { + assert(flags.isSet(STATIC_DATA|DYNAMIC_DATA)); + return (const T*)data; + } + /** - * return the value of what is pointed to in the packet. + * Get the data in the packet byte swapped from big endian to + * host endian. */ template - T get(); + T getBE() const; + + /** + * Get the data in the packet byte swapped from little endian to + * host endian. + */ + template + T getLE() const; + + /** + * Get the data in the packet byte swapped from the specified + * endianness. + */ + template + T get(ByteOrder endian) const; + +#if THE_ISA != NULL_ISA + /** + * Get the data in the packet byte swapped from guest to host + * endian. + */ + template + T get() const + M5_DEPRECATED_MSG("The memory system should be ISA independent."); +#endif + + /** Set the value in the data pointer to v as big endian. */ + template + void setBE(T v); + + /** Set the value in the data pointer to v as little endian. */ + template + void setLE(T v); /** - * set the value in the data pointer to v. + * Set the value in the data pointer to v using the specified + * endianness. */ template - void set(T v); + void set(T v, ByteOrder endian); + +#if THE_ISA != NULL_ISA + /** Set the value in the data pointer to v as guest endian. */ + template + void set(T v) + M5_DEPRECATED_MSG("The memory system should be ISA independent."); +#endif + + /** + * Get the data in the packet byte swapped from the specified + * endianness and zero-extended to 64 bits. + */ + uint64_t getUintX(ByteOrder endian) const; + + /** + * Set the value in the word w after truncating it to the length + * of the packet and then byteswapping it to the desired + * endianness. + */ + void setUintX(uint64_t w, ByteOrder endian); /** * Copy data into the packet from the provided pointer. */ void - setData(uint8_t *p) + setData(const uint8_t *p) { - if (p != getPtr()) + // we should never be copying data onto itself, which means we + // must idenfity packets with static data, as they carry the + // same pointer from source to destination and back + assert(p != getPtr() || flags.isSet(STATIC_DATA)); + + if (p != getPtr()) { + // for packet with allocated dynamic data, we copy data from + // one to the other, e.g. a forwarded response to a response std::memcpy(getPtr(), p, getSize()); + } } /** @@ -880,26 +1193,41 @@ class Packet : public Printable * which is aligned to the given block size. */ void - setDataFromBlock(uint8_t *blk_data, int blkSize) + setDataFromBlock(const uint8_t *blk_data, int blkSize) { setData(blk_data + getOffset(blkSize)); } /** - * Copy data from the packet to the provided block pointer, which - * is aligned to the given block size. + * Copy data from the packet to the memory at the provided pointer. + * @param p Pointer to which data will be copied. */ void - writeData(uint8_t *p) + writeData(uint8_t *p) const { - std::memcpy(p, getPtr(), getSize()); + if (!isMaskedWrite()) { + std::memcpy(p, getConstPtr(), getSize()); + } else { + assert(req->getByteEnable().size() == getSize()); + // Write only the enabled bytes + const uint8_t *base = getConstPtr(); + for (int i = 0; i < getSize(); i++) { + if (req->getByteEnable()[i]) { + p[i] = *(base + i); + } + // Disabled bytes stay untouched + } + } } /** - * Copy data from the packet to the memory at the provided pointer. + * Copy data from the packet to the provided block pointer, which + * is aligned to the given block size. + * @param blk_data Pointer to block to which data will be copied. + * @param blkSize Block size in bytes. */ void - writeDataToBlock(uint8_t *blk_data, int blkSize) + writeDataToBlock(uint8_t *blk_data, int blkSize) const { writeData(blk_data + getOffset(blkSize)); } @@ -911,51 +1239,105 @@ class Packet : public Printable void deleteData() { - if (flags.isSet(ARRAY_DATA)) + if (flags.isSet(DYNAMIC_DATA)) delete [] data; - else if (flags.isSet(DYNAMIC_DATA)) - delete data; - flags.clear(STATIC_DATA|DYNAMIC_DATA|ARRAY_DATA); + flags.clear(STATIC_DATA|DYNAMIC_DATA); data = NULL; } - /** If there isn't data in the packet, allocate some. */ + /** Allocate memory for the packet. */ void allocate() { - if (data) { - assert(flags.isSet(STATIC_DATA|DYNAMIC_DATA)); - return; + // if either this command or the response command has a data + // payload, actually allocate space + if (hasData() || hasRespData()) { + assert(flags.noneSet(STATIC_DATA|DYNAMIC_DATA)); + flags.set(DYNAMIC_DATA); + data = new uint8_t[getSize()]; } - - assert(flags.noneSet(STATIC_DATA|DYNAMIC_DATA)); - flags.set(DYNAMIC_DATA|ARRAY_DATA); - data = new uint8_t[getSize()]; } + /** @} */ + + /** Get the data in the packet without byte swapping. */ + template + T getRaw() const; + + /** Set the value in the data pointer to v without byte swapping. */ + template + void setRaw(T v); + + public: /** - * Check a functional request against a memory value represented - * by a base/size pair and an associated data array. If the - * functional request is a read, it may be satisfied by the memory - * value. If the functional request is a write, it may update the - * memory value. + * Check a functional request against a memory value stored in + * another packet (i.e. an in-transit request or + * response). Returns true if the current packet is a read, and + * the other packet provides the data, which is then copied to the + * current packet. If the current packet is a write, and the other + * packet intersects this one, then we update the data + * accordingly. */ - bool checkFunctional(Printable *obj, Addr base, bool is_secure, int size, - uint8_t *data); + bool + trySatisfyFunctional(PacketPtr other) + { + if (other->isMaskedWrite()) { + // Do not forward data if overlapping with a masked write + if (_isSecure == other->isSecure() && + getAddr() <= (other->getAddr() + other->getSize() - 1) && + other->getAddr() <= (getAddr() + getSize() - 1)) { + warn("Trying to check against a masked write, skipping." + " (addr: 0x%x, other addr: 0x%x)", getAddr(), + other->getAddr()); + } + return false; + } + // all packets that are carrying a payload should have a valid + // data pointer + return trySatisfyFunctional(other, other->getAddr(), other->isSecure(), + other->getSize(), + other->hasData() ? + other->getPtr() : NULL); + } /** - * Check a functional request against a memory value stored in - * another packet (i.e. an in-transit request or response). + * Does the request need to check for cached copies of the same block + * in the memory hierarchy above. + **/ + bool + mustCheckAbove() const + { + return cmd == MemCmd::HardPFReq || isEviction(); + } + + /** + * Is this packet a clean eviction, including both actual clean + * evict packets, but also clean writebacks. */ bool - checkFunctional(PacketPtr other) + isCleanEviction() const + { + return cmd == MemCmd::CleanEvict || cmd == MemCmd::WritebackClean; + } + + bool + isMaskedWrite() const { - uint8_t *data = other->hasData() ? other->getPtr() : NULL; - return checkFunctional(other, other->getAddr(), other->isSecure(), - other->getSize(), data); + return (cmd == MemCmd::WriteReq && !req->getByteEnable().empty()); } + /** + * Check a functional request against a memory value represented + * by a base/size pair and an associated data array. If the + * current packet is a read, it may be satisfied by the memory + * value. If the current packet is a write, it may update the + * memory value. + */ + bool + trySatisfyFunctional(Printable *obj, Addr base, bool is_secure, int size, + uint8_t *_data); + /** * Push label for PrintReq (safe to call unconditionally). */