src/dev/dma_device.hh

   1 /*
   2  * Copyright (c) 2012-2013, 2015, 2017, 2019 ARM Limited
   3  * All rights reserved.
   4  *
   5  * The license below extends only to copyright in the software and shall
   6  * not be construed as granting a license to any other intellectual
   7  * property including but not limited to intellectual property relating
   8  * to a hardware implementation of the functionality of the software
   9  * licensed hereunder.  You may use the software subject to the license
  10  * terms below provided that you ensure that this notice is replicated
  11  * unmodified and in its entirety in all distributions of the software,
  12  * modified or unmodified, in source code or in binary form.
  13  *
  14  * Copyright (c) 2004-2005 The Regents of The University of Michigan
  15  * All rights reserved.
  16  *
  17  * Redistribution and use in source and binary forms, with or without
  18  * modification, are permitted provided that the following conditions are
  19  * met: redistributions of source code must retain the above copyright
  20  * notice, this list of conditions and the following disclaimer;
  21  * redistributions in binary form must reproduce the above copyright
  22  * notice, this list of conditions and the following disclaimer in the
  23  * documentation and/or other materials provided with the distribution;
  24  * neither the name of the copyright holders nor the names of its
  25  * contributors may be used to endorse or promote products derived from
  26  * this software without specific prior written permission.
  27  *
  28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  33  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  34  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  35  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  36  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  37  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  38  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  39  */
  40
  41 #ifndef __DEV_DMA_DEVICE_HH__
  42 #define __DEV_DMA_DEVICE_HH__
  43
  44 #include <deque>
  45 #include <memory>
  46
  47 #include "base/circlebuf.hh"
  48 #include "dev/io_device.hh"
  49 #include "mem/port_proxy.hh"
  50 #include "params/DmaDevice.hh"
  51 #include "sim/drain.hh"
  52 #include "sim/system.hh"
  53
  54 class ClockedObject;
  55
  56 class DmaPort : public RequestPort, public Drainable
  57 {
  58   private:
  59
  60     /**
  61      * Take the first packet of the transmit list and attempt to send
  62      * it as a timing request. If it is successful, schedule the
  63      * sending of the next packet, otherwise remember that we are
  64      * waiting for a retry.
  65      */
  66     void trySendTimingReq();
  67
  68     /**
  69      * For timing, attempt to send the first item on the transmit
  70      * list, and if it is successful and there are more packets
  71      * waiting, then schedule the sending of the next packet. For
  72      * atomic, simply send and process everything on the transmit
  73      * list.
  74      */
  75     void sendDma();
  76
  77     /**
  78      * Handle a response packet by updating the corresponding DMA
  79      * request state to reflect the bytes received, and also update
  80      * the pending request counter. If the DMA request that this
  81      * packet is part of is complete, then signal the completion event
  82      * if present, potentially with a delay added to it.
  83      *
  84      * @param pkt Response packet to handler
  85      * @param delay Additional delay for scheduling the completion event
  86      */
  87     void handleResp(PacketPtr pkt, Tick delay=0);
  88
  89     struct DmaReqState : public Packet::SenderState
  90     {
  91         /** Event to call on the device when this transaction (all packets)
  92          * complete. */
  93         Event *completionEvent;
  94
  95         /** Total number of bytes that this transaction involves. */
  96         const Addr totBytes;
  97
  98         /** Number of bytes that have been acked for this transaction. */
  99         Addr numBytes = 0;
 100
 101         /** Amount to delay completion of dma by */
 102         const Tick delay;
 103
 104         DmaReqState(Event *ce, Addr tb, Tick _delay)
 105             : completionEvent(ce), totBytes(tb), delay(_delay)
 106         {}
 107     };
 108
 109   public:
 110     /** The device that owns this port. */
 111     ClockedObject *const device;
 112
 113     /** The system that device/port are in. This is used to select which mode
 114      * we are currently operating in. */
 115     System *const sys;
 116
 117     /** Id for all requests */
 118     const RequestorID requestorId;
 119
 120   protected:
 121     /** Use a deque as we never do any insertion or removal in the middle */
 122     std::deque<PacketPtr> transmitList;
 123
 124     /** Event used to schedule a future sending from the transmit list. */
 125     EventFunctionWrapper sendEvent;
 126
 127     /** Number of outstanding packets the dma port has. */
 128     uint32_t pendingCount = 0;
 129
 130     /** If the port is currently waiting for a retry before it can
 131      * send whatever it is that it's sending. */
 132     bool inRetry = false;
 133
 134     /** Default streamId */
 135     const uint32_t defaultSid;
 136
 137     /** Default substreamId */
 138     const uint32_t defaultSSid;
 139
 140     const int cacheLineSize;
 141
 142   protected:
 143
 144     bool recvTimingResp(PacketPtr pkt) override;
 145     void recvReqRetry() override;
 146
 147     void queueDma(PacketPtr pkt);
 148
 149   public:
 150
 151     DmaPort(ClockedObject *dev, System *s, uint32_t sid=0, uint32_t ssid=0);
 152
 153     void
 154     dmaAction(Packet::Command cmd, Addr addr, int size, Event *event,
 155               uint8_t *data, Tick delay, Request::Flags flag=0);
 156
 157     void
 158     dmaAction(Packet::Command cmd, Addr addr, int size, Event *event,
 159               uint8_t *data, uint32_t sid, uint32_t ssid, Tick delay,
 160               Request::Flags flag=0);
 161
 162     bool dmaPending() const { return pendingCount > 0; }
 163
 164     DrainState drain() override;
 165 };
 166
 167 class DmaDevice : public PioDevice
 168 {
 169    protected:
 170     DmaPort dmaPort;
 171
 172   public:
 173     typedef DmaDeviceParams Params;
 174     DmaDevice(const Params &p);
 175     virtual ~DmaDevice() = default;
 176
 177     void
 178     dmaWrite(Addr addr, int size, Event *event, uint8_t *data,
 179              uint32_t sid, uint32_t ssid, Tick delay=0)
 180     {
 181         dmaPort.dmaAction(MemCmd::WriteReq, addr, size, event, data,
 182                           sid, ssid, delay);
 183     }
 184
 185     void
 186     dmaWrite(Addr addr, int size, Event *event, uint8_t *data, Tick delay=0)
 187     {
 188         dmaPort.dmaAction(MemCmd::WriteReq, addr, size, event, data, delay);
 189     }
 190
 191     void
 192     dmaRead(Addr addr, int size, Event *event, uint8_t *data,
 193             uint32_t sid, uint32_t ssid, Tick delay=0)
 194     {
 195         dmaPort.dmaAction(MemCmd::ReadReq, addr, size, event, data,
 196                           sid, ssid, delay);
 197     }
 198
 199     void
 200     dmaRead(Addr addr, int size, Event *event, uint8_t *data, Tick delay=0)
 201     {
 202         dmaPort.dmaAction(MemCmd::ReadReq, addr, size, event, data, delay);
 203     }
 204
 205     bool dmaPending() const { return dmaPort.dmaPending(); }
 206
 207     void init() override;
 208
 209     unsigned int cacheBlockSize() const { return sys->cacheLineSize(); }
 210
 211     Port &getPort(const std::string &if_name,
 212                   PortID idx=InvalidPortID) override;
 213
 214 };
 215
 216 /**
 217  * DMA callback class.
 218  *
 219  * Allows one to register for a callback event after a sequence of (potentially
 220  * non-contiguous) DMA transfers on a DmaPort completes.  Derived classes must
 221  * implement the process() method and use getChunkEvent() to allocate a
 222  * callback event for each participating DMA.
 223  */
 224 class DmaCallback : public Drainable
 225 {
 226   public:
 227     virtual const std::string name() const { return "DmaCallback"; }
 228
 229     /**
 230      * DmaPort ensures that all oustanding DMA accesses have completed before
 231      * it finishes draining.  However, DmaChunkEvents scheduled with a delay
 232      * might still be sitting on the event queue.  Therefore, draining is not
 233      * complete until count is 0, which ensures that all outstanding
 234      * DmaChunkEvents associated with this DmaCallback have fired.
 235      */
 236     DrainState
 237     drain() override
 238     {
 239         return count ? DrainState::Draining : DrainState::Drained;
 240     }
 241
 242   protected:
 243     int count = 0;
 244
 245     virtual ~DmaCallback() = default;
 246
 247     /**
 248      * Callback function invoked on completion of all chunks.
 249      */
 250     virtual void process() = 0;
 251
 252   private:
 253     /**
 254      * Called by DMA engine completion event on each chunk completion.
 255      * Since the object may delete itself here, callers should not use
 256      * the object pointer after calling this function.
 257      */
 258     void
 259     chunkComplete()
 260     {
 261         if (--count == 0) {
 262             process();
 263             // Need to notify DrainManager that this object is finished
 264             // draining, even though it is immediately deleted.
 265             signalDrainDone();
 266             delete this;
 267         }
 268     }
 269
 270   public:
 271
 272     /**
 273      * Request a chunk event.  Chunks events should be provided to each DMA
 274      * request that wishes to participate in this DmaCallback.
 275      */
 276     Event *
 277     getChunkEvent()
 278     {
 279         ++count;
 280         return new EventFunctionWrapper([this]{ chunkComplete(); }, name(),
 281                                         true);
 282     }
 283 };
 284
 285 /**
 286  * Buffered DMA engine helper class
 287  *
 288  * This class implements a simple DMA engine that feeds a FIFO
 289  * buffer. The size of the buffer, the maximum number of pending
 290  * requests and the maximum request size are all set when the engine
 291  * is instantiated.
 292  *
 293  * An <i>asynchronous</i> transfer of a <i>block</i> of data
 294  * (designated by a start address and a size) is started by calling
 295  * the startFill() method. The DMA engine will aggressively try to
 296  * keep the internal FIFO full. As soon as there is room in the FIFO
 297  * for more data <i>and</i> there are free request slots, a new fill
 298  * will be started.
 299  *
 300  * Data in the FIFO can be read back using the get() and tryGet()
 301  * methods. Both request a block of data from the FIFO. However, get()
 302  * panics if the block cannot be satisfied, while tryGet() simply
 303  * returns false. The latter call makes it possible to implement
 304  * custom buffer underrun handling.
 305  *
 306  * A simple use case would be something like this:
 307  * \code{.cpp}
 308  *     // Create a DMA engine with a 1KiB buffer. Issue up to 8 concurrent
 309  *     // uncacheable 64 byte (maximum) requests.
 310  *     DmaReadFifo *dma = new DmaReadFifo(port, 1024, 64, 8,
 311  *                                        Request::UNCACHEABLE);
 312  *
 313  *     // Start copying 4KiB data from 0xFF000000
 314  *     dma->startFill(0xFF000000, 0x1000);
 315  *
 316  *     // Some time later when there is data in the FIFO.
 317  *     uint8_t data[8];
 318  *     dma->get(data, sizeof(data))
 319  * \endcode
 320  *
 321  *
 322  * The DMA engine allows new blocks to be requested as soon as the
 323  * last request for a block has been sent (i.e., there is no need to
 324  * wait for pending requests to complete). This can be queried with
 325  * the atEndOfBlock() method and more advanced implementations may
 326  * override the onEndOfBlock() callback.
 327  */
 328 class DmaReadFifo : public Drainable, public Serializable
 329 {
 330   public:
 331     DmaReadFifo(DmaPort &port, size_t size,
 332                 unsigned max_req_size,
 333                 unsigned max_pending,
 334                 Request::Flags flags=0);
 335
 336     ~DmaReadFifo();
 337
 338   public: // Serializable
 339     void serialize(CheckpointOut &cp) const override;
 340     void unserialize(CheckpointIn &cp) override;
 341
 342   public: // Drainable
 343     DrainState drain() override;
 344
 345   public: // FIFO access
 346     /**
 347      * @{
 348      * @name FIFO access
 349      */
 350     /**
 351      * Try to read data from the FIFO.
 352      *
 353      * This method reads len bytes of data from the FIFO and stores
 354      * them in the memory location pointed to by dst. The method
 355      * fails, and no data is written to the buffer, if the FIFO
 356      * doesn't contain enough data to satisfy the request.
 357      *
 358      * @param dst Pointer to a destination buffer
 359      * @param len Amount of data to read.
 360      * @return true on success, false otherwise.
 361      */
 362     bool tryGet(uint8_t *dst, size_t len);
 363
 364     template<typename T>
 365     bool
 366     tryGet(T &value)
 367     {
 368         return tryGet(static_cast<T *>(&value), sizeof(T));
 369     };
 370
 371     /**
 372      * Read data from the FIFO and panic on failure.
 373      *
 374      * @see tryGet()
 375      *
 376      * @param dst Pointer to a destination buffer
 377      * @param len Amount of data to read.
 378      */
 379     void get(uint8_t *dst, size_t len);
 380
 381     template<typename T>
 382     T
 383     get()
 384     {
 385         T value;
 386         get(static_cast<uint8_t *>(&value), sizeof(T));
 387         return value;
 388     };
 389
 390     /** Get the amount of data stored in the FIFO */
 391     size_t size() const { return buffer.size(); }
 392     /** Flush the FIFO */
 393     void flush() { buffer.flush(); }
 394
 395     /** @} */
 396   public: // FIFO fill control
 397     /**
 398      * @{
 399      * @name FIFO fill control
 400      */
 401     /**
 402      * Start filling the FIFO.
 403      *
 404      * @warn It's considered an error to call start on an active DMA
 405      * engine unless the last request from the active block has been
 406      * sent (i.e., atEndOfBlock() is true).
 407      *
 408      * @param start Physical address to copy from.
 409      * @param size Size of the block to copy.
 410      */
 411     void startFill(Addr start, size_t size);
 412
 413     /**
 414      * Stop the DMA engine.
 415      *
 416      * Stop filling the FIFO and ignore incoming responses for pending
 417      * requests. The onEndOfBlock() callback will not be called after
 418      * this method has been invoked. However, once the last response
 419      * has been received, the onIdle() callback will still be called.
 420      */
 421     void stopFill();
 422
 423     /**
 424      * Has the DMA engine sent out the last request for the active
 425      * block?
 426      */
 427     bool atEndOfBlock() const { return nextAddr == endAddr; }
 428
 429     /**
 430      * Is the DMA engine active (i.e., are there still in-flight
 431      * accesses)?
 432      */
 433     bool
 434     isActive() const
 435     {
 436         return !(pendingRequests.empty() && atEndOfBlock());
 437     }
 438
 439     /** @} */
 440   protected: // Callbacks
 441     /**
 442      * @{
 443      * @name Callbacks
 444      */
 445     /**
 446      * End of block callback
 447      *
 448      * This callback is called <i>once</i> after the last access in a
 449      * block has been sent. It is legal for a derived class to call
 450      * startFill() from this method to initiate a transfer.
 451      */
 452     virtual void onEndOfBlock() {};
 453
 454     /**
 455      * Last response received callback
 456      *
 457      * This callback is called when the DMA engine becomes idle (i.e.,
 458      * there are no pending requests).
 459      *
 460      * It is possible for a DMA engine to reach the end of block and
 461      * become idle at the same tick. In such a case, the
 462      * onEndOfBlock() callback will be called first. This callback
 463      * will <i>NOT</i> be called if that callback initiates a new DMA transfer.
 464      */
 465     virtual void onIdle() {};
 466
 467     /** @} */
 468   private: // Configuration
 469     /** Maximum request size in bytes */
 470     const Addr maxReqSize;
 471     /** Maximum FIFO size in bytes */
 472     const size_t fifoSize;
 473     /** Request flags */
 474     const Request::Flags reqFlags;
 475
 476     DmaPort &port;
 477     PortProxy proxy;
 478
 479     const int cacheLineSize;
 480
 481   private:
 482     class DmaDoneEvent : public Event
 483     {
 484       public:
 485         DmaDoneEvent(DmaReadFifo *_parent, size_t max_size);
 486
 487         void kill();
 488         void cancel();
 489         bool canceled() const { return _canceled; }
 490         void reset(size_t size);
 491         void process();
 492
 493         bool done() const { return _done; }
 494         size_t requestSize() const { return _requestSize; }
 495         const uint8_t *data() const { return _data.data(); }
 496         uint8_t *data() { return _data.data(); }
 497
 498       private:
 499         DmaReadFifo *parent;
 500         bool _done = false;
 501         bool _canceled = false;
 502         size_t _requestSize;
 503         std::vector<uint8_t> _data;
 504     };
 505
 506     typedef std::unique_ptr<DmaDoneEvent> DmaDoneEventUPtr;
 507
 508     /**
 509      * DMA request done, handle incoming data and issue new
 510      * request.
 511      */
 512     void dmaDone();
 513
 514     /** Handle pending requests that have been flagged as done. */
 515     void handlePending();
 516
 517     /** Try to issue new DMA requests or bypass DMA requests*/
 518     void resumeFill();
 519
 520     /** Try to issue new DMA requests during normal execution*/
 521     void resumeFillTiming();
 522
 523     /** Try to bypass DMA requests in KVM execution mode */
 524     void resumeFillFunctional();
 525
 526   private: // Internal state
 527     Fifo<uint8_t> buffer;
 528
 529     Addr nextAddr = 0;
 530     Addr endAddr = 0;
 531
 532     std::deque<DmaDoneEventUPtr> pendingRequests;
 533     std::deque<DmaDoneEventUPtr> freeRequests;
 534 };
 535
 536 #endif // __DEV_DMA_DEVICE_HH__