#include "dev/dma_device.hh"
+#include <algorithm>
+#include <cassert>
+#include <cstring>
#include <utility>
+#include "base/logging.hh"
+#include "base/trace.hh"
#include "debug/DMA.hh"
#include "debug/Drain.hh"
#include "sim/clocked_object.hh"
{ }
void
-DmaPort::handleResp(PacketPtr pkt, Tick delay)
+DmaPort::handleRespPacket(PacketPtr pkt, Tick delay)
{
// Should always see a response with a sender state.
assert(pkt->isResponse());
auto *state = dynamic_cast<DmaReqState*>(pkt->senderState);
assert(state);
+ handleResp(state, pkt->getAddr(), pkt->req->getSize(), delay);
+
+ delete pkt;
+}
+
+void
+DmaPort::handleResp(DmaReqState *state, Addr addr, Addr size, Tick delay)
+{
DPRINTF(DMA, "Received response %s for addr: %#x size: %d nb: %d," \
" tot: %d sched %d\n",
- pkt->cmdString(), pkt->getAddr(), pkt->req->getSize(),
+ MemCmd(state->cmd).toString(), addr, size,
state->numBytes, state->totBytes,
state->completionEvent ?
state->completionEvent->scheduled() : 0);
// Update the number of bytes received based on the request rather
// than the packet as the latter could be rounded up to line sizes.
- state->numBytes += pkt->req->getSize();
+ state->numBytes += size;
assert(state->totBytes >= state->numBytes);
// If we have reached the total number of bytes for this DMA request,
delete state;
}
- delete pkt;
-
// We might be drained at this point, if so signal the drain event.
if (pendingCount == 0)
signalDrainDone();
assert(pkt->req->isUncacheable() ||
!(pkt->cacheResponding() && !pkt->hasSharers()));
- handleResp(pkt);
+ handleRespPacket(pkt);
return true;
}
transmitList.size(), inRetry ? 1 : 0);
}
+bool
+DmaPort::sendAtomicReq(DmaReqState *state)
+{
+ PacketPtr pkt = state->createPacket();
+ DPRINTF(DMA, "Sending DMA for addr: %#x size: %d\n",
+ state->gen.addr(), state->gen.size());
+ Tick lat = sendAtomic(pkt);
+
+ // Check if we're done, since handleResp may delete state.
+ bool done = !state->gen.next();
+ handleRespPacket(pkt, lat);
+ return done;
+}
+
+bool
+DmaPort::sendAtomicBdReq(DmaReqState *state)
+{
+ bool done = false;
+
+ auto bd_it = memBackdoors.contains(state->gen.addr());
+ if (bd_it == memBackdoors.end()) {
+ // We don't have a backdoor for this address, so use a packet.
+
+ PacketPtr pkt = state->createPacket();
+ DPRINTF(DMA, "Sending DMA for addr: %#x size: %d\n",
+ state->gen.addr(), state->gen.size());
+
+ MemBackdoorPtr bd = nullptr;
+ Tick lat = sendAtomicBackdoor(pkt, bd);
+
+ // If we got a backdoor, record it.
+ if (bd && memBackdoors.insert(bd->range(), bd) != memBackdoors.end()) {
+ // Invalidation callback which finds this backdoor and removes it.
+ auto callback = [this](const MemBackdoor &backdoor) {
+ for (auto it = memBackdoors.begin();
+ it != memBackdoors.end(); it++) {
+ if (it->second == &backdoor) {
+ memBackdoors.erase(it);
+ return;
+ }
+ }
+ panic("Got invalidation for unknown memory backdoor.");
+ };
+ bd->addInvalidationCallback(callback);
+ }
+
+ // Check if we're done now, since handleResp may delete state.
+ done = !state->gen.next();
+ handleRespPacket(pkt, lat);
+ } else {
+ // We have a backdoor that can at least partially satisfy this request.
+ DPRINTF(DMA, "Handling DMA for addr: %#x size %d through backdoor\n",
+ state->gen.addr(), state->gen.size());
+
+ const auto *bd = bd_it->second;
+ // Offset of this access into the backdoor.
+ const Addr offset = state->gen.addr() - bd->range().start();
+ // How many bytes we still need.
+ const Addr remaining = state->totBytes - state->gen.complete();
+ // How many bytes this backdoor can provide, starting from offset.
+ const Addr available = bd->range().size() - offset;
+
+ // How many bytes we're going to handle through this backdoor.
+ const Addr handled = std::min(remaining, available);
+
+ // If there's a buffer for data, read/write it.
+ if (state->data) {
+ uint8_t *bd_data = bd->ptr() + offset;
+ uint8_t *state_data = state->data + state->gen.complete();
+ if (MemCmd(state->cmd).isRead())
+ memcpy(state_data, bd_data, handled);
+ else
+ memcpy(bd_data, state_data, handled);
+ }
+
+ // Advance the chunk generator past this region of memory.
+ state->gen.setNext(state->gen.addr() + handled);
+
+ // Check if we're done now, since handleResp may delete state.
+ done = !state->gen.next();
+ handleResp(state, state->gen.addr(), handled);
+ }
+
+ return done;
+}
+
void
DmaPort::sendDma()
{
assert(transmitList.size());
if (sys->isTimingMode()) {
- // if we are either waiting for a retry or are still waiting
- // after sending the last packet, then do not proceed
+ // If we are either waiting for a retry or are still waiting after
+ // sending the last packet, then do not proceed.
if (inRetry || sendEvent.scheduled()) {
DPRINTF(DMA, "Can't send immediately, waiting to send\n");
return;
trySendTimingReq();
} else if (sys->isAtomicMode()) {
- // send everything there is to send in zero time
+ const bool bypass = sys->bypassCaches();
+
+ // Send everything there is to send in zero time.
while (!transmitList.empty()) {
DmaReqState *state = transmitList.front();
transmitList.pop_front();
bool done = state->gen.done();
- while (!done) {
- PacketPtr pkt = state->createPacket();
- DPRINTF(DMA, "Sending DMA for addr: %#x size: %d\n",
- pkt->req->getPaddr(), pkt->req->getSize());
- Tick lat = sendAtomic(pkt);
-
- // Check if we're done now, since handleResp may delete state.
- done = !state->gen.next();
- handleResp(pkt, lat);
- }
+ while (!done)
+ done = bypass ? sendAtomicBdReq(state) : sendAtomicReq(state);
}
} else {
panic("Unknown memory mode.");
#include <deque>
#include <memory>
+#include "base/addr_range_map.hh"
#include "base/chunk_generator.hh"
#include "base/circlebuf.hh"
#include "dev/io_device.hh"
+#include "mem/backdoor.hh"
#include "mem/port_proxy.hh"
#include "params/DmaDevice.hh"
#include "sim/drain.hh"
class DmaPort : public RequestPort, public Drainable
{
private:
+ AddrRangeMap<MemBackdoorPtr, 1> memBackdoors;
/**
- * Take the first packet of the transmit list and attempt to send
- * it as a timing request. If it is successful, schedule the
- * sending of the next packet, otherwise remember that we are
- * waiting for a retry.
+ * Take the first request on the transmit list and attempt to send a timing
+ * packet from it. If it is successful, schedule the sending of the next
+ * packet. Otherwise remember that we are waiting for a retry.
*/
void trySendTimingReq();
*/
void sendDma();
- /**
- * Handle a response packet by updating the corresponding DMA
- * request state to reflect the bytes received, and also update
- * the pending request counter. If the DMA request that this
- * packet is part of is complete, then signal the completion event
- * if present, potentially with a delay added to it.
- *
- * @param pkt Response packet to handler
- * @param delay Additional delay for scheduling the completion event
- */
- void handleResp(PacketPtr pkt, Tick delay=0);
-
struct DmaReqState : public Packet::SenderState
{
/** Event to call on the device when this transaction (all packets)
PacketPtr createPacket();
};
+ /** Send the next packet from a DMA request in atomic mode. */
+ bool sendAtomicReq(DmaReqState *state);
+ /**
+ * Send the next packet from a DMA request in atomic mode, and request
+ * and/or use memory backdoors if possible.
+ */
+ bool sendAtomicBdReq(DmaReqState *state);
+
+ /**
+ * Handle a response packet by updating the corresponding DMA
+ * request state to reflect the bytes received, and also update
+ * the pending request counter. If the DMA request that this
+ * packet is part of is complete, then signal the completion event
+ * if present, potentially with a delay added to it.
+ *
+ * @param pkt Response packet to handler
+ * @param delay Additional delay for scheduling the completion event
+ */
+ void handleRespPacket(PacketPtr pkt, Tick delay=0);
+ void handleResp(DmaReqState *state, Addr addr, Addr size, Tick delay=0);
+
public:
/** The device that owns this port. */
ClockedObject *const device;
projects / gem5.git / commitdiff