63642c905187c782c29ecf4b9bfb269b301d4456
2 * Copyright (c) 2012, 2015, 2017, 2019 ARM Limited
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
14 * Copyright (c) 2006 The Regents of The University of Michigan
15 * All rights reserved.
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions are
19 * met: redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer;
21 * redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution;
24 * neither the name of the copyright holders nor the names of its
25 * contributors may be used to endorse or promote products derived from
26 * this software without specific prior written permission.
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
41 #include "dev/dma_device.hh"
45 #include "base/chunk_generator.hh"
46 #include "debug/DMA.hh"
47 #include "debug/Drain.hh"
48 #include "mem/port_proxy.hh"
49 #include "sim/clocked_object.hh"
50 #include "sim/system.hh"
52 DmaPort::DmaPort(ClockedObject
*dev
, System
*s
,
53 uint32_t sid
, uint32_t ssid
)
54 : MasterPort(dev
->name() + ".dma", dev
),
55 device(dev
), sys(s
), masterId(s
->getMasterId(dev
)),
56 sendEvent([this]{ sendDma(); }, dev
->name()),
57 pendingCount(0), inRetry(false),
63 DmaPort::handleResp(PacketPtr pkt
, Tick delay
)
65 // should always see a response with a sender state
66 assert(pkt
->isResponse());
68 // get the DMA sender state
69 DmaReqState
*state
= dynamic_cast<DmaReqState
*>(pkt
->senderState
);
72 DPRINTF(DMA
, "Received response %s for addr: %#x size: %d nb: %d," \
73 " tot: %d sched %d\n",
74 pkt
->cmdString(), pkt
->getAddr(), pkt
->req
->getSize(),
75 state
->numBytes
, state
->totBytes
,
76 state
->completionEvent
?
77 state
->completionEvent
->scheduled() : 0);
79 assert(pendingCount
!= 0);
82 // update the number of bytes received based on the request rather
83 // than the packet as the latter could be rounded up to line sizes
84 state
->numBytes
+= pkt
->req
->getSize();
85 assert(state
->totBytes
>= state
->numBytes
);
87 // if we have reached the total number of bytes for this DMA
88 // request, then signal the completion and delete the sate
89 if (state
->totBytes
== state
->numBytes
) {
90 if (state
->completionEvent
) {
91 delay
+= state
->delay
;
92 device
->schedule(state
->completionEvent
, curTick() + delay
);
100 // we might be drained at this point, if so signal the drain event
101 if (pendingCount
== 0)
106 DmaPort::recvTimingResp(PacketPtr pkt
)
108 // We shouldn't ever get a cacheable block in Modified state
109 assert(pkt
->req
->isUncacheable() ||
110 !(pkt
->cacheResponding() && !pkt
->hasSharers()));
117 DmaDevice::DmaDevice(const Params
*p
)
118 : PioDevice(p
), dmaPort(this, sys
, p
->sid
, p
->ssid
)
124 if (!dmaPort
.isConnected())
125 panic("DMA port of %s not connected to anything!", name());
132 if (pendingCount
== 0) {
133 return DrainState::Drained
;
135 DPRINTF(Drain
, "DmaPort not drained\n");
136 return DrainState::Draining
;
141 DmaPort::recvReqRetry()
143 assert(transmitList
.size());
148 DmaPort::dmaAction(Packet::Command cmd
, Addr addr
, int size
, Event
*event
,
149 uint8_t *data
, uint32_t sid
, uint32_t ssid
, Tick delay
,
152 // one DMA request sender state for every action, that is then
153 // split into many requests and packets based on the block size,
154 // i.e. cache line size
155 DmaReqState
*reqState
= new DmaReqState(event
, size
, delay
);
157 // (functionality added for Table Walker statistics)
158 // We're only interested in this when there will only be one request.
159 // For simplicity, we return the last request, which would also be
160 // the only request in that case.
161 RequestPtr req
= NULL
;
163 DPRINTF(DMA
, "Starting DMA for addr: %#x size: %d sched: %d\n", addr
, size
,
164 event
? event
->scheduled() : -1);
165 for (ChunkGenerator
gen(addr
, size
, sys
->cacheLineSize());
166 !gen
.done(); gen
.next()) {
168 req
= std::make_shared
<Request
>(
169 gen
.addr(), gen
.size(), flag
, masterId
);
171 req
->setStreamId(sid
);
172 req
->setSubStreamId(ssid
);
174 req
->taskId(ContextSwitchTaskId::DMA
);
175 PacketPtr pkt
= new Packet(req
, cmd
);
177 // Increment the data pointer on a write
179 pkt
->dataStatic(data
+ gen
.complete());
181 pkt
->senderState
= reqState
;
183 DPRINTF(DMA
, "--Queuing DMA for addr: %#x size: %d\n", gen
.addr(),
188 // in zero time also initiate the sending of the packets we have
189 // just created, for atomic this involves actually completing all
197 DmaPort::dmaAction(Packet::Command cmd
, Addr addr
, int size
, Event
*event
,
198 uint8_t *data
, Tick delay
, Request::Flags flag
)
200 return dmaAction(cmd
, addr
, size
, event
, data
,
201 defaultSid
, defaultSSid
, delay
, flag
);
205 DmaPort::queueDma(PacketPtr pkt
)
207 transmitList
.push_back(pkt
);
209 // remember that we have another packet pending, this will only be
210 // decremented once a response comes back
215 DmaPort::trySendTimingReq()
217 // send the first packet on the transmit list and schedule the
218 // following send if it is successful
219 PacketPtr pkt
= transmitList
.front();
221 DPRINTF(DMA
, "Trying to send %s addr %#x\n", pkt
->cmdString(),
224 inRetry
= !sendTimingReq(pkt
);
226 transmitList
.pop_front();
227 DPRINTF(DMA
, "-- Done\n");
228 // if there is more to do, then do so
229 if (!transmitList
.empty())
230 // this should ultimately wait for as many cycles as the
231 // device needs to send the packet, but currently the port
232 // does not have any known width so simply wait a single
234 device
->schedule(sendEvent
, device
->clockEdge(Cycles(1)));
236 DPRINTF(DMA
, "-- Failed, waiting for retry\n");
239 DPRINTF(DMA
, "TransmitList: %d, inRetry: %d\n",
240 transmitList
.size(), inRetry
);
246 // some kind of selcetion between access methods
247 // more work is going to have to be done to make
248 // switching actually work
249 assert(transmitList
.size());
251 if (sys
->isTimingMode()) {
252 // if we are either waiting for a retry or are still waiting
253 // after sending the last packet, then do not proceed
254 if (inRetry
|| sendEvent
.scheduled()) {
255 DPRINTF(DMA
, "Can't send immediately, waiting to send\n");
260 } else if (sys
->isAtomicMode()) {
261 // send everything there is to send in zero time
262 while (!transmitList
.empty()) {
263 PacketPtr pkt
= transmitList
.front();
264 transmitList
.pop_front();
266 DPRINTF(DMA
, "Sending DMA for addr: %#x size: %d\n",
267 pkt
->req
->getPaddr(), pkt
->req
->getSize());
268 Tick lat
= sendAtomic(pkt
);
270 handleResp(pkt
, lat
);
273 panic("Unknown memory mode.");
277 DmaDevice::getPort(const std::string
&if_name
, PortID idx
)
279 if (if_name
== "dma") {
282 return PioDevice::getPort(if_name
, idx
);
285 DmaReadFifo::DmaReadFifo(DmaPort
&_port
, size_t size
,
286 unsigned max_req_size
,
287 unsigned max_pending
,
288 Request::Flags flags
)
289 : maxReqSize(max_req_size
), fifoSize(size
),
290 reqFlags(flags
), port(_port
),
292 nextAddr(0), endAddr(0)
294 freeRequests
.resize(max_pending
);
295 for (auto &e
: freeRequests
)
296 e
.reset(new DmaDoneEvent(this, max_req_size
));
300 DmaReadFifo::~DmaReadFifo()
302 for (auto &p
: pendingRequests
) {
303 DmaDoneEvent
*e(p
.release());
308 // We can't kill in-flight DMAs, so we'll just transfer
309 // ownership to the event queue so that they get freed
310 // when they are done.
317 DmaReadFifo::serialize(CheckpointOut
&cp
) const
319 assert(pendingRequests
.empty());
321 SERIALIZE_CONTAINER(buffer
);
322 SERIALIZE_SCALAR(endAddr
);
323 SERIALIZE_SCALAR(nextAddr
);
327 DmaReadFifo::unserialize(CheckpointIn
&cp
)
329 UNSERIALIZE_CONTAINER(buffer
);
330 UNSERIALIZE_SCALAR(endAddr
);
331 UNSERIALIZE_SCALAR(nextAddr
);
335 DmaReadFifo::tryGet(uint8_t *dst
, size_t len
)
337 if (buffer
.size() >= len
) {
338 buffer
.read(dst
, len
);
347 DmaReadFifo::get(uint8_t *dst
, size_t len
)
349 const bool success(tryGet(dst
, len
));
350 panic_if(!success
, "Buffer underrun in DmaReadFifo::get()\n");
354 DmaReadFifo::startFill(Addr start
, size_t size
)
356 assert(atEndOfBlock());
359 endAddr
= start
+ size
;
364 DmaReadFifo::stopFill()
366 // Prevent new DMA requests by setting the next address to the end
367 // address. Pending requests will still complete.
370 // Flag in-flight accesses as canceled. This prevents their data
371 // from being written to the FIFO.
372 for (auto &p
: pendingRequests
)
377 DmaReadFifo::resumeFill()
379 // Don't try to fetch more data if we are draining. This ensures
380 // that the DMA engine settles down before we checkpoint it.
381 if (drainState() == DrainState::Draining
)
384 const bool old_eob(atEndOfBlock());
386 if (port
.sys
->bypassCaches())
387 resumeFillFunctional();
391 if (!old_eob
&& atEndOfBlock())
396 DmaReadFifo::resumeFillFunctional()
398 const size_t fifo_space
= buffer
.capacity() - buffer
.size();
399 const size_t kvm_watermark
= port
.sys
->cacheLineSize();
400 if (fifo_space
>= kvm_watermark
|| buffer
.capacity() < kvm_watermark
) {
401 const size_t block_remaining
= endAddr
- nextAddr
;
402 const size_t xfer_size
= std::min(fifo_space
, block_remaining
);
403 std::vector
<uint8_t> tmp_buffer(xfer_size
);
405 assert(pendingRequests
.empty());
406 DPRINTF(DMA
, "KVM Bypassing startAddr=%#x xfer_size=%#x " \
407 "fifo_space=%#x block_remaining=%#x\n",
408 nextAddr
, xfer_size
, fifo_space
, block_remaining
);
410 port
.sys
->physProxy
.readBlob(nextAddr
, tmp_buffer
.data(), xfer_size
);
411 buffer
.write(tmp_buffer
.begin(), xfer_size
);
412 nextAddr
+= xfer_size
;
417 DmaReadFifo::resumeFillTiming()
419 size_t size_pending(0);
420 for (auto &e
: pendingRequests
)
421 size_pending
+= e
->requestSize();
423 while (!freeRequests
.empty() && !atEndOfBlock()) {
424 const size_t req_size(std::min(maxReqSize
, endAddr
- nextAddr
));
425 if (buffer
.size() + size_pending
+ req_size
> fifoSize
)
428 DmaDoneEventUPtr
event(std::move(freeRequests
.front()));
429 freeRequests
.pop_front();
432 event
->reset(req_size
);
433 port
.dmaAction(MemCmd::ReadReq
, nextAddr
, req_size
, event
.get(),
434 event
->data(), 0, reqFlags
);
435 nextAddr
+= req_size
;
436 size_pending
+= req_size
;
438 pendingRequests
.emplace_back(std::move(event
));
443 DmaReadFifo::dmaDone()
445 const bool old_active(isActive());
450 if (old_active
&& !isActive())
455 DmaReadFifo::handlePending()
457 while (!pendingRequests
.empty() && pendingRequests
.front()->done()) {
458 // Get the first finished pending request
459 DmaDoneEventUPtr
event(std::move(pendingRequests
.front()));
460 pendingRequests
.pop_front();
462 if (!event
->canceled())
463 buffer
.write(event
->data(), event
->requestSize());
465 // Move the event to the list of free requests
466 freeRequests
.emplace_back(std::move(event
));
469 if (pendingRequests
.empty())
476 return pendingRequests
.empty() ? DrainState::Drained
: DrainState::Draining
;
480 DmaReadFifo::DmaDoneEvent::DmaDoneEvent(DmaReadFifo
*_parent
,
482 : parent(_parent
), _done(false), _canceled(false), _data(max_size
, 0)
487 DmaReadFifo::DmaDoneEvent::kill()
490 setFlags(AutoDelete
);
494 DmaReadFifo::DmaDoneEvent::cancel()
500 DmaReadFifo::DmaDoneEvent::reset(size_t size
)
502 assert(size
<= _data
.size());
509 DmaReadFifo::DmaDoneEvent::process()