src/dev/dma_device.cc

   1 /*
   2  * Copyright (c) 2012, 2015, 2017, 2019 ARM Limited
   3  * All rights reserved.
   4  *
   5  * The license below extends only to copyright in the software and shall
   6  * not be construed as granting a license to any other intellectual
   7  * property including but not limited to intellectual property relating
   8  * to a hardware implementation of the functionality of the software
   9  * licensed hereunder.  You may use the software subject to the license
  10  * terms below provided that you ensure that this notice is replicated
  11  * unmodified and in its entirety in all distributions of the software,
  12  * modified or unmodified, in source code or in binary form.
  13  *
  14  * Copyright (c) 2006 The Regents of The University of Michigan
  15  * All rights reserved.
  16  *
  17  * Redistribution and use in source and binary forms, with or without
  18  * modification, are permitted provided that the following conditions are
  19  * met: redistributions of source code must retain the above copyright
  20  * notice, this list of conditions and the following disclaimer;
  21  * redistributions in binary form must reproduce the above copyright
  22  * notice, this list of conditions and the following disclaimer in the
  23  * documentation and/or other materials provided with the distribution;
  24  * neither the name of the copyright holders nor the names of its
  25  * contributors may be used to endorse or promote products derived from
  26  * this software without specific prior written permission.
  27  *
  28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  33  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  34  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  35  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  36  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  37  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  38  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  39  *
  40  * Authors: Ali Saidi
  41  *          Nathan Binkert
  42  *          Andreas Hansson
  43  *          Andreas Sandberg
  44  */
  45
  46 #include "dev/dma_device.hh"
  47
  48 #include <utility>
  49
  50 #include "base/chunk_generator.hh"
  51 #include "debug/DMA.hh"
  52 #include "debug/Drain.hh"
  53 #include "mem/port_proxy.hh"
  54 #include "sim/clocked_object.hh"
  55 #include "sim/system.hh"
  56
  57 DmaPort::DmaPort(ClockedObject *dev, System *s,
  58                  uint32_t sid, uint32_t ssid)
  59     : MasterPort(dev->name() + ".dma", dev),
  60       device(dev), sys(s), masterId(s->getMasterId(dev)),
  61       sendEvent([this]{ sendDma(); }, dev->name()),
  62       pendingCount(0), inRetry(false),
  63       defaultSid(sid),
  64       defaultSSid(ssid)
  65 { }
  66
  67 void
  68 DmaPort::handleResp(PacketPtr pkt, Tick delay)
  69 {
  70     // should always see a response with a sender state
  71     assert(pkt->isResponse());
  72
  73     // get the DMA sender state
  74     DmaReqState *state = dynamic_cast<DmaReqState*>(pkt->senderState);
  75     assert(state);
  76
  77     DPRINTF(DMA, "Received response %s for addr: %#x size: %d nb: %d,"  \
  78             " tot: %d sched %d\n",
  79             pkt->cmdString(), pkt->getAddr(), pkt->req->getSize(),
  80             state->numBytes, state->totBytes,
  81             state->completionEvent ?
  82             state->completionEvent->scheduled() : 0);
  83
  84     assert(pendingCount != 0);
  85     pendingCount--;
  86
  87     // update the number of bytes received based on the request rather
  88     // than the packet as the latter could be rounded up to line sizes
  89     state->numBytes += pkt->req->getSize();
  90     assert(state->totBytes >= state->numBytes);
  91
  92     // if we have reached the total number of bytes for this DMA
  93     // request, then signal the completion and delete the sate
  94     if (state->totBytes == state->numBytes) {
  95         if (state->completionEvent) {
  96             delay += state->delay;
  97             device->schedule(state->completionEvent, curTick() + delay);
  98         }
  99         delete state;
 100     }
 101
 102     // delete the packet
 103     delete pkt;
 104
 105     // we might be drained at this point, if so signal the drain event
 106     if (pendingCount == 0)
 107         signalDrainDone();
 108 }
 109
 110 bool
 111 DmaPort::recvTimingResp(PacketPtr pkt)
 112 {
 113     // We shouldn't ever get a cacheable block in Modified state
 114     assert(pkt->req->isUncacheable() ||
 115            !(pkt->cacheResponding() && !pkt->hasSharers()));
 116
 117     handleResp(pkt);
 118
 119     return true;
 120 }
 121
 122 DmaDevice::DmaDevice(const Params *p)
 123     : PioDevice(p), dmaPort(this, sys, p->sid, p->ssid)
 124 { }
 125
 126 void
 127 DmaDevice::init()
 128 {
 129     if (!dmaPort.isConnected())
 130         panic("DMA port of %s not connected to anything!", name());
 131     PioDevice::init();
 132 }
 133
 134 DrainState
 135 DmaPort::drain()
 136 {
 137     if (pendingCount == 0) {
 138         return DrainState::Drained;
 139     } else {
 140         DPRINTF(Drain, "DmaPort not drained\n");
 141         return DrainState::Draining;
 142     }
 143 }
 144
 145 void
 146 DmaPort::recvReqRetry()
 147 {
 148     assert(transmitList.size());
 149     trySendTimingReq();
 150 }
 151
 152 RequestPtr
 153 DmaPort::dmaAction(Packet::Command cmd, Addr addr, int size, Event *event,
 154                    uint8_t *data, uint32_t sid, uint32_t ssid, Tick delay,
 155                    Request::Flags flag)
 156 {
 157     // one DMA request sender state for every action, that is then
 158     // split into many requests and packets based on the block size,
 159     // i.e. cache line size
 160     DmaReqState *reqState = new DmaReqState(event, size, delay);
 161
 162     // (functionality added for Table Walker statistics)
 163     // We're only interested in this when there will only be one request.
 164     // For simplicity, we return the last request, which would also be
 165     // the only request in that case.
 166     RequestPtr req = NULL;
 167
 168     DPRINTF(DMA, "Starting DMA for addr: %#x size: %d sched: %d\n", addr, size,
 169             event ? event->scheduled() : -1);
 170     for (ChunkGenerator gen(addr, size, sys->cacheLineSize());
 171          !gen.done(); gen.next()) {
 172
 173         req = std::make_shared<Request>(
 174             gen.addr(), gen.size(), flag, masterId);
 175
 176         req->setStreamId(sid);
 177         req->setSubStreamId(ssid);
 178
 179         req->taskId(ContextSwitchTaskId::DMA);
 180         PacketPtr pkt = new Packet(req, cmd);
 181
 182         // Increment the data pointer on a write
 183         if (data)
 184             pkt->dataStatic(data + gen.complete());
 185
 186         pkt->senderState = reqState;
 187
 188         DPRINTF(DMA, "--Queuing DMA for addr: %#x size: %d\n", gen.addr(),
 189                 gen.size());
 190         queueDma(pkt);
 191     }
 192
 193     // in zero time also initiate the sending of the packets we have
 194     // just created, for atomic this involves actually completing all
 195     // the requests
 196     sendDma();
 197
 198     return req;
 199 }
 200
 201 RequestPtr
 202 DmaPort::dmaAction(Packet::Command cmd, Addr addr, int size, Event *event,
 203                    uint8_t *data, Tick delay, Request::Flags flag)
 204 {
 205     return dmaAction(cmd, addr, size, event, data,
 206                      defaultSid, defaultSSid, delay, flag);
 207 }
 208
 209 void
 210 DmaPort::queueDma(PacketPtr pkt)
 211 {
 212     transmitList.push_back(pkt);
 213
 214     // remember that we have another packet pending, this will only be
 215     // decremented once a response comes back
 216     pendingCount++;
 217 }
 218
 219 void
 220 DmaPort::trySendTimingReq()
 221 {
 222     // send the first packet on the transmit list and schedule the
 223     // following send if it is successful
 224     PacketPtr pkt = transmitList.front();
 225
 226     DPRINTF(DMA, "Trying to send %s addr %#x\n", pkt->cmdString(),
 227             pkt->getAddr());
 228
 229     inRetry = !sendTimingReq(pkt);
 230     if (!inRetry) {
 231         transmitList.pop_front();
 232         DPRINTF(DMA, "-- Done\n");
 233         // if there is more to do, then do so
 234         if (!transmitList.empty())
 235             // this should ultimately wait for as many cycles as the
 236             // device needs to send the packet, but currently the port
 237             // does not have any known width so simply wait a single
 238             // cycle
 239             device->schedule(sendEvent, device->clockEdge(Cycles(1)));
 240     } else {
 241         DPRINTF(DMA, "-- Failed, waiting for retry\n");
 242     }
 243
 244     DPRINTF(DMA, "TransmitList: %d, inRetry: %d\n",
 245             transmitList.size(), inRetry);
 246 }
 247
 248 void
 249 DmaPort::sendDma()
 250 {
 251     // some kind of selcetion between access methods
 252     // more work is going to have to be done to make
 253     // switching actually work
 254     assert(transmitList.size());
 255
 256     if (sys->isTimingMode()) {
 257         // if we are either waiting for a retry or are still waiting
 258         // after sending the last packet, then do not proceed
 259         if (inRetry || sendEvent.scheduled()) {
 260             DPRINTF(DMA, "Can't send immediately, waiting to send\n");
 261             return;
 262         }
 263
 264         trySendTimingReq();
 265     } else if (sys->isAtomicMode()) {
 266         // send everything there is to send in zero time
 267         while (!transmitList.empty()) {
 268             PacketPtr pkt = transmitList.front();
 269             transmitList.pop_front();
 270
 271             DPRINTF(DMA, "Sending  DMA for addr: %#x size: %d\n",
 272                     pkt->req->getPaddr(), pkt->req->getSize());
 273             Tick lat = sendAtomic(pkt);
 274
 275             handleResp(pkt, lat);
 276         }
 277     } else
 278         panic("Unknown memory mode.");
 279 }
 280
 281 Port &
 282 DmaDevice::getPort(const std::string &if_name, PortID idx)
 283 {
 284     if (if_name == "dma") {
 285         return dmaPort;
 286     }
 287     return PioDevice::getPort(if_name, idx);
 288 }
 289
 290 DmaReadFifo::DmaReadFifo(DmaPort &_port, size_t size,
 291                          unsigned max_req_size,
 292                          unsigned max_pending,
 293                          Request::Flags flags)
 294     : maxReqSize(max_req_size), fifoSize(size),
 295       reqFlags(flags), port(_port),
 296       buffer(size),
 297       nextAddr(0), endAddr(0)
 298 {
 299     freeRequests.resize(max_pending);
 300     for (auto &e : freeRequests)
 301         e.reset(new DmaDoneEvent(this, max_req_size));
 302
 303 }
 304
 305 DmaReadFifo::~DmaReadFifo()
 306 {
 307     for (auto &p : pendingRequests) {
 308         DmaDoneEvent *e(p.release());
 309
 310         if (e->done()) {
 311             delete e;
 312         } else {
 313             // We can't kill in-flight DMAs, so we'll just transfer
 314             // ownership to the event queue so that they get freed
 315             // when they are done.
 316             e->kill();
 317         }
 318     }
 319 }
 320
 321 void
 322 DmaReadFifo::serialize(CheckpointOut &cp) const
 323 {
 324     assert(pendingRequests.empty());
 325
 326     SERIALIZE_CONTAINER(buffer);
 327     SERIALIZE_SCALAR(endAddr);
 328     SERIALIZE_SCALAR(nextAddr);
 329 }
 330
 331 void
 332 DmaReadFifo::unserialize(CheckpointIn &cp)
 333 {
 334     UNSERIALIZE_CONTAINER(buffer);
 335     UNSERIALIZE_SCALAR(endAddr);
 336     UNSERIALIZE_SCALAR(nextAddr);
 337 }
 338
 339 bool
 340 DmaReadFifo::tryGet(uint8_t *dst, size_t len)
 341 {
 342     if (buffer.size() >= len) {
 343         buffer.read(dst, len);
 344         resumeFill();
 345         return true;
 346     } else {
 347         return false;
 348     }
 349 }
 350
 351 void
 352 DmaReadFifo::get(uint8_t *dst, size_t len)
 353 {
 354     const bool success(tryGet(dst, len));
 355     panic_if(!success, "Buffer underrun in DmaReadFifo::get()\n");
 356 }
 357
 358 void
 359 DmaReadFifo::startFill(Addr start, size_t size)
 360 {
 361     assert(atEndOfBlock());
 362
 363     nextAddr = start;
 364     endAddr = start + size;
 365     resumeFill();
 366 }
 367
 368 void
 369 DmaReadFifo::stopFill()
 370 {
 371     // Prevent new DMA requests by setting the next address to the end
 372     // address. Pending requests will still complete.
 373     nextAddr = endAddr;
 374
 375     // Flag in-flight accesses as canceled. This prevents their data
 376     // from being written to the FIFO.
 377     for (auto &p : pendingRequests)
 378         p->cancel();
 379 }
 380
 381 void
 382 DmaReadFifo::resumeFill()
 383 {
 384     // Don't try to fetch more data if we are draining. This ensures
 385     // that the DMA engine settles down before we checkpoint it.
 386     if (drainState() == DrainState::Draining)
 387         return;
 388
 389     const bool old_eob(atEndOfBlock());
 390
 391     if (port.sys->bypassCaches())
 392         resumeFillFunctional();
 393     else
 394         resumeFillTiming();
 395
 396     if (!old_eob && atEndOfBlock())
 397         onEndOfBlock();
 398 }
 399
 400 void
 401 DmaReadFifo::resumeFillFunctional()
 402 {
 403     const size_t fifo_space = buffer.capacity() - buffer.size();
 404     const size_t kvm_watermark = port.sys->cacheLineSize();
 405     if (fifo_space >= kvm_watermark || buffer.capacity() < kvm_watermark) {
 406         const size_t block_remaining = endAddr - nextAddr;
 407         const size_t xfer_size = std::min(fifo_space, block_remaining);
 408         std::vector<uint8_t> tmp_buffer(xfer_size);
 409
 410         assert(pendingRequests.empty());
 411         DPRINTF(DMA, "KVM Bypassing startAddr=%#x xfer_size=%#x " \
 412                 "fifo_space=%#x block_remaining=%#x\n",
 413                 nextAddr, xfer_size, fifo_space, block_remaining);
 414
 415         port.sys->physProxy.readBlob(nextAddr, tmp_buffer.data(), xfer_size);
 416         buffer.write(tmp_buffer.begin(), xfer_size);
 417         nextAddr += xfer_size;
 418     }
 419 }
 420
 421 void
 422 DmaReadFifo::resumeFillTiming()
 423 {
 424     size_t size_pending(0);
 425     for (auto &e : pendingRequests)
 426         size_pending += e->requestSize();
 427
 428     while (!freeRequests.empty() && !atEndOfBlock()) {
 429         const size_t req_size(std::min(maxReqSize, endAddr - nextAddr));
 430         if (buffer.size() + size_pending + req_size > fifoSize)
 431             break;
 432
 433         DmaDoneEventUPtr event(std::move(freeRequests.front()));
 434         freeRequests.pop_front();
 435         assert(event);
 436
 437         event->reset(req_size);
 438         port.dmaAction(MemCmd::ReadReq, nextAddr, req_size, event.get(),
 439                        event->data(), 0, reqFlags);
 440         nextAddr += req_size;
 441         size_pending += req_size;
 442
 443         pendingRequests.emplace_back(std::move(event));
 444     }
 445 }
 446
 447 void
 448 DmaReadFifo::dmaDone()
 449 {
 450     const bool old_active(isActive());
 451
 452     handlePending();
 453     resumeFill();
 454
 455     if (old_active && !isActive())
 456         onIdle();
 457 }
 458
 459 void
 460 DmaReadFifo::handlePending()
 461 {
 462     while (!pendingRequests.empty() && pendingRequests.front()->done()) {
 463         // Get the first finished pending request
 464         DmaDoneEventUPtr event(std::move(pendingRequests.front()));
 465         pendingRequests.pop_front();
 466
 467         if (!event->canceled())
 468             buffer.write(event->data(), event->requestSize());
 469
 470         // Move the event to the list of free requests
 471         freeRequests.emplace_back(std::move(event));
 472     }
 473
 474     if (pendingRequests.empty())
 475         signalDrainDone();
 476 }
 477
 478 DrainState
 479 DmaReadFifo::drain()
 480 {
 481     return pendingRequests.empty() ? DrainState::Drained : DrainState::Draining;
 482 }
 483
 484
 485 DmaReadFifo::DmaDoneEvent::DmaDoneEvent(DmaReadFifo *_parent,
 486                                         size_t max_size)
 487     : parent(_parent), _done(false), _canceled(false), _data(max_size, 0)
 488 {
 489 }
 490
 491 void
 492 DmaReadFifo::DmaDoneEvent::kill()
 493 {
 494     parent = nullptr;
 495     setFlags(AutoDelete);
 496 }
 497
 498 void
 499 DmaReadFifo::DmaDoneEvent::cancel()
 500 {
 501     _canceled = true;
 502 }
 503
 504 void
 505 DmaReadFifo::DmaDoneEvent::reset(size_t size)
 506 {
 507     assert(size <= _data.size());
 508     _done = false;
 509     _canceled = false;
 510     _requestSize = size;
 511 }
 512
 513 void
 514 DmaReadFifo::DmaDoneEvent::process()
 515 {
 516     if (!parent)
 517         return;
 518
 519     assert(!_done);
 520     _done = true;
 521     parent->dmaDone();
 522 }