src/dev/dma_device.cc

   1 /*
   2  * Copyright (c) 2012, 2015, 2017, 2019 ARM Limited
   3  * All rights reserved.
   4  *
   5  * The license below extends only to copyright in the software and shall
   6  * not be construed as granting a license to any other intellectual
   7  * property including but not limited to intellectual property relating
   8  * to a hardware implementation of the functionality of the software
   9  * licensed hereunder.  You may use the software subject to the license
  10  * terms below provided that you ensure that this notice is replicated
  11  * unmodified and in its entirety in all distributions of the software,
  12  * modified or unmodified, in source code or in binary form.
  13  *
  14  * Copyright (c) 2006 The Regents of The University of Michigan
  15  * All rights reserved.
  16  *
  17  * Redistribution and use in source and binary forms, with or without
  18  * modification, are permitted provided that the following conditions are
  19  * met: redistributions of source code must retain the above copyright
  20  * notice, this list of conditions and the following disclaimer;
  21  * redistributions in binary form must reproduce the above copyright
  22  * notice, this list of conditions and the following disclaimer in the
  23  * documentation and/or other materials provided with the distribution;
  24  * neither the name of the copyright holders nor the names of its
  25  * contributors may be used to endorse or promote products derived from
  26  * this software without specific prior written permission.
  27  *
  28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  33  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  34  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  35  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  36  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  37  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  38  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  39  */
  40
  41 #include "dev/dma_device.hh"
  42
  43 #include <utility>
  44
  45 #include "base/chunk_generator.hh"
  46 #include "debug/DMA.hh"
  47 #include "debug/Drain.hh"
  48 #include "mem/port_proxy.hh"
  49 #include "sim/clocked_object.hh"
  50 #include "sim/system.hh"
  51
  52 DmaPort::DmaPort(ClockedObject *dev, System *s,
  53                  uint32_t sid, uint32_t ssid)
  54     : RequestPort(dev->name() + ".dma", dev),
  55       device(dev), sys(s), masterId(s->getMasterId(dev)),
  56       sendEvent([this]{ sendDma(); }, dev->name()),
  57       pendingCount(0), inRetry(false),
  58       defaultSid(sid),
  59       defaultSSid(ssid)
  60 { }
  61
  62 void
  63 DmaPort::handleResp(PacketPtr pkt, Tick delay)
  64 {
  65     // should always see a response with a sender state
  66     assert(pkt->isResponse());
  67
  68     // get the DMA sender state
  69     DmaReqState *state = dynamic_cast<DmaReqState*>(pkt->senderState);
  70     assert(state);
  71
  72     DPRINTF(DMA, "Received response %s for addr: %#x size: %d nb: %d,"  \
  73             " tot: %d sched %d\n",
  74             pkt->cmdString(), pkt->getAddr(), pkt->req->getSize(),
  75             state->numBytes, state->totBytes,
  76             state->completionEvent ?
  77             state->completionEvent->scheduled() : 0);
  78
  79     assert(pendingCount != 0);
  80     pendingCount--;
  81
  82     // update the number of bytes received based on the request rather
  83     // than the packet as the latter could be rounded up to line sizes
  84     state->numBytes += pkt->req->getSize();
  85     assert(state->totBytes >= state->numBytes);
  86
  87     // if we have reached the total number of bytes for this DMA
  88     // request, then signal the completion and delete the sate
  89     if (state->totBytes == state->numBytes) {
  90         if (state->completionEvent) {
  91             delay += state->delay;
  92             device->schedule(state->completionEvent, curTick() + delay);
  93         }
  94         delete state;
  95     }
  96
  97     // delete the packet
  98     delete pkt;
  99
 100     // we might be drained at this point, if so signal the drain event
 101     if (pendingCount == 0)
 102         signalDrainDone();
 103 }
 104
 105 bool
 106 DmaPort::recvTimingResp(PacketPtr pkt)
 107 {
 108     // We shouldn't ever get a cacheable block in Modified state
 109     assert(pkt->req->isUncacheable() ||
 110            !(pkt->cacheResponding() && !pkt->hasSharers()));
 111
 112     handleResp(pkt);
 113
 114     return true;
 115 }
 116
 117 DmaDevice::DmaDevice(const Params *p)
 118     : PioDevice(p), dmaPort(this, sys, p->sid, p->ssid)
 119 { }
 120
 121 void
 122 DmaDevice::init()
 123 {
 124     if (!dmaPort.isConnected())
 125         panic("DMA port of %s not connected to anything!", name());
 126     PioDevice::init();
 127 }
 128
 129 DrainState
 130 DmaPort::drain()
 131 {
 132     if (pendingCount == 0) {
 133         return DrainState::Drained;
 134     } else {
 135         DPRINTF(Drain, "DmaPort not drained\n");
 136         return DrainState::Draining;
 137     }
 138 }
 139
 140 void
 141 DmaPort::recvReqRetry()
 142 {
 143     assert(transmitList.size());
 144     trySendTimingReq();
 145 }
 146
 147 RequestPtr
 148 DmaPort::dmaAction(Packet::Command cmd, Addr addr, int size, Event *event,
 149                    uint8_t *data, uint32_t sid, uint32_t ssid, Tick delay,
 150                    Request::Flags flag)
 151 {
 152     // one DMA request sender state for every action, that is then
 153     // split into many requests and packets based on the block size,
 154     // i.e. cache line size
 155     DmaReqState *reqState = new DmaReqState(event, size, delay);
 156
 157     // (functionality added for Table Walker statistics)
 158     // We're only interested in this when there will only be one request.
 159     // For simplicity, we return the last request, which would also be
 160     // the only request in that case.
 161     RequestPtr req = NULL;
 162
 163     DPRINTF(DMA, "Starting DMA for addr: %#x size: %d sched: %d\n", addr, size,
 164             event ? event->scheduled() : -1);
 165     for (ChunkGenerator gen(addr, size, sys->cacheLineSize());
 166          !gen.done(); gen.next()) {
 167
 168         req = std::make_shared<Request>(
 169             gen.addr(), gen.size(), flag, masterId);
 170
 171         req->setStreamId(sid);
 172         req->setSubStreamId(ssid);
 173
 174         req->taskId(ContextSwitchTaskId::DMA);
 175         PacketPtr pkt = new Packet(req, cmd);
 176
 177         // Increment the data pointer on a write
 178         if (data)
 179             pkt->dataStatic(data + gen.complete());
 180
 181         pkt->senderState = reqState;
 182
 183         DPRINTF(DMA, "--Queuing DMA for addr: %#x size: %d\n", gen.addr(),
 184                 gen.size());
 185         queueDma(pkt);
 186     }
 187
 188     // in zero time also initiate the sending of the packets we have
 189     // just created, for atomic this involves actually completing all
 190     // the requests
 191     sendDma();
 192
 193     return req;
 194 }
 195
 196 RequestPtr
 197 DmaPort::dmaAction(Packet::Command cmd, Addr addr, int size, Event *event,
 198                    uint8_t *data, Tick delay, Request::Flags flag)
 199 {
 200     return dmaAction(cmd, addr, size, event, data,
 201                      defaultSid, defaultSSid, delay, flag);
 202 }
 203
 204 void
 205 DmaPort::queueDma(PacketPtr pkt)
 206 {
 207     transmitList.push_back(pkt);
 208
 209     // remember that we have another packet pending, this will only be
 210     // decremented once a response comes back
 211     pendingCount++;
 212 }
 213
 214 void
 215 DmaPort::trySendTimingReq()
 216 {
 217     // send the first packet on the transmit list and schedule the
 218     // following send if it is successful
 219     PacketPtr pkt = transmitList.front();
 220
 221     DPRINTF(DMA, "Trying to send %s addr %#x\n", pkt->cmdString(),
 222             pkt->getAddr());
 223
 224     inRetry = !sendTimingReq(pkt);
 225     if (!inRetry) {
 226         transmitList.pop_front();
 227         DPRINTF(DMA, "-- Done\n");
 228         // if there is more to do, then do so
 229         if (!transmitList.empty())
 230             // this should ultimately wait for as many cycles as the
 231             // device needs to send the packet, but currently the port
 232             // does not have any known width so simply wait a single
 233             // cycle
 234             device->schedule(sendEvent, device->clockEdge(Cycles(1)));
 235     } else {
 236         DPRINTF(DMA, "-- Failed, waiting for retry\n");
 237     }
 238
 239     DPRINTF(DMA, "TransmitList: %d, inRetry: %d\n",
 240             transmitList.size(), inRetry);
 241 }
 242
 243 void
 244 DmaPort::sendDma()
 245 {
 246     // some kind of selcetion between access methods
 247     // more work is going to have to be done to make
 248     // switching actually work
 249     assert(transmitList.size());
 250
 251     if (sys->isTimingMode()) {
 252         // if we are either waiting for a retry or are still waiting
 253         // after sending the last packet, then do not proceed
 254         if (inRetry || sendEvent.scheduled()) {
 255             DPRINTF(DMA, "Can't send immediately, waiting to send\n");
 256             return;
 257         }
 258
 259         trySendTimingReq();
 260     } else if (sys->isAtomicMode()) {
 261         // send everything there is to send in zero time
 262         while (!transmitList.empty()) {
 263             PacketPtr pkt = transmitList.front();
 264             transmitList.pop_front();
 265
 266             DPRINTF(DMA, "Sending  DMA for addr: %#x size: %d\n",
 267                     pkt->req->getPaddr(), pkt->req->getSize());
 268             Tick lat = sendAtomic(pkt);
 269
 270             handleResp(pkt, lat);
 271         }
 272     } else
 273         panic("Unknown memory mode.");
 274 }
 275
 276 Port &
 277 DmaDevice::getPort(const std::string &if_name, PortID idx)
 278 {
 279     if (if_name == "dma") {
 280         return dmaPort;
 281     }
 282     return PioDevice::getPort(if_name, idx);
 283 }
 284
 285 DmaReadFifo::DmaReadFifo(DmaPort &_port, size_t size,
 286                          unsigned max_req_size,
 287                          unsigned max_pending,
 288                          Request::Flags flags)
 289     : maxReqSize(max_req_size), fifoSize(size),
 290       reqFlags(flags), port(_port),
 291       buffer(size),
 292       nextAddr(0), endAddr(0)
 293 {
 294     freeRequests.resize(max_pending);
 295     for (auto &e : freeRequests)
 296         e.reset(new DmaDoneEvent(this, max_req_size));
 297
 298 }
 299
 300 DmaReadFifo::~DmaReadFifo()
 301 {
 302     for (auto &p : pendingRequests) {
 303         DmaDoneEvent *e(p.release());
 304
 305         if (e->done()) {
 306             delete e;
 307         } else {
 308             // We can't kill in-flight DMAs, so we'll just transfer
 309             // ownership to the event queue so that they get freed
 310             // when they are done.
 311             e->kill();
 312         }
 313     }
 314 }
 315
 316 void
 317 DmaReadFifo::serialize(CheckpointOut &cp) const
 318 {
 319     assert(pendingRequests.empty());
 320
 321     SERIALIZE_CONTAINER(buffer);
 322     SERIALIZE_SCALAR(endAddr);
 323     SERIALIZE_SCALAR(nextAddr);
 324 }
 325
 326 void
 327 DmaReadFifo::unserialize(CheckpointIn &cp)
 328 {
 329     UNSERIALIZE_CONTAINER(buffer);
 330     UNSERIALIZE_SCALAR(endAddr);
 331     UNSERIALIZE_SCALAR(nextAddr);
 332 }
 333
 334 bool
 335 DmaReadFifo::tryGet(uint8_t *dst, size_t len)
 336 {
 337     if (buffer.size() >= len) {
 338         buffer.read(dst, len);
 339         resumeFill();
 340         return true;
 341     } else {
 342         return false;
 343     }
 344 }
 345
 346 void
 347 DmaReadFifo::get(uint8_t *dst, size_t len)
 348 {
 349     const bool success(tryGet(dst, len));
 350     panic_if(!success, "Buffer underrun in DmaReadFifo::get()\n");
 351 }
 352
 353 void
 354 DmaReadFifo::startFill(Addr start, size_t size)
 355 {
 356     assert(atEndOfBlock());
 357
 358     nextAddr = start;
 359     endAddr = start + size;
 360     resumeFill();
 361 }
 362
 363 void
 364 DmaReadFifo::stopFill()
 365 {
 366     // Prevent new DMA requests by setting the next address to the end
 367     // address. Pending requests will still complete.
 368     nextAddr = endAddr;
 369
 370     // Flag in-flight accesses as canceled. This prevents their data
 371     // from being written to the FIFO.
 372     for (auto &p : pendingRequests)
 373         p->cancel();
 374 }
 375
 376 void
 377 DmaReadFifo::resumeFill()
 378 {
 379     // Don't try to fetch more data if we are draining. This ensures
 380     // that the DMA engine settles down before we checkpoint it.
 381     if (drainState() == DrainState::Draining)
 382         return;
 383
 384     const bool old_eob(atEndOfBlock());
 385
 386     if (port.sys->bypassCaches())
 387         resumeFillFunctional();
 388     else
 389         resumeFillTiming();
 390
 391     if (!old_eob && atEndOfBlock())
 392         onEndOfBlock();
 393 }
 394
 395 void
 396 DmaReadFifo::resumeFillFunctional()
 397 {
 398     const size_t fifo_space = buffer.capacity() - buffer.size();
 399     const size_t kvm_watermark = port.sys->cacheLineSize();
 400     if (fifo_space >= kvm_watermark || buffer.capacity() < kvm_watermark) {
 401         const size_t block_remaining = endAddr - nextAddr;
 402         const size_t xfer_size = std::min(fifo_space, block_remaining);
 403         std::vector<uint8_t> tmp_buffer(xfer_size);
 404
 405         assert(pendingRequests.empty());
 406         DPRINTF(DMA, "KVM Bypassing startAddr=%#x xfer_size=%#x " \
 407                 "fifo_space=%#x block_remaining=%#x\n",
 408                 nextAddr, xfer_size, fifo_space, block_remaining);
 409
 410         port.sys->physProxy.readBlob(nextAddr, tmp_buffer.data(), xfer_size);
 411         buffer.write(tmp_buffer.begin(), xfer_size);
 412         nextAddr += xfer_size;
 413     }
 414 }
 415
 416 void
 417 DmaReadFifo::resumeFillTiming()
 418 {
 419     size_t size_pending(0);
 420     for (auto &e : pendingRequests)
 421         size_pending += e->requestSize();
 422
 423     while (!freeRequests.empty() && !atEndOfBlock()) {
 424         const size_t req_size(std::min(maxReqSize, endAddr - nextAddr));
 425         if (buffer.size() + size_pending + req_size > fifoSize)
 426             break;
 427
 428         DmaDoneEventUPtr event(std::move(freeRequests.front()));
 429         freeRequests.pop_front();
 430         assert(event);
 431
 432         event->reset(req_size);
 433         port.dmaAction(MemCmd::ReadReq, nextAddr, req_size, event.get(),
 434                        event->data(), 0, reqFlags);
 435         nextAddr += req_size;
 436         size_pending += req_size;
 437
 438         pendingRequests.emplace_back(std::move(event));
 439     }
 440 }
 441
 442 void
 443 DmaReadFifo::dmaDone()
 444 {
 445     const bool old_active(isActive());
 446
 447     handlePending();
 448     resumeFill();
 449
 450     if (old_active && !isActive())
 451         onIdle();
 452 }
 453
 454 void
 455 DmaReadFifo::handlePending()
 456 {
 457     while (!pendingRequests.empty() && pendingRequests.front()->done()) {
 458         // Get the first finished pending request
 459         DmaDoneEventUPtr event(std::move(pendingRequests.front()));
 460         pendingRequests.pop_front();
 461
 462         if (!event->canceled())
 463             buffer.write(event->data(), event->requestSize());
 464
 465         // Move the event to the list of free requests
 466         freeRequests.emplace_back(std::move(event));
 467     }
 468
 469     if (pendingRequests.empty())
 470         signalDrainDone();
 471 }
 472
 473 DrainState
 474 DmaReadFifo::drain()
 475 {
 476     return pendingRequests.empty() ? DrainState::Drained : DrainState::Draining;
 477 }
 478
 479
 480 DmaReadFifo::DmaDoneEvent::DmaDoneEvent(DmaReadFifo *_parent,
 481                                         size_t max_size)
 482     : parent(_parent), _done(false), _canceled(false), _data(max_size, 0)
 483 {
 484 }
 485
 486 void
 487 DmaReadFifo::DmaDoneEvent::kill()
 488 {
 489     parent = nullptr;
 490     setFlags(AutoDelete);
 491 }
 492
 493 void
 494 DmaReadFifo::DmaDoneEvent::cancel()
 495 {
 496     _canceled = true;
 497 }
 498
 499 void
 500 DmaReadFifo::DmaDoneEvent::reset(size_t size)
 501 {
 502     assert(size <= _data.size());
 503     _done = false;
 504     _canceled = false;
 505     _requestSize = size;
 506 }
 507
 508 void
 509 DmaReadFifo::DmaDoneEvent::process()
 510 {
 511     if (!parent)
 512         return;
 513
 514     assert(!_done);
 515     _done = true;
 516     parent->dmaDone();
 517 }