src/dev/dma_device.cc

   1 /*
   2  * Copyright (c) 2012, 2015, 2017 ARM Limited
   3  * All rights reserved.
   4  *
   5  * The license below extends only to copyright in the software and shall
   6  * not be construed as granting a license to any other intellectual
   7  * property including but not limited to intellectual property relating
   8  * to a hardware implementation of the functionality of the software
   9  * licensed hereunder.  You may use the software subject to the license
  10  * terms below provided that you ensure that this notice is replicated
  11  * unmodified and in its entirety in all distributions of the software,
  12  * modified or unmodified, in source code or in binary form.
  13  *
  14  * Copyright (c) 2006 The Regents of The University of Michigan
  15  * All rights reserved.
  16  *
  17  * Redistribution and use in source and binary forms, with or without
  18  * modification, are permitted provided that the following conditions are
  19  * met: redistributions of source code must retain the above copyright
  20  * notice, this list of conditions and the following disclaimer;
  21  * redistributions in binary form must reproduce the above copyright
  22  * notice, this list of conditions and the following disclaimer in the
  23  * documentation and/or other materials provided with the distribution;
  24  * neither the name of the copyright holders nor the names of its
  25  * contributors may be used to endorse or promote products derived from
  26  * this software without specific prior written permission.
  27  *
  28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  33  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  34  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  35  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  36  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  37  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  38  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  39  *
  40  * Authors: Ali Saidi
  41  *          Nathan Binkert
  42  *          Andreas Hansson
  43  *          Andreas Sandberg
  44  */
  45
  46 #include "dev/dma_device.hh"
  47
  48 #include <utility>
  49
  50 #include "base/chunk_generator.hh"
  51 #include "debug/DMA.hh"
  52 #include "debug/Drain.hh"
  53 #include "mem/port_proxy.hh"
  54 #include "sim/system.hh"
  55
  56 DmaPort::DmaPort(MemObject *dev, System *s)
  57     : MasterPort(dev->name() + ".dma", dev),
  58       device(dev), sys(s), masterId(s->getMasterId(dev->name())),
  59       sendEvent([this]{ sendDma(); }, dev->name()),
  60       pendingCount(0), inRetry(false)
  61 { }
  62
  63 void
  64 DmaPort::handleResp(PacketPtr pkt, Tick delay)
  65 {
  66     // should always see a response with a sender state
  67     assert(pkt->isResponse());
  68
  69     // get the DMA sender state
  70     DmaReqState *state = dynamic_cast<DmaReqState*>(pkt->senderState);
  71     assert(state);
  72
  73     DPRINTF(DMA, "Received response %s for addr: %#x size: %d nb: %d,"  \
  74             " tot: %d sched %d\n",
  75             pkt->cmdString(), pkt->getAddr(), pkt->req->getSize(),
  76             state->numBytes, state->totBytes,
  77             state->completionEvent ?
  78             state->completionEvent->scheduled() : 0);
  79
  80     assert(pendingCount != 0);
  81     pendingCount--;
  82
  83     // update the number of bytes received based on the request rather
  84     // than the packet as the latter could be rounded up to line sizes
  85     state->numBytes += pkt->req->getSize();
  86     assert(state->totBytes >= state->numBytes);
  87
  88     // if we have reached the total number of bytes for this DMA
  89     // request, then signal the completion and delete the sate
  90     if (state->totBytes == state->numBytes) {
  91         if (state->completionEvent) {
  92             delay += state->delay;
  93             device->schedule(state->completionEvent, curTick() + delay);
  94         }
  95         delete state;
  96     }
  97
  98     // delete the request that we created and also the packet
  99     delete pkt->req;
 100     delete pkt;
 101
 102     // we might be drained at this point, if so signal the drain event
 103     if (pendingCount == 0)
 104         signalDrainDone();
 105 }
 106
 107 bool
 108 DmaPort::recvTimingResp(PacketPtr pkt)
 109 {
 110     // We shouldn't ever get a cacheable block in Modified state
 111     assert(pkt->req->isUncacheable() ||
 112            !(pkt->cacheResponding() && !pkt->hasSharers()));
 113
 114     handleResp(pkt);
 115
 116     return true;
 117 }
 118
 119 DmaDevice::DmaDevice(const Params *p)
 120     : PioDevice(p), dmaPort(this, sys)
 121 { }
 122
 123 void
 124 DmaDevice::init()
 125 {
 126     if (!dmaPort.isConnected())
 127         panic("DMA port of %s not connected to anything!", name());
 128     PioDevice::init();
 129 }
 130
 131 DrainState
 132 DmaPort::drain()
 133 {
 134     if (pendingCount == 0) {
 135         return DrainState::Drained;
 136     } else {
 137         DPRINTF(Drain, "DmaPort not drained\n");
 138         return DrainState::Draining;
 139     }
 140 }
 141
 142 void
 143 DmaPort::recvReqRetry()
 144 {
 145     assert(transmitList.size());
 146     trySendTimingReq();
 147 }
 148
 149 RequestPtr
 150 DmaPort::dmaAction(Packet::Command cmd, Addr addr, int size, Event *event,
 151                    uint8_t *data, Tick delay, Request::Flags flag)
 152 {
 153     // one DMA request sender state for every action, that is then
 154     // split into many requests and packets based on the block size,
 155     // i.e. cache line size
 156     DmaReqState *reqState = new DmaReqState(event, size, delay);
 157
 158     // (functionality added for Table Walker statistics)
 159     // We're only interested in this when there will only be one request.
 160     // For simplicity, we return the last request, which would also be
 161     // the only request in that case.
 162     RequestPtr req = NULL;
 163
 164     DPRINTF(DMA, "Starting DMA for addr: %#x size: %d sched: %d\n", addr, size,
 165             event ? event->scheduled() : -1);
 166     for (ChunkGenerator gen(addr, size, sys->cacheLineSize());
 167          !gen.done(); gen.next()) {
 168         req = new Request(gen.addr(), gen.size(), flag, masterId);
 169         req->taskId(ContextSwitchTaskId::DMA);
 170         PacketPtr pkt = new Packet(req, cmd);
 171
 172         // Increment the data pointer on a write
 173         if (data)
 174             pkt->dataStatic(data + gen.complete());
 175
 176         pkt->senderState = reqState;
 177
 178         DPRINTF(DMA, "--Queuing DMA for addr: %#x size: %d\n", gen.addr(),
 179                 gen.size());
 180         queueDma(pkt);
 181     }
 182
 183     // in zero time also initiate the sending of the packets we have
 184     // just created, for atomic this involves actually completing all
 185     // the requests
 186     sendDma();
 187
 188     return req;
 189 }
 190
 191 void
 192 DmaPort::queueDma(PacketPtr pkt)
 193 {
 194     transmitList.push_back(pkt);
 195
 196     // remember that we have another packet pending, this will only be
 197     // decremented once a response comes back
 198     pendingCount++;
 199 }
 200
 201 void
 202 DmaPort::trySendTimingReq()
 203 {
 204     // send the first packet on the transmit list and schedule the
 205     // following send if it is successful
 206     PacketPtr pkt = transmitList.front();
 207
 208     DPRINTF(DMA, "Trying to send %s addr %#x\n", pkt->cmdString(),
 209             pkt->getAddr());
 210
 211     inRetry = !sendTimingReq(pkt);
 212     if (!inRetry) {
 213         transmitList.pop_front();
 214         DPRINTF(DMA, "-- Done\n");
 215         // if there is more to do, then do so
 216         if (!transmitList.empty())
 217             // this should ultimately wait for as many cycles as the
 218             // device needs to send the packet, but currently the port
 219             // does not have any known width so simply wait a single
 220             // cycle
 221             device->schedule(sendEvent, device->clockEdge(Cycles(1)));
 222     } else {
 223         DPRINTF(DMA, "-- Failed, waiting for retry\n");
 224     }
 225
 226     DPRINTF(DMA, "TransmitList: %d, inRetry: %d\n",
 227             transmitList.size(), inRetry);
 228 }
 229
 230 void
 231 DmaPort::sendDma()
 232 {
 233     // some kind of selcetion between access methods
 234     // more work is going to have to be done to make
 235     // switching actually work
 236     assert(transmitList.size());
 237
 238     if (sys->isTimingMode()) {
 239         // if we are either waiting for a retry or are still waiting
 240         // after sending the last packet, then do not proceed
 241         if (inRetry || sendEvent.scheduled()) {
 242             DPRINTF(DMA, "Can't send immediately, waiting to send\n");
 243             return;
 244         }
 245
 246         trySendTimingReq();
 247     } else if (sys->isAtomicMode()) {
 248         // send everything there is to send in zero time
 249         while (!transmitList.empty()) {
 250             PacketPtr pkt = transmitList.front();
 251             transmitList.pop_front();
 252
 253             DPRINTF(DMA, "Sending  DMA for addr: %#x size: %d\n",
 254                     pkt->req->getPaddr(), pkt->req->getSize());
 255             Tick lat = sendAtomic(pkt);
 256
 257             handleResp(pkt, lat);
 258         }
 259     } else
 260         panic("Unknown memory mode.");
 261 }
 262
 263 BaseMasterPort &
 264 DmaDevice::getMasterPort(const std::string &if_name, PortID idx)
 265 {
 266     if (if_name == "dma") {
 267         return dmaPort;
 268     }
 269     return PioDevice::getMasterPort(if_name, idx);
 270 }
 271
 272
 273
 274
 275
 276 DmaReadFifo::DmaReadFifo(DmaPort &_port, size_t size,
 277                          unsigned max_req_size,
 278                          unsigned max_pending,
 279                          Request::Flags flags)
 280     : maxReqSize(max_req_size), fifoSize(size),
 281       reqFlags(flags), port(_port),
 282       buffer(size),
 283       nextAddr(0), endAddr(0)
 284 {
 285     freeRequests.resize(max_pending);
 286     for (auto &e : freeRequests)
 287         e.reset(new DmaDoneEvent(this, max_req_size));
 288
 289 }
 290
 291 DmaReadFifo::~DmaReadFifo()
 292 {
 293     for (auto &p : pendingRequests) {
 294         DmaDoneEvent *e(p.release());
 295
 296         if (e->done()) {
 297             delete e;
 298         } else {
 299             // We can't kill in-flight DMAs, so we'll just transfer
 300             // ownership to the event queue so that they get freed
 301             // when they are done.
 302             e->kill();
 303         }
 304     }
 305 }
 306
 307 void
 308 DmaReadFifo::serialize(CheckpointOut &cp) const
 309 {
 310     assert(pendingRequests.empty());
 311
 312     SERIALIZE_CONTAINER(buffer);
 313     SERIALIZE_SCALAR(endAddr);
 314     SERIALIZE_SCALAR(nextAddr);
 315 }
 316
 317 void
 318 DmaReadFifo::unserialize(CheckpointIn &cp)
 319 {
 320     UNSERIALIZE_CONTAINER(buffer);
 321     UNSERIALIZE_SCALAR(endAddr);
 322     UNSERIALIZE_SCALAR(nextAddr);
 323 }
 324
 325 bool
 326 DmaReadFifo::tryGet(uint8_t *dst, size_t len)
 327 {
 328     if (buffer.size() >= len) {
 329         buffer.read(dst, len);
 330         resumeFill();
 331         return true;
 332     } else {
 333         return false;
 334     }
 335 }
 336
 337 void
 338 DmaReadFifo::get(uint8_t *dst, size_t len)
 339 {
 340     const bool success(tryGet(dst, len));
 341     panic_if(!success, "Buffer underrun in DmaReadFifo::get()\n");
 342 }
 343
 344 void
 345 DmaReadFifo::startFill(Addr start, size_t size)
 346 {
 347     assert(atEndOfBlock());
 348
 349     nextAddr = start;
 350     endAddr = start + size;
 351     resumeFill();
 352 }
 353
 354 void
 355 DmaReadFifo::stopFill()
 356 {
 357     // Prevent new DMA requests by setting the next address to the end
 358     // address. Pending requests will still complete.
 359     nextAddr = endAddr;
 360
 361     // Flag in-flight accesses as canceled. This prevents their data
 362     // from being written to the FIFO.
 363     for (auto &p : pendingRequests)
 364         p->cancel();
 365 }
 366
 367 void
 368 DmaReadFifo::resumeFill()
 369 {
 370     // Don't try to fetch more data if we are draining. This ensures
 371     // that the DMA engine settles down before we checkpoint it.
 372     if (drainState() == DrainState::Draining)
 373         return;
 374
 375     const bool old_eob(atEndOfBlock());
 376
 377     if (port.sys->bypassCaches())
 378         resumeFillFunctional();
 379     else
 380         resumeFillTiming();
 381
 382     if (!old_eob && atEndOfBlock())
 383         onEndOfBlock();
 384 }
 385
 386 void
 387 DmaReadFifo::resumeFillFunctional()
 388 {
 389     const size_t fifo_space = buffer.capacity() - buffer.size();
 390     const size_t kvm_watermark = port.sys->cacheLineSize();
 391     if (fifo_space >= kvm_watermark || buffer.capacity() < kvm_watermark) {
 392         const size_t block_remaining = endAddr - nextAddr;
 393         const size_t xfer_size = std::min(fifo_space, block_remaining);
 394         std::vector<uint8_t> tmp_buffer(xfer_size);
 395
 396         assert(pendingRequests.empty());
 397         DPRINTF(DMA, "KVM Bypassing startAddr=%#x xfer_size=%#x " \
 398                 "fifo_space=%#x block_remaining=%#x\n",
 399                 nextAddr, xfer_size, fifo_space, block_remaining);
 400
 401         port.sys->physProxy.readBlob(nextAddr, tmp_buffer.data(), xfer_size);
 402         buffer.write(tmp_buffer.begin(), xfer_size);
 403         nextAddr += xfer_size;
 404     }
 405 }
 406
 407 void
 408 DmaReadFifo::resumeFillTiming()
 409 {
 410     size_t size_pending(0);
 411     for (auto &e : pendingRequests)
 412         size_pending += e->requestSize();
 413
 414     while (!freeRequests.empty() && !atEndOfBlock()) {
 415         const size_t req_size(std::min(maxReqSize, endAddr - nextAddr));
 416         if (buffer.size() + size_pending + req_size > fifoSize)
 417             break;
 418
 419         DmaDoneEventUPtr event(std::move(freeRequests.front()));
 420         freeRequests.pop_front();
 421         assert(event);
 422
 423         event->reset(req_size);
 424         port.dmaAction(MemCmd::ReadReq, nextAddr, req_size, event.get(),
 425                        event->data(), 0, reqFlags);
 426         nextAddr += req_size;
 427         size_pending += req_size;
 428
 429         pendingRequests.emplace_back(std::move(event));
 430     }
 431 }
 432
 433 void
 434 DmaReadFifo::dmaDone()
 435 {
 436     const bool old_active(isActive());
 437
 438     handlePending();
 439     resumeFill();
 440
 441     if (old_active && !isActive())
 442         onIdle();
 443 }
 444
 445 void
 446 DmaReadFifo::handlePending()
 447 {
 448     while (!pendingRequests.empty() && pendingRequests.front()->done()) {
 449         // Get the first finished pending request
 450         DmaDoneEventUPtr event(std::move(pendingRequests.front()));
 451         pendingRequests.pop_front();
 452
 453         if (!event->canceled())
 454             buffer.write(event->data(), event->requestSize());
 455
 456         // Move the event to the list of free requests
 457         freeRequests.emplace_back(std::move(event));
 458     }
 459
 460     if (pendingRequests.empty())
 461         signalDrainDone();
 462 }
 463
 464 DrainState
 465 DmaReadFifo::drain()
 466 {
 467     return pendingRequests.empty() ? DrainState::Drained : DrainState::Draining;
 468 }
 469
 470
 471 DmaReadFifo::DmaDoneEvent::DmaDoneEvent(DmaReadFifo *_parent,
 472                                         size_t max_size)
 473     : parent(_parent), _done(false), _canceled(false), _data(max_size, 0)
 474 {
 475 }
 476
 477 void
 478 DmaReadFifo::DmaDoneEvent::kill()
 479 {
 480     parent = nullptr;
 481     setFlags(AutoDelete);
 482 }
 483
 484 void
 485 DmaReadFifo::DmaDoneEvent::cancel()
 486 {
 487     _canceled = true;
 488 }
 489
 490 void
 491 DmaReadFifo::DmaDoneEvent::reset(size_t size)
 492 {
 493     assert(size <= _data.size());
 494     _done = false;
 495     _canceled = false;
 496     _requestSize = size;
 497 }
 498
 499 void
 500 DmaReadFifo::DmaDoneEvent::process()
 501 {
 502     if (!parent)
 503         return;
 504
 505     assert(!_done);
 506     _done = true;
 507     parent->dmaDone();
 508 }