src/dev/dma_device.cc

   1 /*
   2  * Copyright (c) 2012, 2015, 2017 ARM Limited
   3  * All rights reserved.
   4  *
   5  * The license below extends only to copyright in the software and shall
   6  * not be construed as granting a license to any other intellectual
   7  * property including but not limited to intellectual property relating
   8  * to a hardware implementation of the functionality of the software
   9  * licensed hereunder.  You may use the software subject to the license
  10  * terms below provided that you ensure that this notice is replicated
  11  * unmodified and in its entirety in all distributions of the software,
  12  * modified or unmodified, in source code or in binary form.
  13  *
  14  * Copyright (c) 2006 The Regents of The University of Michigan
  15  * All rights reserved.
  16  *
  17  * Redistribution and use in source and binary forms, with or without
  18  * modification, are permitted provided that the following conditions are
  19  * met: redistributions of source code must retain the above copyright
  20  * notice, this list of conditions and the following disclaimer;
  21  * redistributions in binary form must reproduce the above copyright
  22  * notice, this list of conditions and the following disclaimer in the
  23  * documentation and/or other materials provided with the distribution;
  24  * neither the name of the copyright holders nor the names of its
  25  * contributors may be used to endorse or promote products derived from
  26  * this software without specific prior written permission.
  27  *
  28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  33  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  34  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  35  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  36  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  37  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  38  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  39  *
  40  * Authors: Ali Saidi
  41  *          Nathan Binkert
  42  *          Andreas Hansson
  43  *          Andreas Sandberg
  44  */
  45
  46 #include "dev/dma_device.hh"
  47
  48 #include <utility>
  49
  50 #include "base/chunk_generator.hh"
  51 #include "debug/DMA.hh"
  52 #include "debug/Drain.hh"
  53 #include "mem/port_proxy.hh"
  54 #include "sim/system.hh"
  55
  56 DmaPort::DmaPort(MemObject *dev, System *s)
  57     : MasterPort(dev->name() + ".dma", dev),
  58       device(dev), sys(s), masterId(s->getMasterId(dev->name())),
  59       sendEvent(this), pendingCount(0), inRetry(false)
  60 { }
  61
  62 void
  63 DmaPort::handleResp(PacketPtr pkt, Tick delay)
  64 {
  65     // should always see a response with a sender state
  66     assert(pkt->isResponse());
  67
  68     // get the DMA sender state
  69     DmaReqState *state = dynamic_cast<DmaReqState*>(pkt->senderState);
  70     assert(state);
  71
  72     DPRINTF(DMA, "Received response %s for addr: %#x size: %d nb: %d,"  \
  73             " tot: %d sched %d\n",
  74             pkt->cmdString(), pkt->getAddr(), pkt->req->getSize(),
  75             state->numBytes, state->totBytes,
  76             state->completionEvent ?
  77             state->completionEvent->scheduled() : 0);
  78
  79     assert(pendingCount != 0);
  80     pendingCount--;
  81
  82     // update the number of bytes received based on the request rather
  83     // than the packet as the latter could be rounded up to line sizes
  84     state->numBytes += pkt->req->getSize();
  85     assert(state->totBytes >= state->numBytes);
  86
  87     // if we have reached the total number of bytes for this DMA
  88     // request, then signal the completion and delete the sate
  89     if (state->totBytes == state->numBytes) {
  90         if (state->completionEvent) {
  91             delay += state->delay;
  92             device->schedule(state->completionEvent, curTick() + delay);
  93         }
  94         delete state;
  95     }
  96
  97     // delete the request that we created and also the packet
  98     delete pkt->req;
  99     delete pkt;
 100
 101     // we might be drained at this point, if so signal the drain event
 102     if (pendingCount == 0)
 103         signalDrainDone();
 104 }
 105
 106 bool
 107 DmaPort::recvTimingResp(PacketPtr pkt)
 108 {
 109     // We shouldn't ever get a cacheable block in Modified state
 110     assert(pkt->req->isUncacheable() ||
 111            !(pkt->cacheResponding() && !pkt->hasSharers()));
 112
 113     handleResp(pkt);
 114
 115     return true;
 116 }
 117
 118 DmaDevice::DmaDevice(const Params *p)
 119     : PioDevice(p), dmaPort(this, sys)
 120 { }
 121
 122 void
 123 DmaDevice::init()
 124 {
 125     if (!dmaPort.isConnected())
 126         panic("DMA port of %s not connected to anything!", name());
 127     PioDevice::init();
 128 }
 129
 130 DrainState
 131 DmaPort::drain()
 132 {
 133     if (pendingCount == 0) {
 134         return DrainState::Drained;
 135     } else {
 136         DPRINTF(Drain, "DmaPort not drained\n");
 137         return DrainState::Draining;
 138     }
 139 }
 140
 141 void
 142 DmaPort::recvReqRetry()
 143 {
 144     assert(transmitList.size());
 145     trySendTimingReq();
 146 }
 147
 148 RequestPtr
 149 DmaPort::dmaAction(Packet::Command cmd, Addr addr, int size, Event *event,
 150                    uint8_t *data, Tick delay, Request::Flags flag)
 151 {
 152     // one DMA request sender state for every action, that is then
 153     // split into many requests and packets based on the block size,
 154     // i.e. cache line size
 155     DmaReqState *reqState = new DmaReqState(event, size, delay);
 156
 157     // (functionality added for Table Walker statistics)
 158     // We're only interested in this when there will only be one request.
 159     // For simplicity, we return the last request, which would also be
 160     // the only request in that case.
 161     RequestPtr req = NULL;
 162
 163     DPRINTF(DMA, "Starting DMA for addr: %#x size: %d sched: %d\n", addr, size,
 164             event ? event->scheduled() : -1);
 165     for (ChunkGenerator gen(addr, size, sys->cacheLineSize());
 166          !gen.done(); gen.next()) {
 167         req = new Request(gen.addr(), gen.size(), flag, masterId);
 168         req->taskId(ContextSwitchTaskId::DMA);
 169         PacketPtr pkt = new Packet(req, cmd);
 170
 171         // Increment the data pointer on a write
 172         if (data)
 173             pkt->dataStatic(data + gen.complete());
 174
 175         pkt->senderState = reqState;
 176
 177         DPRINTF(DMA, "--Queuing DMA for addr: %#x size: %d\n", gen.addr(),
 178                 gen.size());
 179         queueDma(pkt);
 180     }
 181
 182     // in zero time also initiate the sending of the packets we have
 183     // just created, for atomic this involves actually completing all
 184     // the requests
 185     sendDma();
 186
 187     return req;
 188 }
 189
 190 void
 191 DmaPort::queueDma(PacketPtr pkt)
 192 {
 193     transmitList.push_back(pkt);
 194
 195     // remember that we have another packet pending, this will only be
 196     // decremented once a response comes back
 197     pendingCount++;
 198 }
 199
 200 void
 201 DmaPort::trySendTimingReq()
 202 {
 203     // send the first packet on the transmit list and schedule the
 204     // following send if it is successful
 205     PacketPtr pkt = transmitList.front();
 206
 207     DPRINTF(DMA, "Trying to send %s addr %#x\n", pkt->cmdString(),
 208             pkt->getAddr());
 209
 210     inRetry = !sendTimingReq(pkt);
 211     if (!inRetry) {
 212         transmitList.pop_front();
 213         DPRINTF(DMA, "-- Done\n");
 214         // if there is more to do, then do so
 215         if (!transmitList.empty())
 216             // this should ultimately wait for as many cycles as the
 217             // device needs to send the packet, but currently the port
 218             // does not have any known width so simply wait a single
 219             // cycle
 220             device->schedule(sendEvent, device->clockEdge(Cycles(1)));
 221     } else {
 222         DPRINTF(DMA, "-- Failed, waiting for retry\n");
 223     }
 224
 225     DPRINTF(DMA, "TransmitList: %d, inRetry: %d\n",
 226             transmitList.size(), inRetry);
 227 }
 228
 229 void
 230 DmaPort::sendDma()
 231 {
 232     // some kind of selcetion between access methods
 233     // more work is going to have to be done to make
 234     // switching actually work
 235     assert(transmitList.size());
 236
 237     if (sys->isTimingMode()) {
 238         // if we are either waiting for a retry or are still waiting
 239         // after sending the last packet, then do not proceed
 240         if (inRetry || sendEvent.scheduled()) {
 241             DPRINTF(DMA, "Can't send immediately, waiting to send\n");
 242             return;
 243         }
 244
 245         trySendTimingReq();
 246     } else if (sys->isAtomicMode()) {
 247         // send everything there is to send in zero time
 248         while (!transmitList.empty()) {
 249             PacketPtr pkt = transmitList.front();
 250             transmitList.pop_front();
 251
 252             DPRINTF(DMA, "Sending  DMA for addr: %#x size: %d\n",
 253                     pkt->req->getPaddr(), pkt->req->getSize());
 254             Tick lat = sendAtomic(pkt);
 255
 256             handleResp(pkt, lat);
 257         }
 258     } else
 259         panic("Unknown memory mode.");
 260 }
 261
 262 BaseMasterPort &
 263 DmaDevice::getMasterPort(const std::string &if_name, PortID idx)
 264 {
 265     if (if_name == "dma") {
 266         return dmaPort;
 267     }
 268     return PioDevice::getMasterPort(if_name, idx);
 269 }
 270
 271
 272
 273
 274
 275 DmaReadFifo::DmaReadFifo(DmaPort &_port, size_t size,
 276                          unsigned max_req_size,
 277                          unsigned max_pending,
 278                          Request::Flags flags)
 279     : maxReqSize(max_req_size), fifoSize(size),
 280       reqFlags(flags), port(_port),
 281       buffer(size),
 282       nextAddr(0), endAddr(0)
 283 {
 284     freeRequests.resize(max_pending);
 285     for (auto &e : freeRequests)
 286         e.reset(new DmaDoneEvent(this, max_req_size));
 287
 288 }
 289
 290 DmaReadFifo::~DmaReadFifo()
 291 {
 292     for (auto &p : pendingRequests) {
 293         DmaDoneEvent *e(p.release());
 294
 295         if (e->done()) {
 296             delete e;
 297         } else {
 298             // We can't kill in-flight DMAs, so we'll just transfer
 299             // ownership to the event queue so that they get freed
 300             // when they are done.
 301             e->kill();
 302         }
 303     }
 304 }
 305
 306 void
 307 DmaReadFifo::serialize(CheckpointOut &cp) const
 308 {
 309     assert(pendingRequests.empty());
 310
 311     SERIALIZE_CONTAINER(buffer);
 312     SERIALIZE_SCALAR(endAddr);
 313     SERIALIZE_SCALAR(nextAddr);
 314 }
 315
 316 void
 317 DmaReadFifo::unserialize(CheckpointIn &cp)
 318 {
 319     UNSERIALIZE_CONTAINER(buffer);
 320     UNSERIALIZE_SCALAR(endAddr);
 321     UNSERIALIZE_SCALAR(nextAddr);
 322 }
 323
 324 bool
 325 DmaReadFifo::tryGet(uint8_t *dst, size_t len)
 326 {
 327     if (buffer.size() >= len) {
 328         buffer.read(dst, len);
 329         resumeFill();
 330         return true;
 331     } else {
 332         return false;
 333     }
 334 }
 335
 336 void
 337 DmaReadFifo::get(uint8_t *dst, size_t len)
 338 {
 339     const bool success(tryGet(dst, len));
 340     panic_if(!success, "Buffer underrun in DmaReadFifo::get()\n");
 341 }
 342
 343 void
 344 DmaReadFifo::startFill(Addr start, size_t size)
 345 {
 346     assert(atEndOfBlock());
 347
 348     nextAddr = start;
 349     endAddr = start + size;
 350     resumeFill();
 351 }
 352
 353 void
 354 DmaReadFifo::stopFill()
 355 {
 356     // Prevent new DMA requests by setting the next address to the end
 357     // address. Pending requests will still complete.
 358     nextAddr = endAddr;
 359
 360     // Flag in-flight accesses as canceled. This prevents their data
 361     // from being written to the FIFO.
 362     for (auto &p : pendingRequests)
 363         p->cancel();
 364 }
 365
 366 void
 367 DmaReadFifo::resumeFill()
 368 {
 369     // Don't try to fetch more data if we are draining. This ensures
 370     // that the DMA engine settles down before we checkpoint it.
 371     if (drainState() == DrainState::Draining)
 372         return;
 373
 374     const bool old_eob(atEndOfBlock());
 375
 376     if (port.sys->bypassCaches())
 377         resumeFillFunctional();
 378     else
 379         resumeFillTiming();
 380
 381     if (!old_eob && atEndOfBlock())
 382         onEndOfBlock();
 383 }
 384
 385 void
 386 DmaReadFifo::resumeFillFunctional()
 387 {
 388     const size_t fifo_space = buffer.capacity() - buffer.size();
 389     const size_t kvm_watermark = port.sys->cacheLineSize();
 390     if (fifo_space >= kvm_watermark || buffer.capacity() < kvm_watermark) {
 391         const size_t block_remaining = endAddr - nextAddr;
 392         const size_t xfer_size = std::min(fifo_space, block_remaining);
 393         std::vector<uint8_t> tmp_buffer(xfer_size);
 394
 395         assert(pendingRequests.empty());
 396         DPRINTF(DMA, "KVM Bypassing startAddr=%#x xfer_size=%#x " \
 397                 "fifo_space=%#x block_remaining=%#x\n",
 398                 nextAddr, xfer_size, fifo_space, block_remaining);
 399
 400         port.sys->physProxy.readBlob(nextAddr, tmp_buffer.data(), xfer_size);
 401         buffer.write(tmp_buffer.begin(), xfer_size);
 402         nextAddr += xfer_size;
 403     }
 404 }
 405
 406 void
 407 DmaReadFifo::resumeFillTiming()
 408 {
 409     size_t size_pending(0);
 410     for (auto &e : pendingRequests)
 411         size_pending += e->requestSize();
 412
 413     while (!freeRequests.empty() && !atEndOfBlock()) {
 414         const size_t req_size(std::min(maxReqSize, endAddr - nextAddr));
 415         if (buffer.size() + size_pending + req_size > fifoSize)
 416             break;
 417
 418         DmaDoneEventUPtr event(std::move(freeRequests.front()));
 419         freeRequests.pop_front();
 420         assert(event);
 421
 422         event->reset(req_size);
 423         port.dmaAction(MemCmd::ReadReq, nextAddr, req_size, event.get(),
 424                        event->data(), 0, reqFlags);
 425         nextAddr += req_size;
 426         size_pending += req_size;
 427
 428         pendingRequests.emplace_back(std::move(event));
 429     }
 430 }
 431
 432 void
 433 DmaReadFifo::dmaDone()
 434 {
 435     const bool old_active(isActive());
 436
 437     handlePending();
 438     resumeFill();
 439
 440     if (!old_active && isActive())
 441         onIdle();
 442 }
 443
 444 void
 445 DmaReadFifo::handlePending()
 446 {
 447     while (!pendingRequests.empty() && pendingRequests.front()->done()) {
 448         // Get the first finished pending request
 449         DmaDoneEventUPtr event(std::move(pendingRequests.front()));
 450         pendingRequests.pop_front();
 451
 452         if (!event->canceled())
 453             buffer.write(event->data(), event->requestSize());
 454
 455         // Move the event to the list of free requests
 456         freeRequests.emplace_back(std::move(event));
 457     }
 458
 459     if (pendingRequests.empty())
 460         signalDrainDone();
 461 }
 462
 463 DrainState
 464 DmaReadFifo::drain()
 465 {
 466     return pendingRequests.empty() ? DrainState::Drained : DrainState::Draining;
 467 }
 468
 469
 470 DmaReadFifo::DmaDoneEvent::DmaDoneEvent(DmaReadFifo *_parent,
 471                                         size_t max_size)
 472     : parent(_parent), _done(false), _canceled(false), _data(max_size, 0)
 473 {
 474 }
 475
 476 void
 477 DmaReadFifo::DmaDoneEvent::kill()
 478 {
 479     parent = nullptr;
 480     setFlags(AutoDelete);
 481 }
 482
 483 void
 484 DmaReadFifo::DmaDoneEvent::cancel()
 485 {
 486     _canceled = true;
 487 }
 488
 489 void
 490 DmaReadFifo::DmaDoneEvent::reset(size_t size)
 491 {
 492     assert(size <= _data.size());
 493     _done = false;
 494     _canceled = false;
 495     _requestSize = size;
 496 }
 497
 498 void
 499 DmaReadFifo::DmaDoneEvent::process()
 500 {
 501     if (!parent)
 502         return;
 503
 504     assert(!_done);
 505     _done = true;
 506     parent->dmaDone();
 507 }