--- /dev/null
+/*
+ * Copyright (c) 2008 The Regents of The University of Michigan
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met: redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer;
+ * redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution;
+ * neither the name of the copyright holders nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Authors: Ali Saidi
+ */
+
+/* @file
+ * Device model for Intel's I/O AT DMA copy engine.
+ */
+
+#include <algorithm>
+
+#include "base/trace.hh"
+#include "dev/copy_engine.hh"
+#include "mem/packet.hh"
+#include "mem/packet_access.hh"
+#include "params/CopyEngine.hh"
+#include "sim/stats.hh"
+#include "sim/system.hh"
+
+using namespace CopyEngineReg;
+using namespace std;
+
+CopyEngine::CopyEngine(const Params *p)
+ : PciDev(p)
+{
+ // All Reg regs are initialized to 0 by default
+ regs.chanCount = p->ChanCnt;
+ regs.xferCap = findMsbSet(p->XferCap);
+ regs.attnStatus = 0;
+
+ if (regs.chanCount > 64)
+ fatal("CopyEngine interface doesn't support more than 64 DMA engines\n");
+
+ for (int x = 0; x < regs.chanCount; x++) {
+ CopyEngineChannel *ch = new CopyEngineChannel(this, x);
+ chan.push_back(ch);
+ }
+}
+
+
+CopyEngine::CopyEngineChannel::CopyEngineChannel(CopyEngine *_ce, int cid)
+ : ce(_ce), channelId(cid), busy(false), underReset(false),
+ refreshNext(false), latBeforeBegin(ce->params()->latBeforeBegin),
+ latAfterCompletion(ce->params()->latAfterCompletion),
+ completionDataReg(0), nextState(Idle), drainEvent(NULL),
+ fetchCompleteEvent(this), addrCompleteEvent(this),
+ readCompleteEvent(this), writeCompleteEvent(this),
+ statusCompleteEvent(this)
+
+{
+ cr.status.dma_transfer_status(3);
+ cr.descChainAddr = 0;
+ cr.completionAddr = 0;
+
+ curDmaDesc = new DmaDesc;
+ memset(curDmaDesc, 0, sizeof(DmaDesc));
+ copyBuffer = new uint8_t[ce->params()->XferCap];
+}
+
+CopyEngine::~CopyEngine()
+{
+ for (int x = 0; x < chan.size(); x++) {
+ delete chan[x];
+ }
+}
+
+CopyEngine::CopyEngineChannel::~CopyEngineChannel()
+{
+ delete curDmaDesc;
+ delete [] copyBuffer;
+ delete cePort;
+}
+
+void
+CopyEngine::init()
+{
+ PciDev::init();
+ for (int x = 0; x < chan.size(); x++)
+ chan[x]->init();
+}
+
+void
+CopyEngine::CopyEngineChannel::init()
+{
+ Port *peer;
+
+ cePort = new DmaPort(ce, ce->sys);
+ peer = ce->dmaPort->getPeer()->getOwner()->getPort("");
+ peer->setPeer(cePort);
+ cePort->setPeer(peer);
+}
+
+void
+CopyEngine::CopyEngineChannel::recvCommand()
+{
+ if (cr.command.start_dma()) {
+ assert(!busy);
+ cr.status.dma_transfer_status(0);
+ nextState = DescriptorFetch;
+ fetchAddress = cr.descChainAddr;
+ if (ce->getState() == SimObject::Running)
+ fetchDescriptor(cr.descChainAddr);
+ } else if (cr.command.append_dma()) {
+ if (!busy) {
+ nextState = AddressFetch;
+ if (ce->getState() == SimObject::Running)
+ fetchNextAddr(lastDescriptorAddr);
+ } else
+ refreshNext = true;
+ } else if (cr.command.reset_dma()) {
+ if (busy)
+ underReset = true;
+ else {
+ cr.status.dma_transfer_status(3);
+ nextState = Idle;
+ }
+ } else if (cr.command.resume_dma() || cr.command.abort_dma() ||
+ cr.command.suspend_dma())
+ panic("Resume, Abort, and Suspend are not supported\n");
+ cr.command(0);
+}
+
+Tick
+CopyEngine::read(PacketPtr pkt)
+{
+ int bar;
+ Addr daddr;
+
+ if (!getBAR(pkt->getAddr(), bar, daddr))
+ panic("Invalid PCI memory access to unmapped memory.\n");
+
+ // Only Memory register BAR is allowed
+ assert(bar == 0);
+
+ int size = pkt->getSize();
+ if (size != sizeof(uint64_t) && size != sizeof(uint32_t) &&
+ size != sizeof(uint16_t) && size != sizeof(uint8_t)) {
+ panic("Unknown size for MMIO access: %d\n", pkt->getSize());
+ }
+
+ DPRINTF(DMACopyEngine, "Read device register %#X size: %d\n", daddr, size);
+
+ pkt->allocate();
+
+ ///
+ /// Handle read of register here
+ ///
+
+ if (daddr < 0x80) {
+ switch (daddr) {
+ case GEN_CHANCOUNT:
+ assert(size == sizeof(regs.chanCount));
+ pkt->set<uint8_t>(regs.chanCount);
+ break;
+ case GEN_XFERCAP:
+ assert(size == sizeof(regs.xferCap));
+ pkt->set<uint8_t>(regs.xferCap);
+ break;
+ case GEN_INTRCTRL:
+ assert(size == sizeof(uint8_t));
+ pkt->set<uint8_t>(regs.intrctrl());
+ regs.intrctrl.master_int_enable(0);
+ break;
+ case GEN_ATTNSTATUS:
+ assert(size == sizeof(regs.attnStatus));
+ pkt->set<uint32_t>(regs.attnStatus);
+ regs.attnStatus = 0;
+ break;
+ default:
+ panic("Read request to unknown register number: %#x\n", daddr);
+ }
+ pkt->makeAtomicResponse();
+ return pioDelay;
+ }
+
+
+ // Find which channel we're accessing
+ int chanid = 0;
+ daddr -= 0x80;
+ while (daddr >= 0x80) {
+ chanid++;
+ daddr -= 0x80;
+ }
+
+ if (chanid >= regs.chanCount)
+ panic("Access to channel %d (device only configured for %d channels)",
+ chanid, regs.chanCount);
+
+ ///
+ /// Channel registers are handled here
+ ///
+ chan[chanid]->channelRead(pkt, daddr, size);
+
+ pkt->makeAtomicResponse();
+ return pioDelay;
+}
+
+void
+CopyEngine::CopyEngineChannel::channelRead(Packet *pkt, Addr daddr, int size)
+{
+ switch (daddr) {
+ case CHAN_CONTROL:
+ assert(size == sizeof(uint16_t));
+ pkt->set<uint16_t>(cr.ctrl());
+ cr.ctrl.in_use(1);
+ break;
+ case CHAN_STATUS:
+ assert(size == sizeof(uint64_t));
+ pkt->set<uint64_t>(cr.status() | ~busy);
+ break;
+ case CHAN_CHAINADDR:
+ assert(size == sizeof(uint64_t) || size == sizeof(uint32_t));
+ if (size == sizeof(uint64_t))
+ pkt->set<uint64_t>(cr.descChainAddr);
+ else
+ pkt->set<uint32_t>(bits(cr.descChainAddr,0,31));
+ break;
+ case CHAN_CHAINADDR_HIGH:
+ assert(size == sizeof(uint32_t));
+ pkt->set<uint32_t>(bits(cr.descChainAddr,32,63));
+ break;
+ case CHAN_COMMAND:
+ assert(size == sizeof(uint8_t));
+ pkt->set<uint32_t>(cr.command());
+ break;
+ case CHAN_CMPLNADDR:
+ assert(size == sizeof(uint64_t) || size == sizeof(uint32_t));
+ if (size == sizeof(uint64_t))
+ pkt->set<uint64_t>(cr.completionAddr);
+ else
+ pkt->set<uint32_t>(bits(cr.completionAddr,0,31));
+ break;
+ case CHAN_CMPLNADDR_HIGH:
+ assert(size == sizeof(uint32_t));
+ pkt->set<uint32_t>(bits(cr.completionAddr,32,63));
+ break;
+ case CHAN_ERROR:
+ assert(size == sizeof(uint32_t));
+ pkt->set<uint32_t>(cr.error());
+ break;
+ default:
+ panic("Read request to unknown channel register number: (%d)%#x\n",
+ channelId, daddr);
+ }
+}
+
+
+Tick
+CopyEngine::write(PacketPtr pkt)
+{
+ int bar;
+ Addr daddr;
+
+
+ if (!getBAR(pkt->getAddr(), bar, daddr))
+ panic("Invalid PCI memory access to unmapped memory.\n");
+
+ // Only Memory register BAR is allowed
+ assert(bar == 0);
+
+ int size = pkt->getSize();
+
+ ///
+ /// Handle write of register here
+ ///
+
+ if (size == sizeof(uint64_t)) {
+ uint64_t val M5_VAR_USED = pkt->get<uint64_t>();
+ DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", daddr, val);
+ } else if (size == sizeof(uint32_t)) {
+ uint32_t val M5_VAR_USED = pkt->get<uint32_t>();
+ DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", daddr, val);
+ } else if (size == sizeof(uint16_t)) {
+ uint16_t val M5_VAR_USED = pkt->get<uint16_t>();
+ DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", daddr, val);
+ } else if (size == sizeof(uint8_t)) {
+ uint8_t val M5_VAR_USED = pkt->get<uint8_t>();
+ DPRINTF(DMACopyEngine, "Wrote device register %#X value %#X\n", daddr, val);
+ } else {
+ panic("Unknown size for MMIO access: %d\n", size);
+ }
+
+ if (daddr < 0x80) {
+ switch (daddr) {
+ case GEN_CHANCOUNT:
+ case GEN_XFERCAP:
+ case GEN_ATTNSTATUS:
+ DPRINTF(DMACopyEngine, "Warning, ignorning write to register %x\n",
+ daddr);
+ break;
+ case GEN_INTRCTRL:
+ regs.intrctrl.master_int_enable(bits(pkt->get<uint8_t>(),0,1));
+ break;
+ default:
+ panic("Read request to unknown register number: %#x\n", daddr);
+ }
+ pkt->makeAtomicResponse();
+ return pioDelay;
+ }
+
+ // Find which channel we're accessing
+ int chanid = 0;
+ daddr -= 0x80;
+ while (daddr >= 0x80) {
+ chanid++;
+ daddr -= 0x80;
+ }
+
+ if (chanid >= regs.chanCount)
+ panic("Access to channel %d (device only configured for %d channels)",
+ chanid, regs.chanCount);
+
+ ///
+ /// Channel registers are handled here
+ ///
+ chan[chanid]->channelWrite(pkt, daddr, size);
+
+ pkt->makeAtomicResponse();
+ return pioDelay;
+}
+
+void
+CopyEngine::CopyEngineChannel::channelWrite(Packet *pkt, Addr daddr, int size)
+{
+ switch (daddr) {
+ case CHAN_CONTROL:
+ assert(size == sizeof(uint16_t));
+ int old_int_disable;
+ old_int_disable = cr.ctrl.interrupt_disable();
+ cr.ctrl(pkt->get<uint16_t>());
+ if (cr.ctrl.interrupt_disable())
+ cr.ctrl.interrupt_disable(0);
+ else
+ cr.ctrl.interrupt_disable(old_int_disable);
+ break;
+ case CHAN_STATUS:
+ assert(size == sizeof(uint64_t));
+ DPRINTF(DMACopyEngine, "Warning, ignorning write to register %x\n",
+ daddr);
+ break;
+ case CHAN_CHAINADDR:
+ assert(size == sizeof(uint64_t) || size == sizeof(uint32_t));
+ if (size == sizeof(uint64_t))
+ cr.descChainAddr = pkt->get<uint64_t>();
+ else
+ cr.descChainAddr = (uint64_t)pkt->get<uint32_t>() |
+ (cr.descChainAddr & ~mask(32));
+ DPRINTF(DMACopyEngine, "Chain Address %x\n", cr.descChainAddr);
+ break;
+ case CHAN_CHAINADDR_HIGH:
+ assert(size == sizeof(uint32_t));
+ cr.descChainAddr = ((uint64_t)pkt->get<uint32_t>() <<32) |
+ (cr.descChainAddr & mask(32));
+ DPRINTF(DMACopyEngine, "Chain Address %x\n", cr.descChainAddr);
+ break;
+ case CHAN_COMMAND:
+ assert(size == sizeof(uint8_t));
+ cr.command(pkt->get<uint8_t>());
+ recvCommand();
+ break;
+ case CHAN_CMPLNADDR:
+ assert(size == sizeof(uint64_t) || size == sizeof(uint32_t));
+ if (size == sizeof(uint64_t))
+ cr.completionAddr = pkt->get<uint64_t>();
+ else
+ cr.completionAddr = pkt->get<uint32_t>() |
+ (cr.completionAddr & ~mask(32));
+ break;
+ case CHAN_CMPLNADDR_HIGH:
+ assert(size == sizeof(uint32_t));
+ cr.completionAddr = ((uint64_t)pkt->get<uint32_t>() <<32) |
+ (cr.completionAddr & mask(32));
+ break;
+ case CHAN_ERROR:
+ assert(size == sizeof(uint32_t));
+ cr.error(~pkt->get<uint32_t>() & cr.error());
+ break;
+ default:
+ panic("Read request to unknown channel register number: (%d)%#x\n",
+ channelId, daddr);
+ }
+}
+
+void
+CopyEngine::regStats()
+{
+ using namespace Stats;
+ bytesCopied
+ .init(regs.chanCount)
+ .name(name() + ".bytes_copied")
+ .desc("Number of bytes copied by each engine")
+ .flags(total)
+ ;
+ copiesProcessed
+ .init(regs.chanCount)
+ .name(name() + ".copies_processed")
+ .desc("Number of copies processed by each engine")
+ .flags(total)
+ ;
+}
+
+void
+CopyEngine::CopyEngineChannel::fetchDescriptor(Addr address)
+{
+ DPRINTF(DMACopyEngine, "Reading descriptor from at memory location %#x(%#x)\n",
+ address, ce->platform->pciToDma(address));
+ assert(address);
+ busy = true;
+
+ DPRINTF(DMACopyEngine, "dmaAction: %#x, %d bytes, to addr %#x\n",
+ ce->platform->pciToDma(address), sizeof(DmaDesc), curDmaDesc);
+
+ cePort->dmaAction(MemCmd::ReadReq, ce->platform->pciToDma(address),
+ sizeof(DmaDesc), &fetchCompleteEvent, (uint8_t*)curDmaDesc,
+ latBeforeBegin);
+ lastDescriptorAddr = address;
+}
+
+void
+CopyEngine::CopyEngineChannel::fetchDescComplete()
+{
+ DPRINTF(DMACopyEngine, "Read of descriptor complete\n");
+
+ if ((curDmaDesc->command & DESC_CTRL_NULL)) {
+ DPRINTF(DMACopyEngine, "Got NULL descriptor, skipping\n");
+ assert(!(curDmaDesc->command & DESC_CTRL_CP_STS));
+ if (curDmaDesc->command & DESC_CTRL_CP_STS) {
+ panic("Shouldn't be able to get here\n");
+ nextState = CompletionWrite;
+ if (inDrain()) return;
+ writeCompletionStatus();
+ } else {
+ busy = false;
+ nextState = Idle;
+ inDrain();
+ }
+ return;
+ }
+
+ if (curDmaDesc->command & ~DESC_CTRL_CP_STS)
+ panic("Descriptor has flag other that completion status set\n");
+
+ nextState = DMARead;
+ if (inDrain()) return;
+ readCopyBytes();
+}
+
+void
+CopyEngine::CopyEngineChannel::readCopyBytes()
+{
+ DPRINTF(DMACopyEngine, "Reading %d bytes from buffer to memory location %#x(%#x)\n",
+ curDmaDesc->len, curDmaDesc->dest,
+ ce->platform->pciToDma(curDmaDesc->src));
+ cePort->dmaAction(MemCmd::ReadReq, ce->platform->pciToDma(curDmaDesc->src),
+ curDmaDesc->len, &readCompleteEvent, copyBuffer, 0);
+}
+
+void
+CopyEngine::CopyEngineChannel::readCopyBytesComplete()
+{
+ DPRINTF(DMACopyEngine, "Read of bytes to copy complete\n");
+
+ nextState = DMAWrite;
+ if (inDrain()) return;
+ writeCopyBytes();
+}
+
+void
+CopyEngine::CopyEngineChannel::writeCopyBytes()
+{
+ DPRINTF(DMACopyEngine, "Writing %d bytes from buffer to memory location %#x(%#x)\n",
+ curDmaDesc->len, curDmaDesc->dest,
+ ce->platform->pciToDma(curDmaDesc->dest));
+
+ cePort->dmaAction(MemCmd::WriteReq, ce->platform->pciToDma(curDmaDesc->dest),
+ curDmaDesc->len, &writeCompleteEvent, copyBuffer, 0);
+
+ ce->bytesCopied[channelId] += curDmaDesc->len;
+ ce->copiesProcessed[channelId]++;
+}
+
+void
+CopyEngine::CopyEngineChannel::writeCopyBytesComplete()
+{
+ DPRINTF(DMACopyEngine, "Write of bytes to copy complete user1: %#x\n",
+ curDmaDesc->user1);
+
+ cr.status.compl_desc_addr(lastDescriptorAddr >> 6);
+ completionDataReg = cr.status() | 1;
+
+ if (curDmaDesc->command & DESC_CTRL_CP_STS) {
+ nextState = CompletionWrite;
+ if (inDrain()) return;
+ writeCompletionStatus();
+ return;
+ }
+
+ continueProcessing();
+}
+
+void
+CopyEngine::CopyEngineChannel::continueProcessing()
+{
+ busy = false;
+
+ if (underReset) {
+ underReset = false;
+ refreshNext = false;
+ busy = false;
+ nextState = Idle;
+ return;
+ }
+
+ if (curDmaDesc->next) {
+ nextState = DescriptorFetch;
+ fetchAddress = curDmaDesc->next;
+ if (inDrain()) return;
+ fetchDescriptor(curDmaDesc->next);
+ } else if (refreshNext) {
+ nextState = AddressFetch;
+ refreshNext = false;
+ if (inDrain()) return;
+ fetchNextAddr(lastDescriptorAddr);
+ } else {
+ inDrain();
+ nextState = Idle;
+ }
+}
+
+void
+CopyEngine::CopyEngineChannel::writeCompletionStatus()
+{
+ DPRINTF(DMACopyEngine, "Writing completion status %#x to address %#x(%#x)\n",
+ completionDataReg, cr.completionAddr,
+ ce->platform->pciToDma(cr.completionAddr));
+
+ cePort->dmaAction(MemCmd::WriteReq, ce->platform->pciToDma(cr.completionAddr),
+ sizeof(completionDataReg), &statusCompleteEvent,
+ (uint8_t*)&completionDataReg, latAfterCompletion);
+}
+
+void
+CopyEngine::CopyEngineChannel::writeStatusComplete()
+{
+ DPRINTF(DMACopyEngine, "Writing completion status complete\n");
+ continueProcessing();
+}
+
+void
+CopyEngine::CopyEngineChannel::fetchNextAddr(Addr address)
+{
+ DPRINTF(DMACopyEngine, "Fetching next address...\n");
+ busy = true;
+ cePort->dmaAction(MemCmd::ReadReq, ce->platform->pciToDma(address +
+ offsetof(DmaDesc, next)), sizeof(Addr), &addrCompleteEvent,
+ (uint8_t*)curDmaDesc + offsetof(DmaDesc, next), 0);
+}
+
+void
+CopyEngine::CopyEngineChannel::fetchAddrComplete()
+{
+ DPRINTF(DMACopyEngine, "Fetching next address complete: %#x\n",
+ curDmaDesc->next);
+ if (!curDmaDesc->next) {
+ DPRINTF(DMACopyEngine, "Got NULL descriptor, nothing more to do\n");
+ busy = false;
+ nextState = Idle;
+ inDrain();
+ return;
+ }
+ nextState = DescriptorFetch;
+ fetchAddress = curDmaDesc->next;
+ if (inDrain()) return;
+ fetchDescriptor(curDmaDesc->next);
+}
+
+bool
+CopyEngine::CopyEngineChannel::inDrain()
+{
+ if (ce->getState() == SimObject::Draining) {
+ DPRINTF(DMACopyEngine, "processing drain\n");
+ assert(drainEvent);
+ drainEvent->process();
+ drainEvent = NULL;
+ }
+
+ return ce->getState() != SimObject::Running;
+}
+
+unsigned int
+CopyEngine::CopyEngineChannel::drain(Event *de)
+{
+ if (nextState == Idle || ce->getState() != SimObject::Running)
+ return 0;
+ unsigned int count = 1;
+ count += cePort->drain(de);
+
+ DPRINTF(DMACopyEngine, "unable to drain, returning %d\n", count);
+ drainEvent = de;
+ return count;
+}
+
+unsigned int
+CopyEngine::drain(Event *de)
+{
+ unsigned int count;
+ count = pioPort->drain(de) + dmaPort->drain(de) + configPort->drain(de);
+ for (int x = 0;x < chan.size(); x++)
+ count += chan[x]->drain(de);
+
+ if (count)
+ changeState(Draining);
+ else
+ changeState(Drained);
+
+ DPRINTF(DMACopyEngine, "call to CopyEngine::drain() returning %d\n", count);
+ return count;
+}
+
+void
+CopyEngine::serialize(std::ostream &os)
+{
+ PciDev::serialize(os);
+ regs.serialize(os);
+ for (int x =0; x < chan.size(); x++) {
+ nameOut(os, csprintf("%s.channel%d", name(), x));
+ chan[x]->serialize(os);
+ }
+}
+
+void
+CopyEngine::unserialize(Checkpoint *cp, const std::string §ion)
+{
+ PciDev::unserialize(cp, section);
+ regs.unserialize(cp, section);
+ for (int x = 0; x < chan.size(); x++)
+ chan[x]->unserialize(cp, csprintf("%s.channel%d", section, x));
+}
+
+void
+CopyEngine::CopyEngineChannel::serialize(std::ostream &os)
+{
+ SERIALIZE_SCALAR(channelId);
+ SERIALIZE_SCALAR(busy);
+ SERIALIZE_SCALAR(underReset);
+ SERIALIZE_SCALAR(refreshNext);
+ SERIALIZE_SCALAR(lastDescriptorAddr);
+ SERIALIZE_SCALAR(completionDataReg);
+ SERIALIZE_SCALAR(fetchAddress);
+ int nextState = this->nextState;
+ SERIALIZE_SCALAR(nextState);
+ arrayParamOut(os, "curDmaDesc", (uint8_t*)curDmaDesc, sizeof(DmaDesc));
+ SERIALIZE_ARRAY(copyBuffer, ce->params()->XferCap);
+ cr.serialize(os);
+
+}
+void
+CopyEngine::CopyEngineChannel::unserialize(Checkpoint *cp, const std::string §ion)
+{
+ UNSERIALIZE_SCALAR(channelId);
+ UNSERIALIZE_SCALAR(busy);
+ UNSERIALIZE_SCALAR(underReset);
+ UNSERIALIZE_SCALAR(refreshNext);
+ UNSERIALIZE_SCALAR(lastDescriptorAddr);
+ UNSERIALIZE_SCALAR(completionDataReg);
+ UNSERIALIZE_SCALAR(fetchAddress);
+ int nextState;
+ UNSERIALIZE_SCALAR(nextState);
+ this->nextState = (ChannelState)nextState;
+ arrayParamIn(cp, section, "curDmaDesc", (uint8_t*)curDmaDesc, sizeof(DmaDesc));
+ UNSERIALIZE_ARRAY(copyBuffer, ce->params()->XferCap);
+ cr.unserialize(cp, section);
+
+}
+
+void
+CopyEngine::CopyEngineChannel::restartStateMachine()
+{
+ switch(nextState) {
+ case AddressFetch:
+ fetchNextAddr(lastDescriptorAddr);
+ break;
+ case DescriptorFetch:
+ fetchDescriptor(fetchAddress);
+ break;
+ case DMARead:
+ readCopyBytes();
+ break;
+ case DMAWrite:
+ writeCopyBytes();
+ break;
+ case CompletionWrite:
+ writeCompletionStatus();
+ break;
+ case Idle:
+ break;
+ default:
+ panic("Unknown state for CopyEngineChannel\n");
+ }
+}
+
+void
+CopyEngine::resume()
+{
+ SimObject::resume();
+ for (int x = 0;x < chan.size(); x++)
+ chan[x]->resume();
+}
+
+
+void
+CopyEngine::CopyEngineChannel::resume()
+{
+ DPRINTF(DMACopyEngine, "Restarting state machine at state %d\n", nextState);
+ restartStateMachine();
+}
+
+CopyEngine *
+CopyEngineParams::create()
+{
+ return new CopyEngine(this);
+}
--- /dev/null
+/*
+ * Copyright (c) 2008 The Regents of The University of Michigan
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met: redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer;
+ * redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution;
+ * neither the name of the copyright holders nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Authors: Ali Saidi
+ */
+
+/* @file
+ * Device model for Intel's I/O Acceleration Technology (I/OAT).
+ * A DMA asyncronous copy engine
+ */
+
+#ifndef __DEV_COPY_ENGINE_HH__
+#define __DEV_COPY_ENGINE_HH__
+
+#include <vector>
+
+#include "base/statistics.hh"
+#include "dev/copy_engine_defs.hh"
+#include "dev/pcidev.hh"
+#include "params/CopyEngine.hh"
+#include "sim/eventq.hh"
+
+class CopyEngine : public PciDev
+{
+ class CopyEngineChannel
+ {
+ private:
+ DmaPort *cePort;
+ CopyEngine *ce;
+ CopyEngineReg::ChanRegs cr;
+ int channelId;
+ CopyEngineReg::DmaDesc *curDmaDesc;
+ uint8_t *copyBuffer;
+
+ bool busy;
+ bool underReset;
+ bool refreshNext;
+ Addr lastDescriptorAddr;
+ Addr fetchAddress;
+
+ Tick latBeforeBegin;
+ Tick latAfterCompletion;
+
+ uint64_t completionDataReg;
+
+ enum ChannelState {
+ Idle,
+ AddressFetch,
+ DescriptorFetch,
+ DMARead,
+ DMAWrite,
+ CompletionWrite
+ };
+
+ ChannelState nextState;
+
+ Event *drainEvent;
+ public:
+ CopyEngineChannel(CopyEngine *_ce, int cid);
+ ~CopyEngineChannel();
+ void init();
+
+ std::string name() { assert(ce); return ce->name() + csprintf("-chan%d", channelId); }
+ virtual void addressRanges(AddrRangeList &range_list) { range_list.clear(); }
+ virtual Tick read(PacketPtr pkt)
+ { panic("CopyEngineChannel has no I/O access\n");}
+ virtual Tick write(PacketPtr pkt)
+ { panic("CopyEngineChannel has no I/O access\n"); }
+
+ void channelRead(PacketPtr pkt, Addr daddr, int size);
+ void channelWrite(PacketPtr pkt, Addr daddr, int size);
+
+ unsigned int drain(Event *de);
+ void resume();
+ void serialize(std::ostream &os);
+ void unserialize(Checkpoint *cp, const std::string §ion);
+
+ private:
+ void fetchDescriptor(Addr address);
+ void fetchDescComplete();
+ EventWrapper<CopyEngineChannel, &CopyEngineChannel::fetchDescComplete>
+ fetchCompleteEvent;
+
+ void fetchNextAddr(Addr address);
+ void fetchAddrComplete();
+ EventWrapper<CopyEngineChannel, &CopyEngineChannel::fetchAddrComplete>
+ addrCompleteEvent;
+
+ void readCopyBytes();
+ void readCopyBytesComplete();
+ EventWrapper<CopyEngineChannel, &CopyEngineChannel::readCopyBytesComplete>
+ readCompleteEvent;
+
+ void writeCopyBytes();
+ void writeCopyBytesComplete();
+ EventWrapper <CopyEngineChannel, &CopyEngineChannel::writeCopyBytesComplete>
+ writeCompleteEvent;
+
+ void writeCompletionStatus();
+ void writeStatusComplete();
+ EventWrapper <CopyEngineChannel, &CopyEngineChannel::writeStatusComplete>
+ statusCompleteEvent;
+
+
+ void continueProcessing();
+ void recvCommand();
+ bool inDrain();
+ void restartStateMachine();
+ };
+
+ private:
+
+ Stats::Vector<> bytesCopied;
+ Stats::Vector<> copiesProcessed;
+
+ // device registers
+ CopyEngineReg::Regs regs;
+
+ // Array of channels each one with regs/dma port/etc
+ std::vector<CopyEngineChannel*> chan;
+
+ public:
+ typedef CopyEngineParams Params;
+ const Params *
+ params() const
+ {
+ return dynamic_cast<const Params *>(_params);
+ }
+ CopyEngine(const Params *params);
+ ~CopyEngine();
+
+ void regStats();
+ void init();
+
+ virtual Tick read(PacketPtr pkt);
+ virtual Tick write(PacketPtr pkt);
+
+ virtual void serialize(std::ostream &os);
+ virtual void unserialize(Checkpoint *cp, const std::string §ion);
+ virtual unsigned int drain(Event *de);
+ virtual void resume();
+};
+
+#endif //__DEV_COPY_ENGINE_HH__
+
--- /dev/null
+/*
+ * Copyright (c) 2008 The Regents of The University of Michigan
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met: redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer;
+ * redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution;
+ * neither the name of the copyright holders nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Authors: Ali Saidi
+ */
+
+/* @file
+ * Register and structure descriptions for Intel's I/O AT DMA Engine
+ */
+#include "base/bitfield.hh"
+#include "sim/serialize.hh"
+
+namespace CopyEngineReg {
+
+
+// General Channel independant registers, 128 bytes starting at 0x00
+const uint32_t GEN_CHANCOUNT = 0x00;
+const uint32_t GEN_XFERCAP = 0x01;
+const uint32_t GEN_INTRCTRL = 0x03;
+const uint32_t GEN_ATTNSTATUS = 0x04;
+
+
+// Channel specific registers, each block is 128 bytes, starting at 0x80
+const uint32_t CHAN_CONTROL = 0x00;
+const uint32_t CHAN_STATUS = 0x04;
+const uint32_t CHAN_CHAINADDR = 0x0C;
+const uint32_t CHAN_CHAINADDR_LOW = 0x0C;
+const uint32_t CHAN_CHAINADDR_HIGH = 0x10;
+const uint32_t CHAN_COMMAND = 0x14;
+const uint32_t CHAN_CMPLNADDR = 0x18;
+const uint32_t CHAN_CMPLNADDR_LOW = 0x18;
+const uint32_t CHAN_CMPLNADDR_HIGH = 0x1C;
+const uint32_t CHAN_ERROR = 0x28;
+
+
+const uint32_t DESC_CTRL_INT_GEN = 0x00000001;
+const uint32_t DESC_CTRL_SRC_SN = 0x00000002;
+const uint32_t DESC_CTRL_DST_SN = 0x00000004;
+const uint32_t DESC_CTRL_CP_STS = 0x00000008;
+const uint32_t DESC_CTRL_FRAME = 0x00000010;
+const uint32_t DESC_CTRL_NULL = 0x00000020;
+
+struct DmaDesc {
+ uint32_t len;
+ uint32_t command;
+ Addr src;
+ Addr dest;
+ Addr next;
+ uint64_t reserved1;
+ uint64_t reserved2;
+ uint64_t user1;
+ uint64_t user2;
+};
+
+#define ADD_FIELD8(NAME, OFFSET, BITS) \
+ inline uint8_t NAME() { return bits(_data, OFFSET+BITS-1, OFFSET); } \
+ inline void NAME(uint8_t d) { replaceBits(_data, OFFSET+BITS-1, OFFSET,d); }
+
+#define ADD_FIELD16(NAME, OFFSET, BITS) \
+ inline uint16_t NAME() { return bits(_data, OFFSET+BITS-1, OFFSET); } \
+ inline void NAME(uint16_t d) { replaceBits(_data, OFFSET+BITS-1, OFFSET,d); }
+
+#define ADD_FIELD32(NAME, OFFSET, BITS) \
+ inline uint32_t NAME() { return bits(_data, OFFSET+BITS-1, OFFSET); } \
+ inline void NAME(uint32_t d) { replaceBits(_data, OFFSET+BITS-1, OFFSET,d); }
+
+#define ADD_FIELD64(NAME, OFFSET, BITS) \
+ inline uint64_t NAME() { return bits(_data, OFFSET+BITS-1, OFFSET); } \
+ inline void NAME(uint64_t d) { replaceBits(_data, OFFSET+BITS-1, OFFSET,d); }
+
+template<class T>
+struct Reg {
+ T _data;
+ T operator()() { return _data; }
+ const Reg<T> &operator=(T d) { _data = d; return *this;}
+ bool operator==(T d) { return d == _data; }
+ void operator()(T d) { _data = d; }
+ Reg() { _data = 0; }
+ void serialize(std::ostream &os)
+ {
+ SERIALIZE_SCALAR(_data);
+ }
+ void unserialize(Checkpoint *cp, const std::string §ion)
+ {
+ UNSERIALIZE_SCALAR(_data);
+ }
+};
+
+
+struct Regs {
+ uint8_t chanCount;
+ uint8_t xferCap;
+
+ struct INTRCTRL : public Reg<uint8_t> { // 0x03
+ using Reg<uint8_t>::operator =;
+ ADD_FIELD8(master_int_enable,0,1);
+ ADD_FIELD8(interrupt_status,1,1);
+ ADD_FIELD8(interrupt,2,1);
+ };
+ INTRCTRL intrctrl;
+
+ uint32_t attnStatus; // Read clears
+
+ void serialize(std::ostream &os)
+ {
+ SERIALIZE_SCALAR(chanCount);
+ SERIALIZE_SCALAR(xferCap);
+ paramOut(os, "intrctrl", intrctrl._data);
+ SERIALIZE_SCALAR(attnStatus);
+ }
+
+ void unserialize(Checkpoint *cp, const std::string §ion)
+ {
+ UNSERIALIZE_SCALAR(chanCount);
+ UNSERIALIZE_SCALAR(xferCap);
+ paramIn(cp, section, "intrctrl", intrctrl._data);
+ UNSERIALIZE_SCALAR(attnStatus);
+ }
+
+};
+
+struct ChanRegs {
+ struct CHANCTRL : public Reg<uint16_t> { // channelX + 0x00
+ using Reg<uint16_t>::operator =;
+ ADD_FIELD16(interrupt_disable,0,1);
+ ADD_FIELD16(error_completion_enable, 2,1);
+ ADD_FIELD16(any_error_abort_enable,3,1);
+ ADD_FIELD16(error_int_enable,4,1);
+ ADD_FIELD16(desc_addr_snoop_control,5,1);
+ ADD_FIELD16(in_use, 8,1);
+ };
+ CHANCTRL ctrl;
+
+ struct CHANSTS : public Reg<uint64_t> { // channelX + 0x04
+ ADD_FIELD64(dma_transfer_status, 0, 3);
+ ADD_FIELD64(unaffiliated_error, 3, 1);
+ ADD_FIELD64(soft_error, 4, 1);
+ ADD_FIELD64(compl_desc_addr, 6, 58);
+ };
+ CHANSTS status;
+
+ uint64_t descChainAddr;
+
+ struct CHANCMD : public Reg<uint8_t> { // channelX + 0x14
+ ADD_FIELD8(start_dma,0,1);
+ ADD_FIELD8(append_dma,1,1);
+ ADD_FIELD8(suspend_dma,2,1);
+ ADD_FIELD8(abort_dma,3,1);
+ ADD_FIELD8(resume_dma,4,1);
+ ADD_FIELD8(reset_dma,5,1);
+ };
+ CHANCMD command;
+
+ uint64_t completionAddr;
+
+ struct CHANERR : public Reg<uint32_t> { // channel X + 0x28
+ ADD_FIELD32(source_addr_error,0,1);
+ ADD_FIELD32(dest_addr_error,1,1);
+ ADD_FIELD32(ndesc_addr_error,2,1);
+ ADD_FIELD32(desc_error,3,1);
+ ADD_FIELD32(chain_addr_error,4,1);
+ ADD_FIELD32(chain_cmd_error,5,1);
+ ADD_FIELD32(chipset_parity_error,6,1);
+ ADD_FIELD32(dma_parity_error,7,1);
+ ADD_FIELD32(read_data_error,8,1);
+ ADD_FIELD32(write_data_error,9,1);
+ ADD_FIELD32(desc_control_error,10,1);
+ ADD_FIELD32(desc_len_error,11,1);
+ ADD_FIELD32(completion_addr_error,12,1);
+ ADD_FIELD32(interrupt_config_error,13,1);
+ ADD_FIELD32(soft_error,14,1);
+ ADD_FIELD32(unaffiliated_error,15,1);
+ };
+ CHANERR error;
+
+ void serialize(std::ostream &os)
+ {
+ paramOut(os, "ctrl", ctrl._data);
+ paramOut(os, "status", status._data);
+ SERIALIZE_SCALAR(descChainAddr);
+ paramOut(os, "command", command._data);
+ SERIALIZE_SCALAR(completionAddr);
+ paramOut(os, "error", error._data);
+ }
+
+ void unserialize(Checkpoint *cp, const std::string §ion)
+ {
+ paramIn(cp, section, "ctrl", ctrl._data);
+ paramIn(cp, section, "status", status._data);
+ UNSERIALIZE_SCALAR(descChainAddr);
+ paramIn(cp, section, "command", command._data);
+ UNSERIALIZE_SCALAR(completionAddr);
+ paramIn(cp, section, "error", error._data);
+ }
+
+
+};
+
+} //namespace CopyEngineReg
+
+
projects / gem5.git / commitdiff