2 * Copyright (c) 2012 ARM Limited
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
14 * Copyright (c) 2008 The Regents of The University of Michigan
15 * All rights reserved.
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions are
19 * met: redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer;
21 * redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution;
24 * neither the name of the copyright holders nor the names of its
25 * contributors may be used to endorse or promote products derived from
26 * this software without specific prior written permission.
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
44 * Device model for Intel's I/O AT DMA copy engine.
47 #include "dev/pci/copy_engine.hh"
51 #include "base/cp_annotate.hh"
52 #include "base/trace.hh"
53 #include "debug/DMACopyEngine.hh"
54 #include "debug/Drain.hh"
55 #include "mem/packet.hh"
56 #include "mem/packet_access.hh"
57 #include "params/CopyEngine.hh"
58 #include "sim/stats.hh"
59 #include "sim/system.hh"
61 using namespace CopyEngineReg
;
63 CopyEngine::CopyEngine(const Params
*p
)
66 // All Reg regs are initialized to 0 by default
67 regs
.chanCount
= p
->ChanCnt
;
68 regs
.xferCap
= findMsbSet(p
->XferCap
);
71 if (regs
.chanCount
> 64)
72 fatal("CopyEngine interface doesn't support more than 64 DMA engines\n");
74 for (int x
= 0; x
< regs
.chanCount
; x
++) {
75 CopyEngineChannel
*ch
= new CopyEngineChannel(this, x
);
81 CopyEngine::CopyEngineChannel::CopyEngineChannel(CopyEngine
*_ce
, int cid
)
82 : cePort(_ce
, _ce
->sys
),
83 ce(_ce
), channelId(cid
), busy(false), underReset(false),
84 refreshNext(false), latBeforeBegin(ce
->params()->latBeforeBegin
),
85 latAfterCompletion(ce
->params()->latAfterCompletion
),
86 completionDataReg(0), nextState(Idle
),
87 fetchCompleteEvent([this]{ fetchDescComplete(); }, name()),
88 addrCompleteEvent([this]{ fetchAddrComplete(); }, name()),
89 readCompleteEvent([this]{ readCopyBytesComplete(); }, name()),
90 writeCompleteEvent([this]{ writeCopyBytesComplete(); }, name()),
91 statusCompleteEvent([this]{ writeStatusComplete(); }, name())
94 cr
.status
.dma_transfer_status(3);
96 cr
.completionAddr
= 0;
98 curDmaDesc
= new DmaDesc
;
99 memset(curDmaDesc
, 0, sizeof(DmaDesc
));
100 copyBuffer
= new uint8_t[ce
->params()->XferCap
];
103 CopyEngine::~CopyEngine()
105 for (int x
= 0; x
< chan
.size(); x
++) {
110 CopyEngine::CopyEngineChannel::~CopyEngineChannel()
113 delete [] copyBuffer
;
117 CopyEngine::getMasterPort(const std::string
&if_name
, PortID idx
)
119 if (if_name
!= "dma") {
120 // pass it along to our super class
121 return PciDevice::getMasterPort(if_name
, idx
);
123 if (idx
>= static_cast<int>(chan
.size())) {
124 panic("CopyEngine::getMasterPort: unknown index %d\n", idx
);
127 return chan
[idx
]->getMasterPort();
133 CopyEngine::CopyEngineChannel::getMasterPort()
139 CopyEngine::CopyEngineChannel::recvCommand()
141 if (cr
.command
.start_dma()) {
143 cr
.status
.dma_transfer_status(0);
144 nextState
= DescriptorFetch
;
145 fetchAddress
= cr
.descChainAddr
;
146 if (ce
->drainState() == DrainState::Running
)
147 fetchDescriptor(cr
.descChainAddr
);
148 } else if (cr
.command
.append_dma()) {
150 nextState
= AddressFetch
;
151 if (ce
->drainState() == DrainState::Running
)
152 fetchNextAddr(lastDescriptorAddr
);
155 } else if (cr
.command
.reset_dma()) {
159 cr
.status
.dma_transfer_status(3);
162 } else if (cr
.command
.resume_dma() || cr
.command
.abort_dma() ||
163 cr
.command
.suspend_dma())
164 panic("Resume, Abort, and Suspend are not supported\n");
169 CopyEngine::read(PacketPtr pkt
)
174 if (!getBAR(pkt
->getAddr(), bar
, daddr
))
175 panic("Invalid PCI memory access to unmapped memory.\n");
177 // Only Memory register BAR is allowed
180 int size
= pkt
->getSize();
181 if (size
!= sizeof(uint64_t) && size
!= sizeof(uint32_t) &&
182 size
!= sizeof(uint16_t) && size
!= sizeof(uint8_t)) {
183 panic("Unknown size for MMIO access: %d\n", pkt
->getSize());
186 DPRINTF(DMACopyEngine
, "Read device register %#X size: %d\n", daddr
, size
);
189 /// Handle read of register here
195 assert(size
== sizeof(regs
.chanCount
));
196 pkt
->set
<uint8_t>(regs
.chanCount
);
199 assert(size
== sizeof(regs
.xferCap
));
200 pkt
->set
<uint8_t>(regs
.xferCap
);
203 assert(size
== sizeof(uint8_t));
204 pkt
->set
<uint8_t>(regs
.intrctrl());
205 regs
.intrctrl
.master_int_enable(0);
208 assert(size
== sizeof(regs
.attnStatus
));
209 pkt
->set
<uint32_t>(regs
.attnStatus
);
213 panic("Read request to unknown register number: %#x\n", daddr
);
215 pkt
->makeAtomicResponse();
220 // Find which channel we're accessing
223 while (daddr
>= 0x80) {
228 if (chanid
>= regs
.chanCount
)
229 panic("Access to channel %d (device only configured for %d channels)",
230 chanid
, regs
.chanCount
);
233 /// Channel registers are handled here
235 chan
[chanid
]->channelRead(pkt
, daddr
, size
);
237 pkt
->makeAtomicResponse();
242 CopyEngine::CopyEngineChannel::channelRead(Packet
*pkt
, Addr daddr
, int size
)
246 assert(size
== sizeof(uint16_t));
247 pkt
->set
<uint16_t>(cr
.ctrl());
251 assert(size
== sizeof(uint64_t));
252 pkt
->set
<uint64_t>(cr
.status() | ~busy
);
255 assert(size
== sizeof(uint64_t) || size
== sizeof(uint32_t));
256 if (size
== sizeof(uint64_t))
257 pkt
->set
<uint64_t>(cr
.descChainAddr
);
259 pkt
->set
<uint32_t>(bits(cr
.descChainAddr
,0,31));
261 case CHAN_CHAINADDR_HIGH
:
262 assert(size
== sizeof(uint32_t));
263 pkt
->set
<uint32_t>(bits(cr
.descChainAddr
,32,63));
266 assert(size
== sizeof(uint8_t));
267 pkt
->set
<uint32_t>(cr
.command());
270 assert(size
== sizeof(uint64_t) || size
== sizeof(uint32_t));
271 if (size
== sizeof(uint64_t))
272 pkt
->set
<uint64_t>(cr
.completionAddr
);
274 pkt
->set
<uint32_t>(bits(cr
.completionAddr
,0,31));
276 case CHAN_CMPLNADDR_HIGH
:
277 assert(size
== sizeof(uint32_t));
278 pkt
->set
<uint32_t>(bits(cr
.completionAddr
,32,63));
281 assert(size
== sizeof(uint32_t));
282 pkt
->set
<uint32_t>(cr
.error());
285 panic("Read request to unknown channel register number: (%d)%#x\n",
292 CopyEngine::write(PacketPtr pkt
)
298 if (!getBAR(pkt
->getAddr(), bar
, daddr
))
299 panic("Invalid PCI memory access to unmapped memory.\n");
301 // Only Memory register BAR is allowed
304 int size
= pkt
->getSize();
307 /// Handle write of register here
310 if (size
== sizeof(uint64_t)) {
311 uint64_t val M5_VAR_USED
= pkt
->get
<uint64_t>();
312 DPRINTF(DMACopyEngine
, "Wrote device register %#X value %#X\n", daddr
, val
);
313 } else if (size
== sizeof(uint32_t)) {
314 uint32_t val M5_VAR_USED
= pkt
->get
<uint32_t>();
315 DPRINTF(DMACopyEngine
, "Wrote device register %#X value %#X\n", daddr
, val
);
316 } else if (size
== sizeof(uint16_t)) {
317 uint16_t val M5_VAR_USED
= pkt
->get
<uint16_t>();
318 DPRINTF(DMACopyEngine
, "Wrote device register %#X value %#X\n", daddr
, val
);
319 } else if (size
== sizeof(uint8_t)) {
320 uint8_t val M5_VAR_USED
= pkt
->get
<uint8_t>();
321 DPRINTF(DMACopyEngine
, "Wrote device register %#X value %#X\n", daddr
, val
);
323 panic("Unknown size for MMIO access: %d\n", size
);
331 DPRINTF(DMACopyEngine
, "Warning, ignorning write to register %x\n",
335 regs
.intrctrl
.master_int_enable(bits(pkt
->get
<uint8_t>(),0,1));
338 panic("Read request to unknown register number: %#x\n", daddr
);
340 pkt
->makeAtomicResponse();
344 // Find which channel we're accessing
347 while (daddr
>= 0x80) {
352 if (chanid
>= regs
.chanCount
)
353 panic("Access to channel %d (device only configured for %d channels)",
354 chanid
, regs
.chanCount
);
357 /// Channel registers are handled here
359 chan
[chanid
]->channelWrite(pkt
, daddr
, size
);
361 pkt
->makeAtomicResponse();
366 CopyEngine::CopyEngineChannel::channelWrite(Packet
*pkt
, Addr daddr
, int size
)
370 assert(size
== sizeof(uint16_t));
372 old_int_disable
= cr
.ctrl
.interrupt_disable();
373 cr
.ctrl(pkt
->get
<uint16_t>());
374 if (cr
.ctrl
.interrupt_disable())
375 cr
.ctrl
.interrupt_disable(0);
377 cr
.ctrl
.interrupt_disable(old_int_disable
);
380 assert(size
== sizeof(uint64_t));
381 DPRINTF(DMACopyEngine
, "Warning, ignorning write to register %x\n",
385 assert(size
== sizeof(uint64_t) || size
== sizeof(uint32_t));
386 if (size
== sizeof(uint64_t))
387 cr
.descChainAddr
= pkt
->get
<uint64_t>();
389 cr
.descChainAddr
= (uint64_t)pkt
->get
<uint32_t>() |
390 (cr
.descChainAddr
& ~mask(32));
391 DPRINTF(DMACopyEngine
, "Chain Address %x\n", cr
.descChainAddr
);
393 case CHAN_CHAINADDR_HIGH
:
394 assert(size
== sizeof(uint32_t));
395 cr
.descChainAddr
= ((uint64_t)pkt
->get
<uint32_t>() <<32) |
396 (cr
.descChainAddr
& mask(32));
397 DPRINTF(DMACopyEngine
, "Chain Address %x\n", cr
.descChainAddr
);
400 assert(size
== sizeof(uint8_t));
401 cr
.command(pkt
->get
<uint8_t>());
405 assert(size
== sizeof(uint64_t) || size
== sizeof(uint32_t));
406 if (size
== sizeof(uint64_t))
407 cr
.completionAddr
= pkt
->get
<uint64_t>();
409 cr
.completionAddr
= pkt
->get
<uint32_t>() |
410 (cr
.completionAddr
& ~mask(32));
412 case CHAN_CMPLNADDR_HIGH
:
413 assert(size
== sizeof(uint32_t));
414 cr
.completionAddr
= ((uint64_t)pkt
->get
<uint32_t>() <<32) |
415 (cr
.completionAddr
& mask(32));
418 assert(size
== sizeof(uint32_t));
419 cr
.error(~pkt
->get
<uint32_t>() & cr
.error());
422 panic("Read request to unknown channel register number: (%d)%#x\n",
428 CopyEngine::regStats()
430 PciDevice::regStats();
432 using namespace Stats
;
434 .init(regs
.chanCount
)
435 .name(name() + ".bytes_copied")
436 .desc("Number of bytes copied by each engine")
440 .init(regs
.chanCount
)
441 .name(name() + ".copies_processed")
442 .desc("Number of copies processed by each engine")
448 CopyEngine::CopyEngineChannel::fetchDescriptor(Addr address
)
451 anBegin("FetchDescriptor");
452 DPRINTF(DMACopyEngine
, "Reading descriptor from at memory location %#x(%#x)\n",
453 address
, ce
->pciToDma(address
));
457 DPRINTF(DMACopyEngine
, "dmaAction: %#x, %d bytes, to addr %#x\n",
458 ce
->pciToDma(address
), sizeof(DmaDesc
), curDmaDesc
);
460 cePort
.dmaAction(MemCmd::ReadReq
, ce
->pciToDma(address
),
461 sizeof(DmaDesc
), &fetchCompleteEvent
,
462 (uint8_t*)curDmaDesc
, latBeforeBegin
);
463 lastDescriptorAddr
= address
;
467 CopyEngine::CopyEngineChannel::fetchDescComplete()
469 DPRINTF(DMACopyEngine
, "Read of descriptor complete\n");
471 if ((curDmaDesc
->command
& DESC_CTRL_NULL
)) {
472 DPRINTF(DMACopyEngine
, "Got NULL descriptor, skipping\n");
473 assert(!(curDmaDesc
->command
& DESC_CTRL_CP_STS
));
474 if (curDmaDesc
->command
& DESC_CTRL_CP_STS
) {
475 panic("Shouldn't be able to get here\n");
476 nextState
= CompletionWrite
;
477 if (inDrain()) return;
478 writeCompletionStatus();
489 if (curDmaDesc
->command
& ~DESC_CTRL_CP_STS
)
490 panic("Descriptor has flag other that completion status set\n");
493 if (inDrain()) return;
498 CopyEngine::CopyEngineChannel::readCopyBytes()
500 anBegin("ReadCopyBytes");
501 DPRINTF(DMACopyEngine
, "Reading %d bytes from buffer to memory location %#x(%#x)\n",
502 curDmaDesc
->len
, curDmaDesc
->dest
,
503 ce
->pciToDma(curDmaDesc
->src
));
504 cePort
.dmaAction(MemCmd::ReadReq
, ce
->pciToDma(curDmaDesc
->src
),
505 curDmaDesc
->len
, &readCompleteEvent
, copyBuffer
, 0);
509 CopyEngine::CopyEngineChannel::readCopyBytesComplete()
511 DPRINTF(DMACopyEngine
, "Read of bytes to copy complete\n");
513 nextState
= DMAWrite
;
514 if (inDrain()) return;
519 CopyEngine::CopyEngineChannel::writeCopyBytes()
521 anBegin("WriteCopyBytes");
522 DPRINTF(DMACopyEngine
, "Writing %d bytes from buffer to memory location %#x(%#x)\n",
523 curDmaDesc
->len
, curDmaDesc
->dest
,
524 ce
->pciToDma(curDmaDesc
->dest
));
526 cePort
.dmaAction(MemCmd::WriteReq
, ce
->pciToDma(curDmaDesc
->dest
),
527 curDmaDesc
->len
, &writeCompleteEvent
, copyBuffer
, 0);
529 ce
->bytesCopied
[channelId
] += curDmaDesc
->len
;
530 ce
->copiesProcessed
[channelId
]++;
534 CopyEngine::CopyEngineChannel::writeCopyBytesComplete()
536 DPRINTF(DMACopyEngine
, "Write of bytes to copy complete user1: %#x\n",
539 cr
.status
.compl_desc_addr(lastDescriptorAddr
>> 6);
540 completionDataReg
= cr
.status() | 1;
542 anQ("DMAUsedDescQ", channelId
, 1);
543 anQ("AppRecvQ", curDmaDesc
->user1
, curDmaDesc
->len
);
544 if (curDmaDesc
->command
& DESC_CTRL_CP_STS
) {
545 nextState
= CompletionWrite
;
546 if (inDrain()) return;
547 writeCompletionStatus();
551 continueProcessing();
555 CopyEngine::CopyEngineChannel::continueProcessing()
569 if (curDmaDesc
->next
) {
570 nextState
= DescriptorFetch
;
571 fetchAddress
= curDmaDesc
->next
;
572 if (inDrain()) return;
573 fetchDescriptor(curDmaDesc
->next
);
574 } else if (refreshNext
) {
575 nextState
= AddressFetch
;
577 if (inDrain()) return;
578 fetchNextAddr(lastDescriptorAddr
);
588 CopyEngine::CopyEngineChannel::writeCompletionStatus()
590 anBegin("WriteCompletionStatus");
591 DPRINTF(DMACopyEngine
, "Writing completion status %#x to address %#x(%#x)\n",
592 completionDataReg
, cr
.completionAddr
,
593 ce
->pciToDma(cr
.completionAddr
));
595 cePort
.dmaAction(MemCmd::WriteReq
,
596 ce
->pciToDma(cr
.completionAddr
),
597 sizeof(completionDataReg
), &statusCompleteEvent
,
598 (uint8_t*)&completionDataReg
, latAfterCompletion
);
602 CopyEngine::CopyEngineChannel::writeStatusComplete()
604 DPRINTF(DMACopyEngine
, "Writing completion status complete\n");
605 continueProcessing();
609 CopyEngine::CopyEngineChannel::fetchNextAddr(Addr address
)
611 anBegin("FetchNextAddr");
612 DPRINTF(DMACopyEngine
, "Fetching next address...\n");
614 cePort
.dmaAction(MemCmd::ReadReq
,
615 ce
->pciToDma(address
+ offsetof(DmaDesc
, next
)),
616 sizeof(Addr
), &addrCompleteEvent
,
617 (uint8_t*)curDmaDesc
+ offsetof(DmaDesc
, next
), 0);
621 CopyEngine::CopyEngineChannel::fetchAddrComplete()
623 DPRINTF(DMACopyEngine
, "Fetching next address complete: %#x\n",
625 if (!curDmaDesc
->next
) {
626 DPRINTF(DMACopyEngine
, "Got NULL descriptor, nothing more to do\n");
634 nextState
= DescriptorFetch
;
635 fetchAddress
= curDmaDesc
->next
;
636 if (inDrain()) return;
637 fetchDescriptor(curDmaDesc
->next
);
641 CopyEngine::CopyEngineChannel::inDrain()
643 if (drainState() == DrainState::Draining
) {
644 DPRINTF(Drain
, "CopyEngine done draining, processing drain event\n");
648 return ce
->drainState() != DrainState::Running
;
652 CopyEngine::CopyEngineChannel::drain()
654 if (nextState
== Idle
|| ce
->drainState() != DrainState::Running
) {
655 return DrainState::Drained
;
657 DPRINTF(Drain
, "CopyEngineChannel not drained\n");
658 return DrainState::Draining
;
663 CopyEngine::serialize(CheckpointOut
&cp
) const
665 PciDevice::serialize(cp
);
667 for (int x
=0; x
< chan
.size(); x
++)
668 chan
[x
]->serializeSection(cp
, csprintf("channel%d", x
));
672 CopyEngine::unserialize(CheckpointIn
&cp
)
674 PciDevice::unserialize(cp
);
675 regs
.unserialize(cp
);
676 for (int x
= 0; x
< chan
.size(); x
++)
677 chan
[x
]->unserializeSection(cp
, csprintf("channel%d", x
));
681 CopyEngine::CopyEngineChannel::serialize(CheckpointOut
&cp
) const
683 SERIALIZE_SCALAR(channelId
);
684 SERIALIZE_SCALAR(busy
);
685 SERIALIZE_SCALAR(underReset
);
686 SERIALIZE_SCALAR(refreshNext
);
687 SERIALIZE_SCALAR(lastDescriptorAddr
);
688 SERIALIZE_SCALAR(completionDataReg
);
689 SERIALIZE_SCALAR(fetchAddress
);
690 int nextState
= this->nextState
;
691 SERIALIZE_SCALAR(nextState
);
692 arrayParamOut(cp
, "curDmaDesc", (uint8_t*)curDmaDesc
, sizeof(DmaDesc
));
693 SERIALIZE_ARRAY(copyBuffer
, ce
->params()->XferCap
);
698 CopyEngine::CopyEngineChannel::unserialize(CheckpointIn
&cp
)
700 UNSERIALIZE_SCALAR(channelId
);
701 UNSERIALIZE_SCALAR(busy
);
702 UNSERIALIZE_SCALAR(underReset
);
703 UNSERIALIZE_SCALAR(refreshNext
);
704 UNSERIALIZE_SCALAR(lastDescriptorAddr
);
705 UNSERIALIZE_SCALAR(completionDataReg
);
706 UNSERIALIZE_SCALAR(fetchAddress
);
708 UNSERIALIZE_SCALAR(nextState
);
709 this->nextState
= (ChannelState
)nextState
;
710 arrayParamIn(cp
, "curDmaDesc", (uint8_t*)curDmaDesc
, sizeof(DmaDesc
));
711 UNSERIALIZE_ARRAY(copyBuffer
, ce
->params()->XferCap
);
717 CopyEngine::CopyEngineChannel::restartStateMachine()
721 fetchNextAddr(lastDescriptorAddr
);
723 case DescriptorFetch
:
724 fetchDescriptor(fetchAddress
);
732 case CompletionWrite
:
733 writeCompletionStatus();
738 panic("Unknown state for CopyEngineChannel\n");
743 CopyEngine::CopyEngineChannel::drainResume()
745 DPRINTF(DMACopyEngine
, "Restarting state machine at state %d\n", nextState
);
746 restartStateMachine();
750 CopyEngineParams::create()
752 return new CopyEngine(this);