2 * Copyright (c) 2008 The Regents of The University of Michigan
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 * Device model for Intel's I/O AT DMA copy engine.
37 #include "base/cp_annotate.hh"
38 #include "base/trace.hh"
39 #include "debug/DMACopyEngine.hh"
40 #include "dev/copy_engine.hh"
41 #include "mem/packet.hh"
42 #include "mem/packet_access.hh"
43 #include "params/CopyEngine.hh"
44 #include "sim/stats.hh"
45 #include "sim/system.hh"
47 using namespace CopyEngineReg
;
50 CopyEngine::CopyEngine(const Params
*p
)
53 // All Reg regs are initialized to 0 by default
54 regs
.chanCount
= p
->ChanCnt
;
55 regs
.xferCap
= findMsbSet(p
->XferCap
);
58 if (regs
.chanCount
> 64)
59 fatal("CopyEngine interface doesn't support more than 64 DMA engines\n");
61 for (int x
= 0; x
< regs
.chanCount
; x
++) {
62 CopyEngineChannel
*ch
= new CopyEngineChannel(this, x
);
68 CopyEngine::CopyEngineChannel::CopyEngineChannel(CopyEngine
*_ce
, int cid
)
69 : ce(_ce
), channelId(cid
), busy(false), underReset(false),
70 refreshNext(false), latBeforeBegin(ce
->params()->latBeforeBegin
),
71 latAfterCompletion(ce
->params()->latAfterCompletion
),
72 completionDataReg(0), nextState(Idle
), drainEvent(NULL
),
73 fetchCompleteEvent(this), addrCompleteEvent(this),
74 readCompleteEvent(this), writeCompleteEvent(this),
75 statusCompleteEvent(this)
78 cr
.status
.dma_transfer_status(3);
80 cr
.completionAddr
= 0;
82 curDmaDesc
= new DmaDesc
;
83 memset(curDmaDesc
, 0, sizeof(DmaDesc
));
84 copyBuffer
= new uint8_t[ce
->params()->XferCap
];
87 CopyEngine::~CopyEngine()
89 for (int x
= 0; x
< chan
.size(); x
++) {
94 CopyEngine::CopyEngineChannel::~CopyEngineChannel()
105 for (int x
= 0; x
< chan
.size(); x
++)
110 CopyEngine::CopyEngineChannel::init()
114 cePort
= new DmaPort(ce
, ce
->sys
, ce
->params()->min_backoff_delay
,
115 ce
->params()->max_backoff_delay
);
116 peer
= ce
->dmaPort
->getPeer()->getOwner()->getPort("");
117 peer
->setPeer(cePort
);
118 cePort
->setPeer(peer
);
122 CopyEngine::CopyEngineChannel::recvCommand()
124 if (cr
.command
.start_dma()) {
126 cr
.status
.dma_transfer_status(0);
127 nextState
= DescriptorFetch
;
128 fetchAddress
= cr
.descChainAddr
;
129 if (ce
->getState() == SimObject::Running
)
130 fetchDescriptor(cr
.descChainAddr
);
131 } else if (cr
.command
.append_dma()) {
133 nextState
= AddressFetch
;
134 if (ce
->getState() == SimObject::Running
)
135 fetchNextAddr(lastDescriptorAddr
);
138 } else if (cr
.command
.reset_dma()) {
142 cr
.status
.dma_transfer_status(3);
145 } else if (cr
.command
.resume_dma() || cr
.command
.abort_dma() ||
146 cr
.command
.suspend_dma())
147 panic("Resume, Abort, and Suspend are not supported\n");
152 CopyEngine::read(PacketPtr pkt
)
157 if (!getBAR(pkt
->getAddr(), bar
, daddr
))
158 panic("Invalid PCI memory access to unmapped memory.\n");
160 // Only Memory register BAR is allowed
163 int size
= pkt
->getSize();
164 if (size
!= sizeof(uint64_t) && size
!= sizeof(uint32_t) &&
165 size
!= sizeof(uint16_t) && size
!= sizeof(uint8_t)) {
166 panic("Unknown size for MMIO access: %d\n", pkt
->getSize());
169 DPRINTF(DMACopyEngine
, "Read device register %#X size: %d\n", daddr
, size
);
174 /// Handle read of register here
180 assert(size
== sizeof(regs
.chanCount
));
181 pkt
->set
<uint8_t>(regs
.chanCount
);
184 assert(size
== sizeof(regs
.xferCap
));
185 pkt
->set
<uint8_t>(regs
.xferCap
);
188 assert(size
== sizeof(uint8_t));
189 pkt
->set
<uint8_t>(regs
.intrctrl());
190 regs
.intrctrl
.master_int_enable(0);
193 assert(size
== sizeof(regs
.attnStatus
));
194 pkt
->set
<uint32_t>(regs
.attnStatus
);
198 panic("Read request to unknown register number: %#x\n", daddr
);
200 pkt
->makeAtomicResponse();
205 // Find which channel we're accessing
208 while (daddr
>= 0x80) {
213 if (chanid
>= regs
.chanCount
)
214 panic("Access to channel %d (device only configured for %d channels)",
215 chanid
, regs
.chanCount
);
218 /// Channel registers are handled here
220 chan
[chanid
]->channelRead(pkt
, daddr
, size
);
222 pkt
->makeAtomicResponse();
227 CopyEngine::CopyEngineChannel::channelRead(Packet
*pkt
, Addr daddr
, int size
)
231 assert(size
== sizeof(uint16_t));
232 pkt
->set
<uint16_t>(cr
.ctrl());
236 assert(size
== sizeof(uint64_t));
237 pkt
->set
<uint64_t>(cr
.status() | ~busy
);
240 assert(size
== sizeof(uint64_t) || size
== sizeof(uint32_t));
241 if (size
== sizeof(uint64_t))
242 pkt
->set
<uint64_t>(cr
.descChainAddr
);
244 pkt
->set
<uint32_t>(bits(cr
.descChainAddr
,0,31));
246 case CHAN_CHAINADDR_HIGH
:
247 assert(size
== sizeof(uint32_t));
248 pkt
->set
<uint32_t>(bits(cr
.descChainAddr
,32,63));
251 assert(size
== sizeof(uint8_t));
252 pkt
->set
<uint32_t>(cr
.command());
255 assert(size
== sizeof(uint64_t) || size
== sizeof(uint32_t));
256 if (size
== sizeof(uint64_t))
257 pkt
->set
<uint64_t>(cr
.completionAddr
);
259 pkt
->set
<uint32_t>(bits(cr
.completionAddr
,0,31));
261 case CHAN_CMPLNADDR_HIGH
:
262 assert(size
== sizeof(uint32_t));
263 pkt
->set
<uint32_t>(bits(cr
.completionAddr
,32,63));
266 assert(size
== sizeof(uint32_t));
267 pkt
->set
<uint32_t>(cr
.error());
270 panic("Read request to unknown channel register number: (%d)%#x\n",
277 CopyEngine::write(PacketPtr pkt
)
283 if (!getBAR(pkt
->getAddr(), bar
, daddr
))
284 panic("Invalid PCI memory access to unmapped memory.\n");
286 // Only Memory register BAR is allowed
289 int size
= pkt
->getSize();
292 /// Handle write of register here
295 if (size
== sizeof(uint64_t)) {
296 uint64_t val M5_VAR_USED
= pkt
->get
<uint64_t>();
297 DPRINTF(DMACopyEngine
, "Wrote device register %#X value %#X\n", daddr
, val
);
298 } else if (size
== sizeof(uint32_t)) {
299 uint32_t val M5_VAR_USED
= pkt
->get
<uint32_t>();
300 DPRINTF(DMACopyEngine
, "Wrote device register %#X value %#X\n", daddr
, val
);
301 } else if (size
== sizeof(uint16_t)) {
302 uint16_t val M5_VAR_USED
= pkt
->get
<uint16_t>();
303 DPRINTF(DMACopyEngine
, "Wrote device register %#X value %#X\n", daddr
, val
);
304 } else if (size
== sizeof(uint8_t)) {
305 uint8_t val M5_VAR_USED
= pkt
->get
<uint8_t>();
306 DPRINTF(DMACopyEngine
, "Wrote device register %#X value %#X\n", daddr
, val
);
308 panic("Unknown size for MMIO access: %d\n", size
);
316 DPRINTF(DMACopyEngine
, "Warning, ignorning write to register %x\n",
320 regs
.intrctrl
.master_int_enable(bits(pkt
->get
<uint8_t>(),0,1));
323 panic("Read request to unknown register number: %#x\n", daddr
);
325 pkt
->makeAtomicResponse();
329 // Find which channel we're accessing
332 while (daddr
>= 0x80) {
337 if (chanid
>= regs
.chanCount
)
338 panic("Access to channel %d (device only configured for %d channels)",
339 chanid
, regs
.chanCount
);
342 /// Channel registers are handled here
344 chan
[chanid
]->channelWrite(pkt
, daddr
, size
);
346 pkt
->makeAtomicResponse();
351 CopyEngine::CopyEngineChannel::channelWrite(Packet
*pkt
, Addr daddr
, int size
)
355 assert(size
== sizeof(uint16_t));
357 old_int_disable
= cr
.ctrl
.interrupt_disable();
358 cr
.ctrl(pkt
->get
<uint16_t>());
359 if (cr
.ctrl
.interrupt_disable())
360 cr
.ctrl
.interrupt_disable(0);
362 cr
.ctrl
.interrupt_disable(old_int_disable
);
365 assert(size
== sizeof(uint64_t));
366 DPRINTF(DMACopyEngine
, "Warning, ignorning write to register %x\n",
370 assert(size
== sizeof(uint64_t) || size
== sizeof(uint32_t));
371 if (size
== sizeof(uint64_t))
372 cr
.descChainAddr
= pkt
->get
<uint64_t>();
374 cr
.descChainAddr
= (uint64_t)pkt
->get
<uint32_t>() |
375 (cr
.descChainAddr
& ~mask(32));
376 DPRINTF(DMACopyEngine
, "Chain Address %x\n", cr
.descChainAddr
);
378 case CHAN_CHAINADDR_HIGH
:
379 assert(size
== sizeof(uint32_t));
380 cr
.descChainAddr
= ((uint64_t)pkt
->get
<uint32_t>() <<32) |
381 (cr
.descChainAddr
& mask(32));
382 DPRINTF(DMACopyEngine
, "Chain Address %x\n", cr
.descChainAddr
);
385 assert(size
== sizeof(uint8_t));
386 cr
.command(pkt
->get
<uint8_t>());
390 assert(size
== sizeof(uint64_t) || size
== sizeof(uint32_t));
391 if (size
== sizeof(uint64_t))
392 cr
.completionAddr
= pkt
->get
<uint64_t>();
394 cr
.completionAddr
= pkt
->get
<uint32_t>() |
395 (cr
.completionAddr
& ~mask(32));
397 case CHAN_CMPLNADDR_HIGH
:
398 assert(size
== sizeof(uint32_t));
399 cr
.completionAddr
= ((uint64_t)pkt
->get
<uint32_t>() <<32) |
400 (cr
.completionAddr
& mask(32));
403 assert(size
== sizeof(uint32_t));
404 cr
.error(~pkt
->get
<uint32_t>() & cr
.error());
407 panic("Read request to unknown channel register number: (%d)%#x\n",
413 CopyEngine::regStats()
415 using namespace Stats
;
417 .init(regs
.chanCount
)
418 .name(name() + ".bytes_copied")
419 .desc("Number of bytes copied by each engine")
423 .init(regs
.chanCount
)
424 .name(name() + ".copies_processed")
425 .desc("Number of copies processed by each engine")
431 CopyEngine::CopyEngineChannel::fetchDescriptor(Addr address
)
434 anBegin("FetchDescriptor");
435 DPRINTF(DMACopyEngine
, "Reading descriptor from at memory location %#x(%#x)\n",
436 address
, ce
->platform
->pciToDma(address
));
440 DPRINTF(DMACopyEngine
, "dmaAction: %#x, %d bytes, to addr %#x\n",
441 ce
->platform
->pciToDma(address
), sizeof(DmaDesc
), curDmaDesc
);
443 cePort
->dmaAction(MemCmd::ReadReq
, ce
->platform
->pciToDma(address
),
444 sizeof(DmaDesc
), &fetchCompleteEvent
, (uint8_t*)curDmaDesc
,
446 lastDescriptorAddr
= address
;
450 CopyEngine::CopyEngineChannel::fetchDescComplete()
452 DPRINTF(DMACopyEngine
, "Read of descriptor complete\n");
454 if ((curDmaDesc
->command
& DESC_CTRL_NULL
)) {
455 DPRINTF(DMACopyEngine
, "Got NULL descriptor, skipping\n");
456 assert(!(curDmaDesc
->command
& DESC_CTRL_CP_STS
));
457 if (curDmaDesc
->command
& DESC_CTRL_CP_STS
) {
458 panic("Shouldn't be able to get here\n");
459 nextState
= CompletionWrite
;
460 if (inDrain()) return;
461 writeCompletionStatus();
472 if (curDmaDesc
->command
& ~DESC_CTRL_CP_STS
)
473 panic("Descriptor has flag other that completion status set\n");
476 if (inDrain()) return;
481 CopyEngine::CopyEngineChannel::readCopyBytes()
483 anBegin("ReadCopyBytes");
484 DPRINTF(DMACopyEngine
, "Reading %d bytes from buffer to memory location %#x(%#x)\n",
485 curDmaDesc
->len
, curDmaDesc
->dest
,
486 ce
->platform
->pciToDma(curDmaDesc
->src
));
487 cePort
->dmaAction(MemCmd::ReadReq
, ce
->platform
->pciToDma(curDmaDesc
->src
),
488 curDmaDesc
->len
, &readCompleteEvent
, copyBuffer
, 0);
492 CopyEngine::CopyEngineChannel::readCopyBytesComplete()
494 DPRINTF(DMACopyEngine
, "Read of bytes to copy complete\n");
496 nextState
= DMAWrite
;
497 if (inDrain()) return;
502 CopyEngine::CopyEngineChannel::writeCopyBytes()
504 anBegin("WriteCopyBytes");
505 DPRINTF(DMACopyEngine
, "Writing %d bytes from buffer to memory location %#x(%#x)\n",
506 curDmaDesc
->len
, curDmaDesc
->dest
,
507 ce
->platform
->pciToDma(curDmaDesc
->dest
));
509 cePort
->dmaAction(MemCmd::WriteReq
, ce
->platform
->pciToDma(curDmaDesc
->dest
),
510 curDmaDesc
->len
, &writeCompleteEvent
, copyBuffer
, 0);
512 ce
->bytesCopied
[channelId
] += curDmaDesc
->len
;
513 ce
->copiesProcessed
[channelId
]++;
517 CopyEngine::CopyEngineChannel::writeCopyBytesComplete()
519 DPRINTF(DMACopyEngine
, "Write of bytes to copy complete user1: %#x\n",
522 cr
.status
.compl_desc_addr(lastDescriptorAddr
>> 6);
523 completionDataReg
= cr
.status() | 1;
525 anQ("DMAUsedDescQ", channelId
, 1);
526 anQ("AppRecvQ", curDmaDesc
->user1
, curDmaDesc
->len
);
527 if (curDmaDesc
->command
& DESC_CTRL_CP_STS
) {
528 nextState
= CompletionWrite
;
529 if (inDrain()) return;
530 writeCompletionStatus();
534 continueProcessing();
538 CopyEngine::CopyEngineChannel::continueProcessing()
552 if (curDmaDesc
->next
) {
553 nextState
= DescriptorFetch
;
554 fetchAddress
= curDmaDesc
->next
;
555 if (inDrain()) return;
556 fetchDescriptor(curDmaDesc
->next
);
557 } else if (refreshNext
) {
558 nextState
= AddressFetch
;
560 if (inDrain()) return;
561 fetchNextAddr(lastDescriptorAddr
);
571 CopyEngine::CopyEngineChannel::writeCompletionStatus()
573 anBegin("WriteCompletionStatus");
574 DPRINTF(DMACopyEngine
, "Writing completion status %#x to address %#x(%#x)\n",
575 completionDataReg
, cr
.completionAddr
,
576 ce
->platform
->pciToDma(cr
.completionAddr
));
578 cePort
->dmaAction(MemCmd::WriteReq
, ce
->platform
->pciToDma(cr
.completionAddr
),
579 sizeof(completionDataReg
), &statusCompleteEvent
,
580 (uint8_t*)&completionDataReg
, latAfterCompletion
);
584 CopyEngine::CopyEngineChannel::writeStatusComplete()
586 DPRINTF(DMACopyEngine
, "Writing completion status complete\n");
587 continueProcessing();
591 CopyEngine::CopyEngineChannel::fetchNextAddr(Addr address
)
593 anBegin("FetchNextAddr");
594 DPRINTF(DMACopyEngine
, "Fetching next address...\n");
596 cePort
->dmaAction(MemCmd::ReadReq
, ce
->platform
->pciToDma(address
+
597 offsetof(DmaDesc
, next
)), sizeof(Addr
), &addrCompleteEvent
,
598 (uint8_t*)curDmaDesc
+ offsetof(DmaDesc
, next
), 0);
602 CopyEngine::CopyEngineChannel::fetchAddrComplete()
604 DPRINTF(DMACopyEngine
, "Fetching next address complete: %#x\n",
606 if (!curDmaDesc
->next
) {
607 DPRINTF(DMACopyEngine
, "Got NULL descriptor, nothing more to do\n");
615 nextState
= DescriptorFetch
;
616 fetchAddress
= curDmaDesc
->next
;
617 if (inDrain()) return;
618 fetchDescriptor(curDmaDesc
->next
);
622 CopyEngine::CopyEngineChannel::inDrain()
624 if (ce
->getState() == SimObject::Draining
) {
625 DPRINTF(DMACopyEngine
, "processing drain\n");
627 drainEvent
->process();
631 return ce
->getState() != SimObject::Running
;
635 CopyEngine::CopyEngineChannel::drain(Event
*de
)
637 if (nextState
== Idle
|| ce
->getState() != SimObject::Running
)
639 unsigned int count
= 1;
640 count
+= cePort
->drain(de
);
642 DPRINTF(DMACopyEngine
, "unable to drain, returning %d\n", count
);
648 CopyEngine::drain(Event
*de
)
651 count
= pioPort
->drain(de
) + dmaPort
->drain(de
) + configPort
->drain(de
);
652 for (int x
= 0;x
< chan
.size(); x
++)
653 count
+= chan
[x
]->drain(de
);
656 changeState(Draining
);
658 changeState(Drained
);
660 DPRINTF(DMACopyEngine
, "call to CopyEngine::drain() returning %d\n", count
);
665 CopyEngine::serialize(std::ostream
&os
)
667 PciDev::serialize(os
);
669 for (int x
=0; x
< chan
.size(); x
++) {
670 nameOut(os
, csprintf("%s.channel%d", name(), x
));
671 chan
[x
]->serialize(os
);
676 CopyEngine::unserialize(Checkpoint
*cp
, const std::string
§ion
)
678 PciDev::unserialize(cp
, section
);
679 regs
.unserialize(cp
, section
);
680 for (int x
= 0; x
< chan
.size(); x
++)
681 chan
[x
]->unserialize(cp
, csprintf("%s.channel%d", section
, x
));
685 CopyEngine::CopyEngineChannel::serialize(std::ostream
&os
)
687 SERIALIZE_SCALAR(channelId
);
688 SERIALIZE_SCALAR(busy
);
689 SERIALIZE_SCALAR(underReset
);
690 SERIALIZE_SCALAR(refreshNext
);
691 SERIALIZE_SCALAR(lastDescriptorAddr
);
692 SERIALIZE_SCALAR(completionDataReg
);
693 SERIALIZE_SCALAR(fetchAddress
);
694 int nextState
= this->nextState
;
695 SERIALIZE_SCALAR(nextState
);
696 arrayParamOut(os
, "curDmaDesc", (uint8_t*)curDmaDesc
, sizeof(DmaDesc
));
697 SERIALIZE_ARRAY(copyBuffer
, ce
->params()->XferCap
);
702 CopyEngine::CopyEngineChannel::unserialize(Checkpoint
*cp
, const std::string
§ion
)
704 UNSERIALIZE_SCALAR(channelId
);
705 UNSERIALIZE_SCALAR(busy
);
706 UNSERIALIZE_SCALAR(underReset
);
707 UNSERIALIZE_SCALAR(refreshNext
);
708 UNSERIALIZE_SCALAR(lastDescriptorAddr
);
709 UNSERIALIZE_SCALAR(completionDataReg
);
710 UNSERIALIZE_SCALAR(fetchAddress
);
712 UNSERIALIZE_SCALAR(nextState
);
713 this->nextState
= (ChannelState
)nextState
;
714 arrayParamIn(cp
, section
, "curDmaDesc", (uint8_t*)curDmaDesc
, sizeof(DmaDesc
));
715 UNSERIALIZE_ARRAY(copyBuffer
, ce
->params()->XferCap
);
716 cr
.unserialize(cp
, section
);
721 CopyEngine::CopyEngineChannel::restartStateMachine()
725 fetchNextAddr(lastDescriptorAddr
);
727 case DescriptorFetch
:
728 fetchDescriptor(fetchAddress
);
736 case CompletionWrite
:
737 writeCompletionStatus();
742 panic("Unknown state for CopyEngineChannel\n");
750 for (int x
= 0;x
< chan
.size(); x
++)
756 CopyEngine::CopyEngineChannel::resume()
758 DPRINTF(DMACopyEngine
, "Restarting state machine at state %d\n", nextState
);
759 restartStateMachine();
763 CopyEngineParams::create()
765 return new CopyEngine(this);