2 * Copyright (c) 2012, 2015, 2017 ARM Limited
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 #include "cpu/kvm/base.hh"
40 #include <linux/kvm.h>
41 #include <sys/ioctl.h>
49 #include "arch/utility.hh"
50 #include "debug/Checkpoint.hh"
51 #include "debug/Drain.hh"
52 #include "debug/Kvm.hh"
53 #include "debug/KvmIO.hh"
54 #include "debug/KvmRun.hh"
55 #include "params/BaseKvmCPU.hh"
56 #include "sim/process.hh"
57 #include "sim/system.hh"
59 /* Used by some KVM macros */
60 #define PAGE_SIZE pageSize
62 BaseKvmCPU::BaseKvmCPU(BaseKvmCPUParams
*params
)
64 vm(*params
->system
->getKvmVM()),
66 dataPort(name() + ".dcache_port", this),
67 instPort(name() + ".icache_port", this),
68 alwaysSyncTC(params
->alwaysSyncTC
),
69 threadContextDirty(true),
71 vcpuID(vm
.allocVCPUID()), vcpuFD(-1), vcpuMMapSize(0),
72 _kvmRun(NULL
), mmioRing(NULL
),
73 pageSize(sysconf(_SC_PAGE_SIZE
)),
74 tickEvent([this]{ tick(); }, "BaseKvmCPU tick",
75 false, Event::CPU_Tick_Pri
),
77 perfControlledByTimer(params
->usePerfOverflow
),
78 hostFactor(params
->hostFactor
), stats(this),
82 panic("KVM: Failed to determine host page size (%i)\n",
86 thread
= new SimpleThread(this, 0, params
->system
, params
->itb
, params
->dtb
,
89 thread
= new SimpleThread(this, /* thread_num */ 0, params
->system
,
90 params
->workload
[0], params
->itb
,
91 params
->dtb
, params
->isa
[0]);
93 thread
->setStatus(ThreadContext::Halted
);
95 threadContexts
.push_back(tc
);
98 BaseKvmCPU::~BaseKvmCPU()
101 munmap(_kvmRun
, vcpuMMapSize
);
111 fatal("KVM: Multithreading not supported");
113 tc
->initMemProxies(tc
);
117 BaseKvmCPU::startup()
119 const BaseKvmCPUParams
* const p(
120 dynamic_cast<const BaseKvmCPUParams
*>(params()));
126 assert(vcpuFD
== -1);
128 // Tell the VM that a CPU is about to start.
131 // We can't initialize KVM CPUs in BaseKvmCPU::init() since we are
132 // not guaranteed that the parent KVM VM has initialized at that
133 // point. Initialize virtual CPUs here instead.
134 vcpuFD
= vm
.createVCPU(vcpuID
);
136 // Map the KVM run structure */
137 vcpuMMapSize
= kvm
.getVCPUMMapSize();
138 _kvmRun
= (struct kvm_run
*)mmap(0, vcpuMMapSize
,
139 PROT_READ
| PROT_WRITE
, MAP_SHARED
,
141 if (_kvmRun
== MAP_FAILED
)
142 panic("KVM: Failed to map run data structure\n");
144 // Setup a pointer to the MMIO ring buffer if coalesced MMIO is
145 // available. The offset into the KVM's communication page is
146 // provided by the coalesced MMIO capability.
147 int mmioOffset(kvm
.capCoalescedMMIO());
148 if (!p
->useCoalescedMMIO
) {
149 inform("KVM: Coalesced MMIO disabled by config.\n");
150 } else if (mmioOffset
) {
151 inform("KVM: Coalesced IO available\n");
152 mmioRing
= (struct kvm_coalesced_mmio_ring
*)(
153 (char *)_kvmRun
+ (mmioOffset
* pageSize
));
155 inform("KVM: Coalesced not supported by host OS\n");
159 new EventFunctionWrapper([this]{ startupThread(); }, name(), true));
160 schedule(startupEvent
, curTick());
164 BaseKvmCPU::KVMCpuPort::nextIOState() const
166 return (activeMMIOReqs
|| pendingMMIOPkts
.size())
167 ? RunningMMIOPending
: RunningServiceCompletion
;
171 BaseKvmCPU::KVMCpuPort::submitIO(PacketPtr pkt
)
173 if (cpu
->system
->isAtomicMode()) {
174 Tick delay
= sendAtomic(pkt
);
178 if (pendingMMIOPkts
.empty() && sendTimingReq(pkt
)) {
181 pendingMMIOPkts
.push(pkt
);
183 // Return value is irrelevant for timing-mode accesses.
189 BaseKvmCPU::KVMCpuPort::recvTimingResp(PacketPtr pkt
)
191 DPRINTF(KvmIO
, "KVM: Finished timing request\n");
196 // We can switch back into KVM when all pending and in-flight MMIO
197 // operations have completed.
198 if (!(activeMMIOReqs
|| pendingMMIOPkts
.size())) {
199 DPRINTF(KvmIO
, "KVM: Finished all outstanding timing requests\n");
200 cpu
->finishMMIOPending();
206 BaseKvmCPU::KVMCpuPort::recvReqRetry()
208 DPRINTF(KvmIO
, "KVM: Retry for timing request\n");
210 assert(pendingMMIOPkts
.size());
212 // Assuming that we can issue infinite requests this cycle is a bit
213 // unrealistic, but it's not worth modeling something more complex in
215 while (pendingMMIOPkts
.size() && sendTimingReq(pendingMMIOPkts
.front())) {
216 pendingMMIOPkts
.pop();
222 BaseKvmCPU::finishMMIOPending()
224 assert(_status
== RunningMMIOPending
);
225 assert(!tickEvent
.scheduled());
227 _status
= RunningServiceCompletion
;
228 schedule(tickEvent
, nextCycle());
232 BaseKvmCPU::startupThread()
234 // Do thread-specific initialization. We need to setup signal
235 // delivery for counters and timers from within the thread that
236 // will execute the event queue to ensure that signals are
237 // delivered to the right threads.
238 const BaseKvmCPUParams
* const p(
239 dynamic_cast<const BaseKvmCPUParams
*>(params()));
241 vcpuThread
= pthread_self();
243 // Setup signal handlers. This has to be done after the vCPU is
244 // created since it manipulates the vCPU signal mask.
245 setupSignalHandler();
249 if (p
->usePerfOverflow
)
250 runTimer
.reset(new PerfKvmTimer(hwCycles
,
255 runTimer
.reset(new PosixKvmTimer(KVM_KICK_SIGNAL
, CLOCK_MONOTONIC
,
261 BaseKvmCPU::StatGroup::StatGroup(Stats::Group
*parent
)
262 : Stats::Group(parent
),
263 ADD_STAT(committedInsts
, "Number of instructions committed"),
264 ADD_STAT(numVMExits
, "total number of KVM exits"),
265 ADD_STAT(numVMHalfEntries
,
266 "number of KVM entries to finalize pending operations"),
267 ADD_STAT(numExitSignal
, "exits due to signal delivery"),
268 ADD_STAT(numMMIO
, "number of VM exits due to memory mapped IO"),
269 ADD_STAT(numCoalescedMMIO
,
270 "number of coalesced memory mapped IO requests"),
271 ADD_STAT(numIO
, "number of VM exits due to legacy IO"),
273 "number of VM exits due to wait for interrupt instructions"),
274 ADD_STAT(numInterrupts
, "number of interrupts delivered"),
275 ADD_STAT(numHypercalls
, "number of hypercalls")
280 BaseKvmCPU::serializeThread(CheckpointOut
&cp
, ThreadID tid
) const
282 if (DTRACE(Checkpoint
)) {
283 DPRINTF(Checkpoint
, "KVM: Serializing thread %i:\n", tid
);
288 assert(_status
== Idle
);
289 thread
->serialize(cp
);
293 BaseKvmCPU::unserializeThread(CheckpointIn
&cp
, ThreadID tid
)
295 DPRINTF(Checkpoint
, "KVM: Unserialize thread %i:\n", tid
);
298 assert(_status
== Idle
);
299 thread
->unserialize(cp
);
300 threadContextDirty
= true;
307 return DrainState::Drained
;
309 DPRINTF(Drain
, "BaseKvmCPU::drain\n");
311 // The event queue won't be locked when calling drain since that's
312 // not done from an event. Lock the event queue here to make sure
313 // that scoped migrations continue to work if we need to
314 // synchronize the thread context.
315 std::lock_guard
<EventQueue
> lock(*this->eventQueue());
319 // The base KVM code is normally ready when it is in the
320 // Running state, but the architecture specific code might be
321 // of a different opinion. This may happen when the CPU been
322 // notified of an event that hasn't been accepted by the vCPU
324 if (!archIsDrained())
325 return DrainState::Draining
;
327 // The state of the CPU is consistent, so we don't need to do
328 // anything special to drain it. We simply de-schedule the
329 // tick event and enter the Idle state to prevent nasty things
330 // like MMIOs from happening.
331 if (tickEvent
.scheduled())
332 deschedule(tickEvent
);
337 // Idle, no need to drain
338 assert(!tickEvent
.scheduled());
340 // Sync the thread context here since we'll need it when we
341 // switch CPUs or checkpoint the CPU.
344 return DrainState::Drained
;
346 case RunningServiceCompletion
:
347 // The CPU has just requested a service that was handled in
348 // the RunningService state, but the results have still not
349 // been reported to the CPU. Now, we /could/ probably just
350 // update the register state ourselves instead of letting KVM
351 // handle it, but that would be tricky. Instead, we enter KVM
352 // and let it do its stuff.
353 DPRINTF(Drain
, "KVM CPU is waiting for service completion, "
354 "requesting drain.\n");
355 return DrainState::Draining
;
357 case RunningMMIOPending
:
358 // We need to drain since there are in-flight timing accesses
359 DPRINTF(Drain
, "KVM CPU is waiting for timing accesses to complete, "
360 "requesting drain.\n");
361 return DrainState::Draining
;
364 // We need to drain since the CPU is waiting for service (e.g., MMIOs)
365 DPRINTF(Drain
, "KVM CPU is waiting for service, requesting drain.\n");
366 return DrainState::Draining
;
369 panic("KVM: Unhandled CPU state in drain()\n");
370 return DrainState::Drained
;
375 BaseKvmCPU::drainResume()
377 assert(!tickEvent
.scheduled());
379 // We might have been switched out. In that case, we don't need to
384 DPRINTF(Kvm
, "drainResume\n");
387 // The tick event is de-scheduled as a part of the draining
388 // process. Re-schedule it if the thread context is active.
389 if (tc
->status() == ThreadContext::Active
) {
390 schedule(tickEvent
, nextCycle());
398 BaseKvmCPU::notifyFork()
400 // We should have drained prior to forking, which means that the
401 // tick event shouldn't be scheduled and the CPU is idle.
402 assert(!tickEvent
.scheduled());
403 assert(_status
== Idle
);
406 if (close(vcpuFD
) == -1)
407 warn("kvm CPU: notifyFork failed to close vcpuFD\n");
410 munmap(_kvmRun
, vcpuMMapSize
);
415 hwInstructions
.detach();
421 BaseKvmCPU::switchOut()
423 DPRINTF(Kvm
, "switchOut\n");
425 BaseCPU::switchOut();
427 // We should have drained prior to executing a switchOut, which
428 // means that the tick event shouldn't be scheduled and the CPU is
430 assert(!tickEvent
.scheduled());
431 assert(_status
== Idle
);
435 BaseKvmCPU::takeOverFrom(BaseCPU
*cpu
)
437 DPRINTF(Kvm
, "takeOverFrom\n");
439 BaseCPU::takeOverFrom(cpu
);
441 // We should have drained prior to executing a switchOut, which
442 // means that the tick event shouldn't be scheduled and the CPU is
444 assert(!tickEvent
.scheduled());
445 assert(_status
== Idle
);
446 assert(threadContexts
.size() == 1);
448 // Force an update of the KVM state here instead of flagging the
449 // TC as dirty. This is not ideal from a performance point of
450 // view, but it makes debugging easier as it allows meaningful KVM
451 // state to be dumped before and after a takeover.
453 threadContextDirty
= false;
457 BaseKvmCPU::verifyMemoryMode() const
459 if (!(system
->bypassCaches())) {
460 fatal("The KVM-based CPUs requires the memory system to be in the "
461 "'noncaching' mode.\n");
466 BaseKvmCPU::wakeup(ThreadID tid
)
468 DPRINTF(Kvm
, "wakeup()\n");
469 // This method might have been called from another
470 // context. Migrate to this SimObject's event queue when
471 // delivering the wakeup signal.
472 EventQueue::ScopedMigration
migrate(eventQueue());
474 // Kick the vCPU to get it to come out of KVM.
477 if (thread
->status() != ThreadContext::Suspended
)
484 BaseKvmCPU::activateContext(ThreadID thread_num
)
486 DPRINTF(Kvm
, "ActivateContext %d\n", thread_num
);
488 assert(thread_num
== 0);
491 assert(_status
== Idle
);
492 assert(!tickEvent
.scheduled());
494 numCycles
+= ticksToCycles(thread
->lastActivate
- thread
->lastSuspend
);
496 schedule(tickEvent
, clockEdge(Cycles(0)));
502 BaseKvmCPU::suspendContext(ThreadID thread_num
)
504 DPRINTF(Kvm
, "SuspendContext %d\n", thread_num
);
506 assert(thread_num
== 0);
512 assert(_status
== Running
|| _status
== RunningServiceCompletion
);
514 // The tick event may no be scheduled if the quest has requested
515 // the monitor to wait for interrupts. The normal CPU models can
516 // get their tick events descheduled by quiesce instructions, but
517 // that can't happen here.
518 if (tickEvent
.scheduled())
519 deschedule(tickEvent
);
525 BaseKvmCPU::deallocateContext(ThreadID thread_num
)
527 // for now, these are equivalent
528 suspendContext(thread_num
);
532 BaseKvmCPU::haltContext(ThreadID thread_num
)
534 // for now, these are equivalent
535 suspendContext(thread_num
);
536 updateCycleCounters(BaseCPU::CPU_STATE_SLEEP
);
540 BaseKvmCPU::getContext(int tn
)
549 BaseKvmCPU::totalInsts() const
555 BaseKvmCPU::totalOps() const
557 hack_once("Pretending totalOps is equivalent to totalInsts()\n");
562 BaseKvmCPU::dump() const
564 inform("State dumping not implemented.");
571 assert(_status
!= Idle
&& _status
!= RunningMMIOPending
);
575 // handleKvmExit() will determine the next state of the CPU
576 delay
= handleKvmExit();
582 case RunningServiceCompletion
:
584 auto &queue
= thread
->comInstEventQueue
;
585 const uint64_t nextInstEvent(
586 queue
.empty() ? MaxTick
: queue
.nextTick());
587 // Enter into KVM and complete pending IO instructions if we
588 // have an instruction event pending.
589 const Tick
ticksToExecute(
590 nextInstEvent
> ctrInsts
?
591 curEventQueue()->nextTick() - curTick() : 0);
594 threadContextDirty
= true;
596 // We might need to update the KVM state.
599 // Setup any pending instruction count breakpoints using
600 // PerfEvent if we are going to execute more than just an IO
602 if (ticksToExecute
> 0)
605 DPRINTF(KvmRun
, "Entering KVM...\n");
606 if (drainState() == DrainState::Draining
) {
607 // Force an immediate exit from KVM after completing
608 // pending operations. The architecture-specific code
609 // takes care to run until it is in a state where it can
610 // safely be drained.
611 delay
= kvmRunDrain();
613 delay
= kvmRun(ticksToExecute
);
616 // The CPU might have been suspended before entering into
617 // KVM. Assume that the CPU was suspended /before/ entering
618 // into KVM and skip the exit handling.
622 // Entering into KVM implies that we'll have to reload the thread
623 // context from KVM if we want to access it. Flag the KVM state as
624 // dirty with respect to the cached thread context.
625 kvmStateDirty
= true;
630 // Enter into the RunningService state unless the
631 // simulation was stopped by a timer.
632 if (_kvmRun
->exit_reason
!= KVM_EXIT_INTR
) {
633 _status
= RunningService
;
635 ++stats
.numExitSignal
;
639 // Service any pending instruction events. The vCPU should
640 // have exited in time for the event using the instruction
641 // counter configured by setupInstStop().
642 queue
.serviceEvents(ctrInsts
);
649 panic("BaseKvmCPU entered tick() in an illegal state (%i)\n",
653 // Schedule a new tick if we are still running
654 if (_status
!= Idle
&& _status
!= RunningMMIOPending
) {
655 if (_kvmRun
->exit_reason
== KVM_EXIT_INTR
&& runTimer
->expired())
656 schedule(tickEvent
, clockEdge(ticksToCycles(
657 curEventQueue()->nextTick() - curTick() + 1)));
659 schedule(tickEvent
, clockEdge(ticksToCycles(delay
)));
664 BaseKvmCPU::kvmRunDrain()
666 // By default, the only thing we need to drain is a pending IO
667 // operation which assumes that we are in the
668 // RunningServiceCompletion or RunningMMIOPending state.
669 assert(_status
== RunningServiceCompletion
||
670 _status
== RunningMMIOPending
);
672 // Deliver the data from the pending IO operation and immediately
678 BaseKvmCPU::getHostCycles() const
680 return hwCycles
.read();
684 BaseKvmCPU::kvmRun(Tick ticks
)
687 fatal_if(vcpuFD
== -1,
688 "Trying to run a KVM CPU in a forked child process. "
689 "This is not supported.\n");
690 DPRINTF(KvmRun
, "KVM: Executing for %i ticks\n", ticks
);
693 // Settings ticks == 0 is a special case which causes an entry
694 // into KVM that finishes pending operations (e.g., IO) and
695 // then immediately exits.
696 DPRINTF(KvmRun
, "KVM: Delivering IO without full guest entry\n");
698 ++stats
.numVMHalfEntries
;
700 // Send a KVM_KICK_SIGNAL to the vCPU thread (i.e., this
701 // thread). The KVM control signal is masked while executing
702 // in gem5 and gets unmasked temporarily as when entering
703 // KVM. See setSignalMask() and setupSignalHandler().
706 // Start the vCPU. KVM will check for signals after completing
707 // pending operations (IO). Since the KVM_KICK_SIGNAL is
708 // pending, this forces an immediate exit to gem5 again. We
709 // don't bother to setup timers since this shouldn't actually
710 // execute any code (other than completing half-executed IO
711 // instructions) in the guest.
714 // We always execute at least one cycle to prevent the
715 // BaseKvmCPU::tick() to be rescheduled on the same tick
717 ticksExecuted
= clockPeriod();
719 // This method is executed as a result of a tick event. That
720 // means that the event queue will be locked when entering the
721 // method. We temporarily unlock the event queue to allow
722 // other threads to steal control of this thread to inject
723 // interrupts. They will typically lock the queue and then
724 // force an exit from KVM by kicking the vCPU.
725 EventQueue::ScopedRelease
release(curEventQueue());
727 if (ticks
< runTimer
->resolution()) {
728 DPRINTF(KvmRun
, "KVM: Adjusting tick count (%i -> %i)\n",
729 ticks
, runTimer
->resolution());
730 ticks
= runTimer
->resolution();
733 // Get hardware statistics after synchronizing contexts. The KVM
734 // state update might affect guest cycle counters.
735 uint64_t baseCycles(getHostCycles());
736 uint64_t baseInstrs(hwInstructions
.read());
738 // Arm the run timer and start the cycle timer if it isn't
739 // controlled by the overflow timer. Starting/stopping the cycle
740 // timer automatically starts the other perf timers as they are in
741 // the same counter group.
742 runTimer
->arm(ticks
);
743 if (!perfControlledByTimer
)
749 if (!perfControlledByTimer
)
752 // The control signal may have been delivered after we exited
753 // from KVM. It will be pending in that case since it is
754 // masked when we aren't executing in KVM. Discard it to make
755 // sure we don't deliver it immediately next time we try to
757 discardPendingSignal(KVM_KICK_SIGNAL
);
759 const uint64_t hostCyclesExecuted(getHostCycles() - baseCycles
);
760 const uint64_t simCyclesExecuted(hostCyclesExecuted
* hostFactor
);
761 const uint64_t instsExecuted(hwInstructions
.read() - baseInstrs
);
762 ticksExecuted
= runTimer
->ticksFromHostCycles(hostCyclesExecuted
);
764 /* Update statistics */
765 numCycles
+= simCyclesExecuted
;;
766 stats
.committedInsts
+= instsExecuted
;
767 ctrInsts
+= instsExecuted
;
768 system
->totalNumInsts
+= instsExecuted
;
771 "KVM: Executed %i instructions in %i cycles "
772 "(%i ticks, sim cycles: %i).\n",
773 instsExecuted
, hostCyclesExecuted
, ticksExecuted
, simCyclesExecuted
);
778 return ticksExecuted
+ flushCoalescedMMIO();
782 BaseKvmCPU::kvmNonMaskableInterrupt()
784 ++stats
.numInterrupts
;
785 if (ioctl(KVM_NMI
) == -1)
786 panic("KVM: Failed to deliver NMI to virtual CPU\n");
790 BaseKvmCPU::kvmInterrupt(const struct kvm_interrupt
&interrupt
)
792 ++stats
.numInterrupts
;
793 if (ioctl(KVM_INTERRUPT
, (void *)&interrupt
) == -1)
794 panic("KVM: Failed to deliver interrupt to virtual CPU\n");
798 BaseKvmCPU::getRegisters(struct kvm_regs
®s
) const
800 if (ioctl(KVM_GET_REGS
, ®s
) == -1)
801 panic("KVM: Failed to get guest registers\n");
805 BaseKvmCPU::setRegisters(const struct kvm_regs
®s
)
807 if (ioctl(KVM_SET_REGS
, (void *)®s
) == -1)
808 panic("KVM: Failed to set guest registers\n");
812 BaseKvmCPU::getSpecialRegisters(struct kvm_sregs
®s
) const
814 if (ioctl(KVM_GET_SREGS
, ®s
) == -1)
815 panic("KVM: Failed to get guest special registers\n");
819 BaseKvmCPU::setSpecialRegisters(const struct kvm_sregs
®s
)
821 if (ioctl(KVM_SET_SREGS
, (void *)®s
) == -1)
822 panic("KVM: Failed to set guest special registers\n");
826 BaseKvmCPU::getFPUState(struct kvm_fpu
&state
) const
828 if (ioctl(KVM_GET_FPU
, &state
) == -1)
829 panic("KVM: Failed to get guest FPU state\n");
833 BaseKvmCPU::setFPUState(const struct kvm_fpu
&state
)
835 if (ioctl(KVM_SET_FPU
, (void *)&state
) == -1)
836 panic("KVM: Failed to set guest FPU state\n");
841 BaseKvmCPU::setOneReg(uint64_t id
, const void *addr
)
843 #ifdef KVM_SET_ONE_REG
844 struct kvm_one_reg reg
;
846 reg
.addr
= (uint64_t)addr
;
848 if (ioctl(KVM_SET_ONE_REG
, ®
) == -1) {
849 panic("KVM: Failed to set register (0x%x) value (errno: %i)\n",
853 panic("KVM_SET_ONE_REG is unsupported on this platform.\n");
858 BaseKvmCPU::getOneReg(uint64_t id
, void *addr
) const
860 #ifdef KVM_GET_ONE_REG
861 struct kvm_one_reg reg
;
863 reg
.addr
= (uint64_t)addr
;
865 if (ioctl(KVM_GET_ONE_REG
, ®
) == -1) {
866 panic("KVM: Failed to get register (0x%x) value (errno: %i)\n",
870 panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
875 BaseKvmCPU::getAndFormatOneReg(uint64_t id
) const
877 #ifdef KVM_GET_ONE_REG
878 std::ostringstream ss
;
880 ss
.setf(std::ios::hex
, std::ios::basefield
);
881 ss
.setf(std::ios::showbase
);
882 #define HANDLE_INTTYPE(len) \
883 case KVM_REG_SIZE_U ## len: { \
884 uint ## len ## _t value; \
885 getOneReg(id, &value); \
889 #define HANDLE_ARRAY(len) \
890 case KVM_REG_SIZE_U ## len: { \
891 uint8_t value[len / 8]; \
892 getOneReg(id, value); \
893 ccprintf(ss, "[0x%x", value[0]); \
894 for (int i = 1; i < len / 8; ++i) \
895 ccprintf(ss, ", 0x%x", value[i]); \
899 switch (id
& KVM_REG_SIZE_MASK
) {
912 #undef HANDLE_INTTYPE
917 panic("KVM_GET_ONE_REG is unsupported on this platform.\n");
922 BaseKvmCPU::syncThreadContext()
927 assert(!threadContextDirty
);
929 updateThreadContext();
930 kvmStateDirty
= false;
934 BaseKvmCPU::syncKvmState()
936 if (!threadContextDirty
)
939 assert(!kvmStateDirty
);
942 threadContextDirty
= false;
946 BaseKvmCPU::handleKvmExit()
948 DPRINTF(KvmRun
, "handleKvmExit (exit_reason: %i)\n", _kvmRun
->exit_reason
);
949 assert(_status
== RunningService
);
951 // Switch into the running state by default. Individual handlers
952 // can override this.
954 switch (_kvmRun
->exit_reason
) {
955 case KVM_EXIT_UNKNOWN
:
956 return handleKvmExitUnknown();
958 case KVM_EXIT_EXCEPTION
:
959 return handleKvmExitException();
964 Tick ticks
= handleKvmExitIO();
965 _status
= dataPort
.nextIOState();
969 case KVM_EXIT_HYPERCALL
:
970 ++stats
.numHypercalls
;
971 return handleKvmExitHypercall();
974 /* The guest has halted and is waiting for interrupts */
975 DPRINTF(Kvm
, "handleKvmExitHalt\n");
978 // Suspend the thread until the next interrupt arrives
981 // This is actually ignored since the thread is suspended.
986 /* Service memory mapped IO requests */
987 DPRINTF(KvmIO
, "KVM: Handling MMIO (w: %u, addr: 0x%x, len: %u)\n",
988 _kvmRun
->mmio
.is_write
,
989 _kvmRun
->mmio
.phys_addr
, _kvmRun
->mmio
.len
);
992 Tick ticks
= doMMIOAccess(_kvmRun
->mmio
.phys_addr
, _kvmRun
->mmio
.data
,
993 _kvmRun
->mmio
.len
, _kvmRun
->mmio
.is_write
);
994 // doMMIOAccess could have triggered a suspend, in which case we don't
995 // want to overwrite the _status.
997 _status
= dataPort
.nextIOState();
1001 case KVM_EXIT_IRQ_WINDOW_OPEN
:
1002 return handleKvmExitIRQWindowOpen();
1004 case KVM_EXIT_FAIL_ENTRY
:
1005 return handleKvmExitFailEntry();
1008 /* KVM was interrupted by a signal, restart it in the next
1012 case KVM_EXIT_INTERNAL_ERROR
:
1013 panic("KVM: Internal error (suberror: %u)\n",
1014 _kvmRun
->internal
.suberror
);
1018 panic("KVM: Unexpected exit (exit_reason: %u)\n", _kvmRun
->exit_reason
);
1023 BaseKvmCPU::handleKvmExitIO()
1025 panic("KVM: Unhandled guest IO (dir: %i, size: %i, port: 0x%x, count: %i)\n",
1026 _kvmRun
->io
.direction
, _kvmRun
->io
.size
,
1027 _kvmRun
->io
.port
, _kvmRun
->io
.count
);
1031 BaseKvmCPU::handleKvmExitHypercall()
1033 panic("KVM: Unhandled hypercall\n");
1037 BaseKvmCPU::handleKvmExitIRQWindowOpen()
1039 warn("KVM: Unhandled IRQ window.\n");
1045 BaseKvmCPU::handleKvmExitUnknown()
1048 panic("KVM: Unknown error when starting vCPU (hw reason: 0x%llx)\n",
1049 _kvmRun
->hw
.hardware_exit_reason
);
1053 BaseKvmCPU::handleKvmExitException()
1056 panic("KVM: Got exception when starting vCPU "
1057 "(exception: %u, error_code: %u)\n",
1058 _kvmRun
->ex
.exception
, _kvmRun
->ex
.error_code
);
1062 BaseKvmCPU::handleKvmExitFailEntry()
1065 panic("KVM: Failed to enter virtualized mode (hw reason: 0x%llx)\n",
1066 _kvmRun
->fail_entry
.hardware_entry_failure_reason
);
1070 BaseKvmCPU::doMMIOAccess(Addr paddr
, void *data
, int size
, bool write
)
1072 ThreadContext
*tc(thread
->getTC());
1073 syncThreadContext();
1075 RequestPtr mmio_req
= std::make_shared
<Request
>(
1076 paddr
, size
, Request::UNCACHEABLE
, dataMasterId());
1078 mmio_req
->setContext(tc
->contextId());
1079 // Some architectures do need to massage physical addresses a bit
1080 // before they are inserted into the memory system. This enables
1081 // APIC accesses on x86 and m5ops where supported through a MMIO
1083 BaseTLB::Mode
tlb_mode(write
? BaseTLB::Write
: BaseTLB::Read
);
1084 Fault
fault(tc
->getDTBPtr()->finalizePhysical(mmio_req
, tc
, tlb_mode
));
1085 if (fault
!= NoFault
)
1086 warn("Finalization of MMIO address failed: %s\n", fault
->name());
1089 const MemCmd
cmd(write
? MemCmd::WriteReq
: MemCmd::ReadReq
);
1090 PacketPtr pkt
= new Packet(mmio_req
, cmd
);
1091 pkt
->dataStatic(data
);
1093 if (mmio_req
->isLocalAccess()) {
1094 // We currently assume that there is no need to migrate to a
1095 // different event queue when doing local accesses. Currently, they
1096 // are only used for m5ops, so it should be a valid assumption.
1097 const Cycles ipr_delay
= mmio_req
->localAccessor(tc
, pkt
);
1098 threadContextDirty
= true;
1100 return clockPeriod() * ipr_delay
;
1102 // Temporarily lock and migrate to the device event queue to
1103 // prevent races in multi-core mode.
1104 EventQueue::ScopedMigration
migrate(deviceEventQueue());
1106 return dataPort
.submitIO(pkt
);
1111 BaseKvmCPU::setSignalMask(const sigset_t
*mask
)
1113 std::unique_ptr
<struct kvm_signal_mask
> kvm_mask
;
1116 kvm_mask
.reset((struct kvm_signal_mask
*)operator new(
1117 sizeof(struct kvm_signal_mask
) + sizeof(*mask
)));
1118 // The kernel and the user-space headers have different ideas
1119 // about the size of sigset_t. This seems like a massive hack,
1120 // but is actually what qemu does.
1121 assert(sizeof(*mask
) >= 8);
1123 memcpy(kvm_mask
->sigset
, mask
, kvm_mask
->len
);
1126 if (ioctl(KVM_SET_SIGNAL_MASK
, (void *)kvm_mask
.get()) == -1)
1127 panic("KVM: Failed to set vCPU signal mask (errno: %i)\n",
1132 BaseKvmCPU::ioctl(int request
, long p1
) const
1135 panic("KVM: CPU ioctl called before initialization\n");
1137 return ::ioctl(vcpuFD
, request
, p1
);
1141 BaseKvmCPU::flushCoalescedMMIO()
1146 DPRINTF(KvmIO
, "KVM: Flushing the coalesced MMIO ring buffer\n");
1148 // TODO: We might need to do synchronization when we start to
1149 // support multiple CPUs
1151 while (mmioRing
->first
!= mmioRing
->last
) {
1152 struct kvm_coalesced_mmio
&ent(
1153 mmioRing
->coalesced_mmio
[mmioRing
->first
]);
1155 DPRINTF(KvmIO
, "KVM: Handling coalesced MMIO (addr: 0x%x, len: %u)\n",
1156 ent
.phys_addr
, ent
.len
);
1158 ++stats
.numCoalescedMMIO
;
1159 ticks
+= doMMIOAccess(ent
.phys_addr
, ent
.data
, ent
.len
, true);
1161 mmioRing
->first
= (mmioRing
->first
+ 1) % KVM_COALESCED_MMIO_MAX
;
1168 * Dummy handler for KVM kick signals.
1170 * @note This function is usually not called since the kernel doesn't
1171 * seem to deliver signals when the signal is only unmasked when
1172 * running in KVM. This doesn't matter though since we are only
1173 * interested in getting KVM to exit, which happens as expected. See
1174 * setupSignalHandler() and kvmRun() for details about KVM signal
1178 onKickSignal(int signo
, siginfo_t
*si
, void *data
)
1183 BaseKvmCPU::setupSignalHandler()
1185 struct sigaction sa
;
1187 memset(&sa
, 0, sizeof(sa
));
1188 sa
.sa_sigaction
= onKickSignal
;
1189 sa
.sa_flags
= SA_SIGINFO
| SA_RESTART
;
1190 if (sigaction(KVM_KICK_SIGNAL
, &sa
, NULL
) == -1)
1191 panic("KVM: Failed to setup vCPU timer signal handler\n");
1194 if (pthread_sigmask(SIG_BLOCK
, NULL
, &sigset
) == -1)
1195 panic("KVM: Failed get signal mask\n");
1197 // Request KVM to setup the same signal mask as we're currently
1198 // running with except for the KVM control signal. We'll sometimes
1199 // need to raise the KVM_KICK_SIGNAL to cause immediate exits from
1200 // KVM after servicing IO requests. See kvmRun().
1201 sigdelset(&sigset
, KVM_KICK_SIGNAL
);
1202 setSignalMask(&sigset
);
1204 // Mask our control signals so they aren't delivered unless we're
1205 // actually executing inside KVM.
1206 sigaddset(&sigset
, KVM_KICK_SIGNAL
);
1207 if (pthread_sigmask(SIG_SETMASK
, &sigset
, NULL
) == -1)
1208 panic("KVM: Failed mask the KVM control signals\n");
1212 BaseKvmCPU::discardPendingSignal(int signum
) const
1214 int discardedSignal
;
1216 // Setting the timeout to zero causes sigtimedwait to return
1218 struct timespec timeout
;
1220 timeout
.tv_nsec
= 0;
1223 sigemptyset(&sigset
);
1224 sigaddset(&sigset
, signum
);
1227 discardedSignal
= sigtimedwait(&sigset
, NULL
, &timeout
);
1228 } while (discardedSignal
== -1 && errno
== EINTR
);
1230 if (discardedSignal
== signum
)
1232 else if (discardedSignal
== -1 && errno
== EAGAIN
)
1235 panic("Unexpected return value from sigtimedwait: %i (errno: %i)\n",
1236 discardedSignal
, errno
);
1240 BaseKvmCPU::setupCounters()
1242 DPRINTF(Kvm
, "Attaching cycle counter...\n");
1243 PerfKvmCounterConfig
cfgCycles(PERF_TYPE_HARDWARE
,
1244 PERF_COUNT_HW_CPU_CYCLES
);
1245 cfgCycles
.disabled(true)
1248 // Try to exclude the host. We set both exclude_hv and
1249 // exclude_host since different architectures use slightly
1250 // different APIs in the kernel.
1251 cfgCycles
.exclude_hv(true)
1252 .exclude_host(true);
1254 if (perfControlledByTimer
) {
1255 // We need to configure the cycles counter to send overflows
1256 // since we are going to use it to trigger timer signals that
1257 // trap back into m5 from KVM. In practice, this means that we
1258 // need to set some non-zero sample period that gets
1259 // overridden when the timer is armed.
1260 cfgCycles
.wakeupEvents(1)
1264 hwCycles
.attach(cfgCycles
,
1265 0); // TID (0 => currentThread)
1271 BaseKvmCPU::tryDrain()
1273 if (drainState() != DrainState::Draining
)
1276 if (!archIsDrained()) {
1277 DPRINTF(Drain
, "tryDrain: Architecture code is not ready.\n");
1281 if (_status
== Idle
|| _status
== Running
) {
1283 "tryDrain: CPU transitioned into the Idle state, drain done\n");
1287 DPRINTF(Drain
, "tryDrain: CPU not ready.\n");
1293 BaseKvmCPU::ioctlRun()
1295 if (ioctl(KVM_RUN
) == -1) {
1297 panic("KVM: Failed to start virtual CPU (errno: %i)\n",
1303 BaseKvmCPU::setupInstStop()
1305 if (thread
->comInstEventQueue
.empty()) {
1306 setupInstCounter(0);
1308 Tick next
= thread
->comInstEventQueue
.nextTick();
1309 assert(next
> ctrInsts
);
1310 setupInstCounter(next
- ctrInsts
);
1315 BaseKvmCPU::setupInstCounter(uint64_t period
)
1317 // No need to do anything if we aren't attaching for the first
1318 // time or the period isn't changing.
1319 if (period
== activeInstPeriod
&& hwInstructions
.attached())
1322 PerfKvmCounterConfig
cfgInstructions(PERF_TYPE_HARDWARE
,
1323 PERF_COUNT_HW_INSTRUCTIONS
);
1325 // Try to exclude the host. We set both exclude_hv and
1326 // exclude_host since different architectures use slightly
1327 // different APIs in the kernel.
1328 cfgInstructions
.exclude_hv(true)
1329 .exclude_host(true);
1332 // Setup a sampling counter if that has been requested.
1333 cfgInstructions
.wakeupEvents(1)
1334 .samplePeriod(period
);
1337 // We need to detach and re-attach the counter to reliably change
1338 // sampling settings. See PerfKvmCounter::period() for details.
1339 if (hwInstructions
.attached())
1340 hwInstructions
.detach();
1341 assert(hwCycles
.attached());
1342 hwInstructions
.attach(cfgInstructions
,
1343 0, // TID (0 => currentThread)
1347 hwInstructions
.enableSignals(KVM_KICK_SIGNAL
);
1349 activeInstPeriod
= period
;