From 838bcd3b19d2b6375957be986f1dca1803a2b3ce Mon Sep 17 00:00:00 2001 From: Andreas Sandberg Date: Thu, 3 Apr 2014 11:22:49 +0200 Subject: [PATCH] sim: Add the ability to lock and migrate between event queues We need the ability to lock event queues to enable device accesses across threads. The serviceOne() method now takes a service lock prior to handling a new event. By locking an event queue, a different thread/eq can effectively execute in the context of the locked event queue. To simplify temporary event queue migrations, this changeset introduces the EventQueue::ScopedMigration class that unlocks the current event queue, locks a new event queue, and updates the current event queue variable. In order to prevent deadlocks, event queues need to be released when waiting on barriers. This is implemented using the EventQueue::ScopedRelease class. An instance of this class is, for example, used in the BaseGlobalEvent class to release the event queue when waiting on the synchronization barrier. The intended use for this functionality is when devices need to be accessed across thread boundaries. For example, when fast-forwarding, it might be useful to run devices and CPUs in separate threads. In such a case, the CPU locks the device queue whenever it needs to perform IO. This functionality is primarily intended for KVM. Note: Migrating between event queues can lead to non-deterministic timing. Use with extreme care! --HG-- extra : rebase_source : 23e3a741a1fd73861d1339782dbbe1bc76285315 --- src/sim/eventq.cc | 1 + src/sim/eventq.hh | 137 +++++++++++++++++++++++++++++++++++++++- src/sim/global_event.hh | 9 +++ src/sim/simulate.cc | 3 + 4 files changed, 149 insertions(+), 1 deletion(-) diff --git a/src/sim/eventq.cc b/src/sim/eventq.cc index 0735a011b..b8e45a13e 100644 --- a/src/sim/eventq.cc +++ b/src/sim/eventq.cc @@ -203,6 +203,7 @@ EventQueue::remove(Event *event) Event * EventQueue::serviceOne() { + std::lock_guard lock(*this); Event *event = head; Event *next = head->nextInBin; event->flags.clear(Event::Scheduled); diff --git a/src/sim/eventq.hh b/src/sim/eventq.hh index 66b324c4f..9b3cd9917 100644 --- a/src/sim/eventq.hh +++ b/src/sim/eventq.hh @@ -398,8 +398,43 @@ operator!=(const Event &l, const Event &r) } #endif -/* +/** * Queue of events sorted in time order + * + * Events are scheduled (inserted into the event queue) using the + * schedule() method. This method either inserts a synchronous + * or asynchronous event. + * + * Synchronous events are scheduled using schedule() method with the + * argument 'global' set to false (default). This should only be done + * from a thread holding the event queue lock + * (EventQueue::service_mutex). The lock is always held when an event + * handler is called, it can therefore always insert events into its + * own event queue unless it voluntarily releases the lock. + * + * Events can be scheduled across thread (and event queue borders) by + * either scheduling asynchronous events or taking the target event + * queue's lock. However, the lock should never be taken + * directly since this is likely to cause deadlocks. Instead, code + * that needs to schedule events in other event queues should + * temporarily release its own queue and lock the new queue. This + * prevents deadlocks since a single thread never owns more than one + * event queue lock. This functionality is provided by the + * ScopedMigration helper class. Note that temporarily migrating + * between event queues can make the simulation non-deterministic, it + * should therefore be limited to cases where that can be tolerated + * (e.g., handling asynchronous IO or fast-forwarding in KVM). + * + * Asynchronous events can also be scheduled using the normal + * schedule() method with the 'global' parameter set to true. Unlike + * the previous queue migration strategy, this strategy is fully + * deterministic. This causes the event to be inserted in a separate + * queue of asynchronous events (async_queue), which is merged main + * event queue at the end of each simulation quantum (by calling the + * handleAsyncInsertions() method). Note that this implies that such + * events must happen at least one simulation quantum into the future, + * otherwise they risk being scheduled in the past by + * handleAsyncInsertions(). */ class EventQueue : public Serializable { @@ -414,6 +449,28 @@ class EventQueue : public Serializable //! List of events added by other threads to this event queue. std::list async_queue; + /** + * Lock protecting event handling. + * + * This lock is always taken when servicing events. It is assumed + * that the thread scheduling new events (not asynchronous events + * though) have taken this lock. This is normally done by + * serviceOne() since new events are typically scheduled as a + * response to an earlier event. + * + * This lock is intended to be used to temporarily steal an event + * queue to support inter-thread communication when some + * deterministic timing can be sacrificed for speed. For example, + * the KVM CPU can use this support to access devices running in a + * different thread. + * + * @see EventQueue::ScopedMigration. + * @see EventQueue::ScopedRelease + * @see EventQueue::lock() + * @see EventQueue::unlock() + */ + std::mutex service_mutex; + //! Insert / remove event from the queue. Should only be called //! by thread operating this queue. void insert(Event *event); @@ -427,6 +484,68 @@ class EventQueue : public Serializable EventQueue(const EventQueue &); public: +#ifndef SWIG + /** + * Temporarily migrate execution to a different event queue. + * + * An instance of this class temporarily migrates execution to a + * different event queue by releasing the current queue, locking + * the new queue, and updating curEventQueue(). This can, for + * example, be useful when performing IO across thread event + * queues when timing is not crucial (e.g., during fast + * forwarding). + */ + class ScopedMigration + { + public: + ScopedMigration(EventQueue *_new_eq) + : new_eq(*_new_eq), old_eq(*curEventQueue()) + { + old_eq.unlock(); + new_eq.lock(); + curEventQueue(&new_eq); + } + + ~ScopedMigration() + { + new_eq.unlock(); + old_eq.lock(); + curEventQueue(&old_eq); + } + + private: + EventQueue &new_eq; + EventQueue &old_eq; + }; + + /** + * Temporarily release the event queue service lock. + * + * There are cases where it is desirable to temporarily release + * the event queue lock to prevent deadlocks. For example, when + * waiting on the global barrier, we need to release the lock to + * prevent deadlocks from happening when another thread tries to + * temporarily take over the event queue waiting on the barrier. + */ + class ScopedRelease + { + public: + ScopedRelease(EventQueue *_eq) + : eq(*_eq) + { + eq.unlock(); + } + + ~ScopedRelease() + { + eq.lock(); + } + + private: + EventQueue &eq; + }; +#endif + EventQueue(const std::string &n); virtual const std::string name() const { return objName; } @@ -491,6 +610,22 @@ class EventQueue : public Serializable */ Event* replaceHead(Event* s); + /**@{*/ + /** + * Provide an interface for locking/unlocking the event queue. + * + * @warn Do NOT use these methods directly unless you really know + * what you are doing. Incorrect use can easily lead to simulator + * deadlocks. + * + * @see EventQueue::ScopedMigration. + * @see EventQueue::ScopedRelease + * @see EventQueue + */ + void lock() { service_mutex.lock(); } + void unlock() { service_mutex.unlock(); } + /**@}*/ + #ifndef SWIG virtual void serialize(std::ostream &os); virtual void unserialize(Checkpoint *cp, const std::string §ion); diff --git a/src/sim/global_event.hh b/src/sim/global_event.hh index 7b5fd7485..3a5297e78 100644 --- a/src/sim/global_event.hh +++ b/src/sim/global_event.hh @@ -91,6 +91,15 @@ class BaseGlobalEvent : public EventBase bool globalBarrier() { + // This method will be called from the process() method in + // the local barrier events + // (GlobalSyncEvent::BarrierEvent). The local event + // queues are always locked when servicing events (calling + // the process() method), which means that it will be + // locked when entering this method. We need to unlock it + // while waiting on the barrier to prevent deadlocks if + // another thread wants to lock the event queue. + EventQueue::ScopedRelease release(curEventQueue()); return _globalEvent->barrier->wait(); } diff --git a/src/sim/simulate.cc b/src/sim/simulate.cc index b60b8a783..426c3e662 100644 --- a/src/sim/simulate.cc +++ b/src/sim/simulate.cc @@ -198,6 +198,9 @@ doSimLoop(EventQueue *eventq) } if (async_event && testAndClearAsyncEvent()) { + // Take the event queue lock in case any of the service + // routines want to schedule new events. + std::lock_guard lock(*eventq); async_event = false; if (async_statdump || async_statreset) { Stats::schedStatEvent(async_statdump, async_statreset); -- 2.30.2