# include <condition_variable>
# include <thread>
#endif
-#ifndef _GLIBCXX_HAVE_TLS
-# include <bits/std_function.h>
+#include <ext/atomicity.h> // __gnu_cxx::__is_single_threaded
+
+#if defined _GLIBCXX_HAS_GTHREADS && ! defined _GLIBCXX_HAVE_TLS
+# include <bits/std_function.h> // std::function
#endif
namespace std _GLIBCXX_VISIBILITY(default)
};
#endif // C++17
-#ifdef _GLIBCXX_HAS_GTHREADS
/// Flag type used by std::call_once
struct once_flag
{
- private:
- typedef __gthread_once_t __native_type;
- __native_type _M_once = __GTHREAD_ONCE_INIT;
-
- public:
- /// Constructor
constexpr once_flag() noexcept = default;
/// Deleted copy constructor
/// Deleted assignment operator
once_flag& operator=(const once_flag&) = delete;
+ private:
+ // There are two different std::once_flag interfaces, abstracting four
+ // different implementations.
+ // The preferred interface uses the _M_activate() and _M_finish(bool)
+ // member functions (introduced in GCC 11), which start and finish an
+ // active execution respectively. See [thread.once.callonce] in C++11
+ // for the definition of active/passive/returning/exceptional executions.
+ // This interface is supported for Linux (using atomics and futexes) and
+ // for single-threaded targets with no gthreads support.
+ // For other targets a pthread_once_t is used with pthread_once, but that
+ // doesn't work correctly for exceptional executions. That interface
+ // uses an object of type _Prepare_execution and a lambda expression.
+#if defined _GLIBCXX_HAVE_LINUX_FUTEX || ! defined _GLIBCXX_HAS_GTHREADS
+ enum _Bits : int { _Init = 0, _Active = 1, _Done = 2 };
+
+ int _M_once = _Bits::_Init;
+
+ // Non-blocking check to see if all executions will be passive now.
+ bool
+ _M_passive() const noexcept;
+
+ // Attempts to begin an active execution. Blocks until it either:
+ // - returns true if an active execution has started on this thread, or
+ // - returns false if a returning execution happens on another thread.
+ bool _M_activate();
+
+ // Must be called to complete an active execution.
+ void _M_finish(bool __returning) noexcept;
+
+ // RAII helper to call _M_finish.
+ struct _Active_execution
+ {
+ explicit _Active_execution(once_flag& __flag) : _M_flag(__flag) { }
+
+ ~_Active_execution() { _M_flag._M_finish(_M_returning); }
+
+ _Active_execution(const _Active_execution&) = delete;
+ _Active_execution& operator=(const _Active_execution&) = delete;
+
+ once_flag& _M_flag;
+ bool _M_returning = false;
+ };
+#else
+ __gthread_once_t _M_once = __GTHREAD_ONCE_INIT;
+
+ struct _Prepare_execution;
+#endif // ! GTHREADS
+
template<typename _Callable, typename... _Args>
friend void
call_once(once_flag& __once, _Callable&& __f, _Args&&... __args);
};
+#if ! defined _GLIBCXX_HAS_GTHREADS
+ // Inline definitions of std::once_flag members for single-threaded targets.
+
+ inline bool
+ once_flag::_M_passive() const noexcept
+ { return _M_once == _Bits::_Done; }
+
+ inline bool
+ once_flag::_M_activate()
+ {
+ if (_M_once == _Bits::_Init)
+ {
+ _M_once = _Bits::_Active;
+ return true;
+ }
+ else if (!_M_passive())
+ __throw_system_error(EDEADLK);
+ }
+
+ inline void
+ once_flag::_M_finish(bool returning) noexcept
+ { _M_once = returning ? _Bits::_Done : _Bits::_Init; }
+
+#elif defined _GLIBCXX_HAVE_LINUX_FUTEX
+
+ // Define this inline to make passive executions fast.
+ inline bool
+ once_flag::_M_passive() const noexcept
+ {
+ if (__gnu_cxx::__is_single_threaded())
+ return _M_once == _Bits::_Done;
+ else
+ return __atomic_load_n(&_M_once, __ATOMIC_ACQUIRE) == _Bits::_Done;
+ }
+
+#else // GTHREADS && ! FUTEX
+
/// @cond undocumented
-#ifdef _GLIBCXX_HAVE_TLS
+# ifdef _GLIBCXX_HAVE_TLS
+ // If TLS is available use thread-local state for the type-erased callable
+ // that is being run by std::call_once in the current thread.
extern __thread void* __once_callable;
extern __thread void (*__once_call)();
-#else
+# else
+ // Without TLS use a global std::mutex and store the callable in a
+ // global std::function.
extern function<void()> __once_functor;
extern void
extern mutex&
__get_once_mutex();
-#endif
+# endif
+ // This function is passed to pthread_once by std::call_once.
+ // It runs __once_call() or __once_functor().
extern "C" void __once_proxy(void);
+
+ // RAII type to set up state for pthread_once call.
+ struct once_flag::_Prepare_execution
+ {
+#ifdef _GLIBCXX_HAVE_TLS
+ template<typename _Callable>
+ explicit
+ _Prepare_execution(_Callable& __c)
+ {
+ // Store address in thread-local pointer:
+ __once_callable = std::__addressof(__c);
+ // Trampoline function to invoke the closure via thread-local pointer:
+ __once_call = [] { (*static_cast<_Callable*>(__once_callable))(); };
+ }
+
+ ~_Prepare_execution()
+ {
+ // PR libstdc++/82481
+ __once_callable = nullptr;
+ __once_call = nullptr;
+ }
+#else // ! TLS
+ template<typename _Callable>
+ explicit
+ _Prepare_execution(_Callable& __c)
+ {
+ // Store the callable in the global std::function
+ __once_functor = __c;
+ __set_once_functor_lock_ptr(&_M_functor_lock);
+ }
+
+ ~_Prepare_execution()
+ {
+ if (_M_functor_lock)
+ __set_once_functor_lock_ptr(nullptr);
+ }
+
+ private:
+ unique_lock<mutex> _M_functor_lock{__get_once_mutex()};
+#endif // ! TLS
+
+ _Prepare_execution(const _Prepare_execution&) = delete;
+ _Prepare_execution& operator=(const _Prepare_execution&) = delete;
+ };
/// @endcond
+#endif
/// Invoke a callable and synchronize with other calls using the same flag
template<typename _Callable, typename... _Args>
void
call_once(once_flag& __once, _Callable&& __f, _Args&&... __args)
{
- // _GLIBCXX_RESOLVE_LIB_DEFECTS
- // 2442. call_once() shouldn't DECAY_COPY()
+#if defined _GLIBCXX_HAVE_LINUX_FUTEX || ! defined _GLIBCXX_HAS_GTHREADS
+ if (__once._M_passive())
+ return;
+ else if (__once._M_activate())
+ {
+ once_flag::_Active_execution __exec(__once);
+
+ // _GLIBCXX_RESOLVE_LIB_DEFECTS
+ // 2442. call_once() shouldn't DECAY_COPY()
+ std::__invoke(std::forward<_Callable>(__f),
+ std::forward<_Args>(__args)...);
+
+ // __f(__args...) did not throw
+ __exec._M_returning = true;
+ }
+#else
+ // Closure type that runs the function
auto __callable = [&] {
std::__invoke(std::forward<_Callable>(__f),
std::forward<_Args>(__args)...);
};
-#ifdef _GLIBCXX_HAVE_TLS
- __once_callable = std::__addressof(__callable);
- __once_call = []{ (*(decltype(__callable)*)__once_callable)(); };
-#else
- unique_lock<mutex> __functor_lock(__get_once_mutex());
- __once_functor = __callable;
- __set_once_functor_lock_ptr(&__functor_lock);
-#endif
-
- int __e = __gthread_once(&__once._M_once, &__once_proxy);
-#ifndef _GLIBCXX_HAVE_TLS
- if (__functor_lock)
- __set_once_functor_lock_ptr(0);
-#endif
+ once_flag::_Prepare_execution __exec(__callable);
-#ifdef __clang_analyzer__
- // PR libstdc++/82481
- __once_callable = nullptr;
- __once_call = nullptr;
-#endif
-
- if (__e)
+ // XXX pthread_once does not reset the flag if an exception is thrown.
+ if (int __e = __gthread_once(&__once._M_once, &__once_proxy))
__throw_system_error(__e);
+#endif
}
-#endif // _GLIBCXX_HAS_GTHREADS
// @} group mutexes
_GLIBCXX_END_NAMESPACE_VERSION
#include <mutex>
#ifdef _GLIBCXX_HAS_GTHREADS
+
+#ifdef _GLIBCXX_HAVE_LINUX_FUTEX
+#include <syscall.h>
+#include <unistd.h>
+#include <limits.h>
+
+bool
+std::once_flag::_M_activate()
+{
+ if (__gnu_cxx::__is_single_threaded())
+ {
+ if (_M_once == _Bits::_Done)
+ return false;
+ _M_once = _Bits::_Active;
+ return true;
+ }
+
+ while (true)
+ {
+ int expected = _Bits::_Init;
+ constexpr int active = _Bits::_Active;
+ if (__atomic_compare_exchange_n(&_M_once, &expected, active, false,
+ __ATOMIC_ACQ_REL,
+ __ATOMIC_ACQUIRE))
+ {
+ // This thread is now doing an active execution.
+ return true;
+ }
+
+ if (expected == _Bits::_Done)
+ return false; // A returning execution happened, this is passive.
+
+ // Otherwise, an active execution is happening. Wait for it to finish.
+ constexpr int futex_wait = 128; // FUTEX_WAIT_PRIVATE
+ syscall (SYS_futex, &_M_once, futex_wait, expected, 0);
+ }
+}
+
+void
+std::once_flag::_M_finish(bool returning) noexcept
+{
+ const int newval = returning ? _Bits::_Done : _Bits::_Init;
+ if (__gnu_cxx::__is_single_threaded())
+ {
+ __glibcxx_assert(_M_once == _Bits::_Active);
+ _M_once = newval;
+ }
+ else
+ {
+ int prev = __atomic_exchange_n(&_M_once, newval, __ATOMIC_RELEASE);
+ __glibcxx_assert(prev & _Bits::_Active);
+ // Wake any other threads waiting for this execution to finish.
+ constexpr int futex_wake = 129; // FUTEX_WAKE_PRIVATE
+ syscall (SYS_futex, &_M_once, futex_wake, INT_MAX);
+ }
+}
+
+#endif // ! FUTEX
+
#ifndef _GLIBCXX_HAVE_TLS
namespace
{
projects / gcc.git / commitdiff