-/* Copyright (C) 2008, 2009, 2011 Free Software Foundation, Inc.
+/* Copyright (C) 2008-2013 Free Software Foundation, Inc.
Contributed by Richard Henderson <rth@redhat.com>.
This file is part of the GNU Transactional Memory Library (libitm).
unsigned GTM::gtm_thread::number_of_threads = 0;
gtm_stmlock GTM::gtm_stmlock_array[LOCK_ARRAY_SIZE];
-gtm_version GTM::gtm_clock;
+atomic<gtm_version> GTM::gtm_clock;
/* ??? Move elsewhere when we figure out library initialization. */
uint64_t GTM::gtm_spin_count_var = 1000;
+#ifdef HAVE_64BIT_SYNC_BUILTINS
+static atomic<_ITM_transactionId_t> global_tid;
+#else
static _ITM_transactionId_t global_tid;
+static pthread_mutex_t global_tid_lock = PTHREAD_MUTEX_INITIALIZER;
+#endif
+
// Provides a on-thread-exit callback used to release per-thread data.
static pthread_key_t thr_release_key;
static pthread_once_t thr_release_once = PTHREAD_ONCE_INIT;
+// See gtm_thread::begin_transaction.
+uint32_t GTM::htm_fastpath = 0;
/* Allocate a transaction structure. */
void *
// This object's memory has been set to zero by operator new, so no need
// to initialize any of the other primitive-type members that do not have
// constructors.
- shared_state = ~(typeof shared_state)0;
+ shared_state.store(-1, memory_order_relaxed);
// Register this transaction with the list of all threads' transactions.
serial_lock.write_lock ();
GTM_fatal("Setting thread release TLS key failed.");
}
-
-
-#ifndef HAVE_64BIT_SYNC_BUILTINS
-static pthread_mutex_t global_tid_lock = PTHREAD_MUTEX_INITIALIZER;
-#endif
-
-static inline uint32_t choose_code_path(uint32_t prop, abi_dispatch *disp)
+static inline uint32_t
+choose_code_path(uint32_t prop, abi_dispatch *disp)
{
if ((prop & pr_uninstrumentedCode) && disp->can_run_uninstrumented_code())
return a_runUninstrumentedCode;
if (unlikely(prop & pr_undoLogCode))
GTM_fatal("pr_undoLogCode not supported");
+#if defined(USE_HTM_FASTPATH) && !defined(HTM_CUSTOM_FASTPATH)
+ // HTM fastpath. Only chosen in the absence of transaction_cancel to allow
+ // using an uninstrumented code path.
+ // The fastpath is enabled only by dispatch_htm's method group, which uses
+ // serial-mode methods as fallback. Serial-mode transactions cannot execute
+ // concurrently with HW transactions because the latter monitor the serial
+ // lock's writer flag and thus abort if another thread is or becomes a
+ // serial transaction. Therefore, if the fastpath is enabled, then a
+ // transaction is not executing as a HW transaction iff the serial lock is
+ // write-locked. This allows us to use htm_fastpath and the serial lock's
+ // writer flag to reliable determine whether the current thread runs a HW
+ // transaction, and thus we do not need to maintain this information in
+ // per-thread state.
+ // If an uninstrumented code path is not available, we can still run
+ // instrumented code from a HW transaction because the HTM fastpath kicks
+ // in early in both begin and commit, and the transaction is not canceled.
+ // HW transactions might get requests to switch to serial-irrevocable mode,
+ // but these can be ignored because the HTM provides all necessary
+ // correctness guarantees. Transactions cannot detect whether they are
+ // indeed in serial mode, and HW transactions should never need serial mode
+ // for any internal changes (e.g., they never abort visibly to the STM code
+ // and thus do not trigger the standard retry handling).
+ if (likely(htm_fastpath && (prop & pr_hasNoAbort)))
+ {
+ for (uint32_t t = htm_fastpath; t; t--)
+ {
+ uint32_t ret = htm_begin();
+ if (htm_begin_success(ret))
+ {
+ // We are executing a transaction now.
+ // Monitor the writer flag in the serial-mode lock, and abort
+ // if there is an active or waiting serial-mode transaction.
+ if (unlikely(serial_lock.is_write_locked()))
+ htm_abort();
+ else
+ // We do not need to set a_saveLiveVariables because of HTM.
+ return (prop & pr_uninstrumentedCode) ?
+ a_runUninstrumentedCode : a_runInstrumentedCode;
+ }
+ // The transaction has aborted. Don't retry if it's unlikely that
+ // retrying the transaction will be successful.
+ if (!htm_abort_should_retry(ret))
+ break;
+ // Wait until any concurrent serial-mode transactions have finished.
+ // This is an empty critical section, but won't be elided.
+ if (serial_lock.is_write_locked())
+ {
+ tx = gtm_thr();
+ if (unlikely(tx == NULL))
+ {
+ // See below.
+ tx = new gtm_thread();
+ set_gtm_thr(tx);
+ }
+ serial_lock.read_lock(tx);
+ serial_lock.read_unlock(tx);
+ // TODO We should probably reset the retry count t here, unless
+ // we have retried so often that we should go serial to avoid
+ // starvation.
+ }
+ }
+ }
+#endif
+
tx = gtm_thr();
if (unlikely(tx == NULL))
{
{
// Outermost transaction
disp = tx->decide_begin_dispatch (prop);
- if (disp == dispatch_serialirr() || disp == dispatch_serial())
- {
- tx->state = STATE_SERIAL;
- if (disp == dispatch_serialirr())
- tx->state |= STATE_IRREVOCABLE;
- serial_lock.write_lock ();
- }
- else
- serial_lock.read_lock (tx);
-
set_abi_disp (disp);
}
else
{
#ifdef HAVE_64BIT_SYNC_BUILTINS
- tx->id = __sync_add_and_fetch (&global_tid, tid_block_size);
+ // We don't really care which block of TIDs we get but only that we
+ // acquire one atomically; therefore, relaxed memory order is
+ // sufficient.
+ tx->id = global_tid.fetch_add(tid_block_size, memory_order_relaxed);
tx->local_tid = tx->id + 1;
#else
pthread_mutex_lock (&global_tid_lock);
// data. Because of the latter, we have to roll it back before any
// dispatch-specific rollback (which handles synchronization with other
// transactions).
- rollback_undolog (cp ? cp->undolog_size : 0);
+ undolog.rollback (this, cp ? cp->undolog_size : 0);
// Perform dispatch-specific rollback.
abi_disp()->rollback (cp);
// The transaction is now inactive. Everything that we still have to do
// will not synchronize with other transactions anymore.
if (state & gtm_thread::STATE_SERIAL)
- gtm_thread::serial_lock.write_unlock ();
+ {
+ gtm_thread::serial_lock.write_unlock ();
+ // There are no other active transactions, so there's no need to
+ // enforce privatization safety.
+ priv_time = 0;
+ }
else
gtm_thread::serial_lock.read_unlock (this);
state = 0;
// We can commit the undo log after dispatch-specific commit and after
// making the transaction inactive because we only have to reset
// gtm_thread state.
- commit_undolog ();
+ undolog.commit ();
// Reset further transaction state.
cxa_catch_count = 0;
cxa_unthrown = NULL;
// Ensure privatization safety, if necessary.
if (priv_time)
{
+ // There must be a seq_cst fence between the following loads of the
+ // other transactions' shared_state and the dispatch-specific stores
+ // that signal updates by this transaction (e.g., lock
+ // acquisitions). This ensures that if we read prior to other
+ // reader transactions setting their shared_state to 0, then those
+ // readers will observe our updates. We can reuse the seq_cst fence
+ // in serial_lock.read_unlock() however, so we don't need another
+ // one here.
// TODO Don't just spin but also block using cond vars / futexes
// here. Should probably be integrated with the serial lock code.
- // TODO For C++0x atomics, the loads of other threads' shared_state
- // should have acquire semantics (together with releases for the
- // respective updates). But is this unnecessary overhead because
- // weaker barriers are sufficient?
for (gtm_thread *it = gtm_thread::list_of_threads; it != 0;
it = it->next_thread)
{
if (it == this) continue;
- while (it->shared_state < priv_time)
+ // We need to load other threads' shared_state using acquire
+ // semantics (matching the release semantics of the respective
+ // updates). This is necessary to ensure that the other
+ // threads' memory accesses happen before our actions that
+ // assume privatization safety.
+ // TODO Are there any platform-specific optimizations (e.g.,
+ // merging barriers)?
+ while (it->shared_state.load(memory_order_acquire) < priv_time)
cpu_relax();
}
}
}
void ITM_NORETURN
-GTM::gtm_thread::restart (gtm_restart_reason r)
+GTM::gtm_thread::restart (gtm_restart_reason r, bool finish_serial_upgrade)
{
// Roll back to outermost transaction. Do not reset transaction state because
// we will continue executing this transaction.
rollback ();
+
+ // If we have to restart while an upgrade of the serial lock is happening,
+ // we need to finish this here, after rollback (to ensure privatization
+ // safety despite undo writes) and before deciding about the retry strategy
+ // (which could switch to/from serial mode).
+ if (finish_serial_upgrade)
+ gtm_thread::serial_lock.write_upgrade_finish(this);
+
decide_retry_strategy (r);
// Run dispatch-specific restart code. Retry until we succeed.
void ITM_REGPARM
_ITM_commitTransaction(void)
{
+#if defined(USE_HTM_FASTPATH)
+ // HTM fastpath. If we are not executing a HW transaction, then we will be
+ // a serial-mode transaction. If we are, then there will be no other
+ // concurrent serial-mode transaction.
+ // See gtm_thread::begin_transaction.
+ if (likely(htm_fastpath && !gtm_thread::serial_lock.is_write_locked()))
+ {
+ htm_commit();
+ return;
+ }
+#endif
gtm_thread *tx = gtm_thr();
if (!tx->trycommit ())
tx->restart (RESTART_VALIDATE_COMMIT);
void ITM_REGPARM
_ITM_commitTransactionEH(void *exc_ptr)
{
+#if defined(USE_HTM_FASTPATH)
+ // See _ITM_commitTransaction.
+ if (likely(htm_fastpath && !gtm_thread::serial_lock.is_write_locked()))
+ {
+ htm_commit();
+ return;
+ }
+#endif
gtm_thread *tx = gtm_thr();
if (!tx->trycommit ())
{
projects / gcc.git / blobdiff