libitm/libitm_i.h

   1 /* Copyright (C) 2008-2015 Free Software Foundation, Inc.
   2    Contributed by Richard Henderson <rth@redhat.com>.
   3
   4    This file is part of the GNU Transactional Memory Library (libitm).
   5
   6    Libitm is free software; you can redistribute it and/or modify it
   7    under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 3 of the License, or
   9    (at your option) any later version.
  10
  11    Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
  12    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  13    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14    more details.
  15
  16    Under Section 7 of GPL version 3, you are granted additional
  17    permissions described in the GCC Runtime Library Exception, version
  18    3.1, as published by the Free Software Foundation.
  19
  20    You should have received a copy of the GNU General Public License and
  21    a copy of the GCC Runtime Library Exception along with this program;
  22    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  23    <http://www.gnu.org/licenses/>.  */
  24
  25 /* The following are internal implementation functions and definitions.
  26    To distinguish them from those defined by the Intel ABI, they all
  27    begin with GTM/gtm.  */
  28
  29 #ifndef LIBITM_I_H
  30 #define LIBITM_I_H 1
  31
  32 #include "libitm.h"
  33 #include "config.h"
  34
  35 #include <assert.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <unwind.h>
  39 #include "local_type_traits"
  40 #include "local_atomic"
  41
  42 /* Don't require libgcc_s.so for exceptions.  */
  43 extern void _Unwind_DeleteException (_Unwind_Exception*) __attribute__((weak));
  44
  45
  46 #include "common.h"
  47
  48 namespace GTM HIDDEN {
  49
  50 using namespace std;
  51
  52 // A helper template for accessing an unsigned integral of SIZE bytes.
  53 template<size_t SIZE> struct sized_integral { };
  54 template<> struct sized_integral<1> { typedef uint8_t type; };
  55 template<> struct sized_integral<2> { typedef uint16_t type; };
  56 template<> struct sized_integral<4> { typedef uint32_t type; };
  57 template<> struct sized_integral<8> { typedef uint64_t type; };
  58
  59 typedef unsigned int gtm_word __attribute__((mode (word)));
  60
  61 // These values are given to GTM_restart_transaction and indicate the
  62 // reason for the restart.  The reason is used to decide what STM
  63 // implementation should be used during the next iteration.
  64 enum gtm_restart_reason
  65 {
  66   RESTART_REALLOCATE,
  67   RESTART_LOCKED_READ,
  68   RESTART_LOCKED_WRITE,
  69   RESTART_VALIDATE_READ,
  70   RESTART_VALIDATE_WRITE,
  71   RESTART_VALIDATE_COMMIT,
  72   RESTART_SERIAL_IRR,
  73   RESTART_NOT_READONLY,
  74   RESTART_CLOSED_NESTING,
  75   RESTART_INIT_METHOD_GROUP,
  76   NUM_RESTARTS,
  77   NO_RESTART = NUM_RESTARTS
  78 };
  79
  80 } // namespace GTM
  81
  82 #include "target.h"
  83 #include "rwlock.h"
  84 #include "aatree.h"
  85 #include "cacheline.h"
  86 #include "stmlock.h"
  87 #include "dispatch.h"
  88 #include "containers.h"
  89
  90 #ifdef __USER_LABEL_PREFIX__
  91 # define UPFX UPFX1(__USER_LABEL_PREFIX__)
  92 # define UPFX1(t) UPFX2(t)
  93 # define UPFX2(t) #t
  94 #else
  95 # define UPFX
  96 #endif
  97
  98 namespace GTM HIDDEN {
  99
 100 // This type is private to alloc.c, but needs to be defined so that
 101 // the template used inside gtm_thread can instantiate.
 102 struct gtm_alloc_action
 103 {
 104   void (*free_fn)(void *);
 105   bool allocated;
 106 };
 107
 108 struct gtm_thread;
 109
 110 // A transaction checkpoint: data that has to saved and restored when doing
 111 // closed nesting.
 112 struct gtm_transaction_cp
 113 {
 114   gtm_jmpbuf jb;
 115   size_t undolog_size;
 116   aa_tree<uintptr_t, gtm_alloc_action> alloc_actions;
 117   size_t user_actions_size;
 118   _ITM_transactionId_t id;
 119   uint32_t prop;
 120   uint32_t cxa_catch_count;
 121   void *cxa_unthrown;
 122   // We might want to use a different but compatible dispatch method for
 123   // a nested transaction.
 124   abi_dispatch *disp;
 125   // Nesting level of this checkpoint (1 means that this is a checkpoint of
 126   // the outermost transaction).
 127   uint32_t nesting;
 128
 129   void save(gtm_thread* tx);
 130   void commit(gtm_thread* tx);
 131 };
 132
 133 // An undo log for writes.
 134 struct gtm_undolog
 135 {
 136   vector<gtm_word> undolog;
 137
 138   // Log the previous value at a certain address.
 139   // The easiest way to inline this is to just define this here.
 140   void log(const void *ptr, size_t len)
 141   {
 142     size_t words = (len + sizeof(gtm_word) - 1) / sizeof(gtm_word);
 143     gtm_word *undo = undolog.push(words + 2);
 144     memcpy(undo, ptr, len);
 145     undo[words] = len;
 146     undo[words + 1] = (gtm_word) ptr;
 147   }
 148
 149   void commit () { undolog.clear(); }
 150   size_t size() const { return undolog.size(); }
 151
 152   // In local.cc
 153   void rollback (gtm_thread* tx, size_t until_size = 0);
 154 };
 155
 156 // An entry of a read or write log.  Used by multi-lock TM methods.
 157 struct gtm_rwlog_entry
 158 {
 159   atomic<gtm_word> *orec;
 160   gtm_word value;
 161 };
 162
 163 // Contains all thread-specific data required by the entire library.
 164 // This includes all data relevant to a single transaction. Because most
 165 // thread-specific data is about the current transaction, we also refer to
 166 // the transaction-specific parts of gtm_thread as "the transaction" (the
 167 // same applies to names of variables and arguments).
 168 // All but the shared part of this data structure are thread-local data.
 169 // gtm_thread could be split into transaction-specific structures and other
 170 // per-thread data (with those parts then nested in gtm_thread), but this
 171 // would make it harder to later rearrange individual members to optimize data
 172 // accesses. Thus, for now we keep one flat object, and will only split it if
 173 // the code gets too messy.
 174 struct gtm_thread
 175 {
 176
 177   struct user_action
 178   {
 179     _ITM_userCommitFunction fn;
 180     void *arg;
 181     bool on_commit;
 182     _ITM_transactionId_t resuming_id;
 183   };
 184
 185   // The jump buffer by which GTM_longjmp restarts the transaction.
 186   // This field *must* be at the beginning of the transaction.
 187   gtm_jmpbuf jb;
 188
 189   // Data used by local.c for the undo log for both local and shared memory.
 190   gtm_undolog undolog;
 191
 192   // Read and write logs.  Used by multi-lock TM methods.
 193   vector<gtm_rwlog_entry> readlog;
 194   vector<gtm_rwlog_entry> writelog;
 195
 196   // Data used by alloc.c for the malloc/free undo log.
 197   aa_tree<uintptr_t, gtm_alloc_action> alloc_actions;
 198
 199   // Data used by useraction.c for the user-defined commit/abort handlers.
 200   vector<user_action> user_actions;
 201
 202   // A numerical identifier for this transaction.
 203   _ITM_transactionId_t id;
 204
 205   // The _ITM_codeProperties of this transaction as given by the compiler.
 206   uint32_t prop;
 207
 208   // The nesting depth for subsequently started transactions. This variable
 209   // will be set to 1 when starting an outermost transaction.
 210   uint32_t nesting;
 211
 212   // Set if this transaction owns the serial write lock.
 213   // Can be reset only when restarting the outermost transaction.
 214   static const uint32_t STATE_SERIAL            = 0x0001;
 215   // Set if the serial-irrevocable dispatch table is installed.
 216   // Implies that no logging is being done, and abort is not possible.
 217   // Can be reset only when restarting the outermost transaction.
 218   static const uint32_t STATE_IRREVOCABLE       = 0x0002;
 219
 220   // A bitmask of the above.
 221   uint32_t state;
 222
 223   // In order to reduce cacheline contention on global_tid during
 224   // beginTransaction, we allocate a block of 2**N ids to the thread
 225   // all at once.  This number is the next value to be allocated from
 226   // the block, or 0 % 2**N if no such block is allocated.
 227   _ITM_transactionId_t local_tid;
 228
 229   // Data used by eh_cpp.c for managing exceptions within the transaction.
 230   uint32_t cxa_catch_count;
 231   void *cxa_unthrown;
 232   void *eh_in_flight;
 233
 234   // Checkpoints for closed nesting.
 235   vector<gtm_transaction_cp> parent_txns;
 236
 237   // Data used by retry.c for deciding what STM implementation should
 238   // be used for the next iteration of the transaction.
 239   // Only restart_total is reset to zero when the transaction commits, the
 240   // other counters are total values for all previously executed transactions.
 241   // restart_total is also used by the HTM fastpath in a different way.
 242   uint32_t restart_reason[NUM_RESTARTS];
 243   uint32_t restart_total;
 244
 245   // *** The shared part of gtm_thread starts here. ***
 246   // Shared state is on separate cachelines to avoid false sharing with
 247   // thread-local parts of gtm_thread.
 248
 249   // Points to the next thread in the list of all threads.
 250   gtm_thread *next_thread __attribute__((__aligned__(HW_CACHELINE_SIZE)));
 251
 252   // If this transaction is inactive, shared_state is ~0. Otherwise, this is
 253   // an active or serial transaction.
 254   atomic<gtm_word> shared_state;
 255
 256   // The lock that provides access to serial mode.  Non-serialized
 257   // transactions acquire read locks; a serialized transaction aquires
 258   // a write lock.
 259   // Accessed from assembly language, thus the "asm" specifier on
 260   // the name, avoiding complex name mangling.
 261   static gtm_rwlock serial_lock __asm__(UPFX "gtm_serial_lock");
 262
 263   // The head of the list of all threads' transactions.
 264   static gtm_thread *list_of_threads;
 265   // The number of all registered threads.
 266   static unsigned number_of_threads;
 267
 268   // In alloc.cc
 269   void commit_allocations (bool, aa_tree<uintptr_t, gtm_alloc_action>*);
 270   void record_allocation (void *, void (*)(void *));
 271   void forget_allocation (void *, void (*)(void *));
 272   void drop_references_allocations (const void *ptr)
 273   {
 274     this->alloc_actions.erase((uintptr_t) ptr);
 275   }
 276
 277   // In beginend.cc
 278   void rollback (gtm_transaction_cp *cp = 0, bool aborting = false);
 279   bool trycommit ();
 280   void restart (gtm_restart_reason, bool finish_serial_upgrade = false)
 281         ITM_NORETURN;
 282
 283   gtm_thread();
 284   ~gtm_thread();
 285
 286   static void *operator new(size_t);
 287   static void operator delete(void *);
 288
 289   // Invoked from assembly language, thus the "asm" specifier on
 290   // the name, avoiding complex name mangling.
 291   static uint32_t begin_transaction(uint32_t, const gtm_jmpbuf *)
 292         __asm__(UPFX "GTM_begin_transaction") ITM_REGPARM;
 293   // In eh_cpp.cc
 294   void revert_cpp_exceptions (gtm_transaction_cp *cp = 0);
 295
 296   // In retry.cc
 297   // Must be called outside of transactions (i.e., after rollback).
 298   void decide_retry_strategy (gtm_restart_reason);
 299   abi_dispatch* decide_begin_dispatch (uint32_t prop);
 300   void number_of_threads_changed(unsigned previous, unsigned now);
 301   // Must be called from serial mode. Does not call set_abi_disp().
 302   void set_default_dispatch(abi_dispatch* disp);
 303
 304   // In method-serial.cc
 305   void serialirr_mode ();
 306
 307   // In useraction.cc
 308   void rollback_user_actions (size_t until_size = 0);
 309   void commit_user_actions ();
 310 };
 311
 312 } // namespace GTM
 313
 314 #include "tls.h"
 315
 316 namespace GTM HIDDEN {
 317
 318 // An unscaled count of the number of times we should spin attempting to
 319 // acquire locks before we block the current thread and defer to the OS.
 320 // This variable isn't used when the standard POSIX lock implementations
 321 // are used.
 322 extern uint64_t gtm_spin_count_var;
 323
 324 extern "C" uint32_t GTM_longjmp (uint32_t, const gtm_jmpbuf *, uint32_t)
 325         ITM_NORETURN ITM_REGPARM;
 326
 327 extern "C" void GTM_LB (const void *, size_t) ITM_REGPARM;
 328
 329 extern void GTM_error (const char *fmt, ...)
 330         __attribute__((format (printf, 1, 2)));
 331 extern void GTM_fatal (const char *fmt, ...)
 332         __attribute__((noreturn, format (printf, 1, 2)));
 333
 334 extern abi_dispatch *dispatch_serial();
 335 extern abi_dispatch *dispatch_serialirr();
 336 extern abi_dispatch *dispatch_serialirr_onwrite();
 337 extern abi_dispatch *dispatch_gl_wt();
 338 extern abi_dispatch *dispatch_ml_wt();
 339 extern abi_dispatch *dispatch_htm();
 340
 341 extern gtm_cacheline_mask gtm_mask_stack(gtm_cacheline *, gtm_cacheline_mask);
 342
 343 // Control variable for the HTM fastpath that uses serial mode as fallback.
 344 // Non-zero if the HTM fastpath is enabled. See gtm_thread::begin_transaction.
 345 // Accessed from assembly language, thus the "asm" specifier on
 346 // the name, avoiding complex name mangling.
 347 extern uint32_t htm_fastpath __asm__(UPFX "gtm_htm_fastpath");
 348
 349 } // namespace GTM
 350
 351 #endif // LIBITM_I_H