libitm/barrier.tpl

   1 /* -*- c++ -*- */
   2 /* Copyright (C) 2008, 2009, 2011 Free Software Foundation, Inc.
   3    Contributed by Richard Henderson <rth@redhat.com>.
   4
   5    This file is part of the GNU Transactional Memory Library (libitm).
   6
   7    Libitm is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3 of the License, or
  10    (at your option) any later version.
  11
  12    Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  14    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15    more details.
  16
  17    Under Section 7 of GPL version 3, you are granted additional
  18    permissions described in the GCC Runtime Library Exception, version
  19    3.1, as published by the Free Software Foundation.
  20
  21    You should have received a copy of the GNU General Public License and
  22    a copy of the GCC Runtime Library Exception along with this program;
  23    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  24    <http://www.gnu.org/licenses/>.  */
  25
  26 #include "unaligned.h"
  27
  28 namespace {
  29
  30 using namespace GTM;
  31
  32 template<typename T>
  33 T do_read (const T *ptr, abi_dispatch::lock_type lock)
  34 {
  35   //
  36   // Find the cacheline that holds the current value of *PTR.
  37   //
  38   abi_dispatch *disp = abi_disp();
  39   uintptr_t iptr = reinterpret_cast<uintptr_t>(ptr);
  40   // Normalize PTR by chopping off the bottom bits so we can search
  41   // for PTR in the cacheline hash.
  42   uintptr_t iline = iptr & -CACHELINE_SIZE;
  43   // The position in the resulting cacheline where *PTR is actually stored.
  44   uintptr_t iofs = iptr & (CACHELINE_SIZE - 1);
  45   const gtm_cacheline *pline = reinterpret_cast<const gtm_cacheline *>(iline);
  46   // Search for the actual cacheline that holds the current value of *PTR.
  47   const gtm_cacheline *line = disp->read_lock(pline, lock);
  48
  49   // Point to the position in the cacheline where *PTR is stored.
  50   ptr = reinterpret_cast<const T *>(&line->b[iofs]);
  51
  52   // Straight up loads, because we're either aligned, or we don't care
  53   // about alignment.
  54   //
  55   // If we require alignment on type T, do a straight load if we're
  56   // aligned.  Otherwise do a straight load IFF the load fits entirely
  57   // in this cacheline.  That is, it won't span multiple cachelines.
  58   if (__builtin_expect (strict_alignment<T>::value
  59                         ? (iofs & (sizeof (T) - 1)) == 0
  60                         : iofs + sizeof(T) <= CACHELINE_SIZE, 1))
  61     {
  62     do_normal_load:
  63       return *ptr;
  64     }
  65   // If alignment on T is necessary, but we're unaligned, yet we fit
  66   // entirely in this cacheline... do the unaligned load dance.
  67   else if (__builtin_expect (strict_alignment<T>::value
  68                              && iofs + sizeof(T) <= CACHELINE_SIZE, 1))
  69     {
  70     do_unaligned_load:
  71       return unaligned_load<T>(ptr);
  72     }
  73   // Otherwise, this load will span multiple cachelines.
  74   else
  75     {
  76       // Get the following cacheline for the rest of the data.
  77       const gtm_cacheline *line2 = disp->read_lock(pline + 1, lock);
  78
  79       // If the two cachelines are adjacent, just load it all in one
  80       // swoop.
  81       if (line2 == line + 1)
  82         {
  83           if (!strict_alignment<T>::value)
  84             goto do_normal_load;
  85           else
  86             goto do_unaligned_load;
  87         }
  88       else
  89         {
  90           // Otherwise, ask the backend to load from two different
  91           // cachelines.
  92           return unaligned_load2<T>(line, line2, iofs);
  93         }
  94     }
  95 }
  96
  97 template<typename T>
  98 void do_write (T *ptr, T val, abi_dispatch::lock_type lock)
  99 {
 100   // Note: See comments for do_read() above for hints on this
 101   // function.  Ideally we should abstract out a lot out of these two
 102   // functions, and avoid all this duplication.
 103
 104   abi_dispatch *disp = abi_disp();
 105   uintptr_t iptr = reinterpret_cast<uintptr_t>(ptr);
 106   uintptr_t iline = iptr & -CACHELINE_SIZE;
 107   uintptr_t iofs = iptr & (CACHELINE_SIZE - 1);
 108   gtm_cacheline *pline = reinterpret_cast<gtm_cacheline *>(iline);
 109   gtm_cacheline_mask m = ((gtm_cacheline_mask)2 << (sizeof(T) - 1)) - 1;
 110   abi_dispatch::mask_pair pair = disp->write_lock(pline, lock);
 111
 112   ptr = reinterpret_cast<T *>(&pair.line->b[iofs]);
 113
 114   if (__builtin_expect (strict_alignment<T>::value
 115                         ? (iofs & (sizeof (val) - 1)) == 0
 116                         : iofs + sizeof(val) <= CACHELINE_SIZE, 1))
 117     {
 118       *pair.mask |= m << iofs;
 119     do_normal_store:
 120       *ptr = val;
 121     }
 122   else if (__builtin_expect (strict_alignment<T>::value
 123                              && iofs + sizeof(val) <= CACHELINE_SIZE, 1))
 124     {
 125       *pair.mask |= m << iofs;
 126     do_unaligned_store:
 127       unaligned_store<T>(ptr, val);
 128     }
 129   else
 130     {
 131       *pair.mask |= m << iofs;
 132       abi_dispatch::mask_pair pair2 = disp->write_lock(pline + 1, lock);
 133
 134       uintptr_t ileft = CACHELINE_SIZE - iofs;
 135       *pair2.mask |= m >> ileft;
 136
 137       if (pair2.line == pair.line + 1)
 138         {
 139           if (!strict_alignment<T>::value)
 140             goto do_normal_store;
 141           else
 142             goto do_unaligned_store;
 143         }
 144       else
 145         unaligned_store2<T>(pair.line, pair2.line, iofs, val);
 146     }
 147 }
 148
 149 } /* anonymous namespace */
 150
 151 #define ITM_READ(T, LOCK)                                               \
 152   _ITM_TYPE_##T ITM_REGPARM _ITM_##LOCK##T (const _ITM_TYPE_##T *ptr)   \
 153   {                                                                     \
 154     return do_read (ptr, abi_dispatch::LOCK);                           \
 155   }
 156
 157 #define ITM_WRITE(T, LOCK)                                              \
 158   void ITM_REGPARM _ITM_##LOCK##T (_ITM_TYPE_##T *ptr, _ITM_TYPE_##T val) \
 159   {                                                                     \
 160     do_write (ptr, val, abi_dispatch::LOCK);                            \
 161   }
 162
 163 #define ITM_BARRIERS(T)         \
 164   ITM_READ(T, R)                \
 165   ITM_READ(T, RaR)              \
 166   ITM_READ(T, RaW)              \
 167   ITM_READ(T, RfW)              \
 168   ITM_WRITE(T, W)               \
 169   ITM_WRITE(T, WaR)             \
 170   ITM_WRITE(T, WaW)