src/gallium/auxiliary/util/u_atomic.h

   1 /**
   2  * Many similar implementations exist. See for example libwsbm
   3  * or the linux kernel include/atomic.h
   4  *
   5  * No copyright claimed on this file.
   6  *
   7  */
   8
   9 #ifndef U_ATOMIC_H
  10 #define U_ATOMIC_H
  11
  12 #include "pipe/p_compiler.h"
  13 #include "pipe/p_defines.h"
  14
  15 /* Favor OS-provided implementations.
  16  *
  17  * Where no OS-provided implementation is available, fall back to
  18  * locally coded assembly, compiler intrinsic or ultimately a
  19  * mutex-based implementation.
  20  */
  21 #if (defined(PIPE_SUBSYSTEM_WINDOWS_DISPLAY) || \
  22      defined(PIPE_SUBSYSTEM_WINDOWS_MINIPORT))
  23 #define PIPE_ATOMIC_OS_UNLOCKED
  24 #elif defined(PIPE_OS_SOLARIS)
  25 #define PIPE_ATOMIC_OS_SOLARIS
  26 #elif defined(PIPE_CC_MSVC)
  27 #define PIPE_ATOMIC_MSVC_INTRINSIC
  28 #elif (defined(PIPE_CC_MSVC) && defined(PIPE_ARCH_X86))
  29 #define PIPE_ATOMIC_ASM_MSVC_X86
  30 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86))
  31 #define PIPE_ATOMIC_ASM_GCC_X86
  32 #elif defined(PIPE_CC_GCC) && (PIPE_CC_GCC_VERSION >= 401)
  33 #define PIPE_ATOMIC_GCC_INTRINSIC
  34 #else
  35 #error "Unsupported platform"
  36 #endif
  37
  38
  39
  40 #if defined(PIPE_ATOMIC_ASM_GCC_X86)
  41
  42 #define PIPE_ATOMIC "GCC x86 assembly"
  43
  44 #ifdef __cplusplus
  45 extern "C" {
  46 #endif
  47
  48 #define p_atomic_set(_v, _i) (*(_v) = (_i))
  49 #define p_atomic_read(_v) (*(_v))
  50
  51 static INLINE boolean
  52 p_atomic_dec_zero(int32_t *v)
  53 {
  54    unsigned char c;
  55
  56    __asm__ __volatile__("lock; decl %0; sete %1":"+m"(*v), "=qm"(c)
  57                         ::"memory");
  58
  59    return c != 0;
  60 }
  61
  62 static INLINE void
  63 p_atomic_inc(int32_t *v)
  64 {
  65    __asm__ __volatile__("lock; incl %0":"+m"(*v));
  66 }
  67
  68 static INLINE void
  69 p_atomic_dec(int32_t *v)
  70 {
  71    __asm__ __volatile__("lock; decl %0":"+m"(*v));
  72 }
  73
  74 static INLINE int32_t
  75 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
  76 {
  77    return __sync_val_compare_and_swap(v, old, _new);
  78 }
  79
  80 #ifdef __cplusplus
  81 }
  82 #endif
  83
  84 #endif
  85
  86
  87
  88 /* Implementation using GCC-provided synchronization intrinsics
  89  */
  90 #if defined(PIPE_ATOMIC_GCC_INTRINSIC)
  91
  92 #define PIPE_ATOMIC "GCC Sync Intrinsics"
  93
  94 #ifdef __cplusplus
  95 extern "C" {
  96 #endif
  97
  98 #define p_atomic_set(_v, _i) (*(_v) = (_i))
  99 #define p_atomic_read(_v) (*(_v))
 100
 101 static INLINE boolean
 102 p_atomic_dec_zero(int32_t *v)
 103 {
 104    return (__sync_sub_and_fetch(v, 1) == 0);
 105 }
 106
 107 static INLINE void
 108 p_atomic_inc(int32_t *v)
 109 {
 110    (void) __sync_add_and_fetch(v, 1);
 111 }
 112
 113 static INLINE void
 114 p_atomic_dec(int32_t *v)
 115 {
 116    (void) __sync_sub_and_fetch(v, 1);
 117 }
 118
 119 static INLINE int32_t
 120 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 121 {
 122    return __sync_val_compare_and_swap(v, old, _new);
 123 }
 124
 125 #ifdef __cplusplus
 126 }
 127 #endif
 128
 129 #endif
 130
 131
 132
 133 /* Unlocked version for single threaded environments, such as some
 134  * windows kernel modules.
 135  */
 136 #if defined(PIPE_ATOMIC_OS_UNLOCKED)
 137
 138 #define PIPE_ATOMIC "Unlocked"
 139
 140 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 141 #define p_atomic_read(_v) (*(_v))
 142 #define p_atomic_dec_zero(_v) ((boolean) --(*(_v)))
 143 #define p_atomic_inc(_v) ((void) (*(_v))++)
 144 #define p_atomic_dec(_v) ((void) (*(_v))--)
 145 #define p_atomic_cmpxchg(_v, old, _new) (*(_v) == old ? *(_v) = (_new) : *(_v))
 146
 147 #endif
 148
 149
 150 /* Locally coded assembly for MSVC on x86:
 151  */
 152 #if defined(PIPE_ATOMIC_ASM_MSVC_X86)
 153
 154 #define PIPE_ATOMIC "MSVC x86 assembly"
 155
 156 #ifdef __cplusplus
 157 extern "C" {
 158 #endif
 159
 160 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 161 #define p_atomic_read(_v) (*(_v))
 162
 163 static INLINE boolean
 164 p_atomic_dec_zero(int32_t *v)
 165 {
 166    unsigned char c;
 167
 168    __asm {
 169       mov       eax, [v]
 170       lock dec  dword ptr [eax]
 171       sete      byte ptr [c]
 172    }
 173
 174    return c != 0;
 175 }
 176
 177 static INLINE void
 178 p_atomic_inc(int32_t *v)
 179 {
 180    __asm {
 181       mov       eax, [v]
 182       lock inc  dword ptr [eax]
 183    }
 184 }
 185
 186 static INLINE void
 187 p_atomic_dec(int32_t *v)
 188 {
 189    __asm {
 190       mov       eax, [v]
 191       lock dec  dword ptr [eax]
 192    }
 193 }
 194
 195 static INLINE int32_t
 196 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 197 {
 198    int32_t orig;
 199
 200    __asm {
 201       mov ecx, [v]
 202       mov eax, [old]
 203       mov edx, [_new]
 204       lock cmpxchg [ecx], edx
 205       mov [orig], eax
 206    }
 207
 208    return orig;
 209 }
 210
 211 #ifdef __cplusplus
 212 }
 213 #endif
 214
 215 #endif
 216
 217
 218 #if defined(PIPE_ATOMIC_MSVC_INTRINSIC)
 219
 220 #define PIPE_ATOMIC "MSVC Intrinsics"
 221
 222 #include <intrin.h>
 223
 224 #pragma intrinsic(_InterlockedIncrement)
 225 #pragma intrinsic(_InterlockedDecrement)
 226 #pragma intrinsic(_InterlockedCompareExchange)
 227
 228 #ifdef __cplusplus
 229 extern "C" {
 230 #endif
 231
 232 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 233 #define p_atomic_read(_v) (*(_v))
 234
 235 static INLINE boolean
 236 p_atomic_dec_zero(int32_t *v)
 237 {
 238    return _InterlockedDecrement((long *)v) == 0;
 239 }
 240
 241 static INLINE void
 242 p_atomic_inc(int32_t *v)
 243 {
 244    _InterlockedIncrement((long *)v);
 245 }
 246
 247 static INLINE void
 248 p_atomic_dec(int32_t *v)
 249 {
 250    _InterlockedDecrement((long *)v);
 251 }
 252
 253 static INLINE int32_t
 254 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 255 {
 256    return _InterlockedCompareExchange((long *)v, _new, old);
 257 }
 258
 259 #ifdef __cplusplus
 260 }
 261 #endif
 262
 263 #endif
 264
 265 #if defined(PIPE_ATOMIC_OS_SOLARIS)
 266
 267 #define PIPE_ATOMIC "Solaris OS atomic functions"
 268
 269 #include <atomic.h>
 270
 271 #ifdef __cplusplus
 272 extern "C" {
 273 #endif
 274
 275 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 276 #define p_atomic_read(_v) (*(_v))
 277
 278 static INLINE boolean
 279 p_atomic_dec_zero(int32_t *v)
 280 {
 281    uint32_t n = atomic_dec_32_nv((uint32_t *) v);
 282
 283    return n != 0;
 284 }
 285
 286 #define p_atomic_inc(_v) atomic_inc_32((uint32_t *) _v)
 287 #define p_atomic_dec(_v) atomic_dec_32((uint32_t *) _v)
 288
 289 #define p_atomic_cmpxchg(_v, _old, _new) \
 290         atomic_cas_32( (uint32_t *) _v, (uint32_t) _old, (uint32_t) _new)
 291
 292 #ifdef __cplusplus
 293 }
 294 #endif
 295
 296 #endif
 297
 298
 299 #ifndef PIPE_ATOMIC
 300 #error "No pipe_atomic implementation selected"
 301 #endif
 302
 303
 304
 305 #endif /* U_ATOMIC_H */