src/gallium/auxiliary/util/u_atomic.h

   1 /**
   2  * Many similar implementations exist. See for example libwsbm
   3  * or the linux kernel include/atomic.h
   4  *
   5  * No copyright claimed on this file.
   6  *
   7  */
   8
   9 #ifndef U_ATOMIC_H
  10 #define U_ATOMIC_H
  11
  12 #include "pipe/p_compiler.h"
  13 #include "pipe/p_defines.h"
  14
  15 /* Favor OS-provided implementations.
  16  *
  17  * Where no OS-provided implementation is available, fall back to
  18  * locally coded assembly, compiler intrinsic or ultimately a
  19  * mutex-based implementation.
  20  */
  21 #if defined(PIPE_OS_SOLARIS)
  22 #define PIPE_ATOMIC_OS_SOLARIS
  23 #elif defined(PIPE_CC_MSVC)
  24 #define PIPE_ATOMIC_MSVC_INTRINSIC
  25 #elif (defined(PIPE_CC_MSVC) && defined(PIPE_ARCH_X86))
  26 #define PIPE_ATOMIC_ASM_MSVC_X86
  27 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86))
  28 #define PIPE_ATOMIC_ASM_GCC_X86
  29 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86_64))
  30 #define PIPE_ATOMIC_ASM_GCC_X86_64
  31 #elif defined(PIPE_CC_GCC) && (PIPE_CC_GCC_VERSION >= 401)
  32 #define PIPE_ATOMIC_GCC_INTRINSIC
  33 #else
  34 #error "Unsupported platform"
  35 #endif
  36
  37
  38 #if defined(PIPE_ATOMIC_ASM_GCC_X86_64)
  39 #define PIPE_ATOMIC "GCC x86_64 assembly"
  40
  41 #ifdef __cplusplus
  42 extern "C" {
  43 #endif
  44
  45 #define p_atomic_set(_v, _i) (*(_v) = (_i))
  46 #define p_atomic_read(_v) (*(_v))
  47
  48 static INLINE boolean
  49 p_atomic_dec_zero(int32_t *v)
  50 {
  51    unsigned char c;
  52
  53    __asm__ __volatile__("lock; decl %0; sete %1":"+m"(*v), "=qm"(c)
  54                         ::"memory");
  55
  56    return c != 0;
  57 }
  58
  59 static INLINE void
  60 p_atomic_inc(int32_t *v)
  61 {
  62    __asm__ __volatile__("lock; incl %0":"+m"(*v));
  63 }
  64
  65 static INLINE void
  66 p_atomic_dec(int32_t *v)
  67 {
  68    __asm__ __volatile__("lock; decl %0":"+m"(*v));
  69 }
  70
  71 static INLINE int32_t
  72 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
  73 {
  74    return __sync_val_compare_and_swap(v, old, _new);
  75 }
  76
  77 #ifdef __cplusplus
  78 }
  79 #endif
  80
  81 #endif /* PIPE_ATOMIC_ASM_GCC_X86_64 */
  82
  83
  84 #if defined(PIPE_ATOMIC_ASM_GCC_X86)
  85
  86 #define PIPE_ATOMIC "GCC x86 assembly"
  87
  88 #ifdef __cplusplus
  89 extern "C" {
  90 #endif
  91
  92 #define p_atomic_set(_v, _i) (*(_v) = (_i))
  93 #define p_atomic_read(_v) (*(_v))
  94
  95 static INLINE boolean
  96 p_atomic_dec_zero(int32_t *v)
  97 {
  98    unsigned char c;
  99
 100    __asm__ __volatile__("lock; decl %0; sete %1":"+m"(*v), "=qm"(c)
 101                         ::"memory");
 102
 103    return c != 0;
 104 }
 105
 106 static INLINE void
 107 p_atomic_inc(int32_t *v)
 108 {
 109    __asm__ __volatile__("lock; incl %0":"+m"(*v));
 110 }
 111
 112 static INLINE void
 113 p_atomic_dec(int32_t *v)
 114 {
 115    __asm__ __volatile__("lock; decl %0":"+m"(*v));
 116 }
 117
 118 static INLINE int32_t
 119 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 120 {
 121    return __sync_val_compare_and_swap(v, old, _new);
 122 }
 123
 124 #ifdef __cplusplus
 125 }
 126 #endif
 127
 128 #endif
 129
 130
 131
 132 /* Implementation using GCC-provided synchronization intrinsics
 133  */
 134 #if defined(PIPE_ATOMIC_GCC_INTRINSIC)
 135
 136 #define PIPE_ATOMIC "GCC Sync Intrinsics"
 137
 138 #ifdef __cplusplus
 139 extern "C" {
 140 #endif
 141
 142 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 143 #define p_atomic_read(_v) (*(_v))
 144
 145 static INLINE boolean
 146 p_atomic_dec_zero(int32_t *v)
 147 {
 148    return (__sync_sub_and_fetch(v, 1) == 0);
 149 }
 150
 151 static INLINE void
 152 p_atomic_inc(int32_t *v)
 153 {
 154    (void) __sync_add_and_fetch(v, 1);
 155 }
 156
 157 static INLINE void
 158 p_atomic_dec(int32_t *v)
 159 {
 160    (void) __sync_sub_and_fetch(v, 1);
 161 }
 162
 163 static INLINE int32_t
 164 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 165 {
 166    return __sync_val_compare_and_swap(v, old, _new);
 167 }
 168
 169 #ifdef __cplusplus
 170 }
 171 #endif
 172
 173 #endif
 174
 175
 176
 177 /* Unlocked version for single threaded environments, such as some
 178  * windows kernel modules.
 179  */
 180 #if defined(PIPE_ATOMIC_OS_UNLOCKED)
 181
 182 #define PIPE_ATOMIC "Unlocked"
 183
 184 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 185 #define p_atomic_read(_v) (*(_v))
 186 #define p_atomic_dec_zero(_v) ((boolean) --(*(_v)))
 187 #define p_atomic_inc(_v) ((void) (*(_v))++)
 188 #define p_atomic_dec(_v) ((void) (*(_v))--)
 189 #define p_atomic_cmpxchg(_v, old, _new) (*(_v) == old ? *(_v) = (_new) : *(_v))
 190
 191 #endif
 192
 193
 194 /* Locally coded assembly for MSVC on x86:
 195  */
 196 #if defined(PIPE_ATOMIC_ASM_MSVC_X86)
 197
 198 #define PIPE_ATOMIC "MSVC x86 assembly"
 199
 200 #ifdef __cplusplus
 201 extern "C" {
 202 #endif
 203
 204 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 205 #define p_atomic_read(_v) (*(_v))
 206
 207 static INLINE boolean
 208 p_atomic_dec_zero(int32_t *v)
 209 {
 210    unsigned char c;
 211
 212    __asm {
 213       mov       eax, [v]
 214       lock dec  dword ptr [eax]
 215       sete      byte ptr [c]
 216    }
 217
 218    return c != 0;
 219 }
 220
 221 static INLINE void
 222 p_atomic_inc(int32_t *v)
 223 {
 224    __asm {
 225       mov       eax, [v]
 226       lock inc  dword ptr [eax]
 227    }
 228 }
 229
 230 static INLINE void
 231 p_atomic_dec(int32_t *v)
 232 {
 233    __asm {
 234       mov       eax, [v]
 235       lock dec  dword ptr [eax]
 236    }
 237 }
 238
 239 static INLINE int32_t
 240 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 241 {
 242    int32_t orig;
 243
 244    __asm {
 245       mov ecx, [v]
 246       mov eax, [old]
 247       mov edx, [_new]
 248       lock cmpxchg [ecx], edx
 249       mov [orig], eax
 250    }
 251
 252    return orig;
 253 }
 254
 255 #ifdef __cplusplus
 256 }
 257 #endif
 258
 259 #endif
 260
 261
 262 #if defined(PIPE_ATOMIC_MSVC_INTRINSIC)
 263
 264 #define PIPE_ATOMIC "MSVC Intrinsics"
 265
 266 #include <intrin.h>
 267
 268 #pragma intrinsic(_InterlockedIncrement)
 269 #pragma intrinsic(_InterlockedDecrement)
 270 #pragma intrinsic(_InterlockedCompareExchange)
 271
 272 #ifdef __cplusplus
 273 extern "C" {
 274 #endif
 275
 276 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 277 #define p_atomic_read(_v) (*(_v))
 278
 279 static INLINE boolean
 280 p_atomic_dec_zero(int32_t *v)
 281 {
 282    return _InterlockedDecrement((long *)v) == 0;
 283 }
 284
 285 static INLINE void
 286 p_atomic_inc(int32_t *v)
 287 {
 288    _InterlockedIncrement((long *)v);
 289 }
 290
 291 static INLINE void
 292 p_atomic_dec(int32_t *v)
 293 {
 294    _InterlockedDecrement((long *)v);
 295 }
 296
 297 static INLINE int32_t
 298 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 299 {
 300    return _InterlockedCompareExchange((long *)v, _new, old);
 301 }
 302
 303 #ifdef __cplusplus
 304 }
 305 #endif
 306
 307 #endif
 308
 309 #if defined(PIPE_ATOMIC_OS_SOLARIS)
 310
 311 #define PIPE_ATOMIC "Solaris OS atomic functions"
 312
 313 #include <atomic.h>
 314
 315 #ifdef __cplusplus
 316 extern "C" {
 317 #endif
 318
 319 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 320 #define p_atomic_read(_v) (*(_v))
 321
 322 static INLINE boolean
 323 p_atomic_dec_zero(int32_t *v)
 324 {
 325    uint32_t n = atomic_dec_32_nv((uint32_t *) v);
 326
 327    return n != 0;
 328 }
 329
 330 #define p_atomic_inc(_v) atomic_inc_32((uint32_t *) _v)
 331 #define p_atomic_dec(_v) atomic_dec_32((uint32_t *) _v)
 332
 333 #define p_atomic_cmpxchg(_v, _old, _new) \
 334         atomic_cas_32( (uint32_t *) _v, (uint32_t) _old, (uint32_t) _new)
 335
 336 #ifdef __cplusplus
 337 }
 338 #endif
 339
 340 #endif
 341
 342
 343 #ifndef PIPE_ATOMIC
 344 #error "No pipe_atomic implementation selected"
 345 #endif
 346
 347
 348
 349 #endif /* U_ATOMIC_H */