src/util/u_atomic.h

   1 /**
   2  * Many similar implementations exist. See for example libwsbm
   3  * or the linux kernel include/atomic.h
   4  *
   5  * No copyright claimed on this file.
   6  *
   7  */
   8
   9 #ifndef U_ATOMIC_H
  10 #define U_ATOMIC_H
  11
  12 #include "pipe/p_compiler.h"
  13 #include "pipe/p_defines.h"
  14
  15 /* Favor OS-provided implementations.
  16  *
  17  * Where no OS-provided implementation is available, fall back to
  18  * locally coded assembly, compiler intrinsic or ultimately a
  19  * mutex-based implementation.
  20  */
  21 #if defined(PIPE_OS_SOLARIS)
  22 #define PIPE_ATOMIC_OS_SOLARIS
  23 #elif defined(PIPE_CC_MSVC)
  24 #define PIPE_ATOMIC_MSVC_INTRINSIC
  25 #elif (defined(PIPE_CC_MSVC) && defined(PIPE_ARCH_X86))
  26 #define PIPE_ATOMIC_ASM_MSVC_X86
  27 #elif defined(PIPE_CC_GCC) && (PIPE_CC_GCC_VERSION >= 401)
  28 #define PIPE_ATOMIC_GCC_INTRINSIC
  29 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86))
  30 #define PIPE_ATOMIC_ASM_GCC_X86
  31 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86_64))
  32 #define PIPE_ATOMIC_ASM_GCC_X86_64
  33 #else
  34 #error "Unsupported platform"
  35 #endif
  36
  37
  38 #if defined(PIPE_ATOMIC_ASM_GCC_X86_64)
  39 #define PIPE_ATOMIC "GCC x86_64 assembly"
  40
  41 #ifdef __cplusplus
  42 extern "C" {
  43 #endif
  44
  45 #define p_atomic_set(_v, _i) (*(_v) = (_i))
  46 #define p_atomic_read(_v) (*(_v))
  47
  48 static INLINE boolean
  49 p_atomic_dec_zero(int32_t *v)
  50 {
  51    unsigned char c;
  52
  53    __asm__ __volatile__("lock; decl %0; sete %1":"+m"(*v), "=qm"(c)
  54                         ::"memory");
  55
  56    return c != 0;
  57 }
  58
  59 static INLINE void
  60 p_atomic_inc(int32_t *v)
  61 {
  62    __asm__ __volatile__("lock; incl %0":"+m"(*v));
  63 }
  64
  65 static INLINE void
  66 p_atomic_dec(int32_t *v)
  67 {
  68    __asm__ __volatile__("lock; decl %0":"+m"(*v));
  69 }
  70
  71 static INLINE int32_t
  72 p_atomic_inc_return(int32_t *v)
  73 {
  74    return __sync_add_and_fetch(v, 1);
  75 }
  76
  77 static INLINE int32_t
  78 p_atomic_dec_return(int32_t *v)
  79 {
  80    return __sync_sub_and_fetch(v, 1);
  81 }
  82
  83 static INLINE int32_t
  84 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
  85 {
  86    return __sync_val_compare_and_swap(v, old, _new);
  87 }
  88
  89 #ifdef __cplusplus
  90 }
  91 #endif
  92
  93 #endif /* PIPE_ATOMIC_ASM_GCC_X86_64 */
  94
  95
  96 #if defined(PIPE_ATOMIC_ASM_GCC_X86)
  97
  98 #define PIPE_ATOMIC "GCC x86 assembly"
  99
 100 #ifdef __cplusplus
 101 extern "C" {
 102 #endif
 103
 104 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 105 #define p_atomic_read(_v) (*(_v))
 106
 107 static INLINE boolean
 108 p_atomic_dec_zero(int32_t *v)
 109 {
 110    unsigned char c;
 111
 112    __asm__ __volatile__("lock; decl %0; sete %1":"+m"(*v), "=qm"(c)
 113                         ::"memory");
 114
 115    return c != 0;
 116 }
 117
 118 static INLINE void
 119 p_atomic_inc(int32_t *v)
 120 {
 121    __asm__ __volatile__("lock; incl %0":"+m"(*v));
 122 }
 123
 124 static INLINE void
 125 p_atomic_dec(int32_t *v)
 126 {
 127    __asm__ __volatile__("lock; decl %0":"+m"(*v));
 128 }
 129
 130 static INLINE int32_t
 131 p_atomic_inc_return(int32_t *v)
 132 {
 133    return __sync_add_and_fetch(v, 1);
 134 }
 135
 136 static INLINE int32_t
 137 p_atomic_dec_return(int32_t *v)
 138 {
 139    return __sync_sub_and_fetch(v, 1);
 140 }
 141
 142 static INLINE int32_t
 143 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 144 {
 145    return __sync_val_compare_and_swap(v, old, _new);
 146 }
 147
 148 #ifdef __cplusplus
 149 }
 150 #endif
 151
 152 #endif
 153
 154
 155
 156 /* Implementation using GCC-provided synchronization intrinsics
 157  */
 158 #if defined(PIPE_ATOMIC_GCC_INTRINSIC)
 159
 160 #define PIPE_ATOMIC "GCC Sync Intrinsics"
 161
 162 #ifdef __cplusplus
 163 extern "C" {
 164 #endif
 165
 166 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 167 #define p_atomic_read(_v) (*(_v))
 168
 169 static INLINE boolean
 170 p_atomic_dec_zero(int32_t *v)
 171 {
 172    return (__sync_sub_and_fetch(v, 1) == 0);
 173 }
 174
 175 static INLINE void
 176 p_atomic_inc(int32_t *v)
 177 {
 178    (void) __sync_add_and_fetch(v, 1);
 179 }
 180
 181 static INLINE void
 182 p_atomic_dec(int32_t *v)
 183 {
 184    (void) __sync_sub_and_fetch(v, 1);
 185 }
 186
 187 static INLINE int32_t
 188 p_atomic_inc_return(int32_t *v)
 189 {
 190    return __sync_add_and_fetch(v, 1);
 191 }
 192
 193 static INLINE int32_t
 194 p_atomic_dec_return(int32_t *v)
 195 {
 196    return __sync_sub_and_fetch(v, 1);
 197 }
 198
 199 static INLINE int32_t
 200 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 201 {
 202    return __sync_val_compare_and_swap(v, old, _new);
 203 }
 204
 205 #ifdef __cplusplus
 206 }
 207 #endif
 208
 209 #endif
 210
 211
 212
 213 /* Unlocked version for single threaded environments, such as some
 214  * windows kernel modules.
 215  */
 216 #if defined(PIPE_ATOMIC_OS_UNLOCKED)
 217
 218 #define PIPE_ATOMIC "Unlocked"
 219
 220 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 221 #define p_atomic_read(_v) (*(_v))
 222 #define p_atomic_dec_zero(_v) ((boolean) --(*(_v)))
 223 #define p_atomic_inc(_v) ((void) (*(_v))++)
 224 #define p_atomic_dec(_v) ((void) (*(_v))--)
 225 #define p_atomic_inc_return(_v) ((*(_v))++)
 226 #define p_atomic_dec_return(_v) ((*(_v))--)
 227 #define p_atomic_cmpxchg(_v, old, _new) (*(_v) == old ? *(_v) = (_new) : *(_v))
 228
 229 #endif
 230
 231
 232 /* Locally coded assembly for MSVC on x86:
 233  */
 234 #if defined(PIPE_ATOMIC_ASM_MSVC_X86)
 235
 236 #define PIPE_ATOMIC "MSVC x86 assembly"
 237
 238 #ifdef __cplusplus
 239 extern "C" {
 240 #endif
 241
 242 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 243 #define p_atomic_read(_v) (*(_v))
 244
 245 static INLINE boolean
 246 p_atomic_dec_zero(int32_t *v)
 247 {
 248    unsigned char c;
 249
 250    __asm {
 251       mov       eax, [v]
 252       lock dec  dword ptr [eax]
 253       sete      byte ptr [c]
 254    }
 255
 256    return c != 0;
 257 }
 258
 259 static INLINE void
 260 p_atomic_inc(int32_t *v)
 261 {
 262    __asm {
 263       mov       eax, [v]
 264       lock inc  dword ptr [eax]
 265    }
 266 }
 267
 268 static INLINE void
 269 p_atomic_dec(int32_t *v)
 270 {
 271    __asm {
 272       mov       eax, [v]
 273       lock dec  dword ptr [eax]
 274    }
 275 }
 276
 277 static INLINE int32_t
 278 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 279 {
 280    int32_t orig;
 281
 282    __asm {
 283       mov ecx, [v]
 284       mov eax, [old]
 285       mov edx, [_new]
 286       lock cmpxchg [ecx], edx
 287       mov [orig], eax
 288    }
 289
 290    return orig;
 291 }
 292
 293 #ifdef __cplusplus
 294 }
 295 #endif
 296
 297 #endif
 298
 299
 300 #if defined(PIPE_ATOMIC_MSVC_INTRINSIC)
 301
 302 #define PIPE_ATOMIC "MSVC Intrinsics"
 303
 304 #include <intrin.h>
 305
 306 #pragma intrinsic(_InterlockedIncrement)
 307 #pragma intrinsic(_InterlockedDecrement)
 308 #pragma intrinsic(_InterlockedCompareExchange)
 309
 310 #ifdef __cplusplus
 311 extern "C" {
 312 #endif
 313
 314 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 315 #define p_atomic_read(_v) (*(_v))
 316
 317 static INLINE boolean
 318 p_atomic_dec_zero(int32_t *v)
 319 {
 320    return _InterlockedDecrement((long *)v) == 0;
 321 }
 322
 323 static INLINE void
 324 p_atomic_inc(int32_t *v)
 325 {
 326    _InterlockedIncrement((long *)v);
 327 }
 328
 329 static INLINE int32_t
 330 p_atomic_inc_return(int32_t *v)
 331 {
 332    return _InterlockedIncrement((long *)v);
 333 }
 334
 335 static INLINE void
 336 p_atomic_dec(int32_t *v)
 337 {
 338    _InterlockedDecrement((long *)v);
 339 }
 340
 341 static INLINE int32_t
 342 p_atomic_dec_return(int32_t *v)
 343 {
 344    return _InterlockedDecrement((long *)v);
 345 }
 346
 347 static INLINE int32_t
 348 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 349 {
 350    return _InterlockedCompareExchange((long *)v, _new, old);
 351 }
 352
 353 #ifdef __cplusplus
 354 }
 355 #endif
 356
 357 #endif
 358
 359 #if defined(PIPE_ATOMIC_OS_SOLARIS)
 360
 361 #define PIPE_ATOMIC "Solaris OS atomic functions"
 362
 363 #include <atomic.h>
 364
 365 #ifdef __cplusplus
 366 extern "C" {
 367 #endif
 368
 369 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 370 #define p_atomic_read(_v) (*(_v))
 371
 372 static INLINE boolean
 373 p_atomic_dec_zero(int32_t *v)
 374 {
 375    uint32_t n = atomic_dec_32_nv((uint32_t *) v);
 376
 377    return n != 0;
 378 }
 379
 380 #define p_atomic_inc(_v) atomic_inc_32((uint32_t *) _v)
 381 #define p_atomic_dec(_v) atomic_dec_32((uint32_t *) _v)
 382 #define p_atomic_inc_return(_v) atomic_inc_32_nv((uint32_t *) _v)
 383 #define p_atomic_dec_return(_v) atomic_dec_32_nv((uint32_t *) _v)
 384
 385 #define p_atomic_cmpxchg(_v, _old, _new) \
 386         atomic_cas_32( (uint32_t *) _v, (uint32_t) _old, (uint32_t) _new)
 387
 388 #ifdef __cplusplus
 389 }
 390 #endif
 391
 392 #endif
 393
 394
 395 #ifndef PIPE_ATOMIC
 396 #error "No pipe_atomic implementation selected"
 397 #endif
 398
 399
 400
 401 #endif /* U_ATOMIC_H */