src/gallium/auxiliary/util/u_atomic.h

   1 /**
   2  * Many similar implementations exist. See for example libwsbm
   3  * or the linux kernel include/atomic.h
   4  *
   5  * No copyright claimed on this file.
   6  *
   7  */
   8
   9 #ifndef U_ATOMIC_H
  10 #define U_ATOMIC_H
  11
  12 #include "pipe/p_compiler.h"
  13 #include "pipe/p_defines.h"
  14
  15 /* Favor OS-provided implementations.
  16  *
  17  * Where no OS-provided implementation is available, fall back to
  18  * locally coded assembly, compiler intrinsic or ultimately a
  19  * mutex-based implementation.
  20  */
  21 #if (defined(PIPE_SUBSYSTEM_WINDOWS_DISPLAY) || \
  22      defined(PIPE_SUBSYSTEM_WINDOWS_MINIPORT))
  23 #define PIPE_ATOMIC_OS_UNLOCKED
  24 #elif defined(PIPE_OS_SOLARIS)
  25 #define PIPE_ATOMIC_OS_SOLARIS
  26 #elif defined(PIPE_CC_MSVC)
  27 #define PIPE_ATOMIC_MSVC_INTRINSIC
  28 #elif (defined(PIPE_CC_MSVC) && defined(PIPE_ARCH_X86))
  29 #define PIPE_ATOMIC_ASM_MSVC_X86
  30 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86))
  31 #define PIPE_ATOMIC_ASM_GCC_X86
  32 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86_64))
  33 #define PIPE_ATOMIC_ASM_GCC_X86_64
  34 #elif defined(PIPE_CC_GCC) && (PIPE_CC_GCC_VERSION >= 401)
  35 #define PIPE_ATOMIC_GCC_INTRINSIC
  36 #else
  37 #error "Unsupported platform"
  38 #endif
  39
  40
  41 #if defined(PIPE_ATOMIC_ASM_GCC_X86_64)
  42 #define PIPE_ATOMIC "GCC x86_64 assembly"
  43
  44 #ifdef __cplusplus
  45 extern "C" {
  46 #endif
  47
  48 #define p_atomic_set(_v, _i) (*(_v) = (_i))
  49 #define p_atomic_read(_v) (*(_v))
  50
  51 static INLINE boolean
  52 p_atomic_dec_zero(int32_t *v)
  53 {
  54    unsigned char c;
  55
  56    __asm__ __volatile__("lock; decl %0; sete %1":"+m"(*v), "=qm"(c)
  57                         ::"memory");
  58
  59    return c != 0;
  60 }
  61
  62 static INLINE void
  63 p_atomic_inc(int32_t *v)
  64 {
  65    __asm__ __volatile__("lock; incl %0":"+m"(*v));
  66 }
  67
  68 static INLINE void
  69 p_atomic_dec(int32_t *v)
  70 {
  71    __asm__ __volatile__("lock; decl %0":"+m"(*v));
  72 }
  73
  74 static INLINE int32_t
  75 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
  76 {
  77    return __sync_val_compare_and_swap(v, old, _new);
  78 }
  79
  80 #ifdef __cplusplus
  81 }
  82 #endif
  83
  84 #endif /* PIPE_ATOMIC_ASM_GCC_X86_64 */
  85
  86
  87 #if defined(PIPE_ATOMIC_ASM_GCC_X86)
  88
  89 #define PIPE_ATOMIC "GCC x86 assembly"
  90
  91 #ifdef __cplusplus
  92 extern "C" {
  93 #endif
  94
  95 #define p_atomic_set(_v, _i) (*(_v) = (_i))
  96 #define p_atomic_read(_v) (*(_v))
  97
  98 static INLINE boolean
  99 p_atomic_dec_zero(int32_t *v)
 100 {
 101    unsigned char c;
 102
 103    __asm__ __volatile__("lock; decl %0; sete %1":"+m"(*v), "=qm"(c)
 104                         ::"memory");
 105
 106    return c != 0;
 107 }
 108
 109 static INLINE void
 110 p_atomic_inc(int32_t *v)
 111 {
 112    __asm__ __volatile__("lock; incl %0":"+m"(*v));
 113 }
 114
 115 static INLINE void
 116 p_atomic_dec(int32_t *v)
 117 {
 118    __asm__ __volatile__("lock; decl %0":"+m"(*v));
 119 }
 120
 121 static INLINE int32_t
 122 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 123 {
 124    return __sync_val_compare_and_swap(v, old, _new);
 125 }
 126
 127 #ifdef __cplusplus
 128 }
 129 #endif
 130
 131 #endif
 132
 133
 134
 135 /* Implementation using GCC-provided synchronization intrinsics
 136  */
 137 #if defined(PIPE_ATOMIC_GCC_INTRINSIC)
 138
 139 #define PIPE_ATOMIC "GCC Sync Intrinsics"
 140
 141 #ifdef __cplusplus
 142 extern "C" {
 143 #endif
 144
 145 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 146 #define p_atomic_read(_v) (*(_v))
 147
 148 static INLINE boolean
 149 p_atomic_dec_zero(int32_t *v)
 150 {
 151    return (__sync_sub_and_fetch(v, 1) == 0);
 152 }
 153
 154 static INLINE void
 155 p_atomic_inc(int32_t *v)
 156 {
 157    (void) __sync_add_and_fetch(v, 1);
 158 }
 159
 160 static INLINE void
 161 p_atomic_dec(int32_t *v)
 162 {
 163    (void) __sync_sub_and_fetch(v, 1);
 164 }
 165
 166 static INLINE int32_t
 167 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 168 {
 169    return __sync_val_compare_and_swap(v, old, _new);
 170 }
 171
 172 #ifdef __cplusplus
 173 }
 174 #endif
 175
 176 #endif
 177
 178
 179
 180 /* Unlocked version for single threaded environments, such as some
 181  * windows kernel modules.
 182  */
 183 #if defined(PIPE_ATOMIC_OS_UNLOCKED)
 184
 185 #define PIPE_ATOMIC "Unlocked"
 186
 187 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 188 #define p_atomic_read(_v) (*(_v))
 189 #define p_atomic_dec_zero(_v) ((boolean) --(*(_v)))
 190 #define p_atomic_inc(_v) ((void) (*(_v))++)
 191 #define p_atomic_dec(_v) ((void) (*(_v))--)
 192 #define p_atomic_cmpxchg(_v, old, _new) (*(_v) == old ? *(_v) = (_new) : *(_v))
 193
 194 #endif
 195
 196
 197 /* Locally coded assembly for MSVC on x86:
 198  */
 199 #if defined(PIPE_ATOMIC_ASM_MSVC_X86)
 200
 201 #define PIPE_ATOMIC "MSVC x86 assembly"
 202
 203 #ifdef __cplusplus
 204 extern "C" {
 205 #endif
 206
 207 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 208 #define p_atomic_read(_v) (*(_v))
 209
 210 static INLINE boolean
 211 p_atomic_dec_zero(int32_t *v)
 212 {
 213    unsigned char c;
 214
 215    __asm {
 216       mov       eax, [v]
 217       lock dec  dword ptr [eax]
 218       sete      byte ptr [c]
 219    }
 220
 221    return c != 0;
 222 }
 223
 224 static INLINE void
 225 p_atomic_inc(int32_t *v)
 226 {
 227    __asm {
 228       mov       eax, [v]
 229       lock inc  dword ptr [eax]
 230    }
 231 }
 232
 233 static INLINE void
 234 p_atomic_dec(int32_t *v)
 235 {
 236    __asm {
 237       mov       eax, [v]
 238       lock dec  dword ptr [eax]
 239    }
 240 }
 241
 242 static INLINE int32_t
 243 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 244 {
 245    int32_t orig;
 246
 247    __asm {
 248       mov ecx, [v]
 249       mov eax, [old]
 250       mov edx, [_new]
 251       lock cmpxchg [ecx], edx
 252       mov [orig], eax
 253    }
 254
 255    return orig;
 256 }
 257
 258 #ifdef __cplusplus
 259 }
 260 #endif
 261
 262 #endif
 263
 264
 265 #if defined(PIPE_ATOMIC_MSVC_INTRINSIC)
 266
 267 #define PIPE_ATOMIC "MSVC Intrinsics"
 268
 269 #include <intrin.h>
 270
 271 #pragma intrinsic(_InterlockedIncrement)
 272 #pragma intrinsic(_InterlockedDecrement)
 273 #pragma intrinsic(_InterlockedCompareExchange)
 274
 275 #ifdef __cplusplus
 276 extern "C" {
 277 #endif
 278
 279 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 280 #define p_atomic_read(_v) (*(_v))
 281
 282 static INLINE boolean
 283 p_atomic_dec_zero(int32_t *v)
 284 {
 285    return _InterlockedDecrement((long *)v) == 0;
 286 }
 287
 288 static INLINE void
 289 p_atomic_inc(int32_t *v)
 290 {
 291    _InterlockedIncrement((long *)v);
 292 }
 293
 294 static INLINE void
 295 p_atomic_dec(int32_t *v)
 296 {
 297    _InterlockedDecrement((long *)v);
 298 }
 299
 300 static INLINE int32_t
 301 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 302 {
 303    return _InterlockedCompareExchange((long *)v, _new, old);
 304 }
 305
 306 #ifdef __cplusplus
 307 }
 308 #endif
 309
 310 #endif
 311
 312 #if defined(PIPE_ATOMIC_OS_SOLARIS)
 313
 314 #define PIPE_ATOMIC "Solaris OS atomic functions"
 315
 316 #include <atomic.h>
 317
 318 #ifdef __cplusplus
 319 extern "C" {
 320 #endif
 321
 322 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 323 #define p_atomic_read(_v) (*(_v))
 324
 325 static INLINE boolean
 326 p_atomic_dec_zero(int32_t *v)
 327 {
 328    uint32_t n = atomic_dec_32_nv((uint32_t *) v);
 329
 330    return n != 0;
 331 }
 332
 333 #define p_atomic_inc(_v) atomic_inc_32((uint32_t *) _v)
 334 #define p_atomic_dec(_v) atomic_dec_32((uint32_t *) _v)
 335
 336 #define p_atomic_cmpxchg(_v, _old, _new) \
 337         atomic_cas_32( (uint32_t *) _v, (uint32_t) _old, (uint32_t) _new)
 338
 339 #ifdef __cplusplus
 340 }
 341 #endif
 342
 343 #endif
 344
 345
 346 #ifndef PIPE_ATOMIC
 347 #error "No pipe_atomic implementation selected"
 348 #endif
 349
 350
 351
 352 #endif /* U_ATOMIC_H */