src/util/u_atomic.h

   1 /**
   2  * Many similar implementations exist. See for example libwsbm
   3  * or the linux kernel include/atomic.h
   4  *
   5  * No copyright claimed on this file.
   6  *
   7  */
   8
   9 #ifndef U_ATOMIC_H
  10 #define U_ATOMIC_H
  11
  12 /* Favor OS-provided implementations.
  13  *
  14  * Where no OS-provided implementation is available, fall back to
  15  * locally coded assembly, compiler intrinsic or ultimately a
  16  * mutex-based implementation.
  17  */
  18 #if defined(__sun)
  19 #define PIPE_ATOMIC_OS_SOLARIS
  20 #elif defined(_MSC_VER)
  21 #define PIPE_ATOMIC_MSVC_INTRINSIC
  22 #elif (defined(_MSC_VER) && (defined(__i386__) || defined(_M_IX86))
  23 #define PIPE_ATOMIC_ASM_MSVC_X86
  24 #elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 401)
  25 #define PIPE_ATOMIC_GCC_INTRINSIC
  26 #elif (defined(__GNUC__) && defined(__i386__))
  27 #define PIPE_ATOMIC_ASM_GCC_X86
  28 #elif (defined(__GNUC__) && defined(__x86_64__))
  29 #define PIPE_ATOMIC_ASM_GCC_X86_64
  30 #else
  31 #error "Unsupported platform"
  32 #endif
  33
  34
  35 #if defined(PIPE_ATOMIC_ASM_GCC_X86_64)
  36 #define PIPE_ATOMIC "GCC x86_64 assembly"
  37
  38 #ifdef __cplusplus
  39 extern "C" {
  40 #endif
  41
  42 #define p_atomic_set(_v, _i) (*(_v) = (_i))
  43 #define p_atomic_read(_v) (*(_v))
  44
  45 static inline boolean
  46 p_atomic_dec_zero(int32_t *v)
  47 {
  48    unsigned char c;
  49
  50    __asm__ __volatile__("lock; decl %0; sete %1":"+m"(*v), "=qm"(c)
  51                         ::"memory");
  52
  53    return c != 0;
  54 }
  55
  56 static inline void
  57 p_atomic_inc(int32_t *v)
  58 {
  59    __asm__ __volatile__("lock; incl %0":"+m"(*v));
  60 }
  61
  62 static inline void
  63 p_atomic_dec(int32_t *v)
  64 {
  65    __asm__ __volatile__("lock; decl %0":"+m"(*v));
  66 }
  67
  68 static inline int32_t
  69 p_atomic_inc_return(int32_t *v)
  70 {
  71    return __sync_add_and_fetch(v, 1);
  72 }
  73
  74 static inline int32_t
  75 p_atomic_dec_return(int32_t *v)
  76 {
  77    return __sync_sub_and_fetch(v, 1);
  78 }
  79
  80 static inline int32_t
  81 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
  82 {
  83    return __sync_val_compare_and_swap(v, old, _new);
  84 }
  85
  86 #ifdef __cplusplus
  87 }
  88 #endif
  89
  90 #endif /* PIPE_ATOMIC_ASM_GCC_X86_64 */
  91
  92
  93 #if defined(PIPE_ATOMIC_ASM_GCC_X86)
  94
  95 #define PIPE_ATOMIC "GCC x86 assembly"
  96
  97 #ifdef __cplusplus
  98 extern "C" {
  99 #endif
 100
 101 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 102 #define p_atomic_read(_v) (*(_v))
 103
 104 static inline boolean
 105 p_atomic_dec_zero(int32_t *v)
 106 {
 107    unsigned char c;
 108
 109    __asm__ __volatile__("lock; decl %0; sete %1":"+m"(*v), "=qm"(c)
 110                         ::"memory");
 111
 112    return c != 0;
 113 }
 114
 115 static inline void
 116 p_atomic_inc(int32_t *v)
 117 {
 118    __asm__ __volatile__("lock; incl %0":"+m"(*v));
 119 }
 120
 121 static inline void
 122 p_atomic_dec(int32_t *v)
 123 {
 124    __asm__ __volatile__("lock; decl %0":"+m"(*v));
 125 }
 126
 127 static inline int32_t
 128 p_atomic_inc_return(int32_t *v)
 129 {
 130    return __sync_add_and_fetch(v, 1);
 131 }
 132
 133 static inline int32_t
 134 p_atomic_dec_return(int32_t *v)
 135 {
 136    return __sync_sub_and_fetch(v, 1);
 137 }
 138
 139 static inline int32_t
 140 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 141 {
 142    return __sync_val_compare_and_swap(v, old, _new);
 143 }
 144
 145 #ifdef __cplusplus
 146 }
 147 #endif
 148
 149 #endif
 150
 151
 152
 153 /* Implementation using GCC-provided synchronization intrinsics
 154  */
 155 #if defined(PIPE_ATOMIC_GCC_INTRINSIC)
 156
 157 #define PIPE_ATOMIC "GCC Sync Intrinsics"
 158
 159 #ifdef __cplusplus
 160 extern "C" {
 161 #endif
 162
 163 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 164 #define p_atomic_read(_v) (*(_v))
 165
 166 static inline boolean
 167 p_atomic_dec_zero(int32_t *v)
 168 {
 169    return (__sync_sub_and_fetch(v, 1) == 0);
 170 }
 171
 172 static inline void
 173 p_atomic_inc(int32_t *v)
 174 {
 175    (void) __sync_add_and_fetch(v, 1);
 176 }
 177
 178 static inline void
 179 p_atomic_dec(int32_t *v)
 180 {
 181    (void) __sync_sub_and_fetch(v, 1);
 182 }
 183
 184 static inline int32_t
 185 p_atomic_inc_return(int32_t *v)
 186 {
 187    return __sync_add_and_fetch(v, 1);
 188 }
 189
 190 static inline int32_t
 191 p_atomic_dec_return(int32_t *v)
 192 {
 193    return __sync_sub_and_fetch(v, 1);
 194 }
 195
 196 static inline int32_t
 197 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 198 {
 199    return __sync_val_compare_and_swap(v, old, _new);
 200 }
 201
 202 #ifdef __cplusplus
 203 }
 204 #endif
 205
 206 #endif
 207
 208
 209
 210 /* Unlocked version for single threaded environments, such as some
 211  * windows kernel modules.
 212  */
 213 #if defined(PIPE_ATOMIC_OS_UNLOCKED)
 214
 215 #define PIPE_ATOMIC "Unlocked"
 216
 217 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 218 #define p_atomic_read(_v) (*(_v))
 219 #define p_atomic_dec_zero(_v) ((boolean) --(*(_v)))
 220 #define p_atomic_inc(_v) ((void) (*(_v))++)
 221 #define p_atomic_dec(_v) ((void) (*(_v))--)
 222 #define p_atomic_inc_return(_v) ((*(_v))++)
 223 #define p_atomic_dec_return(_v) ((*(_v))--)
 224 #define p_atomic_cmpxchg(_v, old, _new) (*(_v) == old ? *(_v) = (_new) : *(_v))
 225
 226 #endif
 227
 228
 229 /* Locally coded assembly for MSVC on x86:
 230  */
 231 #if defined(PIPE_ATOMIC_ASM_MSVC_X86)
 232
 233 #define PIPE_ATOMIC "MSVC x86 assembly"
 234
 235 #ifdef __cplusplus
 236 extern "C" {
 237 #endif
 238
 239 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 240 #define p_atomic_read(_v) (*(_v))
 241
 242 static inline boolean
 243 p_atomic_dec_zero(int32_t *v)
 244 {
 245    unsigned char c;
 246
 247    __asm {
 248       mov       eax, [v]
 249       lock dec  dword ptr [eax]
 250       sete      byte ptr [c]
 251    }
 252
 253    return c != 0;
 254 }
 255
 256 static inline void
 257 p_atomic_inc(int32_t *v)
 258 {
 259    __asm {
 260       mov       eax, [v]
 261       lock inc  dword ptr [eax]
 262    }
 263 }
 264
 265 static inline void
 266 p_atomic_dec(int32_t *v)
 267 {
 268    __asm {
 269       mov       eax, [v]
 270       lock dec  dword ptr [eax]
 271    }
 272 }
 273
 274 static inline int32_t
 275 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 276 {
 277    int32_t orig;
 278
 279    __asm {
 280       mov ecx, [v]
 281       mov eax, [old]
 282       mov edx, [_new]
 283       lock cmpxchg [ecx], edx
 284       mov [orig], eax
 285    }
 286
 287    return orig;
 288 }
 289
 290 #ifdef __cplusplus
 291 }
 292 #endif
 293
 294 #endif
 295
 296
 297 #if defined(PIPE_ATOMIC_MSVC_INTRINSIC)
 298
 299 #define PIPE_ATOMIC "MSVC Intrinsics"
 300
 301 #include <intrin.h>
 302
 303 #pragma intrinsic(_InterlockedIncrement)
 304 #pragma intrinsic(_InterlockedDecrement)
 305 #pragma intrinsic(_InterlockedCompareExchange)
 306
 307 #ifdef __cplusplus
 308 extern "C" {
 309 #endif
 310
 311 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 312 #define p_atomic_read(_v) (*(_v))
 313
 314 static inline boolean
 315 p_atomic_dec_zero(int32_t *v)
 316 {
 317    return _InterlockedDecrement((long *)v) == 0;
 318 }
 319
 320 static inline void
 321 p_atomic_inc(int32_t *v)
 322 {
 323    _InterlockedIncrement((long *)v);
 324 }
 325
 326 static inline int32_t
 327 p_atomic_inc_return(int32_t *v)
 328 {
 329    return _InterlockedIncrement((long *)v);
 330 }
 331
 332 static inline void
 333 p_atomic_dec(int32_t *v)
 334 {
 335    _InterlockedDecrement((long *)v);
 336 }
 337
 338 static inline int32_t
 339 p_atomic_dec_return(int32_t *v)
 340 {
 341    return _InterlockedDecrement((long *)v);
 342 }
 343
 344 static inline int32_t
 345 p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new)
 346 {
 347    return _InterlockedCompareExchange((long *)v, _new, old);
 348 }
 349
 350 #ifdef __cplusplus
 351 }
 352 #endif
 353
 354 #endif
 355
 356 #if defined(PIPE_ATOMIC_OS_SOLARIS)
 357
 358 #define PIPE_ATOMIC "Solaris OS atomic functions"
 359
 360 #include <atomic.h>
 361
 362 #ifdef __cplusplus
 363 extern "C" {
 364 #endif
 365
 366 #define p_atomic_set(_v, _i) (*(_v) = (_i))
 367 #define p_atomic_read(_v) (*(_v))
 368
 369 static inline boolean
 370 p_atomic_dec_zero(int32_t *v)
 371 {
 372    uint32_t n = atomic_dec_32_nv((uint32_t *) v);
 373
 374    return n != 0;
 375 }
 376
 377 #define p_atomic_inc(_v) atomic_inc_32((uint32_t *) _v)
 378 #define p_atomic_dec(_v) atomic_dec_32((uint32_t *) _v)
 379 #define p_atomic_inc_return(_v) atomic_inc_32_nv((uint32_t *) _v)
 380 #define p_atomic_dec_return(_v) atomic_dec_32_nv((uint32_t *) _v)
 381
 382 #define p_atomic_cmpxchg(_v, _old, _new) \
 383         atomic_cas_32( (uint32_t *) _v, (uint32_t) _old, (uint32_t) _new)
 384
 385 #ifdef __cplusplus
 386 }
 387 #endif
 388
 389 #endif
 390
 391
 392 #ifndef PIPE_ATOMIC
 393 #error "No pipe_atomic implementation selected"
 394 #endif
 395
 396
 397
 398 #endif /* U_ATOMIC_H */