src/gallium/include/pipe/p_atomic.h

   1 /**
   2  * Many similar implementations exist. See for example libwsbm
   3  * or the linux kernel include/atomic.h
   4  *
   5  * No copyright claimed on this file.
   6  *
   7  */
   8
   9 #ifndef P_ATOMIC_H
  10 #define P_ATOMIC_H
  11
  12 #include "p_compiler.h"
  13 #include "p_defines.h"
  14
  15 #ifdef __cplusplus
  16 extern "C" {
  17 #endif
  18
  19
  20 /* Favor OS-provided implementations.
  21  *
  22  * Where no OS-provided implementation is available, fall back to
  23  * locally coded assembly, compiler intrinsic or ultimately a
  24  * mutex-based implementation.
  25  */
  26 #if (defined(PIPE_SUBSYSTEM_WINDOWS_DISPLAY) || \
  27      defined(PIPE_SUBSYSTEM_WINDOWS_MINIPORT))
  28 #define PIPE_ATOMIC_OS_UNLOCKED
  29 #elif (defined(PIPE_CC_MSVC) && defined(PIPE_SUBSYSTEM_WINDOWS_USER))
  30 #define PIPE_ATOMIC_OS_MS_INTERLOCK
  31 #elif (defined(PIPE_CC_MSVC) && defined(PIPE_ARCH_X86))
  32 #define PIPE_ATOMIC_ASM_MSVC_X86
  33 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86))
  34 #define PIPE_ATOMIC_ASM_GCC_X86
  35 #elif defined(PIPE_CC_GCC)
  36 #define PIPE_ATOMIC_GCC_INTRINSIC
  37 #else
  38 #define PIPE_ATOMIC_MUTEX
  39 #endif
  40
  41
  42
  43 #if defined(PIPE_ATOMIC_ASM_GCC_X86)
  44
  45 #define PIPE_ATOMIC "GCC x86 assembly"
  46
  47 struct pipe_atomic {
  48    int32_t count;
  49 };
  50
  51 #define p_atomic_set(_v, _i) ((_v)->count = (_i))
  52 #define p_atomic_read(_v) ((_v)->count)
  53
  54
  55 static INLINE boolean
  56 p_atomic_dec_zero(struct pipe_atomic *v)
  57 {
  58    unsigned char c;
  59
  60    __asm__ __volatile__("lock; decl %0; sete %1":"+m"(v->count), "=qm"(c)
  61                         ::"memory");
  62
  63    return c != 0;
  64 }
  65
  66 static INLINE void
  67 p_atomic_inc(struct pipe_atomic *v)
  68 {
  69    __asm__ __volatile__("lock; incl %0":"+m"(v->count));
  70 }
  71
  72 static INLINE void
  73 p_atomic_dec(struct pipe_atomic *v)
  74 {
  75    __asm__ __volatile__("lock; decl %0":"+m"(v->count));
  76 }
  77
  78 static INLINE int32_t
  79 p_atomic_cmpxchg(struct pipe_atomic *v, int32_t old, int32_t _new)
  80 {
  81    return __sync_val_compare_and_swap(&v->count, old, _new);
  82 }
  83 #endif
  84
  85
  86
  87 /* Implementation using GCC-provided synchronization intrinsics
  88  */
  89 #if defined(PIPE_ATOMIC_GCC_INTRINSIC)
  90
  91 #define PIPE_ATOMIC "GCC Sync Intrinsics"
  92
  93 struct pipe_atomic {
  94    int32_t count;
  95 };
  96
  97 #define p_atomic_set(_v, _i) ((_v)->count = (_i))
  98 #define p_atomic_read(_v) ((_v)->count)
  99
 100
 101 static INLINE boolean
 102 p_atomic_dec_zero(struct pipe_atomic *v)
 103 {
 104    return (__sync_sub_and_fetch(&v->count, 1) == 0);
 105 }
 106
 107 static INLINE void
 108 p_atomic_inc(struct pipe_atomic *v)
 109 {
 110    (void) __sync_add_and_fetch(&v->count, 1);
 111 }
 112
 113 static INLINE void
 114 p_atomic_dec(struct pipe_atomic *v)
 115 {
 116    (void) __sync_sub_and_fetch(&v->count, 1);
 117 }
 118
 119 static INLINE int32_t
 120 p_atomic_cmpxchg(struct pipe_atomic *v, int32_t old, int32_t _new)
 121 {
 122    return __sync_val_compare_and_swap(&v->count, old, _new);
 123 }
 124 #endif
 125
 126
 127
 128 /* Unlocked version for single threaded environments, such as some
 129  * windows kernel modules.
 130  */
 131 #if defined(PIPE_ATOMIC_OS_UNLOCKED)
 132
 133 #define PIPE_ATOMIC "Unlocked"
 134
 135 struct pipe_atomic
 136 {
 137    int32_t count;
 138 };
 139
 140 #define p_atomic_set(_v, _i) ((_v)->count = (_i))
 141 #define p_atomic_read(_v) ((_v)->count)
 142 #define p_atomic_dec_zero(_v) ((boolean) --(_v)->count)
 143 #define p_atomic_inc(_v) ((void) (_v)->count++)
 144 #define p_atomic_dec(_v) ((void) (_v)->count--)
 145 #define p_atomic_cmpxchg(_v, old, _new) ((_v)->count == old ? (_v)->count = (_new) : (_v)->count)
 146
 147 #endif
 148
 149
 150 /* Locally coded assembly for MSVC on x86:
 151  */
 152 #if defined(PIPE_ATOMIC_ASM_MSVC_X86)
 153
 154 #define PIPE_ATOMIC "MSVC x86 assembly"
 155
 156 struct pipe_atomic
 157 {
 158    int32_t count;
 159 };
 160
 161 #define p_atomic_set(_v, _i) ((_v)->count = (_i))
 162 #define p_atomic_read(_v) ((_v)->count)
 163
 164 static INLINE boolean
 165 p_atomic_dec_zero(struct pipe_atomic *v)
 166 {
 167    int32_t *pcount = &v->count;
 168    unsigned char c;
 169
 170    __asm {
 171       mov       eax, [pcount]
 172       lock dec  dword ptr [eax]
 173       sete      byte ptr [c]
 174    }
 175
 176    return c != 0;
 177 }
 178
 179 static INLINE void
 180 p_atomic_inc(struct pipe_atomic *v)
 181 {
 182    int32_t *pcount = &v->count;
 183
 184    __asm {
 185       mov       eax, [pcount]
 186       lock inc  dword ptr [eax]
 187    }
 188 }
 189
 190 static INLINE void
 191 p_atomic_dec(struct pipe_atomic *v)
 192 {
 193    int32_t *pcount = &v->count;
 194
 195    __asm {
 196       mov       eax, [pcount]
 197       lock dec  dword ptr [eax]
 198    }
 199 }
 200
 201 static INLINE int32_t
 202 p_atomic_cmpxchg(struct pipe_atomic *v, int32_t old, int32_t _new)
 203 {
 204    int32_t *pcount = &v->count;
 205    int32_t orig;
 206
 207    __asm {
 208       mov ecx, [pcount]
 209       mov eax, [old]
 210       mov edx, [_new]
 211       lock cmpxchg [ecx], edx
 212       mov [orig], eax
 213    }
 214
 215    return orig;
 216 }
 217 #endif
 218
 219
 220 #if defined(PIPE_ATOMIC_OS_MS_INTERLOCK)
 221
 222 #define PIPE_ATOMIC "MS userspace interlocks"
 223
 224 #include <windows.h>
 225
 226 struct pipe_atomic
 227 {
 228    volatile long count;
 229 };
 230
 231 #define p_atomic_set(_v, _i) ((_v)->count = (_i))
 232 #define p_atomic_read(_v) ((_v)->count)
 233
 234 static INLINE boolean
 235 p_atomic_dec_zero(struct pipe_atomic *v)
 236 {
 237    return InterlockedDecrement(&v->count) == 0;
 238 }
 239
 240 static INLINE void
 241 p_atomic_inc(struct pipe_atomic *v)
 242 {
 243    InterlockedIncrement(&v->count);
 244 }
 245
 246 static INLINE void
 247 p_atomic_dec(struct pipe_atomic *v)
 248 {
 249    InterlockedDecrement(&v->count);
 250 }
 251
 252 static INLINE int32_t
 253 p_atomic_cmpxchg(struct pipe_atomic *v, int32_t old, int32_t _new)
 254 {
 255    return InterlockedCompareExchange(&v->count, _new, old);
 256 }
 257
 258 #endif
 259
 260
 261
 262 #if defined(PIPE_ATOMIC_MUTEX)
 263
 264 #define PIPE_ATOMIC "mutex-based fallback"
 265
 266 #include "pipe/p_thread.h"
 267
 268 /**
 269  * This implementation should really not be used.
 270  * Add an assembly port instead. It may abort and
 271  * doesn't destroy used mutexes.
 272  */
 273
 274 struct pipe_atomic {
 275    pipe_mutex mutex;
 276    int32_t count;
 277 };
 278
 279 static INLINE void
 280 p_atomic_set(struct pipe_atomic *v, int32_t i)
 281 {
 282    pipe_mutex_init(v->mutex);
 283    pipe_mutex_lock(v->mutex);
 284    v->count = i;
 285    pipe_mutex_unlock(v->mutex);
 286 }
 287
 288 static INLINE int32_t
 289 p_atomic_read(struct pipe_atomic *v)
 290 {
 291    int32_t ret;
 292
 293    pipe_mutex_lock(v->mutex);
 294    ret = v->count;
 295    pipe_mutex_unlock(v->mutex);
 296    return ret;
 297 }
 298
 299 static INLINE void
 300 p_atomic_inc(struct pipe_atomic *v)
 301 {
 302    pipe_mutex_lock(v->mutex);
 303    ++v->count;
 304    pipe_mutex_unlock(v->mutex);
 305 }
 306
 307 static INLINE void
 308 p_atomic_dec(struct pipe_atomic *v)
 309 {
 310    pipe_mutex_lock(v->mutex);
 311    --v->count;
 312    pipe_mutex_unlock(v->mutex);
 313 }
 314
 315 static INLINE boolean
 316 p_atomic_dec_zero(struct pipe_atomic *v)
 317 {
 318    boolean ret;
 319
 320    pipe_mutex_lock(v->mutex);
 321    ret = (--v->count == 0);
 322    pipe_mutex_unlock(v->mutex);
 323    return ret;
 324 }
 325
 326 static INLINE int32_t
 327 p_atomic_cmpxchg(struct pipe_atomic *v, int32_t old, int32_t _new)
 328 {
 329    int32_t ret;
 330
 331    pipe_mutex_lock(v->mutex);
 332    ret = v->count;
 333    if (ret == old)
 334       v->count = _new;
 335    pipe_mutex_unlock(v->mutex);
 336
 337    return ret;
 338 }
 339
 340 #endif
 341
 342
 343 #ifndef PIPE_ATOMIC
 344 #error "No pipe_atomic implementation selected"
 345 #endif
 346
 347
 348
 349 #ifdef __cplusplus
 350 }
 351 #endif
 352
 353 #endif /* P_ATOMIC_H */