src/gallium/include/pipe/p_atomic.h

   1 /**
   2  * Many similar implementations exist. See for example libwsbm
   3  * or the linux kernel include/atomic.h
   4  *
   5  * No copyright claimed on this file.
   6  *
   7  */
   8
   9 #ifndef P_ATOMIC_H
  10 #define P_ATOMIC_H
  11
  12 #include "p_compiler.h"
  13 #include "p_defines.h"
  14
  15 #ifdef __cplusplus
  16 extern "C" {
  17 #endif
  18
  19
  20 /* Favor OS-provided implementations.
  21  */
  22 #define PIPE_ATOMIC_OS_UNLOCKED                 \
  23    (defined(PIPE_SUBSYSTEM_WINDOWS_DISPLAY) ||  \
  24     defined(PIPE_SUBSYSTEM_WINDOWS_MINIPORT))
  25
  26 #define PIPE_ATOMIC_OS_MS_INTERLOCK             \
  27    (!defined(PIPE_CC_GCC) &&                    \
  28     !PIPE_ATOMIC_OS_UNLOCKED &&                 \
  29     defined(PIPE_SUBSYSTEM_WINDOWS_USER))
  30
  31 #define PIPE_ATOMIC_OS_PROVIDED                 \
  32    (PIPE_ATOMIC_OS_UNLOCKED ||                  \
  33     PIPE_ATOMIC_OS_MS_INTERLOCK)
  34
  35 /* Where no OS-provided implementation is available, fall back to
  36  * either locally coded assembly or ultimately a mutex-based
  37  * implementation:
  38  */
  39 #define PIPE_ATOMIC_ASM_GCC_X86                 \
  40    (!PIPE_ATOMIC_OS_PROVIDED &&                 \
  41     defined(PIPE_CC_GCC) &&                     \
  42     defined(PIPE_ARCH_X86))
  43
  44 /* KW: this was originally used when x86 asm wasn't available.
  45  * Maintain that logic here.
  46  */
  47 #define PIPE_ATOMIC_GCC_INTRINISIC              \
  48    (!PIPE_ATOMIC_OS_PROVIDED &&                 \
  49     !PIPE_ATOMIC_ASM_GCC_X86 &&                 \
  50     defined(PIPE_CC_GCC))
  51
  52 #define PIPE_ATOMIC_ASM_MSVC_X86                \
  53    (!PIPE_ATOMIC_OS_PROVIDED &&                 \
  54     defined(PIPE_CC_MSVC) &&                    \
  55     defined(PIPE_ARCH_X86))
  56
  57 #define PIPE_ATOMIC_ASM                         \
  58    (PIPE_ATOMIC_ASM_GCC_X86 ||                  \
  59     PIPE_ATOMIC_ASM_GCC_INTRINSIC ||            \
  60     PIPE_ATOMIC_ASM_MSVC_X86)
  61
  62
  63 /* Where no OS-provided or locally-coded assembly implemenation is
  64  * available, use pipe_mutex:
  65  */
  66 #define PIPE_ATOMIC_MUTEX                       \
  67    (!PIPE_ATOMIC_OS_PROVIDED &&                         \
  68     !PIPE_ATOMIC_ASM)
  69
  70
  71
  72 #if (PIPE_ATOMIC_ASM_GCC_X86)
  73
  74 #define PIPE_ATOMIC "GCC x86 assembly"
  75
  76 struct pipe_atomic {
  77    int32_t count;
  78 };
  79
  80 #define p_atomic_set(_v, _i) ((_v)->count = (_i))
  81 #define p_atomic_read(_v) ((_v)->count)
  82
  83
  84 static INLINE boolean
  85 p_atomic_dec_zero(struct pipe_atomic *v)
  86 {
  87    unsigned char c;
  88
  89    __asm__ __volatile__("lock; decl %0; sete %1":"+m"(v->count), "=qm"(c)
  90                         ::"memory");
  91
  92    return c != 0;
  93 }
  94
  95 static INLINE void
  96 p_atomic_inc(struct pipe_atomic *v)
  97 {
  98    __asm__ __volatile__("lock; incl %0":"+m"(v->count));
  99 }
 100
 101 static INLINE void
 102 p_atomic_dec(struct pipe_atomic *v)
 103 {
 104    __asm__ __volatile__("lock; decl %0":"+m"(v->count));
 105 }
 106
 107 static INLINE int32_t
 108 p_atomic_cmpxchg(struct pipe_atomic *v, int32_t old, int32_t _new)
 109 {
 110    return __sync_val_compare_and_swap(&v->count, old, _new);
 111 }
 112 #endif
 113
 114
 115
 116 /* Implementation using GCC-provided synchronization intrinsics
 117  */
 118 #if (PIPE_ATOMIC_ASM_GCC_INTRINSIC)
 119
 120 #define PIPE_ATOMIC "GCC Sync Intrinsics"
 121
 122 struct pipe_atomic {
 123    int32_t count;
 124 };
 125
 126 #define p_atomic_set(_v, _i) ((_v)->count = (_i))
 127 #define p_atomic_read(_v) ((_v)->count)
 128
 129
 130 static INLINE boolean
 131 p_atomic_dec_zero(struct pipe_atomic *v)
 132 {
 133    return (__sync_sub_and_fetch(&v->count, 1) == 0);
 134 }
 135
 136 static INLINE void
 137 p_atomic_inc(struct pipe_atomic *v)
 138 {
 139    (void) __sync_add_and_fetch(&v->count, 1);
 140 }
 141
 142 static INLINE void
 143 p_atomic_dec(struct pipe_atomic *v)
 144 {
 145    (void) __sync_sub_and_fetch(&v->count, 1);
 146 }
 147
 148 static INLINE int32_t
 149 p_atomic_cmpxchg(struct pipe_atomic *v, int32_t old, int32_t _new)
 150 {
 151    return __sync_val_compare_and_swap(&v->count, old, _new);
 152 }
 153 #endif
 154
 155
 156
 157 /* Unlocked version for single threaded environments, such as some
 158  * windows kernel modules.
 159  */
 160 #if (PIPE_ATOMIC_OS_UNLOCKED)
 161
 162 #define PIPE_ATOMIC "Unlocked"
 163
 164 struct pipe_atomic
 165 {
 166    int32_t count;
 167 };
 168
 169 #define p_atomic_set(_v, _i) ((_v)->count = (_i))
 170 #define p_atomic_read(_v) ((_v)->count)
 171 #define p_atomic_dec_zero(_v) ((boolean) --(_v)->count)
 172 #define p_atomic_inc(_v) ((void) (_v)->count++)
 173 #define p_atomic_dec(_v) ((void) (_v)->count--)
 174 #define p_atomic_cmpxchg(_v, old, _new) ((_v)->count == old ? (_v)->count = (_new) : (_v)->count)
 175
 176 #endif
 177
 178
 179 /* Locally coded assembly for MSVC on x86:
 180  */
 181 #if (PIPE_ATOMIC_ASM_MSVC_X86)
 182
 183 #define PIPE_ATOMIC "MSVC x86 assembly"
 184
 185 struct pipe_atomic
 186 {
 187    int32_t count;
 188 };
 189
 190 #define p_atomic_set(_v, _i) ((_v)->count = (_i))
 191 #define p_atomic_read(_v) ((_v)->count)
 192
 193 static INLINE boolean
 194 p_atomic_dec_zero(struct pipe_atomic *v)
 195 {
 196    int32_t *pcount = &v->count;
 197    unsigned char c;
 198
 199    __asm {
 200       mov       eax, [pcount]
 201       lock dec  dword ptr [eax]
 202       sete      byte ptr [c]
 203    }
 204
 205    return c != 0;
 206 }
 207
 208 static INLINE void
 209 p_atomic_inc(struct pipe_atomic *v)
 210 {
 211    int32_t *pcount = &v->count;
 212
 213    __asm {
 214       mov       eax, [pcount]
 215       lock inc  dword ptr [eax]
 216    }
 217 }
 218
 219 static INLINE void
 220 p_atomic_dec(struct pipe_atomic *v)
 221 {
 222    int32_t *pcount = &v->count;
 223
 224    __asm {
 225       mov       eax, [pcount]
 226       lock dec  dword ptr [eax]
 227    }
 228 }
 229
 230 static INLINE int32_t
 231 p_atomic_cmpxchg(struct pipe_atomic *v, int32_t old, int32_t _new)
 232 {
 233    int32_t *pcount = &v->count;
 234    int32_t orig;
 235
 236    __asm {
 237       mov ecx, [pcount]
 238       mov eax, [old]
 239       mov edx, [_new]
 240       lock cmpxchg [ecx], edx
 241       mov [orig], eax
 242    }
 243
 244    return orig;
 245 }
 246 #endif
 247
 248
 249 #if (PIPE_ATOMIC_OS_MS_INTERLOCK)
 250
 251 #define PIPE_ATOMIC "MS userspace interlocks"
 252
 253 #include <windows.h>
 254
 255 struct pipe_atomic
 256 {
 257    long count;
 258 };
 259
 260 #define p_atomic_set(_v, _i) ((_v)->count = (_i))
 261 #define p_atomic_read(_v) ((_v)->count)
 262
 263 static INLINE boolean
 264 p_atomic_dec_zero(struct pipe_atomic *v)
 265 {
 266    return InterlockedDecrement(&v->count);
 267 }
 268
 269 static INLINE void
 270 p_atomic_inc(struct pipe_atomic *v)
 271 {
 272    InterlockedIncrement(&v->count);
 273 }
 274
 275 static INLINE void
 276 p_atomic_dec(struct pipe_atomic *v)
 277 {
 278    InterlockedDecrement(&v->count);
 279 }
 280
 281 static INLINE int32_t
 282 p_atomic_cmpxchg(struct pipe_atomic *v, int32_t old, int32_t _new)
 283 {
 284    return InterlockedCompareExchange(&v->count, _new, old);
 285 }
 286
 287 #endif
 288
 289
 290
 291 #if (PIPE_ATOMIC_MUTEX)
 292
 293 #define PIPE_ATOMIC "mutex-based fallback"
 294
 295 #include "pipe/p_thread.h"
 296
 297 /**
 298  * This implementation should really not be used.
 299  * Add an assembly port instead. It may abort and
 300  * doesn't destroy used mutexes.
 301  */
 302
 303 struct pipe_atomic {
 304    pipe_mutex mutex;
 305    int32_t count;
 306 };
 307
 308 static INLINE void
 309 p_atomic_set(struct pipe_atomic *v, int32_t i)
 310 {
 311    pipe_mutex_init(v->mutex);
 312    pipe_mutex_lock(v->mutex);
 313    v->count = i;
 314    pipe_mutex_unlock(v->mutex);
 315 }
 316
 317 static INLINE int32_t
 318 p_atomic_read(struct pipe_atomic *v)
 319 {
 320    int32_t ret;
 321
 322    pipe_mutex_lock(v->mutex);
 323    ret = v->count;
 324    pipe_mutex_unlock(v->mutex);
 325    return ret;
 326 }
 327
 328 static INLINE void
 329 p_atomic_inc(struct pipe_atomic *v)
 330 {
 331    pipe_mutex_lock(v->mutex);
 332    ++v->count;
 333    pipe_mutex_unlock(v->mutex);
 334 }
 335
 336 static INLINE void
 337 p_atomic_dec(struct pipe_atomic *v)
 338 {
 339    pipe_mutex_lock(v->mutex);
 340    --v->count;
 341    pipe_mutex_unlock(v->mutex);
 342 }
 343
 344 static INLINE boolean
 345 p_atomic_dec_zero(struct pipe_atomic *v)
 346 {
 347    boolean ret;
 348
 349    pipe_mutex_lock(v->mutex);
 350    ret = (--v->count == 0);
 351    pipe_mutex_unlock(v->mutex);
 352    return ret;
 353 }
 354
 355 static INLINE int32_t
 356 p_atomic_cmpxchg(struct pipe_atomic *v, int32_t old, int32_t _new)
 357 {
 358    int32_t ret;
 359
 360    pipe_mutex_lock(v->mutex);
 361    ret = v->count;
 362    if (ret == old)
 363       v->count = _new;
 364    pipe_mutex_unlock(v->mutex);
 365
 366    return ret;
 367 }
 368
 369 #endif
 370
 371
 372 #ifndef PIPE_ATOMIC
 373 #error "No pipe_atomic implementation selected"
 374 #endif
 375
 376
 377
 378 #ifdef __cplusplus
 379 }
 380 #endif
 381
 382 #endif /* P_ATOMIC_H */