src/gallium/include/pipe/p_atomic.h

   1 /**
   2  * Many similar implementations exist. See for example libwsbm
   3  * or the linux kernel include/atomic.h
   4  *
   5  * No copyright claimed on this file.
   6  *
   7  */
   8
   9 #ifndef P_ATOMIC_H
  10 #define P_ATOMIC_H
  11
  12 #include "p_compiler.h"
  13 #include "p_defines.h"
  14
  15 #ifdef __cplusplus
  16 extern "C" {
  17 #endif
  18
  19
  20 /* Favor OS-provided implementations.
  21  */
  22 #define PIPE_ATOMIC_OS_UNLOCKED                 \
  23    (defined(PIPE_SUBSYSTEM_WINDOWS_DISPLAY) ||  \
  24     defined(PIPE_SUBSYSTEM_WINDOWS_MINIPORT))
  25
  26 #define PIPE_ATOMIC_OS_MS_INTERLOCK             \
  27    (!PIPE_ATOMIC_OS_UNLOCKED &&                 \
  28     defined(PIPE_SUBSYSTEM_WINDOWS_USER))
  29
  30 #define PIPE_ATOMIC_OS_PROVIDED                 \
  31    (PIPE_ATOMIC_OS_UNLOCKED ||                  \
  32     PIPE_ATOMIC_OS_MS_INTERLOCK)
  33
  34 /* Where no OS-provided implementation is available, fall back to
  35  * either locally coded assembly or ultimately a mutex-based
  36  * implementation:
  37  */
  38 #define PIPE_ATOMIC_ASM_GCC_X86                 \
  39    (!PIPE_ATOMIC_OS_PROVIDED &&                 \
  40     defined(PIPE_CC_GCC) &&                     \
  41     defined(PIPE_ARCH_X86))
  42
  43 /* KW: this was originally used when x86 asm wasn't available.
  44  * Maintain that logic here.
  45  */
  46 #define PIPE_ATOMIC_GCC_INTRINISIC              \
  47    (!PIPE_ATOMIC_OS_PROVIDED &&                 \
  48     !PIPE_ATOMIC_ASM_GCC_X86 &&                 \
  49     defined(PIPE_CC_GCC))
  50
  51 #define PIPE_ATOMIC_ASM_MSVC_X86                \
  52    (!PIPE_ATOMIC_OS_PROVIDED &&                 \
  53     defined(PIPE_CC_MSVC) &&                    \
  54     defined(PIPE_ARCH_X86))
  55
  56 #define PIPE_ATOMIC_ASM                         \
  57    (PIPE_ATOMIC_ASM_GCC_X86 ||                  \
  58     PIPE_ATOMIC_ASM_GCC_INTRINSIC ||            \
  59     PIPE_ATOMIC_ASM_MSVC_X86)
  60
  61
  62 /* Where no OS-provided or locally-coded assembly implemenation is
  63  * available, use pipe_mutex:
  64  */
  65 #define PIPE_ATOMIC_MUTEX                       \
  66    (!PIPE_ATOMIC_OS_PROVIDED &&                         \
  67     !PIPE_ATOMIC_ASM)
  68
  69
  70
  71 #if (PIPE_ATOMIC_ASM_GCC_X86)
  72
  73 #define PIPE_ATOMIC "GCC x86 assembly"
  74
  75 struct pipe_atomic {
  76    int32_t count;
  77 };
  78
  79 #define p_atomic_set(_v, _i) ((_v)->count = (_i))
  80 #define p_atomic_read(_v) ((_v)->count)
  81
  82
  83 static INLINE boolean
  84 p_atomic_dec_zero(struct pipe_atomic *v)
  85 {
  86    unsigned char c;
  87
  88    __asm__ __volatile__("lock; decl %0; sete %1":"+m"(v->count), "=qm"(c)
  89                         ::"memory");
  90
  91    return c != 0;
  92 }
  93
  94 static INLINE void
  95 p_atomic_inc(struct pipe_atomic *v)
  96 {
  97    __asm__ __volatile__("lock; incl %0":"+m"(v->count));
  98 }
  99
 100 static INLINE void
 101 p_atomic_dec(struct pipe_atomic *v)
 102 {
 103    __asm__ __volatile__("lock; decl %0":"+m"(v->count));
 104 }
 105
 106 static INLINE int32_t
 107 p_atomic_cmpxchg(struct pipe_atomic *v, int32_t old, int32_t _new)
 108 {
 109    return __sync_val_compare_and_swap(&v->count, old, _new);
 110 }
 111 #endif
 112
 113
 114
 115 /* Implementation using GCC-provided synchronization intrinsics
 116  */
 117 #if (PIPE_ATOMIC_ASM_GCC_INTRINSIC)
 118
 119 #define PIPE_ATOMIC "GCC Sync Intrinsics"
 120
 121 struct pipe_atomic {
 122    int32_t count;
 123 };
 124
 125 #define p_atomic_set(_v, _i) ((_v)->count = (_i))
 126 #define p_atomic_read(_v) ((_v)->count)
 127
 128
 129 static INLINE boolean
 130 p_atomic_dec_zero(struct pipe_atomic *v)
 131 {
 132    return (__sync_sub_and_fetch(&v->count, 1) == 0);
 133 }
 134
 135 static INLINE void
 136 p_atomic_inc(struct pipe_atomic *v)
 137 {
 138    (void) __sync_add_and_fetch(&v->count, 1);
 139 }
 140
 141 static INLINE void
 142 p_atomic_dec(struct pipe_atomic *v)
 143 {
 144    (void) __sync_sub_and_fetch(&v->count, 1);
 145 }
 146
 147 static INLINE int32_t
 148 p_atomic_cmpxchg(struct pipe_atomic *v, int32_t old, int32_t _new)
 149 {
 150    return __sync_val_compare_and_swap(&v->count, old, _new);
 151 }
 152 #endif
 153
 154
 155
 156 /* Unlocked version for single threaded environments, such as some
 157  * windows kernel modules.
 158  */
 159 #if (PIPE_ATOMIC_OS_UNLOCKED)
 160
 161 #define PIPE_ATOMIC "Unlocked"
 162
 163 struct pipe_atomic
 164 {
 165    int32_t count;
 166 };
 167
 168 #define p_atomic_set(_v, _i) ((_v)->count = (_i))
 169 #define p_atomic_read(_v) ((_v)->count)
 170 #define p_atomic_dec_zero(_v) ((boolean) --(_v)->count)
 171 #define p_atomic_inc(_v) ((void) (_v)->count++)
 172 #define p_atomic_dec(_v) ((void) (_v)->count--)
 173 #define p_atomic_cmpxchg(_v, old, _new) ((_v)->count == old ? (_v)->count = (_new) : (_v)->count)
 174
 175 #endif
 176
 177
 178 /* Locally coded assembly for MSVC on x86:
 179  */
 180 #if (PIPE_ATOMIC_ASM_MSVC_X86)
 181
 182 #define PIPE_ATOMIC "MSVC x86 assembly"
 183
 184 struct pipe_atomic
 185 {
 186    int32_t count;
 187 };
 188
 189 #define p_atomic_set(_v, _i) ((_v)->count = (_i))
 190 #define p_atomic_read(_v) ((_v)->count)
 191
 192 static INLINE boolean
 193 p_atomic_dec_zero(struct pipe_atomic *v)
 194 {
 195    int32_t *pcount = &v->count;
 196    unsigned char c;
 197
 198    __asm {
 199       mov       eax, [pcount]
 200       lock dec  dword ptr [eax]
 201       sete      byte ptr [c]
 202    }
 203
 204    return c != 0;
 205 }
 206
 207 static INLINE void
 208 p_atomic_inc(struct pipe_atomic *v)
 209 {
 210    int32_t *pcount = &v->count;
 211
 212    __asm {
 213       mov       eax, [pcount]
 214       lock inc  dword ptr [eax]
 215    }
 216 }
 217
 218 static INLINE void
 219 p_atomic_dec(struct pipe_atomic *v)
 220 {
 221    int32_t *pcount = &v->count;
 222
 223    __asm {
 224       mov       eax, [pcount]
 225       lock dec  dword ptr [eax]
 226    }
 227 }
 228
 229 static INLINE int32_t
 230 p_atomic_cmpxchg(struct pipe_atomic *v, int32_t old, int32_t _new)
 231 {
 232    int32_t *pcount = &v->count;
 233    int32_t orig;
 234
 235    __asm {
 236       mov ecx, [pcount]
 237       mov eax, [old]
 238       mov edx, [_new]
 239       lock cmpxchg [ecx], edx
 240       mov [orig], eax
 241    }
 242
 243    return orig;
 244 }
 245 #endif
 246
 247
 248 #if (PIPE_ATOMIC_OS_MS_INTERLOCK)
 249
 250 #define PIPE_ATOMIC "MS userspace interlocks"
 251
 252 #include <windows.h>
 253
 254 struct pipe_atomic
 255 {
 256    long count;
 257 };
 258
 259 #define p_atomic_set(_v, _i) ((_v)->count = (_i))
 260 #define p_atomic_read(_v) ((_v)->count)
 261
 262 static INLINE boolean
 263 p_atomic_dec_zero(struct pipe_atomic *v)
 264 {
 265    return InterlockedDecrement(&v->count);
 266 }
 267
 268 static INLINE void
 269 p_atomic_inc(struct pipe_atomic *v)
 270 {
 271    InterlockedIncrement(&v->count);
 272 }
 273
 274 static INLINE void
 275 p_atomic_dec(struct pipe_atomic *v)
 276 {
 277    InterlockedDecrement(&v->count);
 278 }
 279
 280 static INLINE int32_t
 281 p_atomic_cmpxchg(struct pipe_atomic *v, int32_t old, int32_t _new)
 282 {
 283    return InterlockedCompareExchange(&v->count, _new, old);
 284 }
 285
 286 #endif
 287
 288
 289
 290 #if (PIPE_ATOMIC_MUTEX)
 291
 292 #define PIPE_ATOMIC "mutex-based fallback"
 293
 294 #include "pipe/p_thread.h"
 295
 296 /**
 297  * This implementation should really not be used.
 298  * Add an assembly port instead. It may abort and
 299  * doesn't destroy used mutexes.
 300  */
 301
 302 struct pipe_atomic {
 303    pipe_mutex mutex;
 304    int32_t count;
 305 };
 306
 307 static INLINE void
 308 p_atomic_set(struct pipe_atomic *v, int32_t i)
 309 {
 310    pipe_mutex_init(v->mutex);
 311    pipe_mutex_lock(v->mutex);
 312    v->count = i;
 313    pipe_mutex_unlock(v->mutex);
 314 }
 315
 316 static INLINE int32_t
 317 p_atomic_read(struct pipe_atomic *v)
 318 {
 319    int32_t ret;
 320
 321    pipe_mutex_lock(v->mutex);
 322    ret = v->count;
 323    pipe_mutex_unlock(v->mutex);
 324    return ret;
 325 }
 326
 327 static INLINE void
 328 p_atomic_inc(struct pipe_atomic *v)
 329 {
 330    pipe_mutex_lock(v->mutex);
 331    ++v->count;
 332    pipe_mutex_unlock(v->mutex);
 333 }
 334
 335 static INLINE void
 336 p_atomic_dec(struct pipe_atomic *v)
 337 {
 338    pipe_mutex_lock(v->mutex);
 339    --v->count;
 340    pipe_mutex_unlock(v->mutex);
 341 }
 342
 343 static INLINE boolean
 344 p_atomic_dec_zero(struct pipe_atomic *v)
 345 {
 346    boolean ret;
 347
 348    pipe_mutex_lock(v->mutex);
 349    ret = (--v->count == 0);
 350    pipe_mutex_unlock(v->mutex);
 351    return ret;
 352 }
 353
 354 static INLINE int32_t
 355 p_atomic_cmpxchg(struct pipe_atomic *v, int32_t old, int32_t _new)
 356 {
 357    int32_t ret;
 358
 359    pipe_mutex_lock(v->mutex);
 360    ret = v->count;
 361    if (ret == old)
 362       v->count = _new;
 363    pipe_mutex_unlock(v->mutex);
 364
 365    return ret;
 366 }
 367
 368 #endif
 369
 370
 371 #ifndef PIPE_ATOMIC
 372 #error "No pipe_atomic implementation selected"
 373 #endif
 374
 375
 376
 377 #ifdef __cplusplus
 378 }
 379 #endif
 380
 381 #endif /* P_ATOMIC_H */