8653e73f4615c12c60f9c860c0ca356c543f54ec
2 * Many similar implementations exist. See for example libwsbm
3 * or the linux kernel include/atomic.h
5 * No copyright claimed on this file.
12 #include "pipe/p_compiler.h"
13 #include "pipe/p_defines.h"
15 /* Favor OS-provided implementations.
17 * Where no OS-provided implementation is available, fall back to
18 * locally coded assembly, compiler intrinsic or ultimately a
19 * mutex-based implementation.
21 #if defined(PIPE_OS_SOLARIS)
22 #define PIPE_ATOMIC_OS_SOLARIS
23 #elif defined(PIPE_CC_MSVC)
24 #define PIPE_ATOMIC_MSVC_INTRINSIC
25 #elif (defined(PIPE_CC_MSVC) && defined(PIPE_ARCH_X86))
26 #define PIPE_ATOMIC_ASM_MSVC_X86
27 #elif defined(PIPE_CC_GCC) && (PIPE_CC_GCC_VERSION >= 401)
28 #define PIPE_ATOMIC_GCC_INTRINSIC
29 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86))
30 #define PIPE_ATOMIC_ASM_GCC_X86
31 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86_64))
32 #define PIPE_ATOMIC_ASM_GCC_X86_64
34 #error "Unsupported platform"
38 #if defined(PIPE_ATOMIC_ASM_GCC_X86_64)
39 #define PIPE_ATOMIC "GCC x86_64 assembly"
45 #define p_atomic_set(_v, _i) (*(_v) = (_i))
46 #define p_atomic_read(_v) (*(_v))
49 p_atomic_dec_zero(int32_t *v
)
53 __asm__
__volatile__("lock; decl %0; sete %1":"+m"(*v
), "=qm"(c
)
60 p_atomic_inc(int32_t *v
)
62 __asm__
__volatile__("lock; incl %0":"+m"(*v
));
66 p_atomic_dec(int32_t *v
)
68 __asm__
__volatile__("lock; decl %0":"+m"(*v
));
72 p_atomic_inc_return(int32_t *v
)
74 return __sync_add_and_fetch(v
, 1);
78 p_atomic_dec_return(int32_t *v
)
80 return __sync_sub_and_fetch(v
, 1);
84 p_atomic_cmpxchg(int32_t *v
, int32_t old
, int32_t _new
)
86 return __sync_val_compare_and_swap(v
, old
, _new
);
93 #endif /* PIPE_ATOMIC_ASM_GCC_X86_64 */
96 #if defined(PIPE_ATOMIC_ASM_GCC_X86)
98 #define PIPE_ATOMIC "GCC x86 assembly"
104 #define p_atomic_set(_v, _i) (*(_v) = (_i))
105 #define p_atomic_read(_v) (*(_v))
107 static inline boolean
108 p_atomic_dec_zero(int32_t *v
)
112 __asm__
__volatile__("lock; decl %0; sete %1":"+m"(*v
), "=qm"(c
)
119 p_atomic_inc(int32_t *v
)
121 __asm__
__volatile__("lock; incl %0":"+m"(*v
));
125 p_atomic_dec(int32_t *v
)
127 __asm__
__volatile__("lock; decl %0":"+m"(*v
));
130 static inline int32_t
131 p_atomic_inc_return(int32_t *v
)
133 return __sync_add_and_fetch(v
, 1);
136 static inline int32_t
137 p_atomic_dec_return(int32_t *v
)
139 return __sync_sub_and_fetch(v
, 1);
142 static inline int32_t
143 p_atomic_cmpxchg(int32_t *v
, int32_t old
, int32_t _new
)
145 return __sync_val_compare_and_swap(v
, old
, _new
);
156 /* Implementation using GCC-provided synchronization intrinsics
158 #if defined(PIPE_ATOMIC_GCC_INTRINSIC)
160 #define PIPE_ATOMIC "GCC Sync Intrinsics"
166 #define p_atomic_set(_v, _i) (*(_v) = (_i))
167 #define p_atomic_read(_v) (*(_v))
169 static inline boolean
170 p_atomic_dec_zero(int32_t *v
)
172 return (__sync_sub_and_fetch(v
, 1) == 0);
176 p_atomic_inc(int32_t *v
)
178 (void) __sync_add_and_fetch(v
, 1);
182 p_atomic_dec(int32_t *v
)
184 (void) __sync_sub_and_fetch(v
, 1);
187 static inline int32_t
188 p_atomic_inc_return(int32_t *v
)
190 return __sync_add_and_fetch(v
, 1);
193 static inline int32_t
194 p_atomic_dec_return(int32_t *v
)
196 return __sync_sub_and_fetch(v
, 1);
199 static inline int32_t
200 p_atomic_cmpxchg(int32_t *v
, int32_t old
, int32_t _new
)
202 return __sync_val_compare_and_swap(v
, old
, _new
);
213 /* Unlocked version for single threaded environments, such as some
214 * windows kernel modules.
216 #if defined(PIPE_ATOMIC_OS_UNLOCKED)
218 #define PIPE_ATOMIC "Unlocked"
220 #define p_atomic_set(_v, _i) (*(_v) = (_i))
221 #define p_atomic_read(_v) (*(_v))
222 #define p_atomic_dec_zero(_v) ((boolean) --(*(_v)))
223 #define p_atomic_inc(_v) ((void) (*(_v))++)
224 #define p_atomic_dec(_v) ((void) (*(_v))--)
225 #define p_atomic_inc_return(_v) ((*(_v))++)
226 #define p_atomic_dec_return(_v) ((*(_v))--)
227 #define p_atomic_cmpxchg(_v, old, _new) (*(_v) == old ? *(_v) = (_new) : *(_v))
232 /* Locally coded assembly for MSVC on x86:
234 #if defined(PIPE_ATOMIC_ASM_MSVC_X86)
236 #define PIPE_ATOMIC "MSVC x86 assembly"
242 #define p_atomic_set(_v, _i) (*(_v) = (_i))
243 #define p_atomic_read(_v) (*(_v))
245 static inline boolean
246 p_atomic_dec_zero(int32_t *v
)
252 lock dec dword ptr
[eax
]
260 p_atomic_inc(int32_t *v
)
264 lock inc dword ptr
[eax
]
269 p_atomic_dec(int32_t *v
)
273 lock dec dword ptr
[eax
]
277 static inline int32_t
278 p_atomic_cmpxchg(int32_t *v
, int32_t old
, int32_t _new
)
286 lock cmpxchg
[ecx
], edx
300 #if defined(PIPE_ATOMIC_MSVC_INTRINSIC)
302 #define PIPE_ATOMIC "MSVC Intrinsics"
306 #pragma intrinsic(_InterlockedIncrement)
307 #pragma intrinsic(_InterlockedDecrement)
308 #pragma intrinsic(_InterlockedCompareExchange)
314 #define p_atomic_set(_v, _i) (*(_v) = (_i))
315 #define p_atomic_read(_v) (*(_v))
317 static inline boolean
318 p_atomic_dec_zero(int32_t *v
)
320 return _InterlockedDecrement((long *)v
) == 0;
324 p_atomic_inc(int32_t *v
)
326 _InterlockedIncrement((long *)v
);
329 static inline int32_t
330 p_atomic_inc_return(int32_t *v
)
332 return _InterlockedIncrement((long *)v
);
336 p_atomic_dec(int32_t *v
)
338 _InterlockedDecrement((long *)v
);
341 static inline int32_t
342 p_atomic_dec_return(int32_t *v
)
344 return _InterlockedDecrement((long *)v
);
347 static inline int32_t
348 p_atomic_cmpxchg(int32_t *v
, int32_t old
, int32_t _new
)
350 return _InterlockedCompareExchange((long *)v
, _new
, old
);
359 #if defined(PIPE_ATOMIC_OS_SOLARIS)
361 #define PIPE_ATOMIC "Solaris OS atomic functions"
369 #define p_atomic_set(_v, _i) (*(_v) = (_i))
370 #define p_atomic_read(_v) (*(_v))
372 static inline boolean
373 p_atomic_dec_zero(int32_t *v
)
375 uint32_t n
= atomic_dec_32_nv((uint32_t *) v
);
380 #define p_atomic_inc(_v) atomic_inc_32((uint32_t *) _v)
381 #define p_atomic_dec(_v) atomic_dec_32((uint32_t *) _v)
382 #define p_atomic_inc_return(_v) atomic_inc_32_nv((uint32_t *) _v)
383 #define p_atomic_dec_return(_v) atomic_dec_32_nv((uint32_t *) _v)
385 #define p_atomic_cmpxchg(_v, _old, _new) \
386 atomic_cas_32( (uint32_t *) _v, (uint32_t) _old, (uint32_t) _new)
396 #error "No pipe_atomic implementation selected"
401 #endif /* U_ATOMIC_H */