2 * Many similar implementations exist. See for example libwsbm
3 * or the linux kernel include/atomic.h
5 * No copyright claimed on this file.
12 #include "pipe/p_compiler.h"
13 #include "pipe/p_defines.h"
15 /* Favor OS-provided implementations.
17 * Where no OS-provided implementation is available, fall back to
18 * locally coded assembly, compiler intrinsic or ultimately a
19 * mutex-based implementation.
21 #if defined(PIPE_OS_SOLARIS)
22 #define PIPE_ATOMIC_OS_SOLARIS
23 #elif defined(PIPE_CC_MSVC)
24 #define PIPE_ATOMIC_MSVC_INTRINSIC
25 #elif (defined(PIPE_CC_MSVC) && defined(PIPE_ARCH_X86))
26 #define PIPE_ATOMIC_ASM_MSVC_X86
27 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86))
28 #define PIPE_ATOMIC_ASM_GCC_X86
29 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86_64))
30 #define PIPE_ATOMIC_ASM_GCC_X86_64
31 #elif defined(PIPE_CC_GCC) && (PIPE_CC_GCC_VERSION >= 401)
32 #define PIPE_ATOMIC_GCC_INTRINSIC
34 #error "Unsupported platform"
38 #if defined(PIPE_ATOMIC_ASM_GCC_X86_64)
39 #define PIPE_ATOMIC "GCC x86_64 assembly"
45 #define p_atomic_set(_v, _i) (*(_v) = (_i))
46 #define p_atomic_read(_v) (*(_v))
49 p_atomic_dec_zero(int32_t *v
)
53 __asm__
__volatile__("lock; decl %0; sete %1":"+m"(*v
), "=qm"(c
)
60 p_atomic_inc(int32_t *v
)
62 __asm__
__volatile__("lock; incl %0":"+m"(*v
));
66 p_atomic_dec(int32_t *v
)
68 __asm__
__volatile__("lock; decl %0":"+m"(*v
));
72 p_atomic_cmpxchg(int32_t *v
, int32_t old
, int32_t _new
)
74 return __sync_val_compare_and_swap(v
, old
, _new
);
81 #endif /* PIPE_ATOMIC_ASM_GCC_X86_64 */
84 #if defined(PIPE_ATOMIC_ASM_GCC_X86)
86 #define PIPE_ATOMIC "GCC x86 assembly"
92 #define p_atomic_set(_v, _i) (*(_v) = (_i))
93 #define p_atomic_read(_v) (*(_v))
96 p_atomic_dec_zero(int32_t *v
)
100 __asm__
__volatile__("lock; decl %0; sete %1":"+m"(*v
), "=qm"(c
)
107 p_atomic_inc(int32_t *v
)
109 __asm__
__volatile__("lock; incl %0":"+m"(*v
));
113 p_atomic_dec(int32_t *v
)
115 __asm__
__volatile__("lock; decl %0":"+m"(*v
));
118 static INLINE
int32_t
119 p_atomic_cmpxchg(int32_t *v
, int32_t old
, int32_t _new
)
121 return __sync_val_compare_and_swap(v
, old
, _new
);
132 /* Implementation using GCC-provided synchronization intrinsics
134 #if defined(PIPE_ATOMIC_GCC_INTRINSIC)
136 #define PIPE_ATOMIC "GCC Sync Intrinsics"
142 #define p_atomic_set(_v, _i) (*(_v) = (_i))
143 #define p_atomic_read(_v) (*(_v))
145 static INLINE boolean
146 p_atomic_dec_zero(int32_t *v
)
148 return (__sync_sub_and_fetch(v
, 1) == 0);
152 p_atomic_inc(int32_t *v
)
154 (void) __sync_add_and_fetch(v
, 1);
158 p_atomic_dec(int32_t *v
)
160 (void) __sync_sub_and_fetch(v
, 1);
163 static INLINE
int32_t
164 p_atomic_cmpxchg(int32_t *v
, int32_t old
, int32_t _new
)
166 return __sync_val_compare_and_swap(v
, old
, _new
);
177 /* Unlocked version for single threaded environments, such as some
178 * windows kernel modules.
180 #if defined(PIPE_ATOMIC_OS_UNLOCKED)
182 #define PIPE_ATOMIC "Unlocked"
184 #define p_atomic_set(_v, _i) (*(_v) = (_i))
185 #define p_atomic_read(_v) (*(_v))
186 #define p_atomic_dec_zero(_v) ((boolean) --(*(_v)))
187 #define p_atomic_inc(_v) ((void) (*(_v))++)
188 #define p_atomic_dec(_v) ((void) (*(_v))--)
189 #define p_atomic_cmpxchg(_v, old, _new) (*(_v) == old ? *(_v) = (_new) : *(_v))
194 /* Locally coded assembly for MSVC on x86:
196 #if defined(PIPE_ATOMIC_ASM_MSVC_X86)
198 #define PIPE_ATOMIC "MSVC x86 assembly"
204 #define p_atomic_set(_v, _i) (*(_v) = (_i))
205 #define p_atomic_read(_v) (*(_v))
207 static INLINE boolean
208 p_atomic_dec_zero(int32_t *v
)
214 lock dec dword ptr
[eax
]
222 p_atomic_inc(int32_t *v
)
226 lock inc dword ptr
[eax
]
231 p_atomic_dec(int32_t *v
)
235 lock dec dword ptr
[eax
]
239 static INLINE
int32_t
240 p_atomic_cmpxchg(int32_t *v
, int32_t old
, int32_t _new
)
248 lock cmpxchg
[ecx
], edx
262 #if defined(PIPE_ATOMIC_MSVC_INTRINSIC)
264 #define PIPE_ATOMIC "MSVC Intrinsics"
268 #pragma intrinsic(_InterlockedIncrement)
269 #pragma intrinsic(_InterlockedDecrement)
270 #pragma intrinsic(_InterlockedCompareExchange)
276 #define p_atomic_set(_v, _i) (*(_v) = (_i))
277 #define p_atomic_read(_v) (*(_v))
279 static INLINE boolean
280 p_atomic_dec_zero(int32_t *v
)
282 return _InterlockedDecrement((long *)v
) == 0;
286 p_atomic_inc(int32_t *v
)
288 _InterlockedIncrement((long *)v
);
292 p_atomic_dec(int32_t *v
)
294 _InterlockedDecrement((long *)v
);
297 static INLINE
int32_t
298 p_atomic_cmpxchg(int32_t *v
, int32_t old
, int32_t _new
)
300 return _InterlockedCompareExchange((long *)v
, _new
, old
);
309 #if defined(PIPE_ATOMIC_OS_SOLARIS)
311 #define PIPE_ATOMIC "Solaris OS atomic functions"
319 #define p_atomic_set(_v, _i) (*(_v) = (_i))
320 #define p_atomic_read(_v) (*(_v))
322 static INLINE boolean
323 p_atomic_dec_zero(int32_t *v
)
325 uint32_t n
= atomic_dec_32_nv((uint32_t *) v
);
330 #define p_atomic_inc(_v) atomic_inc_32((uint32_t *) _v)
331 #define p_atomic_dec(_v) atomic_dec_32((uint32_t *) _v)
333 #define p_atomic_cmpxchg(_v, _old, _new) \
334 atomic_cas_32( (uint32_t *) _v, (uint32_t) _old, (uint32_t) _new)
344 #error "No pipe_atomic implementation selected"
349 #endif /* U_ATOMIC_H */