2 * Many similar implementations exist. See for example libwsbm
3 * or the linux kernel include/atomic.h
5 * No copyright claimed on this file.
12 #include "pipe/p_compiler.h"
13 #include "pipe/p_defines.h"
15 /* Favor OS-provided implementations.
17 * Where no OS-provided implementation is available, fall back to
18 * locally coded assembly, compiler intrinsic or ultimately a
19 * mutex-based implementation.
21 #if (defined(PIPE_SUBSYSTEM_WINDOWS_DISPLAY) || \
22 defined(PIPE_SUBSYSTEM_WINDOWS_MINIPORT))
23 #define PIPE_ATOMIC_OS_UNLOCKED
24 #elif defined(PIPE_OS_SOLARIS)
25 #define PIPE_ATOMIC_OS_SOLARIS
26 #elif defined(PIPE_CC_MSVC)
27 #define PIPE_ATOMIC_MSVC_INTRINSIC
28 #elif (defined(PIPE_CC_MSVC) && defined(PIPE_ARCH_X86))
29 #define PIPE_ATOMIC_ASM_MSVC_X86
30 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86))
31 #define PIPE_ATOMIC_ASM_GCC_X86
32 #elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86_64))
33 #define PIPE_ATOMIC_ASM_GCC_X86_64
34 #elif defined(PIPE_CC_GCC) && (PIPE_CC_GCC_VERSION >= 401)
35 #define PIPE_ATOMIC_GCC_INTRINSIC
37 #error "Unsupported platform"
41 #if defined(PIPE_ATOMIC_ASM_GCC_X86_64)
42 #define PIPE_ATOMIC "GCC x86_64 assembly"
48 #define p_atomic_set(_v, _i) (*(_v) = (_i))
49 #define p_atomic_read(_v) (*(_v))
52 p_atomic_dec_zero(int32_t *v
)
56 __asm__
__volatile__("lock; decl %0; sete %1":"+m"(*v
), "=qm"(c
)
63 p_atomic_inc(int32_t *v
)
65 __asm__
__volatile__("lock; incl %0":"+m"(*v
));
69 p_atomic_dec(int32_t *v
)
71 __asm__
__volatile__("lock; decl %0":"+m"(*v
));
75 p_atomic_cmpxchg(int32_t *v
, int32_t old
, int32_t _new
)
77 return __sync_val_compare_and_swap(v
, old
, _new
);
84 #endif /* PIPE_ATOMIC_ASM_GCC_X86_64 */
87 #if defined(PIPE_ATOMIC_ASM_GCC_X86)
89 #define PIPE_ATOMIC "GCC x86 assembly"
95 #define p_atomic_set(_v, _i) (*(_v) = (_i))
96 #define p_atomic_read(_v) (*(_v))
99 p_atomic_dec_zero(int32_t *v
)
103 __asm__
__volatile__("lock; decl %0; sete %1":"+m"(*v
), "=qm"(c
)
110 p_atomic_inc(int32_t *v
)
112 __asm__
__volatile__("lock; incl %0":"+m"(*v
));
116 p_atomic_dec(int32_t *v
)
118 __asm__
__volatile__("lock; decl %0":"+m"(*v
));
121 static INLINE
int32_t
122 p_atomic_cmpxchg(int32_t *v
, int32_t old
, int32_t _new
)
124 return __sync_val_compare_and_swap(v
, old
, _new
);
135 /* Implementation using GCC-provided synchronization intrinsics
137 #if defined(PIPE_ATOMIC_GCC_INTRINSIC)
139 #define PIPE_ATOMIC "GCC Sync Intrinsics"
145 #define p_atomic_set(_v, _i) (*(_v) = (_i))
146 #define p_atomic_read(_v) (*(_v))
148 static INLINE boolean
149 p_atomic_dec_zero(int32_t *v
)
151 return (__sync_sub_and_fetch(v
, 1) == 0);
155 p_atomic_inc(int32_t *v
)
157 (void) __sync_add_and_fetch(v
, 1);
161 p_atomic_dec(int32_t *v
)
163 (void) __sync_sub_and_fetch(v
, 1);
166 static INLINE
int32_t
167 p_atomic_cmpxchg(int32_t *v
, int32_t old
, int32_t _new
)
169 return __sync_val_compare_and_swap(v
, old
, _new
);
180 /* Unlocked version for single threaded environments, such as some
181 * windows kernel modules.
183 #if defined(PIPE_ATOMIC_OS_UNLOCKED)
185 #define PIPE_ATOMIC "Unlocked"
187 #define p_atomic_set(_v, _i) (*(_v) = (_i))
188 #define p_atomic_read(_v) (*(_v))
189 #define p_atomic_dec_zero(_v) ((boolean) --(*(_v)))
190 #define p_atomic_inc(_v) ((void) (*(_v))++)
191 #define p_atomic_dec(_v) ((void) (*(_v))--)
192 #define p_atomic_cmpxchg(_v, old, _new) (*(_v) == old ? *(_v) = (_new) : *(_v))
197 /* Locally coded assembly for MSVC on x86:
199 #if defined(PIPE_ATOMIC_ASM_MSVC_X86)
201 #define PIPE_ATOMIC "MSVC x86 assembly"
207 #define p_atomic_set(_v, _i) (*(_v) = (_i))
208 #define p_atomic_read(_v) (*(_v))
210 static INLINE boolean
211 p_atomic_dec_zero(int32_t *v
)
217 lock dec dword ptr
[eax
]
225 p_atomic_inc(int32_t *v
)
229 lock inc dword ptr
[eax
]
234 p_atomic_dec(int32_t *v
)
238 lock dec dword ptr
[eax
]
242 static INLINE
int32_t
243 p_atomic_cmpxchg(int32_t *v
, int32_t old
, int32_t _new
)
251 lock cmpxchg
[ecx
], edx
265 #if defined(PIPE_ATOMIC_MSVC_INTRINSIC)
267 #define PIPE_ATOMIC "MSVC Intrinsics"
271 #pragma intrinsic(_InterlockedIncrement)
272 #pragma intrinsic(_InterlockedDecrement)
273 #pragma intrinsic(_InterlockedCompareExchange)
279 #define p_atomic_set(_v, _i) (*(_v) = (_i))
280 #define p_atomic_read(_v) (*(_v))
282 static INLINE boolean
283 p_atomic_dec_zero(int32_t *v
)
285 return _InterlockedDecrement((long *)v
) == 0;
289 p_atomic_inc(int32_t *v
)
291 _InterlockedIncrement((long *)v
);
295 p_atomic_dec(int32_t *v
)
297 _InterlockedDecrement((long *)v
);
300 static INLINE
int32_t
301 p_atomic_cmpxchg(int32_t *v
, int32_t old
, int32_t _new
)
303 return _InterlockedCompareExchange((long *)v
, _new
, old
);
312 #if defined(PIPE_ATOMIC_OS_SOLARIS)
314 #define PIPE_ATOMIC "Solaris OS atomic functions"
322 #define p_atomic_set(_v, _i) (*(_v) = (_i))
323 #define p_atomic_read(_v) (*(_v))
325 static INLINE boolean
326 p_atomic_dec_zero(int32_t *v
)
328 uint32_t n
= atomic_dec_32_nv((uint32_t *) v
);
333 #define p_atomic_inc(_v) atomic_inc_32((uint32_t *) _v)
334 #define p_atomic_dec(_v) atomic_dec_32((uint32_t *) _v)
336 #define p_atomic_cmpxchg(_v, _old, _new) \
337 atomic_cas_32( (uint32_t *) _v, (uint32_t) _old, (uint32_t) _new)
347 #error "No pipe_atomic implementation selected"
352 #endif /* U_ATOMIC_H */