2 * Many similar implementations exist. See for example libwsbm
3 * or the linux kernel include/atomic.h
5 * No copyright claimed on this file.
14 /* Favor OS-provided implementations.
16 * Where no OS-provided implementation is available, fall back to
17 * locally coded assembly, compiler intrinsic or ultimately a
18 * mutex-based implementation.
21 #define PIPE_ATOMIC_OS_SOLARIS
22 #elif defined(_MSC_VER)
23 #define PIPE_ATOMIC_MSVC_INTRINSIC
24 #elif defined(__GNUC__)
25 #define PIPE_ATOMIC_GCC_INTRINSIC
27 #error "Unsupported platform"
31 /* Implementation using GCC-provided synchronization intrinsics
33 #if defined(PIPE_ATOMIC_GCC_INTRINSIC)
35 #define PIPE_ATOMIC "GCC Sync Intrinsics"
37 #define p_atomic_set(_v, _i) (*(_v) = (_i))
38 #define p_atomic_read(_v) (*(_v))
39 #define p_atomic_dec_zero(v) (__sync_sub_and_fetch((v), 1) == 0)
40 #define p_atomic_inc(v) (void) __sync_add_and_fetch((v), 1)
41 #define p_atomic_dec(v) (void) __sync_sub_and_fetch((v), 1)
42 #define p_atomic_add(v, i) (void) __sync_add_and_fetch((v), (i))
43 #define p_atomic_inc_return(v) __sync_add_and_fetch((v), 1)
44 #define p_atomic_dec_return(v) __sync_sub_and_fetch((v), 1)
45 #define p_atomic_cmpxchg(v, old, _new) \
46 __sync_val_compare_and_swap((v), (old), (_new))
52 /* Unlocked version for single threaded environments, such as some
53 * windows kernel modules.
55 #if defined(PIPE_ATOMIC_OS_UNLOCKED)
57 #define PIPE_ATOMIC "Unlocked"
59 #define p_atomic_set(_v, _i) (*(_v) = (_i))
60 #define p_atomic_read(_v) (*(_v))
61 #define p_atomic_dec_zero(_v) (p_atomic_dec_return(_v) == 0)
62 #define p_atomic_inc(_v) ((void) p_atomic_inc_return(_v))
63 #define p_atomic_dec(_v) ((void) p_atomic_dec_return(_v))
64 #define p_atomic_add(_v, _i) (*(_v) = *(_v) + (_i))
65 #define p_atomic_inc_return(_v) (++(*(_v)))
66 #define p_atomic_dec_return(_v) (--(*(_v)))
67 #define p_atomic_cmpxchg(_v, _old, _new) (*(_v) == (_old) ? (*(_v) = (_new), (_old)) : *(_v))
72 #if defined(PIPE_ATOMIC_MSVC_INTRINSIC)
74 #define PIPE_ATOMIC "MSVC Intrinsics"
76 /* We use the Windows header's Interlocked*64 functions instead of the
77 * _Interlocked*64 intrinsics wherever we can, as support for the latter varies
78 * with target CPU, whereas Windows headers take care of all portability
79 * issues: using intrinsics where available, falling back to library
80 * implementations where not.
82 #ifndef WIN32_LEAN_AND_MEAN
83 #define WIN32_LEAN_AND_MEAN 1
91 /* Implement _InterlockedCompareExchange8 in terms of _InterlockedCompareExchange16 */
93 _InterlockedCompareExchange8(char volatile *destination8
, char exchange8
, char comparand8
)
95 INT_PTR destinationAddr
= (INT_PTR
)destination8
;
96 short volatile *destination16
= (short volatile *)(destinationAddr
& ~1);
97 const short shift8
= (destinationAddr
& 1) * 8;
98 const short mask8
= 0xff << shift8
;
99 short initial16
= *destination16
;
100 char initial8
= initial16
>> shift8
;
101 while (initial8
== comparand8
) {
102 /* initial *destination8 matches, so try exchange it while keeping the
103 * neighboring byte untouched */
104 short exchange16
= (initial16
& ~mask8
) | ((short)exchange8
<< shift8
);
105 short comparand16
= initial16
;
106 short initial16
= _InterlockedCompareExchange16(destination16
, exchange16
, comparand16
);
107 if (initial16
== comparand16
) {
111 /* something changed, retry with the new initial value */
112 initial8
= initial16
>> shift8
;
117 /* Implement _InterlockedExchangeAdd16 in terms of _InterlockedCompareExchange16 */
118 static __inline
short
119 _InterlockedExchangeAdd16(short volatile *addend
, short value
)
121 short initial
= *addend
;
124 short exchange
= initial
+ value
;
126 /* if *addend==comparand then *addend=exchange, return original *addend */
127 initial
= _InterlockedCompareExchange16(addend
, exchange
, comparand
);
128 } while(initial
!= comparand
);
132 /* Implement _InterlockedExchangeAdd8 in terms of _InterlockedCompareExchange8 */
134 _InterlockedExchangeAdd8(char volatile *addend
, char value
)
136 char initial
= *addend
;
139 char exchange
= initial
+ value
;
141 initial
= _InterlockedCompareExchange8(addend
, exchange
, comparand
);
142 } while(initial
!= comparand
);
146 #endif /* _MSC_VER < 1600 */
148 /* MSVC supports decltype keyword, but it's only supported on C++ and doesn't
149 * quite work here; and if a C++-only solution is worthwhile, then it would be
150 * better to use templates / function overloading, instead of decltype magic.
151 * Therefore, we rely on implicit casting to LONGLONG for the functions that return
154 #define p_atomic_set(_v, _i) (*(_v) = (_i))
155 #define p_atomic_read(_v) (*(_v))
157 #define p_atomic_dec_zero(_v) \
158 (p_atomic_dec_return(_v) == 0)
160 #define p_atomic_inc(_v) \
161 ((void) p_atomic_inc_return(_v))
163 #define p_atomic_inc_return(_v) (\
164 sizeof *(_v) == sizeof(short) ? _InterlockedIncrement16((short *) (_v)) : \
165 sizeof *(_v) == sizeof(long) ? _InterlockedIncrement ((long *) (_v)) : \
166 sizeof *(_v) == sizeof(__int64) ? InterlockedIncrement64 ((__int64 *)(_v)) : \
167 (assert(!"should not get here"), 0))
169 #define p_atomic_dec(_v) \
170 ((void) p_atomic_dec_return(_v))
172 #define p_atomic_dec_return(_v) (\
173 sizeof *(_v) == sizeof(short) ? _InterlockedDecrement16((short *) (_v)) : \
174 sizeof *(_v) == sizeof(long) ? _InterlockedDecrement ((long *) (_v)) : \
175 sizeof *(_v) == sizeof(__int64) ? InterlockedDecrement64 ((__int64 *)(_v)) : \
176 (assert(!"should not get here"), 0))
178 #define p_atomic_add(_v, _i) (\
179 sizeof *(_v) == sizeof(char) ? _InterlockedExchangeAdd8 ((char *) (_v), (_i)) : \
180 sizeof *(_v) == sizeof(short) ? _InterlockedExchangeAdd16((short *) (_v), (_i)) : \
181 sizeof *(_v) == sizeof(long) ? _InterlockedExchangeAdd ((long *) (_v), (_i)) : \
182 sizeof *(_v) == sizeof(__int64) ? InterlockedExchangeAdd64((__int64 *)(_v), (_i)) : \
183 (assert(!"should not get here"), 0))
185 #define p_atomic_cmpxchg(_v, _old, _new) (\
186 sizeof *(_v) == sizeof(char) ? _InterlockedCompareExchange8 ((char *) (_v), (char) (_new), (char) (_old)) : \
187 sizeof *(_v) == sizeof(short) ? _InterlockedCompareExchange16((short *) (_v), (short) (_new), (short) (_old)) : \
188 sizeof *(_v) == sizeof(long) ? _InterlockedCompareExchange ((long *) (_v), (long) (_new), (long) (_old)) : \
189 sizeof *(_v) == sizeof(__int64) ? InterlockedCompareExchange64 ((__int64 *)(_v), (__int64)(_new), (__int64)(_old)) : \
190 (assert(!"should not get here"), 0))
194 #if defined(PIPE_ATOMIC_OS_SOLARIS)
196 #define PIPE_ATOMIC "Solaris OS atomic functions"
201 #define p_atomic_set(_v, _i) (*(_v) = (_i))
202 #define p_atomic_read(_v) (*(_v))
204 #define p_atomic_dec_zero(v) (\
205 sizeof(*v) == sizeof(uint8_t) ? atomic_dec_8_nv ((uint8_t *)(v)) == 0 : \
206 sizeof(*v) == sizeof(uint16_t) ? atomic_dec_16_nv((uint16_t *)(v)) == 0 : \
207 sizeof(*v) == sizeof(uint32_t) ? atomic_dec_32_nv((uint32_t *)(v)) == 0 : \
208 sizeof(*v) == sizeof(uint64_t) ? atomic_dec_64_nv((uint64_t *)(v)) == 0 : \
209 (assert(!"should not get here"), 0))
211 #define p_atomic_inc(v) (void) (\
212 sizeof(*v) == sizeof(uint8_t) ? atomic_inc_8 ((uint8_t *)(v)) : \
213 sizeof(*v) == sizeof(uint16_t) ? atomic_inc_16((uint16_t *)(v)) : \
214 sizeof(*v) == sizeof(uint32_t) ? atomic_inc_32((uint32_t *)(v)) : \
215 sizeof(*v) == sizeof(uint64_t) ? atomic_inc_64((uint64_t *)(v)) : \
216 (assert(!"should not get here"), 0))
218 #define p_atomic_inc_return(v) ((typeof(*v)) \
219 sizeof(*v) == sizeof(uint8_t) ? atomic_inc_8_nv ((uint8_t *)(v)) : \
220 sizeof(*v) == sizeof(uint16_t) ? atomic_inc_16_nv((uint16_t *)(v)) : \
221 sizeof(*v) == sizeof(uint32_t) ? atomic_inc_32_nv((uint32_t *)(v)) : \
222 sizeof(*v) == sizeof(uint64_t) ? atomic_inc_64_nv((uint64_t *)(v)) : \
223 (assert(!"should not get here"), 0))
225 #define p_atomic_dec(v) ((void) \
226 sizeof(*v) == sizeof(uint8_t) ? atomic_dec_8 ((uint8_t *)(v)) : \
227 sizeof(*v) == sizeof(uint16_t) ? atomic_dec_16((uint16_t *)(v)) : \
228 sizeof(*v) == sizeof(uint32_t) ? atomic_dec_32((uint32_t *)(v)) : \
229 sizeof(*v) == sizeof(uint64_t) ? atomic_dec_64((uint64_t *)(v)) : \
230 (assert(!"should not get here"), 0))
232 #define p_atomic_dec_return(v) ((typeof(*v)) \
233 sizeof(*v) == sizeof(uint8_t) ? atomic_dec_8_nv ((uint8_t *)(v)) : \
234 sizeof(*v) == sizeof(uint16_t) ? atomic_dec_16_nv((uint16_t *)(v)) : \
235 sizeof(*v) == sizeof(uint32_t) ? atomic_dec_32_nv((uint32_t *)(v)) : \
236 sizeof(*v) == sizeof(uint64_t) ? atomic_dec_64_nv((uint64_t *)(v)) : \
237 (assert(!"should not get here"), 0))
239 #define p_atomic_add(v, i) ((void) \
240 sizeof(*v) == sizeof(uint8_t) ? atomic_add_8 ((uint8_t *)(v), (i)) : \
241 sizeof(*v) == sizeof(uint16_t) ? atomic_add_16((uint16_t *)(v), (i)) : \
242 sizeof(*v) == sizeof(uint32_t) ? atomic_add_32((uint32_t *)(v), (i)) : \
243 sizeof(*v) == sizeof(uint64_t) ? atomic_add_64((uint64_t *)(v), (i)) : \
244 (assert(!"should not get here"), 0))
246 #define p_atomic_cmpxchg(v, old, _new) ((typeof(*v)) \
247 sizeof(*v) == sizeof(uint8_t) ? atomic_cas_8 ((uint8_t *)(v), (uint8_t )(old), (uint8_t )(_new)) : \
248 sizeof(*v) == sizeof(uint16_t) ? atomic_cas_16((uint16_t *)(v), (uint16_t)(old), (uint16_t)(_new)) : \
249 sizeof(*v) == sizeof(uint32_t) ? atomic_cas_32((uint32_t *)(v), (uint32_t)(old), (uint32_t)(_new)) : \
250 sizeof(*v) == sizeof(uint64_t) ? atomic_cas_64((uint64_t *)(v), (uint64_t)(old), (uint64_t)(_new)) : \
251 (assert(!"should not get here"), 0))
256 #error "No pipe_atomic implementation selected"
261 #endif /* U_ATOMIC_H */