2 * Many similar implementations exist. See for example libwsbm
3 * or the linux kernel include/atomic.h
5 * No copyright claimed on this file.
14 /* Favor OS-provided implementations.
16 * Where no OS-provided implementation is available, fall back to
17 * locally coded assembly, compiler intrinsic or ultimately a
18 * mutex-based implementation.
21 #define PIPE_ATOMIC_OS_SOLARIS
22 #elif defined(_MSC_VER)
23 #define PIPE_ATOMIC_MSVC_INTRINSIC
24 #elif defined(__GNUC__)
25 #define PIPE_ATOMIC_GCC_INTRINSIC
27 #error "Unsupported platform"
31 /* Implementation using GCC-provided synchronization intrinsics
33 #if defined(PIPE_ATOMIC_GCC_INTRINSIC)
35 #define PIPE_ATOMIC "GCC Sync Intrinsics"
37 #define p_atomic_set(_v, _i) (*(_v) = (_i))
38 #define p_atomic_read(_v) (*(_v))
39 #define p_atomic_dec_zero(v) (__sync_sub_and_fetch((v), 1) == 0)
40 #define p_atomic_inc(v) (void) __sync_add_and_fetch((v), 1)
41 #define p_atomic_dec(v) (void) __sync_sub_and_fetch((v), 1)
42 #define p_atomic_inc_return(v) __sync_add_and_fetch((v), 1)
43 #define p_atomic_dec_return(v) __sync_sub_and_fetch((v), 1)
44 #define p_atomic_cmpxchg(v, old, _new) \
45 __sync_val_compare_and_swap((v), (old), (_new))
51 /* Unlocked version for single threaded environments, such as some
52 * windows kernel modules.
54 #if defined(PIPE_ATOMIC_OS_UNLOCKED)
56 #define PIPE_ATOMIC "Unlocked"
58 #define p_atomic_set(_v, _i) (*(_v) = (_i))
59 #define p_atomic_read(_v) (*(_v))
60 #define p_atomic_dec_zero(_v) (p_atomic_dec_return(_v) == 0)
61 #define p_atomic_inc(_v) ((void) p_atomic_inc_return(_v))
62 #define p_atomic_dec(_v) ((void) p_atomic_dec_return(_v))
63 #define p_atomic_inc_return(_v) (++(*(_v)))
64 #define p_atomic_dec_return(_v) (--(*(_v)))
65 #define p_atomic_cmpxchg(_v, _old, _new) (*(_v) == (_old) ? (*(_v) = (_new), (_old)) : *(_v))
70 #if defined(PIPE_ATOMIC_MSVC_INTRINSIC)
72 #define PIPE_ATOMIC "MSVC Intrinsics"
74 /* We use the Windows header's Interlocked*64 functions instead of the
75 * _Interlocked*64 intrinsics wherever we can, as support for the latter varies
76 * with target CPU, whereas Windows headers take care of all portability
77 * issues: using intrinsics where available, falling back to library
78 * implementations where not.
80 #ifndef WIN32_LEAN_AND_MEAN
81 #define WIN32_LEAN_AND_MEAN 1
89 /* Implement _InterlockedCompareExchange8 in terms of InterlockedCompareExchange16 */
91 char _InterlockedCompareExchange8(char volatile *Destination8
, char Exchange8
, char Comparand8
)
93 INT_PTR DestinationAddr
= (INT_PTR
)Destination8
;
94 short volatile *Destination16
= (short volatile *)(DestinationAddr
& ~1);
95 const short Shift8
= (DestinationAddr
& 1) * 8;
96 const short Mask8
= 0xff << Shift8
;
97 short Initial16
= *Destination16
;
98 char Initial8
= Initial16
>> Shift8
;
99 while (Initial8
== Comparand8
) {
100 /* initial *Destination8 matches, so try exchange it while keeping the
101 * neighboring byte untouched */
102 short Exchange16
= (Initial16
& ~Mask8
) | ((short)Exchange8
<< Shift8
);
103 short Comparand16
= Initial16
;
104 short Initial16
= InterlockedCompareExchange16(Destination16
, Exchange16
, Comparand16
);
105 if (Initial16
== Comparand16
) {
109 /* something changed, retry with the new initial value */
110 Initial8
= Initial16
>> Shift8
;
115 #endif /* _MSC_VER < 1600 */
117 /* MSVC supports decltype keyword, but it's only supported on C++ and doesn't
118 * quite work here; and if a C++-only solution is worthwhile, then it would be
119 * better to use templates / function overloading, instead of decltype magic.
120 * Therefore, we rely on implicit casting to LONGLONG for the functions that return
123 #define p_atomic_set(_v, _i) (*(_v) = (_i))
124 #define p_atomic_read(_v) (*(_v))
126 #define p_atomic_dec_zero(_v) \
127 (p_atomic_dec_return(_v) == 0)
129 #define p_atomic_inc(_v) \
130 ((void) p_atomic_inc_return(_v))
132 #define p_atomic_inc_return(_v) (\
133 sizeof *(_v) == sizeof(short) ? _InterlockedIncrement16((short *) (_v)) : \
134 sizeof *(_v) == sizeof(long) ? _InterlockedIncrement ((long *) (_v)) : \
135 sizeof *(_v) == sizeof(__int64) ? InterlockedIncrement64 ((__int64 *)(_v)) : \
136 (assert(!"should not get here"), 0))
138 #define p_atomic_dec(_v) \
139 ((void) p_atomic_dec_return(_v))
141 #define p_atomic_dec_return(_v) (\
142 sizeof *(_v) == sizeof(short) ? _InterlockedDecrement16((short *) (_v)) : \
143 sizeof *(_v) == sizeof(long) ? _InterlockedDecrement ((long *) (_v)) : \
144 sizeof *(_v) == sizeof(__int64) ? InterlockedDecrement64 ((__int64 *)(_v)) : \
145 (assert(!"should not get here"), 0))
147 #define p_atomic_cmpxchg(_v, _old, _new) (\
148 sizeof *(_v) == sizeof(char) ? _InterlockedCompareExchange8 ((char *) (_v), (char) (_new), (char) (_old)) : \
149 sizeof *(_v) == sizeof(short) ? _InterlockedCompareExchange16((short *) (_v), (short) (_new), (short) (_old)) : \
150 sizeof *(_v) == sizeof(long) ? _InterlockedCompareExchange ((long *) (_v), (long) (_new), (long) (_old)) : \
151 sizeof *(_v) == sizeof(__int64) ? InterlockedCompareExchange64 ((__int64 *)(_v), (__int64)(_new), (__int64)(_old)) : \
152 (assert(!"should not get here"), 0))
156 #if defined(PIPE_ATOMIC_OS_SOLARIS)
158 #define PIPE_ATOMIC "Solaris OS atomic functions"
163 #define p_atomic_set(_v, _i) (*(_v) = (_i))
164 #define p_atomic_read(_v) (*(_v))
166 #define p_atomic_dec_zero(v) (\
167 sizeof(*v) == sizeof(uint8_t) ? atomic_dec_8_nv ((uint8_t *)(v)) == 0 : \
168 sizeof(*v) == sizeof(uint16_t) ? atomic_dec_16_nv((uint16_t *)(v)) == 0 : \
169 sizeof(*v) == sizeof(uint32_t) ? atomic_dec_32_nv((uint32_t *)(v)) == 0 : \
170 sizeof(*v) == sizeof(uint64_t) ? atomic_dec_64_nv((uint64_t *)(v)) == 0 : \
171 (assert(!"should not get here"), 0))
173 #define p_atomic_inc(v) (void) (\
174 sizeof(*v) == sizeof(uint8_t) ? atomic_inc_8 ((uint8_t *)(v)) : \
175 sizeof(*v) == sizeof(uint16_t) ? atomic_inc_16((uint16_t *)(v)) : \
176 sizeof(*v) == sizeof(uint32_t) ? atomic_inc_32((uint32_t *)(v)) : \
177 sizeof(*v) == sizeof(uint64_t) ? atomic_inc_64((uint64_t *)(v)) : \
178 (assert(!"should not get here"), 0))
180 #define p_atomic_inc_return(v) ((typeof(*v)) \
181 sizeof(*v) == sizeof(uint8_t) ? atomic_inc_8_nv ((uint8_t *)(v)) : \
182 sizeof(*v) == sizeof(uint16_t) ? atomic_inc_16_nv((uint16_t *)(v)) : \
183 sizeof(*v) == sizeof(uint32_t) ? atomic_inc_32_nv((uint32_t *)(v)) : \
184 sizeof(*v) == sizeof(uint64_t) ? atomic_inc_64_nv((uint64_t *)(v)) : \
185 (assert(!"should not get here"), 0))
187 #define p_atomic_dec(v) ((void) \
188 sizeof(*v) == sizeof(uint8_t) ? atomic_dec_8 ((uint8_t *)(v)) : \
189 sizeof(*v) == sizeof(uint16_t) ? atomic_dec_16((uint16_t *)(v)) : \
190 sizeof(*v) == sizeof(uint32_t) ? atomic_dec_32((uint32_t *)(v)) : \
191 sizeof(*v) == sizeof(uint64_t) ? atomic_dec_64((uint64_t *)(v)) : \
192 (assert(!"should not get here"), 0))
194 #define p_atomic_dec_return(v) ((typeof(*v)) \
195 sizeof(*v) == sizeof(uint8_t) ? atomic_dec_8_nv ((uint8_t *)(v)) : \
196 sizeof(*v) == sizeof(uint16_t) ? atomic_dec_16_nv((uint16_t *)(v)) : \
197 sizeof(*v) == sizeof(uint32_t) ? atomic_dec_32_nv((uint32_t *)(v)) : \
198 sizeof(*v) == sizeof(uint64_t) ? atomic_dec_64_nv((uint64_t *)(v)) : \
199 (assert(!"should not get here"), 0))
201 #define p_atomic_cmpxchg(v, old, _new) ((typeof(*v)) \
202 sizeof(*v) == sizeof(uint8_t) ? atomic_cas_8 ((uint8_t *)(v), (uint8_t )(old), (uint8_t )(_new)) : \
203 sizeof(*v) == sizeof(uint16_t) ? atomic_cas_16((uint16_t *)(v), (uint16_t)(old), (uint16_t)(_new)) : \
204 sizeof(*v) == sizeof(uint32_t) ? atomic_cas_32((uint32_t *)(v), (uint32_t)(old), (uint32_t)(_new)) : \
205 sizeof(*v) == sizeof(uint64_t) ? atomic_cas_64((uint64_t *)(v), (uint64_t)(old), (uint64_t)(_new)) : \
206 (assert(!"should not get here"), 0))
211 #error "No pipe_atomic implementation selected"
216 #endif /* U_ATOMIC_H */