1 | /* ---------------------------------------------------------------------------- |
2 | Copyright (c) 2018, Microsoft Research, Daan Leijen |
3 | This is free software; you can redistribute it and/or modify it under the |
4 | terms of the MIT license. A copy of the license can be found in the file |
5 | "LICENSE" at the root of this distribution. |
6 | -----------------------------------------------------------------------------*/ |
7 | #pragma once |
8 | #ifndef MIMALLOC_ATOMIC_H |
9 | #define MIMALLOC_ATOMIC_H |
10 | |
11 | // ------------------------------------------------------ |
12 | // Atomics |
13 | // We need to be portable between C, C++, and MSVC. |
14 | // ------------------------------------------------------ |
15 | |
16 | #if defined(_MSC_VER) |
17 | #define _Atomic(tp) tp |
18 | #define ATOMIC_VAR_INIT(x) x |
19 | #elif defined(__cplusplus) |
20 | #include <atomic> |
21 | #define _Atomic(tp) std::atomic<tp> |
22 | #else |
23 | #include <stdatomic.h> |
24 | #endif |
25 | |
26 | #define mi_atomic_cast(tp,x) (volatile _Atomic(tp)*)(x) |
27 | |
28 | // ------------------------------------------------------ |
29 | // Atomic operations specialized for mimalloc |
30 | // ------------------------------------------------------ |
31 | |
32 | // Atomically add a 64-bit value; returns the previous value. |
33 | // Note: not using _Atomic(int64_t) as it is only used for statistics. |
34 | static inline void mi_atomic_add64(volatile int64_t* p, int64_t add); |
35 | |
36 | // Atomically add a value; returns the previous value. Memory ordering is relaxed. |
37 | static inline intptr_t mi_atomic_add(volatile _Atomic(intptr_t)* p, intptr_t add); |
38 | |
39 | // Atomically compare and exchange a value; returns `true` if successful. |
40 | // May fail spuriously. Memory ordering as release on success, and relaxed on failure. |
41 | // (Note: expected and desired are in opposite order from atomic_compare_exchange) |
42 | static inline bool mi_atomic_cas_weak(volatile _Atomic(uintptr_t)* p, uintptr_t desired, uintptr_t expected); |
43 | |
44 | // Atomically compare and exchange a value; returns `true` if successful. |
45 | // Memory ordering is acquire-release |
46 | // (Note: expected and desired are in opposite order from atomic_compare_exchange) |
47 | static inline bool mi_atomic_cas_strong(volatile _Atomic(uintptr_t)* p, uintptr_t desired, uintptr_t expected); |
48 | |
49 | // Atomically exchange a value. Memory ordering is acquire-release. |
50 | static inline uintptr_t mi_atomic_exchange(volatile _Atomic(uintptr_t)* p, uintptr_t exchange); |
51 | |
52 | // Atomically read a value. Memory ordering is relaxed. |
53 | static inline uintptr_t mi_atomic_read_relaxed(const volatile _Atomic(uintptr_t)* p); |
54 | |
55 | // Atomically read a value. Memory ordering is acquire. |
56 | static inline uintptr_t mi_atomic_read(const volatile _Atomic(uintptr_t)* p); |
57 | |
58 | // Atomically write a value. Memory ordering is release. |
59 | static inline void mi_atomic_write(volatile _Atomic(uintptr_t)* p, uintptr_t x); |
60 | |
61 | // Yield |
62 | static inline void mi_atomic_yield(void); |
63 | |
64 | |
65 | |
66 | // Atomically add a value; returns the previous value. |
67 | static inline uintptr_t mi_atomic_addu(volatile _Atomic(uintptr_t)* p, uintptr_t add) { |
68 | return (uintptr_t)mi_atomic_add((volatile _Atomic(intptr_t)*)p, (intptr_t)add); |
69 | } |
70 | // Atomically subtract a value; returns the previous value. |
71 | static inline uintptr_t mi_atomic_subu(volatile _Atomic(uintptr_t)* p, uintptr_t sub) { |
72 | return (uintptr_t)mi_atomic_add((volatile _Atomic(intptr_t)*)p, -((intptr_t)sub)); |
73 | } |
74 | |
75 | // Atomically increment a value; returns the incremented result. |
76 | static inline uintptr_t mi_atomic_increment(volatile _Atomic(uintptr_t)* p) { |
77 | return mi_atomic_addu(p, 1); |
78 | } |
79 | |
80 | // Atomically decrement a value; returns the decremented result. |
81 | static inline uintptr_t mi_atomic_decrement(volatile _Atomic(uintptr_t)* p) { |
82 | return mi_atomic_subu(p, 1); |
83 | } |
84 | |
85 | // Atomically read a pointer; Memory order is relaxed. |
86 | static inline void* mi_atomic_read_ptr_relaxed(volatile _Atomic(void*) const * p) { |
87 | return (void*)mi_atomic_read_relaxed((const volatile _Atomic(uintptr_t)*)p); |
88 | } |
89 | |
90 | // Atomically read a pointer; Memory order is acquire. |
91 | static inline void* mi_atomic_read_ptr(volatile _Atomic(void*) const * p) { |
92 | return (void*)mi_atomic_read((const volatile _Atomic(uintptr_t)*)p); |
93 | } |
94 | |
95 | // Atomically write a pointer |
96 | static inline void mi_atomic_write_ptr(volatile _Atomic(void*)* p, void* x) { |
97 | mi_atomic_write((volatile _Atomic(uintptr_t)*)p, (uintptr_t)x ); |
98 | } |
99 | |
100 | // Atomically compare and exchange a pointer; returns `true` if successful. May fail spuriously. |
101 | // (Note: expected and desired are in opposite order from atomic_compare_exchange) |
102 | static inline bool mi_atomic_cas_ptr_weak(volatile _Atomic(void*)* p, void* desired, void* expected) { |
103 | return mi_atomic_cas_weak((volatile _Atomic(uintptr_t)*)p, (uintptr_t)desired, (uintptr_t)expected); |
104 | } |
105 | |
106 | // Atomically compare and exchange a pointer; returns `true` if successful. |
107 | // (Note: expected and desired are in opposite order from atomic_compare_exchange) |
108 | static inline bool mi_atomic_cas_ptr_strong(volatile _Atomic(void*)* p, void* desired, void* expected) { |
109 | return mi_atomic_cas_strong((volatile _Atomic(uintptr_t)*)p, (uintptr_t)desired, (uintptr_t)expected); |
110 | } |
111 | |
112 | // Atomically exchange a pointer value. |
113 | static inline void* mi_atomic_exchange_ptr(volatile _Atomic(void*)* p, void* exchange) { |
114 | return (void*)mi_atomic_exchange((volatile _Atomic(uintptr_t)*)p, (uintptr_t)exchange); |
115 | } |
116 | |
117 | |
118 | #ifdef _MSC_VER |
119 | #define WIN32_LEAN_AND_MEAN |
120 | #include <windows.h> |
121 | #include <intrin.h> |
122 | #ifdef _WIN64 |
123 | typedef LONG64 msc_intptr_t; |
124 | #define RC64(f) f##64 |
125 | #else |
126 | typedef LONG msc_intptr_t; |
127 | #define RC64(f) f |
128 | #endif |
129 | static inline intptr_t mi_atomic_add(volatile _Atomic(intptr_t)* p, intptr_t add) { |
130 | return (intptr_t)RC64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, (msc_intptr_t)add); |
131 | } |
132 | static inline bool mi_atomic_cas_strong(volatile _Atomic(uintptr_t)* p, uintptr_t desired, uintptr_t expected) { |
133 | return (expected == (uintptr_t)RC64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)desired, (msc_intptr_t)expected)); |
134 | } |
135 | static inline bool mi_atomic_cas_weak(volatile _Atomic(uintptr_t)* p, uintptr_t desired, uintptr_t expected) { |
136 | return mi_atomic_cas_strong(p,desired,expected); |
137 | } |
138 | static inline uintptr_t mi_atomic_exchange(volatile _Atomic(uintptr_t)* p, uintptr_t exchange) { |
139 | return (uintptr_t)RC64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange); |
140 | } |
141 | static inline uintptr_t mi_atomic_read(volatile _Atomic(uintptr_t) const* p) { |
142 | return *p; |
143 | } |
144 | static inline uintptr_t mi_atomic_read_relaxed(volatile _Atomic(uintptr_t) const* p) { |
145 | return mi_atomic_read(p); |
146 | } |
147 | static inline void mi_atomic_write(volatile _Atomic(uintptr_t)* p, uintptr_t x) { |
148 | mi_atomic_exchange(p,x); |
149 | } |
150 | static inline void mi_atomic_yield(void) { |
151 | YieldProcessor(); |
152 | } |
153 | static inline void mi_atomic_add64(volatile _Atomic(int64_t)* p, int64_t add) { |
154 | #ifdef _WIN64 |
155 | mi_atomic_add(p,add); |
156 | #else |
157 | int64_t current; |
158 | int64_t sum; |
159 | do { |
160 | current = *p; |
161 | sum = current + add; |
162 | } while (_InterlockedCompareExchange64(p, sum, current) != current); |
163 | #endif |
164 | } |
165 | |
166 | #else |
167 | #ifdef __cplusplus |
168 | #define MI_USING_STD using namespace std; |
169 | #else |
170 | #define MI_USING_STD |
171 | #endif |
172 | static inline void mi_atomic_add64(volatile int64_t* p, int64_t add) { |
173 | MI_USING_STD |
174 | atomic_fetch_add_explicit((volatile _Atomic(int64_t)*)p, add, memory_order_relaxed); |
175 | } |
176 | static inline intptr_t mi_atomic_add(volatile _Atomic(intptr_t)* p, intptr_t add) { |
177 | MI_USING_STD |
178 | return atomic_fetch_add_explicit(p, add, memory_order_relaxed); |
179 | } |
180 | static inline bool mi_atomic_cas_weak(volatile _Atomic(uintptr_t)* p, uintptr_t desired, uintptr_t expected) { |
181 | MI_USING_STD |
182 | return atomic_compare_exchange_weak_explicit(p, &expected, desired, memory_order_release, memory_order_relaxed); |
183 | } |
184 | static inline bool mi_atomic_cas_strong(volatile _Atomic(uintptr_t)* p, uintptr_t desired, uintptr_t expected) { |
185 | MI_USING_STD |
186 | return atomic_compare_exchange_strong_explicit(p, &expected, desired, memory_order_acq_rel, memory_order_relaxed); |
187 | } |
188 | static inline uintptr_t mi_atomic_exchange(volatile _Atomic(uintptr_t)* p, uintptr_t exchange) { |
189 | MI_USING_STD |
190 | return atomic_exchange_explicit(p, exchange, memory_order_acq_rel); |
191 | } |
192 | static inline uintptr_t mi_atomic_read_relaxed(const volatile _Atomic(uintptr_t)* p) { |
193 | MI_USING_STD |
194 | return atomic_load_explicit((volatile _Atomic(uintptr_t)*) p, memory_order_relaxed); |
195 | } |
196 | static inline uintptr_t mi_atomic_read(const volatile _Atomic(uintptr_t)* p) { |
197 | MI_USING_STD |
198 | return atomic_load_explicit((volatile _Atomic(uintptr_t)*) p, memory_order_acquire); |
199 | } |
200 | static inline void mi_atomic_write(volatile _Atomic(uintptr_t)* p, uintptr_t x) { |
201 | MI_USING_STD |
202 | return atomic_store_explicit(p, x, memory_order_release); |
203 | } |
204 | |
205 | #if defined(__cplusplus) |
206 | #include <thread> |
207 | static inline void mi_atomic_yield(void) { |
208 | std::this_thread::yield(); |
209 | } |
210 | #elif (defined(__GNUC__) || defined(__clang__)) && \ |
211 | (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__)) |
212 | #if defined(__x86_64__) || defined(__i386__) |
213 | static inline void mi_atomic_yield(void) { |
214 | asm volatile ("pause" ::: "memory" ); |
215 | } |
216 | #elif defined(__arm__) || defined(__aarch64__) |
217 | static inline void mi_atomic_yield(void) { |
218 | asm volatile("yield" ); |
219 | } |
220 | #endif |
221 | #elif defined(__wasi__) |
222 | #include <sched.h> |
223 | static inline void mi_atomic_yield(void) { |
224 | sched_yield(); |
225 | } |
226 | #else |
227 | #include <unistd.h> |
228 | static inline void mi_atomic_yield(void) { |
229 | sleep(0); |
230 | } |
231 | #endif |
232 | |
233 | #endif |
234 | |
235 | #endif // __MIMALLOC_ATOMIC_H |
236 | |