1/* ----------------------------------------------------------------------------
2Copyright (c) 2018, Microsoft Research, Daan Leijen
3This is free software; you can redistribute it and/or modify it under the
4terms of the MIT license. A copy of the license can be found in the file
5"LICENSE" at the root of this distribution.
6-----------------------------------------------------------------------------*/
7#pragma once
8#ifndef MIMALLOC_ATOMIC_H
9#define MIMALLOC_ATOMIC_H
10
11// ------------------------------------------------------
12// Atomics
13// We need to be portable between C, C++, and MSVC.
14// ------------------------------------------------------
15
16#if defined(_MSC_VER)
17#define _Atomic(tp) tp
18#define ATOMIC_VAR_INIT(x) x
19#elif defined(__cplusplus)
20#include <atomic>
21#define _Atomic(tp) std::atomic<tp>
22#else
23#include <stdatomic.h>
24#endif
25
26#define mi_atomic_cast(tp,x) (volatile _Atomic(tp)*)(x)
27
28// ------------------------------------------------------
29// Atomic operations specialized for mimalloc
30// ------------------------------------------------------
31
32// Atomically add a 64-bit value; returns the previous value.
33// Note: not using _Atomic(int64_t) as it is only used for statistics.
34static inline void mi_atomic_add64(volatile int64_t* p, int64_t add);
35
36// Atomically add a value; returns the previous value. Memory ordering is relaxed.
37static inline intptr_t mi_atomic_add(volatile _Atomic(intptr_t)* p, intptr_t add);
38
39// Atomically compare and exchange a value; returns `true` if successful.
40// May fail spuriously. Memory ordering as release on success, and relaxed on failure.
41// (Note: expected and desired are in opposite order from atomic_compare_exchange)
42static inline bool mi_atomic_cas_weak(volatile _Atomic(uintptr_t)* p, uintptr_t desired, uintptr_t expected);
43
44// Atomically compare and exchange a value; returns `true` if successful.
45// Memory ordering is acquire-release
46// (Note: expected and desired are in opposite order from atomic_compare_exchange)
47static inline bool mi_atomic_cas_strong(volatile _Atomic(uintptr_t)* p, uintptr_t desired, uintptr_t expected);
48
49// Atomically exchange a value. Memory ordering is acquire-release.
50static inline uintptr_t mi_atomic_exchange(volatile _Atomic(uintptr_t)* p, uintptr_t exchange);
51
52// Atomically read a value. Memory ordering is relaxed.
53static inline uintptr_t mi_atomic_read_relaxed(const volatile _Atomic(uintptr_t)* p);
54
55// Atomically read a value. Memory ordering is acquire.
56static inline uintptr_t mi_atomic_read(const volatile _Atomic(uintptr_t)* p);
57
58// Atomically write a value. Memory ordering is release.
59static inline void mi_atomic_write(volatile _Atomic(uintptr_t)* p, uintptr_t x);
60
61// Yield
62static inline void mi_atomic_yield(void);
63
64
65
66// Atomically add a value; returns the previous value.
67static inline uintptr_t mi_atomic_addu(volatile _Atomic(uintptr_t)* p, uintptr_t add) {
68 return (uintptr_t)mi_atomic_add((volatile _Atomic(intptr_t)*)p, (intptr_t)add);
69}
70// Atomically subtract a value; returns the previous value.
71static inline uintptr_t mi_atomic_subu(volatile _Atomic(uintptr_t)* p, uintptr_t sub) {
72 return (uintptr_t)mi_atomic_add((volatile _Atomic(intptr_t)*)p, -((intptr_t)sub));
73}
74
75// Atomically increment a value; returns the incremented result.
76static inline uintptr_t mi_atomic_increment(volatile _Atomic(uintptr_t)* p) {
77 return mi_atomic_addu(p, 1);
78}
79
80// Atomically decrement a value; returns the decremented result.
81static inline uintptr_t mi_atomic_decrement(volatile _Atomic(uintptr_t)* p) {
82 return mi_atomic_subu(p, 1);
83}
84
85// Atomically read a pointer; Memory order is relaxed.
86static inline void* mi_atomic_read_ptr_relaxed(volatile _Atomic(void*) const * p) {
87 return (void*)mi_atomic_read_relaxed((const volatile _Atomic(uintptr_t)*)p);
88}
89
90// Atomically read a pointer; Memory order is acquire.
91static inline void* mi_atomic_read_ptr(volatile _Atomic(void*) const * p) {
92 return (void*)mi_atomic_read((const volatile _Atomic(uintptr_t)*)p);
93}
94
95// Atomically write a pointer
96static inline void mi_atomic_write_ptr(volatile _Atomic(void*)* p, void* x) {
97 mi_atomic_write((volatile _Atomic(uintptr_t)*)p, (uintptr_t)x );
98}
99
100// Atomically compare and exchange a pointer; returns `true` if successful. May fail spuriously.
101// (Note: expected and desired are in opposite order from atomic_compare_exchange)
102static inline bool mi_atomic_cas_ptr_weak(volatile _Atomic(void*)* p, void* desired, void* expected) {
103 return mi_atomic_cas_weak((volatile _Atomic(uintptr_t)*)p, (uintptr_t)desired, (uintptr_t)expected);
104}
105
106// Atomically compare and exchange a pointer; returns `true` if successful.
107// (Note: expected and desired are in opposite order from atomic_compare_exchange)
108static inline bool mi_atomic_cas_ptr_strong(volatile _Atomic(void*)* p, void* desired, void* expected) {
109 return mi_atomic_cas_strong((volatile _Atomic(uintptr_t)*)p, (uintptr_t)desired, (uintptr_t)expected);
110}
111
112// Atomically exchange a pointer value.
113static inline void* mi_atomic_exchange_ptr(volatile _Atomic(void*)* p, void* exchange) {
114 return (void*)mi_atomic_exchange((volatile _Atomic(uintptr_t)*)p, (uintptr_t)exchange);
115}
116
117
118#ifdef _MSC_VER
119#define WIN32_LEAN_AND_MEAN
120#include <windows.h>
121#include <intrin.h>
122#ifdef _WIN64
123typedef LONG64 msc_intptr_t;
124#define RC64(f) f##64
125#else
126typedef LONG msc_intptr_t;
127#define RC64(f) f
128#endif
129static inline intptr_t mi_atomic_add(volatile _Atomic(intptr_t)* p, intptr_t add) {
130 return (intptr_t)RC64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, (msc_intptr_t)add);
131}
132static inline bool mi_atomic_cas_strong(volatile _Atomic(uintptr_t)* p, uintptr_t desired, uintptr_t expected) {
133 return (expected == (uintptr_t)RC64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)desired, (msc_intptr_t)expected));
134}
135static inline bool mi_atomic_cas_weak(volatile _Atomic(uintptr_t)* p, uintptr_t desired, uintptr_t expected) {
136 return mi_atomic_cas_strong(p,desired,expected);
137}
138static inline uintptr_t mi_atomic_exchange(volatile _Atomic(uintptr_t)* p, uintptr_t exchange) {
139 return (uintptr_t)RC64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange);
140}
141static inline uintptr_t mi_atomic_read(volatile _Atomic(uintptr_t) const* p) {
142 return *p;
143}
144static inline uintptr_t mi_atomic_read_relaxed(volatile _Atomic(uintptr_t) const* p) {
145 return mi_atomic_read(p);
146}
147static inline void mi_atomic_write(volatile _Atomic(uintptr_t)* p, uintptr_t x) {
148 mi_atomic_exchange(p,x);
149}
150static inline void mi_atomic_yield(void) {
151 YieldProcessor();
152}
153static inline void mi_atomic_add64(volatile _Atomic(int64_t)* p, int64_t add) {
154 #ifdef _WIN64
155 mi_atomic_add(p,add);
156 #else
157 int64_t current;
158 int64_t sum;
159 do {
160 current = *p;
161 sum = current + add;
162 } while (_InterlockedCompareExchange64(p, sum, current) != current);
163 #endif
164}
165
166#else
167#ifdef __cplusplus
168#define MI_USING_STD using namespace std;
169#else
170#define MI_USING_STD
171#endif
172static inline void mi_atomic_add64(volatile int64_t* p, int64_t add) {
173 MI_USING_STD
174 atomic_fetch_add_explicit((volatile _Atomic(int64_t)*)p, add, memory_order_relaxed);
175}
176static inline intptr_t mi_atomic_add(volatile _Atomic(intptr_t)* p, intptr_t add) {
177 MI_USING_STD
178 return atomic_fetch_add_explicit(p, add, memory_order_relaxed);
179}
180static inline bool mi_atomic_cas_weak(volatile _Atomic(uintptr_t)* p, uintptr_t desired, uintptr_t expected) {
181 MI_USING_STD
182 return atomic_compare_exchange_weak_explicit(p, &expected, desired, memory_order_release, memory_order_relaxed);
183}
184static inline bool mi_atomic_cas_strong(volatile _Atomic(uintptr_t)* p, uintptr_t desired, uintptr_t expected) {
185 MI_USING_STD
186 return atomic_compare_exchange_strong_explicit(p, &expected, desired, memory_order_acq_rel, memory_order_relaxed);
187}
188static inline uintptr_t mi_atomic_exchange(volatile _Atomic(uintptr_t)* p, uintptr_t exchange) {
189 MI_USING_STD
190 return atomic_exchange_explicit(p, exchange, memory_order_acq_rel);
191}
192static inline uintptr_t mi_atomic_read_relaxed(const volatile _Atomic(uintptr_t)* p) {
193 MI_USING_STD
194 return atomic_load_explicit((volatile _Atomic(uintptr_t)*) p, memory_order_relaxed);
195}
196static inline uintptr_t mi_atomic_read(const volatile _Atomic(uintptr_t)* p) {
197 MI_USING_STD
198 return atomic_load_explicit((volatile _Atomic(uintptr_t)*) p, memory_order_acquire);
199}
200static inline void mi_atomic_write(volatile _Atomic(uintptr_t)* p, uintptr_t x) {
201 MI_USING_STD
202 return atomic_store_explicit(p, x, memory_order_release);
203}
204
205#if defined(__cplusplus)
206 #include <thread>
207 static inline void mi_atomic_yield(void) {
208 std::this_thread::yield();
209 }
210#elif (defined(__GNUC__) || defined(__clang__)) && \
211 (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))
212#if defined(__x86_64__) || defined(__i386__)
213 static inline void mi_atomic_yield(void) {
214 asm volatile ("pause" ::: "memory");
215 }
216#elif defined(__arm__) || defined(__aarch64__)
217 static inline void mi_atomic_yield(void) {
218 asm volatile("yield");
219 }
220#endif
221#elif defined(__wasi__)
222 #include <sched.h>
223 static inline void mi_atomic_yield(void) {
224 sched_yield();
225 }
226#else
227 #include <unistd.h>
228 static inline void mi_atomic_yield(void) {
229 sleep(0);
230 }
231#endif
232
233#endif
234
235#endif // __MIMALLOC_ATOMIC_H
236