1#ifndef JEMALLOC_INTERNAL_MUTEX_H
2#define JEMALLOC_INTERNAL_MUTEX_H
3
4#include "jemalloc/internal/atomic.h"
5#include "jemalloc/internal/mutex_prof.h"
6#include "jemalloc/internal/tsd.h"
7#include "jemalloc/internal/witness.h"
8
9typedef enum {
10 /* Can only acquire one mutex of a given witness rank at a time. */
11 malloc_mutex_rank_exclusive,
12 /*
13 * Can acquire multiple mutexes of the same witness rank, but in
14 * address-ascending order only.
15 */
16 malloc_mutex_address_ordered
17} malloc_mutex_lock_order_t;
18
19typedef struct malloc_mutex_s malloc_mutex_t;
20struct malloc_mutex_s {
21 union {
22 struct {
23 /*
24 * prof_data is defined first to reduce cacheline
25 * bouncing: the data is not touched by the mutex holder
26 * during unlocking, while might be modified by
27 * contenders. Having it before the mutex itself could
28 * avoid prefetching a modified cacheline (for the
29 * unlocking thread).
30 */
31 mutex_prof_data_t prof_data;
32#ifdef _WIN32
33# if _WIN32_WINNT >= 0x0600
34 SRWLOCK lock;
35# else
36 CRITICAL_SECTION lock;
37# endif
38#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
39 os_unfair_lock lock;
40#elif (defined(JEMALLOC_OSSPIN))
41 OSSpinLock lock;
42#elif (defined(JEMALLOC_MUTEX_INIT_CB))
43 pthread_mutex_t lock;
44 malloc_mutex_t *postponed_next;
45#else
46 pthread_mutex_t lock;
47#endif
48 };
49 /*
50 * We only touch witness when configured w/ debug. However we
51 * keep the field in a union when !debug so that we don't have
52 * to pollute the code base with #ifdefs, while avoid paying the
53 * memory cost.
54 */
55#if !defined(JEMALLOC_DEBUG)
56 witness_t witness;
57 malloc_mutex_lock_order_t lock_order;
58#endif
59 };
60
61#if defined(JEMALLOC_DEBUG)
62 witness_t witness;
63 malloc_mutex_lock_order_t lock_order;
64#endif
65};
66
67/*
68 * Based on benchmark results, a fixed spin with this amount of retries works
69 * well for our critical sections.
70 */
71#define MALLOC_MUTEX_MAX_SPIN 250
72
73#ifdef _WIN32
74# if _WIN32_WINNT >= 0x0600
75# define MALLOC_MUTEX_LOCK(m) AcquireSRWLockExclusive(&(m)->lock)
76# define MALLOC_MUTEX_UNLOCK(m) ReleaseSRWLockExclusive(&(m)->lock)
77# define MALLOC_MUTEX_TRYLOCK(m) (!TryAcquireSRWLockExclusive(&(m)->lock))
78# else
79# define MALLOC_MUTEX_LOCK(m) EnterCriticalSection(&(m)->lock)
80# define MALLOC_MUTEX_UNLOCK(m) LeaveCriticalSection(&(m)->lock)
81# define MALLOC_MUTEX_TRYLOCK(m) (!TryEnterCriticalSection(&(m)->lock))
82# endif
83#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
84# define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock)
85# define MALLOC_MUTEX_UNLOCK(m) os_unfair_lock_unlock(&(m)->lock)
86# define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock))
87#elif (defined(JEMALLOC_OSSPIN))
88# define MALLOC_MUTEX_LOCK(m) OSSpinLockLock(&(m)->lock)
89# define MALLOC_MUTEX_UNLOCK(m) OSSpinLockUnlock(&(m)->lock)
90# define MALLOC_MUTEX_TRYLOCK(m) (!OSSpinLockTry(&(m)->lock))
91#else
92# define MALLOC_MUTEX_LOCK(m) pthread_mutex_lock(&(m)->lock)
93# define MALLOC_MUTEX_UNLOCK(m) pthread_mutex_unlock(&(m)->lock)
94# define MALLOC_MUTEX_TRYLOCK(m) (pthread_mutex_trylock(&(m)->lock) != 0)
95#endif
96
97#define LOCK_PROF_DATA_INITIALIZER \
98 {NSTIME_ZERO_INITIALIZER, NSTIME_ZERO_INITIALIZER, 0, 0, 0, \
99 ATOMIC_INIT(0), 0, NULL, 0}
100
101#ifdef _WIN32
102# define MALLOC_MUTEX_INITIALIZER
103#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
104# if defined(JEMALLOC_DEBUG)
105# define MALLOC_MUTEX_INITIALIZER \
106 {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT}}, \
107 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
108# else
109# define MALLOC_MUTEX_INITIALIZER \
110 {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT}}, \
111 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
112# endif
113#elif (defined(JEMALLOC_OSSPIN))
114# define MALLOC_MUTEX_INITIALIZER \
115 {{{LOCK_PROF_DATA_INITIALIZER, 0}}, \
116 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
117#elif (defined(JEMALLOC_MUTEX_INIT_CB))
118# if (defined(JEMALLOC_DEBUG))
119# define MALLOC_MUTEX_INITIALIZER \
120 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL}}, \
121 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
122# else
123# define MALLOC_MUTEX_INITIALIZER \
124 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL}}, \
125 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
126# endif
127
128#else
129# define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT
130# if defined(JEMALLOC_DEBUG)
131# define MALLOC_MUTEX_INITIALIZER \
132 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER}}, \
133 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
134# else
135# define MALLOC_MUTEX_INITIALIZER \
136 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER}}, \
137 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
138# endif
139#endif
140
141#ifdef JEMALLOC_LAZY_LOCK
142extern bool isthreaded;
143#else
144# undef isthreaded /* Undo private_namespace.h definition. */
145# define isthreaded true
146#endif
147
148bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
149 witness_rank_t rank, malloc_mutex_lock_order_t lock_order);
150void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex);
151void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex);
152void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex);
153bool malloc_mutex_boot(void);
154void malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex);
155
156void malloc_mutex_lock_slow(malloc_mutex_t *mutex);
157
158static inline void
159malloc_mutex_lock_final(malloc_mutex_t *mutex) {
160 MALLOC_MUTEX_LOCK(mutex);
161}
162
163static inline bool
164malloc_mutex_trylock_final(malloc_mutex_t *mutex) {
165 return MALLOC_MUTEX_TRYLOCK(mutex);
166}
167
168static inline void
169mutex_owner_stats_update(tsdn_t *tsdn, malloc_mutex_t *mutex) {
170 if (config_stats) {
171 mutex_prof_data_t *data = &mutex->prof_data;
172 data->n_lock_ops++;
173 if (data->prev_owner != tsdn) {
174 data->prev_owner = tsdn;
175 data->n_owner_switches++;
176 }
177 }
178}
179
180/* Trylock: return false if the lock is successfully acquired. */
181static inline bool
182malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
183 witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
184 if (isthreaded) {
185 if (malloc_mutex_trylock_final(mutex)) {
186 return true;
187 }
188 mutex_owner_stats_update(tsdn, mutex);
189 }
190 witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
191
192 return false;
193}
194
195/* Aggregate lock prof data. */
196static inline void
197malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) {
198 nstime_add(&sum->tot_wait_time, &data->tot_wait_time);
199 if (nstime_compare(&sum->max_wait_time, &data->max_wait_time) < 0) {
200 nstime_copy(&sum->max_wait_time, &data->max_wait_time);
201 }
202
203 sum->n_wait_times += data->n_wait_times;
204 sum->n_spin_acquired += data->n_spin_acquired;
205
206 if (sum->max_n_thds < data->max_n_thds) {
207 sum->max_n_thds = data->max_n_thds;
208 }
209 uint32_t cur_n_waiting_thds = atomic_load_u32(&sum->n_waiting_thds,
210 ATOMIC_RELAXED);
211 uint32_t new_n_waiting_thds = cur_n_waiting_thds + atomic_load_u32(
212 &data->n_waiting_thds, ATOMIC_RELAXED);
213 atomic_store_u32(&sum->n_waiting_thds, new_n_waiting_thds,
214 ATOMIC_RELAXED);
215 sum->n_owner_switches += data->n_owner_switches;
216 sum->n_lock_ops += data->n_lock_ops;
217}
218
219static inline void
220malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
221 witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
222 if (isthreaded) {
223 if (malloc_mutex_trylock_final(mutex)) {
224 malloc_mutex_lock_slow(mutex);
225 }
226 mutex_owner_stats_update(tsdn, mutex);
227 }
228 witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
229}
230
231static inline void
232malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
233 witness_unlock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
234 if (isthreaded) {
235 MALLOC_MUTEX_UNLOCK(mutex);
236 }
237}
238
239static inline void
240malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) {
241 witness_assert_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
242}
243
244static inline void
245malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) {
246 witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
247}
248
249/* Copy the prof data from mutex for processing. */
250static inline void
251malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data,
252 malloc_mutex_t *mutex) {
253 mutex_prof_data_t *source = &mutex->prof_data;
254 /* Can only read holding the mutex. */
255 malloc_mutex_assert_owner(tsdn, mutex);
256
257 /*
258 * Not *really* allowed (we shouldn't be doing non-atomic loads of
259 * atomic data), but the mutex protection makes this safe, and writing
260 * a member-for-member copy is tedious for this situation.
261 */
262 *data = *source;
263 /* n_wait_thds is not reported (modified w/o locking). */
264 atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED);
265}
266
267#endif /* JEMALLOC_INTERNAL_MUTEX_H */
268