1 | /******************************************************************************/ |
2 | #ifdef JEMALLOC_H_TYPES |
3 | |
4 | #endif /* JEMALLOC_H_TYPES */ |
5 | /******************************************************************************/ |
6 | #ifdef JEMALLOC_H_STRUCTS |
7 | |
8 | #endif /* JEMALLOC_H_STRUCTS */ |
9 | /******************************************************************************/ |
10 | #ifdef JEMALLOC_H_EXTERNS |
11 | |
12 | #define atomic_read_uint64(p) atomic_add_uint64(p, 0) |
13 | #define atomic_read_uint32(p) atomic_add_uint32(p, 0) |
14 | #define atomic_read_p(p) atomic_add_p(p, NULL) |
15 | #define atomic_read_z(p) atomic_add_z(p, 0) |
16 | #define atomic_read_u(p) atomic_add_u(p, 0) |
17 | |
18 | #endif /* JEMALLOC_H_EXTERNS */ |
19 | /******************************************************************************/ |
20 | #ifdef JEMALLOC_H_INLINES |
21 | |
22 | /* |
23 | * All arithmetic functions return the arithmetic result of the atomic |
24 | * operation. Some atomic operation APIs return the value prior to mutation, in |
25 | * which case the following functions must redundantly compute the result so |
26 | * that it can be returned. These functions are normally inlined, so the extra |
27 | * operations can be optimized away if the return values aren't used by the |
28 | * callers. |
29 | * |
30 | * <t> atomic_read_<t>(<t> *p) { return (*p); } |
31 | * <t> atomic_add_<t>(<t> *p, <t> x) { return (*p += x); } |
32 | * <t> atomic_sub_<t>(<t> *p, <t> x) { return (*p -= x); } |
33 | * bool atomic_cas_<t>(<t> *p, <t> c, <t> s) |
34 | * { |
35 | * if (*p != c) |
36 | * return (true); |
37 | * *p = s; |
38 | * return (false); |
39 | * } |
40 | * void atomic_write_<t>(<t> *p, <t> x) { *p = x; } |
41 | */ |
42 | |
43 | #ifndef JEMALLOC_ENABLE_INLINE |
44 | uint64_t atomic_add_uint64(uint64_t *p, uint64_t x); |
45 | uint64_t atomic_sub_uint64(uint64_t *p, uint64_t x); |
46 | bool atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s); |
47 | void atomic_write_uint64(uint64_t *p, uint64_t x); |
48 | uint32_t atomic_add_uint32(uint32_t *p, uint32_t x); |
49 | uint32_t atomic_sub_uint32(uint32_t *p, uint32_t x); |
50 | bool atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s); |
51 | void atomic_write_uint32(uint32_t *p, uint32_t x); |
52 | void *atomic_add_p(void **p, void *x); |
53 | void *atomic_sub_p(void **p, void *x); |
54 | bool atomic_cas_p(void **p, void *c, void *s); |
55 | void atomic_write_p(void **p, const void *x); |
56 | size_t atomic_add_z(size_t *p, size_t x); |
57 | size_t atomic_sub_z(size_t *p, size_t x); |
58 | bool atomic_cas_z(size_t *p, size_t c, size_t s); |
59 | void atomic_write_z(size_t *p, size_t x); |
60 | unsigned atomic_add_u(unsigned *p, unsigned x); |
61 | unsigned atomic_sub_u(unsigned *p, unsigned x); |
62 | bool atomic_cas_u(unsigned *p, unsigned c, unsigned s); |
63 | void atomic_write_u(unsigned *p, unsigned x); |
64 | #endif |
65 | |
66 | #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_)) |
67 | /******************************************************************************/ |
68 | /* 64-bit operations. */ |
69 | #if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3) |
70 | # if (defined(__amd64__) || defined(__x86_64__)) |
71 | JEMALLOC_INLINE uint64_t |
72 | atomic_add_uint64(uint64_t *p, uint64_t x) |
73 | { |
74 | uint64_t t = x; |
75 | |
76 | asm volatile ( |
77 | "lock; xaddq %0, %1;" |
78 | : "+r" (t), "=m" (*p) /* Outputs. */ |
79 | : "m" (*p) /* Inputs. */ |
80 | ); |
81 | |
82 | return (t + x); |
83 | } |
84 | |
85 | JEMALLOC_INLINE uint64_t |
86 | atomic_sub_uint64(uint64_t *p, uint64_t x) |
87 | { |
88 | uint64_t t; |
89 | |
90 | x = (uint64_t)(-(int64_t)x); |
91 | t = x; |
92 | asm volatile ( |
93 | "lock; xaddq %0, %1;" |
94 | : "+r" (t), "=m" (*p) /* Outputs. */ |
95 | : "m" (*p) /* Inputs. */ |
96 | ); |
97 | |
98 | return (t + x); |
99 | } |
100 | |
101 | JEMALLOC_INLINE bool |
102 | atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s) |
103 | { |
104 | uint8_t success; |
105 | |
106 | asm volatile ( |
107 | "lock; cmpxchgq %4, %0;" |
108 | "sete %1;" |
109 | : "=m" (*p), "=a" (success) /* Outputs. */ |
110 | : "m" (*p), "a" (c), "r" (s) /* Inputs. */ |
111 | : "memory" /* Clobbers. */ |
112 | ); |
113 | |
114 | return (!(bool)success); |
115 | } |
116 | |
117 | JEMALLOC_INLINE void |
118 | atomic_write_uint64(uint64_t *p, uint64_t x) |
119 | { |
120 | |
121 | asm volatile ( |
122 | "xchgq %1, %0;" /* Lock is implied by xchgq. */ |
123 | : "=m" (*p), "+r" (x) /* Outputs. */ |
124 | : "m" (*p) /* Inputs. */ |
125 | : "memory" /* Clobbers. */ |
126 | ); |
127 | } |
128 | # elif (defined(JEMALLOC_C11ATOMICS)) |
129 | JEMALLOC_INLINE uint64_t |
130 | atomic_add_uint64(uint64_t *p, uint64_t x) |
131 | { |
132 | volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p; |
133 | return (atomic_fetch_add(a, x) + x); |
134 | } |
135 | |
136 | JEMALLOC_INLINE uint64_t |
137 | atomic_sub_uint64(uint64_t *p, uint64_t x) |
138 | { |
139 | volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p; |
140 | return (atomic_fetch_sub(a, x) - x); |
141 | } |
142 | |
143 | JEMALLOC_INLINE bool |
144 | atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s) |
145 | { |
146 | volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p; |
147 | return (!atomic_compare_exchange_strong(a, &c, s)); |
148 | } |
149 | |
150 | JEMALLOC_INLINE void |
151 | atomic_write_uint64(uint64_t *p, uint64_t x) |
152 | { |
153 | volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p; |
154 | atomic_store(a, x); |
155 | } |
156 | # elif (defined(JEMALLOC_ATOMIC9)) |
157 | JEMALLOC_INLINE uint64_t |
158 | atomic_add_uint64(uint64_t *p, uint64_t x) |
159 | { |
160 | |
161 | /* |
162 | * atomic_fetchadd_64() doesn't exist, but we only ever use this |
163 | * function on LP64 systems, so atomic_fetchadd_long() will do. |
164 | */ |
165 | assert(sizeof(uint64_t) == sizeof(unsigned long)); |
166 | |
167 | return (atomic_fetchadd_long(p, (unsigned long)x) + x); |
168 | } |
169 | |
170 | JEMALLOC_INLINE uint64_t |
171 | atomic_sub_uint64(uint64_t *p, uint64_t x) |
172 | { |
173 | |
174 | assert(sizeof(uint64_t) == sizeof(unsigned long)); |
175 | |
176 | return (atomic_fetchadd_long(p, (unsigned long)(-(long)x)) - x); |
177 | } |
178 | |
179 | JEMALLOC_INLINE bool |
180 | atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s) |
181 | { |
182 | |
183 | assert(sizeof(uint64_t) == sizeof(unsigned long)); |
184 | |
185 | return (!atomic_cmpset_long(p, (unsigned long)c, (unsigned long)s)); |
186 | } |
187 | |
188 | JEMALLOC_INLINE void |
189 | atomic_write_uint64(uint64_t *p, uint64_t x) |
190 | { |
191 | |
192 | assert(sizeof(uint64_t) == sizeof(unsigned long)); |
193 | |
194 | atomic_store_rel_long(p, x); |
195 | } |
196 | # elif (defined(JEMALLOC_OSATOMIC)) |
197 | JEMALLOC_INLINE uint64_t |
198 | atomic_add_uint64(uint64_t *p, uint64_t x) |
199 | { |
200 | |
201 | return (OSAtomicAdd64((int64_t)x, (int64_t *)p)); |
202 | } |
203 | |
204 | JEMALLOC_INLINE uint64_t |
205 | atomic_sub_uint64(uint64_t *p, uint64_t x) |
206 | { |
207 | |
208 | return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p)); |
209 | } |
210 | |
211 | JEMALLOC_INLINE bool |
212 | atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s) |
213 | { |
214 | |
215 | return (!OSAtomicCompareAndSwap64(c, s, (int64_t *)p)); |
216 | } |
217 | |
218 | JEMALLOC_INLINE void |
219 | atomic_write_uint64(uint64_t *p, uint64_t x) |
220 | { |
221 | uint64_t o; |
222 | |
223 | /*The documented OSAtomic*() API does not expose an atomic exchange. */ |
224 | do { |
225 | o = atomic_read_uint64(p); |
226 | } while (atomic_cas_uint64(p, o, x)); |
227 | } |
228 | # elif (defined(_MSC_VER)) |
229 | JEMALLOC_INLINE uint64_t |
230 | atomic_add_uint64(uint64_t *p, uint64_t x) |
231 | { |
232 | |
233 | return (InterlockedExchangeAdd64(p, x) + x); |
234 | } |
235 | |
236 | JEMALLOC_INLINE uint64_t |
237 | atomic_sub_uint64(uint64_t *p, uint64_t x) |
238 | { |
239 | |
240 | return (InterlockedExchangeAdd64(p, -((int64_t)x)) - x); |
241 | } |
242 | |
243 | JEMALLOC_INLINE bool |
244 | atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s) |
245 | { |
246 | uint64_t o; |
247 | |
248 | o = InterlockedCompareExchange64(p, s, c); |
249 | return (o != c); |
250 | } |
251 | |
252 | JEMALLOC_INLINE void |
253 | atomic_write_uint64(uint64_t *p, uint64_t x) |
254 | { |
255 | |
256 | InterlockedExchange64(p, x); |
257 | } |
258 | # elif (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) || \ |
259 | defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8)) |
260 | JEMALLOC_INLINE uint64_t |
261 | atomic_add_uint64(uint64_t *p, uint64_t x) |
262 | { |
263 | |
264 | return (__sync_add_and_fetch(p, x)); |
265 | } |
266 | |
267 | JEMALLOC_INLINE uint64_t |
268 | atomic_sub_uint64(uint64_t *p, uint64_t x) |
269 | { |
270 | |
271 | return (__sync_sub_and_fetch(p, x)); |
272 | } |
273 | |
274 | JEMALLOC_INLINE bool |
275 | atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s) |
276 | { |
277 | |
278 | return (!__sync_bool_compare_and_swap(p, c, s)); |
279 | } |
280 | |
281 | JEMALLOC_INLINE void |
282 | atomic_write_uint64(uint64_t *p, uint64_t x) |
283 | { |
284 | |
285 | __sync_lock_test_and_set(p, x); |
286 | } |
287 | # else |
288 | # error "Missing implementation for 64-bit atomic operations" |
289 | # endif |
290 | #endif |
291 | |
292 | /******************************************************************************/ |
293 | /* 32-bit operations. */ |
294 | #if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) |
295 | JEMALLOC_INLINE uint32_t |
296 | atomic_add_uint32(uint32_t *p, uint32_t x) |
297 | { |
298 | uint32_t t = x; |
299 | |
300 | asm volatile ( |
301 | "lock; xaddl %0, %1;" |
302 | : "+r" (t), "=m" (*p) /* Outputs. */ |
303 | : "m" (*p) /* Inputs. */ |
304 | ); |
305 | |
306 | return (t + x); |
307 | } |
308 | |
309 | JEMALLOC_INLINE uint32_t |
310 | atomic_sub_uint32(uint32_t *p, uint32_t x) |
311 | { |
312 | uint32_t t; |
313 | |
314 | x = (uint32_t)(-(int32_t)x); |
315 | t = x; |
316 | asm volatile ( |
317 | "lock; xaddl %0, %1;" |
318 | : "+r" (t), "=m" (*p) /* Outputs. */ |
319 | : "m" (*p) /* Inputs. */ |
320 | ); |
321 | |
322 | return (t + x); |
323 | } |
324 | |
325 | JEMALLOC_INLINE bool |
326 | atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s) |
327 | { |
328 | uint8_t success; |
329 | |
330 | asm volatile ( |
331 | "lock; cmpxchgl %4, %0;" |
332 | "sete %1;" |
333 | : "=m" (*p), "=a" (success) /* Outputs. */ |
334 | : "m" (*p), "a" (c), "r" (s) /* Inputs. */ |
335 | : "memory" |
336 | ); |
337 | |
338 | return (!(bool)success); |
339 | } |
340 | |
341 | JEMALLOC_INLINE void |
342 | atomic_write_uint32(uint32_t *p, uint32_t x) |
343 | { |
344 | |
345 | asm volatile ( |
346 | "xchgl %1, %0;" /* Lock is implied by xchgl. */ |
347 | : "=m" (*p), "+r" (x) /* Outputs. */ |
348 | : "m" (*p) /* Inputs. */ |
349 | : "memory" /* Clobbers. */ |
350 | ); |
351 | } |
352 | # elif (defined(JEMALLOC_C11ATOMICS)) |
353 | JEMALLOC_INLINE uint32_t |
354 | atomic_add_uint32(uint32_t *p, uint32_t x) |
355 | { |
356 | volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p; |
357 | return (atomic_fetch_add(a, x) + x); |
358 | } |
359 | |
360 | JEMALLOC_INLINE uint32_t |
361 | atomic_sub_uint32(uint32_t *p, uint32_t x) |
362 | { |
363 | volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p; |
364 | return (atomic_fetch_sub(a, x) - x); |
365 | } |
366 | |
367 | JEMALLOC_INLINE bool |
368 | atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s) |
369 | { |
370 | volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p; |
371 | return (!atomic_compare_exchange_strong(a, &c, s)); |
372 | } |
373 | |
374 | JEMALLOC_INLINE void |
375 | atomic_write_uint32(uint32_t *p, uint32_t x) |
376 | { |
377 | volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p; |
378 | atomic_store(a, x); |
379 | } |
380 | #elif (defined(JEMALLOC_ATOMIC9)) |
381 | JEMALLOC_INLINE uint32_t |
382 | atomic_add_uint32(uint32_t *p, uint32_t x) |
383 | { |
384 | |
385 | return (atomic_fetchadd_32(p, x) + x); |
386 | } |
387 | |
388 | JEMALLOC_INLINE uint32_t |
389 | atomic_sub_uint32(uint32_t *p, uint32_t x) |
390 | { |
391 | |
392 | return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x); |
393 | } |
394 | |
395 | JEMALLOC_INLINE bool |
396 | atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s) |
397 | { |
398 | |
399 | return (!atomic_cmpset_32(p, c, s)); |
400 | } |
401 | |
402 | JEMALLOC_INLINE void |
403 | atomic_write_uint32(uint32_t *p, uint32_t x) |
404 | { |
405 | |
406 | atomic_store_rel_32(p, x); |
407 | } |
408 | #elif (defined(JEMALLOC_OSATOMIC)) |
409 | JEMALLOC_INLINE uint32_t |
410 | atomic_add_uint32(uint32_t *p, uint32_t x) |
411 | { |
412 | |
413 | return (OSAtomicAdd32((int32_t)x, (int32_t *)p)); |
414 | } |
415 | |
416 | JEMALLOC_INLINE uint32_t |
417 | atomic_sub_uint32(uint32_t *p, uint32_t x) |
418 | { |
419 | |
420 | return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p)); |
421 | } |
422 | |
423 | JEMALLOC_INLINE bool |
424 | atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s) |
425 | { |
426 | |
427 | return (!OSAtomicCompareAndSwap32(c, s, (int32_t *)p)); |
428 | } |
429 | |
430 | JEMALLOC_INLINE void |
431 | atomic_write_uint32(uint32_t *p, uint32_t x) |
432 | { |
433 | uint32_t o; |
434 | |
435 | /*The documented OSAtomic*() API does not expose an atomic exchange. */ |
436 | do { |
437 | o = atomic_read_uint32(p); |
438 | } while (atomic_cas_uint32(p, o, x)); |
439 | } |
440 | #elif (defined(_MSC_VER)) |
441 | JEMALLOC_INLINE uint32_t |
442 | atomic_add_uint32(uint32_t *p, uint32_t x) |
443 | { |
444 | |
445 | return (InterlockedExchangeAdd(p, x) + x); |
446 | } |
447 | |
448 | JEMALLOC_INLINE uint32_t |
449 | atomic_sub_uint32(uint32_t *p, uint32_t x) |
450 | { |
451 | |
452 | return (InterlockedExchangeAdd(p, -((int32_t)x)) - x); |
453 | } |
454 | |
455 | JEMALLOC_INLINE bool |
456 | atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s) |
457 | { |
458 | uint32_t o; |
459 | |
460 | o = InterlockedCompareExchange(p, s, c); |
461 | return (o != c); |
462 | } |
463 | |
464 | JEMALLOC_INLINE void |
465 | atomic_write_uint32(uint32_t *p, uint32_t x) |
466 | { |
467 | |
468 | InterlockedExchange(p, x); |
469 | } |
470 | #elif (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) || \ |
471 | defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4)) |
472 | JEMALLOC_INLINE uint32_t |
473 | atomic_add_uint32(uint32_t *p, uint32_t x) |
474 | { |
475 | |
476 | return (__sync_add_and_fetch(p, x)); |
477 | } |
478 | |
479 | JEMALLOC_INLINE uint32_t |
480 | atomic_sub_uint32(uint32_t *p, uint32_t x) |
481 | { |
482 | |
483 | return (__sync_sub_and_fetch(p, x)); |
484 | } |
485 | |
486 | JEMALLOC_INLINE bool |
487 | atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s) |
488 | { |
489 | |
490 | return (!__sync_bool_compare_and_swap(p, c, s)); |
491 | } |
492 | |
493 | JEMALLOC_INLINE void |
494 | atomic_write_uint32(uint32_t *p, uint32_t x) |
495 | { |
496 | |
497 | __sync_lock_test_and_set(p, x); |
498 | } |
499 | #else |
500 | # error "Missing implementation for 32-bit atomic operations" |
501 | #endif |
502 | |
503 | /******************************************************************************/ |
504 | /* Pointer operations. */ |
505 | JEMALLOC_INLINE void * |
506 | atomic_add_p(void **p, void *x) |
507 | { |
508 | |
509 | #if (LG_SIZEOF_PTR == 3) |
510 | return ((void *)atomic_add_uint64((uint64_t *)p, (uint64_t)x)); |
511 | #elif (LG_SIZEOF_PTR == 2) |
512 | return ((void *)atomic_add_uint32((uint32_t *)p, (uint32_t)x)); |
513 | #endif |
514 | } |
515 | |
516 | JEMALLOC_INLINE void * |
517 | atomic_sub_p(void **p, void *x) |
518 | { |
519 | |
520 | #if (LG_SIZEOF_PTR == 3) |
521 | return ((void *)atomic_add_uint64((uint64_t *)p, |
522 | (uint64_t)-((int64_t)x))); |
523 | #elif (LG_SIZEOF_PTR == 2) |
524 | return ((void *)atomic_add_uint32((uint32_t *)p, |
525 | (uint32_t)-((int32_t)x))); |
526 | #endif |
527 | } |
528 | |
529 | JEMALLOC_INLINE bool |
530 | atomic_cas_p(void **p, void *c, void *s) |
531 | { |
532 | |
533 | #if (LG_SIZEOF_PTR == 3) |
534 | return (atomic_cas_uint64((uint64_t *)p, (uint64_t)c, (uint64_t)s)); |
535 | #elif (LG_SIZEOF_PTR == 2) |
536 | return (atomic_cas_uint32((uint32_t *)p, (uint32_t)c, (uint32_t)s)); |
537 | #endif |
538 | } |
539 | |
540 | JEMALLOC_INLINE void |
541 | atomic_write_p(void **p, const void *x) |
542 | { |
543 | |
544 | #if (LG_SIZEOF_PTR == 3) |
545 | atomic_write_uint64((uint64_t *)p, (uint64_t)x); |
546 | #elif (LG_SIZEOF_PTR == 2) |
547 | atomic_write_uint32((uint32_t *)p, (uint32_t)x); |
548 | #endif |
549 | } |
550 | |
551 | /******************************************************************************/ |
552 | /* size_t operations. */ |
553 | JEMALLOC_INLINE size_t |
554 | atomic_add_z(size_t *p, size_t x) |
555 | { |
556 | |
557 | #if (LG_SIZEOF_PTR == 3) |
558 | return ((size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x)); |
559 | #elif (LG_SIZEOF_PTR == 2) |
560 | return ((size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x)); |
561 | #endif |
562 | } |
563 | |
564 | JEMALLOC_INLINE size_t |
565 | atomic_sub_z(size_t *p, size_t x) |
566 | { |
567 | |
568 | #if (LG_SIZEOF_PTR == 3) |
569 | return ((size_t)atomic_add_uint64((uint64_t *)p, |
570 | (uint64_t)-((int64_t)x))); |
571 | #elif (LG_SIZEOF_PTR == 2) |
572 | return ((size_t)atomic_add_uint32((uint32_t *)p, |
573 | (uint32_t)-((int32_t)x))); |
574 | #endif |
575 | } |
576 | |
577 | JEMALLOC_INLINE bool |
578 | atomic_cas_z(size_t *p, size_t c, size_t s) |
579 | { |
580 | |
581 | #if (LG_SIZEOF_PTR == 3) |
582 | return (atomic_cas_uint64((uint64_t *)p, (uint64_t)c, (uint64_t)s)); |
583 | #elif (LG_SIZEOF_PTR == 2) |
584 | return (atomic_cas_uint32((uint32_t *)p, (uint32_t)c, (uint32_t)s)); |
585 | #endif |
586 | } |
587 | |
588 | JEMALLOC_INLINE void |
589 | atomic_write_z(size_t *p, size_t x) |
590 | { |
591 | |
592 | #if (LG_SIZEOF_PTR == 3) |
593 | atomic_write_uint64((uint64_t *)p, (uint64_t)x); |
594 | #elif (LG_SIZEOF_PTR == 2) |
595 | atomic_write_uint32((uint32_t *)p, (uint32_t)x); |
596 | #endif |
597 | } |
598 | |
599 | /******************************************************************************/ |
600 | /* unsigned operations. */ |
601 | JEMALLOC_INLINE unsigned |
602 | atomic_add_u(unsigned *p, unsigned x) |
603 | { |
604 | |
605 | #if (LG_SIZEOF_INT == 3) |
606 | return ((unsigned)atomic_add_uint64((uint64_t *)p, (uint64_t)x)); |
607 | #elif (LG_SIZEOF_INT == 2) |
608 | return ((unsigned)atomic_add_uint32((uint32_t *)p, (uint32_t)x)); |
609 | #endif |
610 | } |
611 | |
612 | JEMALLOC_INLINE unsigned |
613 | atomic_sub_u(unsigned *p, unsigned x) |
614 | { |
615 | |
616 | #if (LG_SIZEOF_INT == 3) |
617 | return ((unsigned)atomic_add_uint64((uint64_t *)p, |
618 | (uint64_t)-((int64_t)x))); |
619 | #elif (LG_SIZEOF_INT == 2) |
620 | return ((unsigned)atomic_add_uint32((uint32_t *)p, |
621 | (uint32_t)-((int32_t)x))); |
622 | #endif |
623 | } |
624 | |
625 | JEMALLOC_INLINE bool |
626 | atomic_cas_u(unsigned *p, unsigned c, unsigned s) |
627 | { |
628 | |
629 | #if (LG_SIZEOF_INT == 3) |
630 | return (atomic_cas_uint64((uint64_t *)p, (uint64_t)c, (uint64_t)s)); |
631 | #elif (LG_SIZEOF_INT == 2) |
632 | return (atomic_cas_uint32((uint32_t *)p, (uint32_t)c, (uint32_t)s)); |
633 | #endif |
634 | } |
635 | |
636 | JEMALLOC_INLINE void |
637 | atomic_write_u(unsigned *p, unsigned x) |
638 | { |
639 | |
640 | #if (LG_SIZEOF_INT == 3) |
641 | atomic_write_uint64((uint64_t *)p, (uint64_t)x); |
642 | #elif (LG_SIZEOF_INT == 2) |
643 | atomic_write_uint32((uint32_t *)p, (uint32_t)x); |
644 | #endif |
645 | } |
646 | |
647 | /******************************************************************************/ |
648 | #endif |
649 | |
650 | #endif /* JEMALLOC_H_INLINES */ |
651 | /******************************************************************************/ |
652 | |