1// -*- C++ -*- header.
2
3// Copyright (C) 2008-2022 Free Software Foundation, Inc.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25/** @file bits/atomic_base.h
26 * This is an internal header file, included by other library headers.
27 * Do not attempt to use it directly. @headername{atomic}
28 */
29
30#ifndef _GLIBCXX_ATOMIC_BASE_H
31#define _GLIBCXX_ATOMIC_BASE_H 1
32
33#pragma GCC system_header
34
35#include <bits/c++config.h>
36#include <stdint.h>
37#include <bits/atomic_lockfree_defines.h>
38#include <bits/move.h>
39
40#if __cplusplus > 201703L && _GLIBCXX_HOSTED
41#include <bits/atomic_wait.h>
42#endif
43
44#ifndef _GLIBCXX_ALWAYS_INLINE
45#define _GLIBCXX_ALWAYS_INLINE inline __attribute__((__always_inline__))
46#endif
47
48namespace std _GLIBCXX_VISIBILITY(default)
49{
50_GLIBCXX_BEGIN_NAMESPACE_VERSION
51
52 /**
53 * @defgroup atomics Atomics
54 *
55 * Components for performing atomic operations.
56 * @{
57 */
58
59 /// Enumeration for memory_order
60#if __cplusplus > 201703L
61 enum class memory_order : int
62 {
63 relaxed,
64 consume,
65 acquire,
66 release,
67 acq_rel,
68 seq_cst
69 };
70
71 inline constexpr memory_order memory_order_relaxed = memory_order::relaxed;
72 inline constexpr memory_order memory_order_consume = memory_order::consume;
73 inline constexpr memory_order memory_order_acquire = memory_order::acquire;
74 inline constexpr memory_order memory_order_release = memory_order::release;
75 inline constexpr memory_order memory_order_acq_rel = memory_order::acq_rel;
76 inline constexpr memory_order memory_order_seq_cst = memory_order::seq_cst;
77#else
78 typedef enum memory_order
79 {
80 memory_order_relaxed,
81 memory_order_consume,
82 memory_order_acquire,
83 memory_order_release,
84 memory_order_acq_rel,
85 memory_order_seq_cst
86 } memory_order;
87#endif
88
89 enum __memory_order_modifier
90 {
91 __memory_order_mask = 0x0ffff,
92 __memory_order_modifier_mask = 0xffff0000,
93 __memory_order_hle_acquire = 0x10000,
94 __memory_order_hle_release = 0x20000
95 };
96
97 constexpr memory_order
98 operator|(memory_order __m, __memory_order_modifier __mod)
99 {
100 return memory_order(int(__m) | int(__mod));
101 }
102
103 constexpr memory_order
104 operator&(memory_order __m, __memory_order_modifier __mod)
105 {
106 return memory_order(int(__m) & int(__mod));
107 }
108
109 // Drop release ordering as per [atomics.types.operations.req]/21
110 constexpr memory_order
111 __cmpexch_failure_order2(memory_order __m) noexcept
112 {
113 return __m == memory_order_acq_rel ? memory_order_acquire
114 : __m == memory_order_release ? memory_order_relaxed : __m;
115 }
116
117 constexpr memory_order
118 __cmpexch_failure_order(memory_order __m) noexcept
119 {
120 return memory_order(__cmpexch_failure_order2(m: __m & __memory_order_mask)
121 | __memory_order_modifier(__m & __memory_order_modifier_mask));
122 }
123
124 constexpr bool
125 __is_valid_cmpexch_failure_order(memory_order __m) noexcept
126 {
127 return (__m & __memory_order_mask) != memory_order_release
128 && (__m & __memory_order_mask) != memory_order_acq_rel;
129 }
130
131 _GLIBCXX_ALWAYS_INLINE void
132 atomic_thread_fence(memory_order __m) noexcept
133 { __atomic_thread_fence(int(__m)); }
134
135 _GLIBCXX_ALWAYS_INLINE void
136 atomic_signal_fence(memory_order __m) noexcept
137 { __atomic_signal_fence(int(__m)); }
138
139 /// kill_dependency
140 template<typename _Tp>
141 inline _Tp
142 kill_dependency(_Tp __y) noexcept
143 {
144 _Tp __ret(__y);
145 return __ret;
146 }
147
148 // Base types for atomics.
149 template<typename _IntTp>
150 struct __atomic_base;
151
152#if __cplusplus <= 201703L
153# define _GLIBCXX20_INIT(I)
154#else
155# define __cpp_lib_atomic_value_initialization 201911L
156# define _GLIBCXX20_INIT(I) = I
157#endif
158
159#define ATOMIC_VAR_INIT(_VI) { _VI }
160
161 template<typename _Tp>
162 struct atomic;
163
164 template<typename _Tp>
165 struct atomic<_Tp*>;
166
167 /* The target's "set" value for test-and-set may not be exactly 1. */
168#if __GCC_ATOMIC_TEST_AND_SET_TRUEVAL == 1
169 typedef bool __atomic_flag_data_type;
170#else
171 typedef unsigned char __atomic_flag_data_type;
172#endif
173
174 /**
175 * @brief Base type for atomic_flag.
176 *
177 * Base type is POD with data, allowing atomic_flag to derive from
178 * it and meet the standard layout type requirement. In addition to
179 * compatibility with a C interface, this allows different
180 * implementations of atomic_flag to use the same atomic operation
181 * functions, via a standard conversion to the __atomic_flag_base
182 * argument.
183 */
184 _GLIBCXX_BEGIN_EXTERN_C
185
186 struct __atomic_flag_base
187 {
188 __atomic_flag_data_type _M_i _GLIBCXX20_INIT({});
189 };
190
191 _GLIBCXX_END_EXTERN_C
192
193#define ATOMIC_FLAG_INIT { 0 }
194
195 /// atomic_flag
196 struct atomic_flag : public __atomic_flag_base
197 {
198 atomic_flag() noexcept = default;
199 ~atomic_flag() noexcept = default;
200 atomic_flag(const atomic_flag&) = delete;
201 atomic_flag& operator=(const atomic_flag&) = delete;
202 atomic_flag& operator=(const atomic_flag&) volatile = delete;
203
204 // Conversion to ATOMIC_FLAG_INIT.
205 constexpr atomic_flag(bool __i) noexcept
206 : __atomic_flag_base{ ._M_i: _S_init(__i) }
207 { }
208
209 _GLIBCXX_ALWAYS_INLINE bool
210 test_and_set(memory_order __m = memory_order_seq_cst) noexcept
211 {
212 return __atomic_test_and_set (&_M_i, int(__m));
213 }
214
215 _GLIBCXX_ALWAYS_INLINE bool
216 test_and_set(memory_order __m = memory_order_seq_cst) volatile noexcept
217 {
218 return __atomic_test_and_set (&_M_i, int(__m));
219 }
220
221#if __cplusplus > 201703L
222#define __cpp_lib_atomic_flag_test 201907L
223
224 _GLIBCXX_ALWAYS_INLINE bool
225 test(memory_order __m = memory_order_seq_cst) const noexcept
226 {
227 __atomic_flag_data_type __v;
228 __atomic_load(&_M_i, &__v, int(__m));
229 return __v == __GCC_ATOMIC_TEST_AND_SET_TRUEVAL;
230 }
231
232 _GLIBCXX_ALWAYS_INLINE bool
233 test(memory_order __m = memory_order_seq_cst) const volatile noexcept
234 {
235 __atomic_flag_data_type __v;
236 __atomic_load(&_M_i, &__v, int(__m));
237 return __v == __GCC_ATOMIC_TEST_AND_SET_TRUEVAL;
238 }
239
240#if __cpp_lib_atomic_wait
241 _GLIBCXX_ALWAYS_INLINE void
242 wait(bool __old,
243 memory_order __m = memory_order_seq_cst) const noexcept
244 {
245 const __atomic_flag_data_type __v
246 = __old ? __GCC_ATOMIC_TEST_AND_SET_TRUEVAL : 0;
247
248 std::__atomic_wait_address_v(&_M_i, __v,
249 [__m, this] { return __atomic_load_n(&_M_i, int(__m)); });
250 }
251
252 // TODO add const volatile overload
253
254 _GLIBCXX_ALWAYS_INLINE void
255 notify_one() noexcept
256 { std::__atomic_notify_address(&_M_i, false); }
257
258 // TODO add const volatile overload
259
260 _GLIBCXX_ALWAYS_INLINE void
261 notify_all() noexcept
262 { std::__atomic_notify_address(&_M_i, true); }
263
264 // TODO add const volatile overload
265#endif // __cpp_lib_atomic_wait
266#endif // C++20
267
268 _GLIBCXX_ALWAYS_INLINE void
269 clear(memory_order __m = memory_order_seq_cst) noexcept
270 {
271 memory_order __b __attribute__ ((__unused__))
272 = __m & __memory_order_mask;
273 __glibcxx_assert(__b != memory_order_consume);
274 __glibcxx_assert(__b != memory_order_acquire);
275 __glibcxx_assert(__b != memory_order_acq_rel);
276
277 __atomic_clear (&_M_i, int(__m));
278 }
279
280 _GLIBCXX_ALWAYS_INLINE void
281 clear(memory_order __m = memory_order_seq_cst) volatile noexcept
282 {
283 memory_order __b __attribute__ ((__unused__))
284 = __m & __memory_order_mask;
285 __glibcxx_assert(__b != memory_order_consume);
286 __glibcxx_assert(__b != memory_order_acquire);
287 __glibcxx_assert(__b != memory_order_acq_rel);
288
289 __atomic_clear (&_M_i, int(__m));
290 }
291
292 private:
293 static constexpr __atomic_flag_data_type
294 _S_init(bool __i)
295 { return __i ? __GCC_ATOMIC_TEST_AND_SET_TRUEVAL : 0; }
296 };
297
298
299 /// Base class for atomic integrals.
300 //
301 // For each of the integral types, define atomic_[integral type] struct
302 //
303 // atomic_bool bool
304 // atomic_char char
305 // atomic_schar signed char
306 // atomic_uchar unsigned char
307 // atomic_short short
308 // atomic_ushort unsigned short
309 // atomic_int int
310 // atomic_uint unsigned int
311 // atomic_long long
312 // atomic_ulong unsigned long
313 // atomic_llong long long
314 // atomic_ullong unsigned long long
315 // atomic_char8_t char8_t
316 // atomic_char16_t char16_t
317 // atomic_char32_t char32_t
318 // atomic_wchar_t wchar_t
319 //
320 // NB: Assuming _ITp is an integral scalar type that is 1, 2, 4, or
321 // 8 bytes, since that is what GCC built-in functions for atomic
322 // memory access expect.
323 template<typename _ITp>
324 struct __atomic_base
325 {
326 using value_type = _ITp;
327 using difference_type = value_type;
328
329 private:
330 typedef _ITp __int_type;
331
332 static constexpr int _S_alignment =
333 sizeof(_ITp) > alignof(_ITp) ? sizeof(_ITp) : alignof(_ITp);
334
335 alignas(_S_alignment) __int_type _M_i _GLIBCXX20_INIT(0);
336
337 public:
338 __atomic_base() noexcept = default;
339 ~__atomic_base() noexcept = default;
340 __atomic_base(const __atomic_base&) = delete;
341 __atomic_base& operator=(const __atomic_base&) = delete;
342 __atomic_base& operator=(const __atomic_base&) volatile = delete;
343
344 // Requires __int_type convertible to _M_i.
345 constexpr __atomic_base(__int_type __i) noexcept : _M_i (__i) { }
346
347 operator __int_type() const noexcept
348 { return load(); }
349
350 operator __int_type() const volatile noexcept
351 { return load(); }
352
353 __int_type
354 operator=(__int_type __i) noexcept
355 {
356 store(__i);
357 return __i;
358 }
359
360 __int_type
361 operator=(__int_type __i) volatile noexcept
362 {
363 store(__i);
364 return __i;
365 }
366
367 __int_type
368 operator++(int) noexcept
369 { return fetch_add(1); }
370
371 __int_type
372 operator++(int) volatile noexcept
373 { return fetch_add(1); }
374
375 __int_type
376 operator--(int) noexcept
377 { return fetch_sub(1); }
378
379 __int_type
380 operator--(int) volatile noexcept
381 { return fetch_sub(1); }
382
383 __int_type
384 operator++() noexcept
385 { return __atomic_add_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
386
387 __int_type
388 operator++() volatile noexcept
389 { return __atomic_add_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
390
391 __int_type
392 operator--() noexcept
393 { return __atomic_sub_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
394
395 __int_type
396 operator--() volatile noexcept
397 { return __atomic_sub_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
398
399 __int_type
400 operator+=(__int_type __i) noexcept
401 { return __atomic_add_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
402
403 __int_type
404 operator+=(__int_type __i) volatile noexcept
405 { return __atomic_add_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
406
407 __int_type
408 operator-=(__int_type __i) noexcept
409 { return __atomic_sub_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
410
411 __int_type
412 operator-=(__int_type __i) volatile noexcept
413 { return __atomic_sub_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
414
415 __int_type
416 operator&=(__int_type __i) noexcept
417 { return __atomic_and_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
418
419 __int_type
420 operator&=(__int_type __i) volatile noexcept
421 { return __atomic_and_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
422
423 __int_type
424 operator|=(__int_type __i) noexcept
425 { return __atomic_or_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
426
427 __int_type
428 operator|=(__int_type __i) volatile noexcept
429 { return __atomic_or_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
430
431 __int_type
432 operator^=(__int_type __i) noexcept
433 { return __atomic_xor_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
434
435 __int_type
436 operator^=(__int_type __i) volatile noexcept
437 { return __atomic_xor_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
438
439 bool
440 is_lock_free() const noexcept
441 {
442 // Use a fake, minimally aligned pointer.
443 return __atomic_is_lock_free(sizeof(_M_i),
444 reinterpret_cast<void *>(-_S_alignment));
445 }
446
447 bool
448 is_lock_free() const volatile noexcept
449 {
450 // Use a fake, minimally aligned pointer.
451 return __atomic_is_lock_free(sizeof(_M_i),
452 reinterpret_cast<void *>(-_S_alignment));
453 }
454
455 _GLIBCXX_ALWAYS_INLINE void
456 store(__int_type __i, memory_order __m = memory_order_seq_cst) noexcept
457 {
458 memory_order __b __attribute__ ((__unused__))
459 = __m & __memory_order_mask;
460 __glibcxx_assert(__b != memory_order_acquire);
461 __glibcxx_assert(__b != memory_order_acq_rel);
462 __glibcxx_assert(__b != memory_order_consume);
463
464 __atomic_store_n(&_M_i, __i, int(__m));
465 }
466
467 _GLIBCXX_ALWAYS_INLINE void
468 store(__int_type __i,
469 memory_order __m = memory_order_seq_cst) volatile noexcept
470 {
471 memory_order __b __attribute__ ((__unused__))
472 = __m & __memory_order_mask;
473 __glibcxx_assert(__b != memory_order_acquire);
474 __glibcxx_assert(__b != memory_order_acq_rel);
475 __glibcxx_assert(__b != memory_order_consume);
476
477 __atomic_store_n(&_M_i, __i, int(__m));
478 }
479
480 _GLIBCXX_ALWAYS_INLINE __int_type
481 load(memory_order __m = memory_order_seq_cst) const noexcept
482 {
483 memory_order __b __attribute__ ((__unused__))
484 = __m & __memory_order_mask;
485 __glibcxx_assert(__b != memory_order_release);
486 __glibcxx_assert(__b != memory_order_acq_rel);
487
488 return __atomic_load_n(&_M_i, int(__m));
489 }
490
491 _GLIBCXX_ALWAYS_INLINE __int_type
492 load(memory_order __m = memory_order_seq_cst) const volatile noexcept
493 {
494 memory_order __b __attribute__ ((__unused__))
495 = __m & __memory_order_mask;
496 __glibcxx_assert(__b != memory_order_release);
497 __glibcxx_assert(__b != memory_order_acq_rel);
498
499 return __atomic_load_n(&_M_i, int(__m));
500 }
501
502 _GLIBCXX_ALWAYS_INLINE __int_type
503 exchange(__int_type __i,
504 memory_order __m = memory_order_seq_cst) noexcept
505 {
506 return __atomic_exchange_n(&_M_i, __i, int(__m));
507 }
508
509
510 _GLIBCXX_ALWAYS_INLINE __int_type
511 exchange(__int_type __i,
512 memory_order __m = memory_order_seq_cst) volatile noexcept
513 {
514 return __atomic_exchange_n(&_M_i, __i, int(__m));
515 }
516
517 _GLIBCXX_ALWAYS_INLINE bool
518 compare_exchange_weak(__int_type& __i1, __int_type __i2,
519 memory_order __m1, memory_order __m2) noexcept
520 {
521 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
522
523 return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1,
524 int(__m1), int(__m2));
525 }
526
527 _GLIBCXX_ALWAYS_INLINE bool
528 compare_exchange_weak(__int_type& __i1, __int_type __i2,
529 memory_order __m1,
530 memory_order __m2) volatile noexcept
531 {
532 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
533
534 return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1,
535 int(__m1), int(__m2));
536 }
537
538 _GLIBCXX_ALWAYS_INLINE bool
539 compare_exchange_weak(__int_type& __i1, __int_type __i2,
540 memory_order __m = memory_order_seq_cst) noexcept
541 {
542 return compare_exchange_weak(__i1, __i2, __m,
543 __cmpexch_failure_order(__m));
544 }
545
546 _GLIBCXX_ALWAYS_INLINE bool
547 compare_exchange_weak(__int_type& __i1, __int_type __i2,
548 memory_order __m = memory_order_seq_cst) volatile noexcept
549 {
550 return compare_exchange_weak(__i1, __i2, __m,
551 __cmpexch_failure_order(__m));
552 }
553
554 _GLIBCXX_ALWAYS_INLINE bool
555 compare_exchange_strong(__int_type& __i1, __int_type __i2,
556 memory_order __m1, memory_order __m2) noexcept
557 {
558 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
559
560 return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0,
561 int(__m1), int(__m2));
562 }
563
564 _GLIBCXX_ALWAYS_INLINE bool
565 compare_exchange_strong(__int_type& __i1, __int_type __i2,
566 memory_order __m1,
567 memory_order __m2) volatile noexcept
568 {
569 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
570
571 return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0,
572 int(__m1), int(__m2));
573 }
574
575 _GLIBCXX_ALWAYS_INLINE bool
576 compare_exchange_strong(__int_type& __i1, __int_type __i2,
577 memory_order __m = memory_order_seq_cst) noexcept
578 {
579 return compare_exchange_strong(__i1, __i2, __m,
580 __cmpexch_failure_order(__m));
581 }
582
583 _GLIBCXX_ALWAYS_INLINE bool
584 compare_exchange_strong(__int_type& __i1, __int_type __i2,
585 memory_order __m = memory_order_seq_cst) volatile noexcept
586 {
587 return compare_exchange_strong(__i1, __i2, __m,
588 __cmpexch_failure_order(__m));
589 }
590
591#if __cpp_lib_atomic_wait
592 _GLIBCXX_ALWAYS_INLINE void
593 wait(__int_type __old,
594 memory_order __m = memory_order_seq_cst) const noexcept
595 {
596 std::__atomic_wait_address_v(&_M_i, __old,
597 [__m, this] { return this->load(__m); });
598 }
599
600 // TODO add const volatile overload
601
602 _GLIBCXX_ALWAYS_INLINE void
603 notify_one() noexcept
604 { std::__atomic_notify_address(&_M_i, false); }
605
606 // TODO add const volatile overload
607
608 _GLIBCXX_ALWAYS_INLINE void
609 notify_all() noexcept
610 { std::__atomic_notify_address(&_M_i, true); }
611
612 // TODO add const volatile overload
613#endif // __cpp_lib_atomic_wait
614
615 _GLIBCXX_ALWAYS_INLINE __int_type
616 fetch_add(__int_type __i,
617 memory_order __m = memory_order_seq_cst) noexcept
618 { return __atomic_fetch_add(&_M_i, __i, int(__m)); }
619
620 _GLIBCXX_ALWAYS_INLINE __int_type
621 fetch_add(__int_type __i,
622 memory_order __m = memory_order_seq_cst) volatile noexcept
623 { return __atomic_fetch_add(&_M_i, __i, int(__m)); }
624
625 _GLIBCXX_ALWAYS_INLINE __int_type
626 fetch_sub(__int_type __i,
627 memory_order __m = memory_order_seq_cst) noexcept
628 { return __atomic_fetch_sub(&_M_i, __i, int(__m)); }
629
630 _GLIBCXX_ALWAYS_INLINE __int_type
631 fetch_sub(__int_type __i,
632 memory_order __m = memory_order_seq_cst) volatile noexcept
633 { return __atomic_fetch_sub(&_M_i, __i, int(__m)); }
634
635 _GLIBCXX_ALWAYS_INLINE __int_type
636 fetch_and(__int_type __i,
637 memory_order __m = memory_order_seq_cst) noexcept
638 { return __atomic_fetch_and(&_M_i, __i, int(__m)); }
639
640 _GLIBCXX_ALWAYS_INLINE __int_type
641 fetch_and(__int_type __i,
642 memory_order __m = memory_order_seq_cst) volatile noexcept
643 { return __atomic_fetch_and(&_M_i, __i, int(__m)); }
644
645 _GLIBCXX_ALWAYS_INLINE __int_type
646 fetch_or(__int_type __i,
647 memory_order __m = memory_order_seq_cst) noexcept
648 { return __atomic_fetch_or(&_M_i, __i, int(__m)); }
649
650 _GLIBCXX_ALWAYS_INLINE __int_type
651 fetch_or(__int_type __i,
652 memory_order __m = memory_order_seq_cst) volatile noexcept
653 { return __atomic_fetch_or(&_M_i, __i, int(__m)); }
654
655 _GLIBCXX_ALWAYS_INLINE __int_type
656 fetch_xor(__int_type __i,
657 memory_order __m = memory_order_seq_cst) noexcept
658 { return __atomic_fetch_xor(&_M_i, __i, int(__m)); }
659
660 _GLIBCXX_ALWAYS_INLINE __int_type
661 fetch_xor(__int_type __i,
662 memory_order __m = memory_order_seq_cst) volatile noexcept
663 { return __atomic_fetch_xor(&_M_i, __i, int(__m)); }
664 };
665
666
667 /// Partial specialization for pointer types.
668 template<typename _PTp>
669 struct __atomic_base<_PTp*>
670 {
671 private:
672 typedef _PTp* __pointer_type;
673
674 __pointer_type _M_p _GLIBCXX20_INIT(nullptr);
675
676 // Factored out to facilitate explicit specialization.
677 constexpr ptrdiff_t
678 _M_type_size(ptrdiff_t __d) const { return __d * sizeof(_PTp); }
679
680 constexpr ptrdiff_t
681 _M_type_size(ptrdiff_t __d) const volatile { return __d * sizeof(_PTp); }
682
683 public:
684 __atomic_base() noexcept = default;
685 ~__atomic_base() noexcept = default;
686 __atomic_base(const __atomic_base&) = delete;
687 __atomic_base& operator=(const __atomic_base&) = delete;
688 __atomic_base& operator=(const __atomic_base&) volatile = delete;
689
690 // Requires __pointer_type convertible to _M_p.
691 constexpr __atomic_base(__pointer_type __p) noexcept : _M_p (__p) { }
692
693 operator __pointer_type() const noexcept
694 { return load(); }
695
696 operator __pointer_type() const volatile noexcept
697 { return load(); }
698
699 __pointer_type
700 operator=(__pointer_type __p) noexcept
701 {
702 store(__p);
703 return __p;
704 }
705
706 __pointer_type
707 operator=(__pointer_type __p) volatile noexcept
708 {
709 store(__p);
710 return __p;
711 }
712
713 __pointer_type
714 operator++(int) noexcept
715 { return fetch_add(1); }
716
717 __pointer_type
718 operator++(int) volatile noexcept
719 { return fetch_add(1); }
720
721 __pointer_type
722 operator--(int) noexcept
723 { return fetch_sub(1); }
724
725 __pointer_type
726 operator--(int) volatile noexcept
727 { return fetch_sub(1); }
728
729 __pointer_type
730 operator++() noexcept
731 { return __atomic_add_fetch(&_M_p, _M_type_size(1),
732 int(memory_order_seq_cst)); }
733
734 __pointer_type
735 operator++() volatile noexcept
736 { return __atomic_add_fetch(&_M_p, _M_type_size(1),
737 int(memory_order_seq_cst)); }
738
739 __pointer_type
740 operator--() noexcept
741 { return __atomic_sub_fetch(&_M_p, _M_type_size(1),
742 int(memory_order_seq_cst)); }
743
744 __pointer_type
745 operator--() volatile noexcept
746 { return __atomic_sub_fetch(&_M_p, _M_type_size(1),
747 int(memory_order_seq_cst)); }
748
749 __pointer_type
750 operator+=(ptrdiff_t __d) noexcept
751 { return __atomic_add_fetch(&_M_p, _M_type_size(__d),
752 int(memory_order_seq_cst)); }
753
754 __pointer_type
755 operator+=(ptrdiff_t __d) volatile noexcept
756 { return __atomic_add_fetch(&_M_p, _M_type_size(__d),
757 int(memory_order_seq_cst)); }
758
759 __pointer_type
760 operator-=(ptrdiff_t __d) noexcept
761 { return __atomic_sub_fetch(&_M_p, _M_type_size(__d),
762 int(memory_order_seq_cst)); }
763
764 __pointer_type
765 operator-=(ptrdiff_t __d) volatile noexcept
766 { return __atomic_sub_fetch(&_M_p, _M_type_size(__d),
767 int(memory_order_seq_cst)); }
768
769 bool
770 is_lock_free() const noexcept
771 {
772 // Produce a fake, minimally aligned pointer.
773 return __atomic_is_lock_free(sizeof(_M_p),
774 reinterpret_cast<void *>(-__alignof(_M_p)));
775 }
776
777 bool
778 is_lock_free() const volatile noexcept
779 {
780 // Produce a fake, minimally aligned pointer.
781 return __atomic_is_lock_free(sizeof(_M_p),
782 reinterpret_cast<void *>(-__alignof(_M_p)));
783 }
784
785 _GLIBCXX_ALWAYS_INLINE void
786 store(__pointer_type __p,
787 memory_order __m = memory_order_seq_cst) noexcept
788 {
789 memory_order __b __attribute__ ((__unused__))
790 = __m & __memory_order_mask;
791
792 __glibcxx_assert(__b != memory_order_acquire);
793 __glibcxx_assert(__b != memory_order_acq_rel);
794 __glibcxx_assert(__b != memory_order_consume);
795
796 __atomic_store_n(&_M_p, __p, int(__m));
797 }
798
799 _GLIBCXX_ALWAYS_INLINE void
800 store(__pointer_type __p,
801 memory_order __m = memory_order_seq_cst) volatile noexcept
802 {
803 memory_order __b __attribute__ ((__unused__))
804 = __m & __memory_order_mask;
805 __glibcxx_assert(__b != memory_order_acquire);
806 __glibcxx_assert(__b != memory_order_acq_rel);
807 __glibcxx_assert(__b != memory_order_consume);
808
809 __atomic_store_n(&_M_p, __p, int(__m));
810 }
811
812 _GLIBCXX_ALWAYS_INLINE __pointer_type
813 load(memory_order __m = memory_order_seq_cst) const noexcept
814 {
815 memory_order __b __attribute__ ((__unused__))
816 = __m & __memory_order_mask;
817 __glibcxx_assert(__b != memory_order_release);
818 __glibcxx_assert(__b != memory_order_acq_rel);
819
820 return __atomic_load_n(&_M_p, int(__m));
821 }
822
823 _GLIBCXX_ALWAYS_INLINE __pointer_type
824 load(memory_order __m = memory_order_seq_cst) const volatile noexcept
825 {
826 memory_order __b __attribute__ ((__unused__))
827 = __m & __memory_order_mask;
828 __glibcxx_assert(__b != memory_order_release);
829 __glibcxx_assert(__b != memory_order_acq_rel);
830
831 return __atomic_load_n(&_M_p, int(__m));
832 }
833
834 _GLIBCXX_ALWAYS_INLINE __pointer_type
835 exchange(__pointer_type __p,
836 memory_order __m = memory_order_seq_cst) noexcept
837 {
838 return __atomic_exchange_n(&_M_p, __p, int(__m));
839 }
840
841
842 _GLIBCXX_ALWAYS_INLINE __pointer_type
843 exchange(__pointer_type __p,
844 memory_order __m = memory_order_seq_cst) volatile noexcept
845 {
846 return __atomic_exchange_n(&_M_p, __p, int(__m));
847 }
848
849 _GLIBCXX_ALWAYS_INLINE bool
850 compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
851 memory_order __m1,
852 memory_order __m2) noexcept
853 {
854 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
855
856 return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 1,
857 int(__m1), int(__m2));
858 }
859
860 _GLIBCXX_ALWAYS_INLINE bool
861 compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
862 memory_order __m1,
863 memory_order __m2) volatile noexcept
864 {
865 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
866
867 return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 1,
868 int(__m1), int(__m2));
869 }
870
871 _GLIBCXX_ALWAYS_INLINE bool
872 compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
873 memory_order __m1,
874 memory_order __m2) noexcept
875 {
876 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
877
878 return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0,
879 int(__m1), int(__m2));
880 }
881
882 _GLIBCXX_ALWAYS_INLINE bool
883 compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
884 memory_order __m1,
885 memory_order __m2) volatile noexcept
886 {
887 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
888
889 return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0,
890 int(__m1), int(__m2));
891 }
892
893#if __cpp_lib_atomic_wait
894 _GLIBCXX_ALWAYS_INLINE void
895 wait(__pointer_type __old,
896 memory_order __m = memory_order_seq_cst) const noexcept
897 {
898 std::__atomic_wait_address_v(&_M_p, __old,
899 [__m, this]
900 { return this->load(__m); });
901 }
902
903 // TODO add const volatile overload
904
905 _GLIBCXX_ALWAYS_INLINE void
906 notify_one() const noexcept
907 { std::__atomic_notify_address(&_M_p, false); }
908
909 // TODO add const volatile overload
910
911 _GLIBCXX_ALWAYS_INLINE void
912 notify_all() const noexcept
913 { std::__atomic_notify_address(&_M_p, true); }
914
915 // TODO add const volatile overload
916#endif // __cpp_lib_atomic_wait
917
918 _GLIBCXX_ALWAYS_INLINE __pointer_type
919 fetch_add(ptrdiff_t __d,
920 memory_order __m = memory_order_seq_cst) noexcept
921 { return __atomic_fetch_add(&_M_p, _M_type_size(__d), int(__m)); }
922
923 _GLIBCXX_ALWAYS_INLINE __pointer_type
924 fetch_add(ptrdiff_t __d,
925 memory_order __m = memory_order_seq_cst) volatile noexcept
926 { return __atomic_fetch_add(&_M_p, _M_type_size(__d), int(__m)); }
927
928 _GLIBCXX_ALWAYS_INLINE __pointer_type
929 fetch_sub(ptrdiff_t __d,
930 memory_order __m = memory_order_seq_cst) noexcept
931 { return __atomic_fetch_sub(&_M_p, _M_type_size(__d), int(__m)); }
932
933 _GLIBCXX_ALWAYS_INLINE __pointer_type
934 fetch_sub(ptrdiff_t __d,
935 memory_order __m = memory_order_seq_cst) volatile noexcept
936 { return __atomic_fetch_sub(&_M_p, _M_type_size(__d), int(__m)); }
937 };
938
939#if __cplusplus > 201703L
940 // Implementation details of atomic_ref and atomic<floating-point>.
941 namespace __atomic_impl
942 {
943 // Remove volatile and create a non-deduced context for value arguments.
944 template<typename _Tp>
945 using _Val = remove_volatile_t<_Tp>;
946
947 // As above, but for difference_type arguments.
948 template<typename _Tp>
949 using _Diff = __conditional_t<is_pointer_v<_Tp>, ptrdiff_t, _Val<_Tp>>;
950
951 template<size_t _Size, size_t _Align>
952 _GLIBCXX_ALWAYS_INLINE bool
953 is_lock_free() noexcept
954 {
955 // Produce a fake, minimally aligned pointer.
956 return __atomic_is_lock_free(_Size, reinterpret_cast<void *>(-_Align));
957 }
958
959 template<typename _Tp>
960 _GLIBCXX_ALWAYS_INLINE void
961 store(_Tp* __ptr, _Val<_Tp> __t, memory_order __m) noexcept
962 { __atomic_store(__ptr, std::__addressof(__t), int(__m)); }
963
964 template<typename _Tp>
965 _GLIBCXX_ALWAYS_INLINE _Val<_Tp>
966 load(const _Tp* __ptr, memory_order __m) noexcept
967 {
968 alignas(_Tp) unsigned char __buf[sizeof(_Tp)];
969 auto* __dest = reinterpret_cast<_Val<_Tp>*>(__buf);
970 __atomic_load(__ptr, __dest, int(__m));
971 return *__dest;
972 }
973
974 template<typename _Tp>
975 _GLIBCXX_ALWAYS_INLINE _Val<_Tp>
976 exchange(_Tp* __ptr, _Val<_Tp> __desired, memory_order __m) noexcept
977 {
978 alignas(_Tp) unsigned char __buf[sizeof(_Tp)];
979 auto* __dest = reinterpret_cast<_Val<_Tp>*>(__buf);
980 __atomic_exchange(__ptr, std::__addressof(__desired), __dest, int(__m));
981 return *__dest;
982 }
983
984 template<typename _Tp>
985 _GLIBCXX_ALWAYS_INLINE bool
986 compare_exchange_weak(_Tp* __ptr, _Val<_Tp>& __expected,
987 _Val<_Tp> __desired, memory_order __success,
988 memory_order __failure) noexcept
989 {
990 __glibcxx_assert(__is_valid_cmpexch_failure_order(__failure));
991
992 return __atomic_compare_exchange(__ptr, std::__addressof(__expected),
993 std::__addressof(__desired), true,
994 int(__success), int(__failure));
995 }
996
997 template<typename _Tp>
998 _GLIBCXX_ALWAYS_INLINE bool
999 compare_exchange_strong(_Tp* __ptr, _Val<_Tp>& __expected,
1000 _Val<_Tp> __desired, memory_order __success,
1001 memory_order __failure) noexcept
1002 {
1003 __glibcxx_assert(__is_valid_cmpexch_failure_order(__failure));
1004
1005 return __atomic_compare_exchange(__ptr, std::__addressof(__expected),
1006 std::__addressof(__desired), false,
1007 int(__success), int(__failure));
1008 }
1009
1010#if __cpp_lib_atomic_wait
1011 template<typename _Tp>
1012 _GLIBCXX_ALWAYS_INLINE void
1013 wait(const _Tp* __ptr, _Val<_Tp> __old,
1014 memory_order __m = memory_order_seq_cst) noexcept
1015 {
1016 std::__atomic_wait_address_v(__ptr, __old,
1017 [__ptr, __m]() { return __atomic_impl::load(__ptr, __m); });
1018 }
1019
1020 // TODO add const volatile overload
1021
1022 template<typename _Tp>
1023 _GLIBCXX_ALWAYS_INLINE void
1024 notify_one(const _Tp* __ptr) noexcept
1025 { std::__atomic_notify_address(__ptr, false); }
1026
1027 // TODO add const volatile overload
1028
1029 template<typename _Tp>
1030 _GLIBCXX_ALWAYS_INLINE void
1031 notify_all(const _Tp* __ptr) noexcept
1032 { std::__atomic_notify_address(__ptr, true); }
1033
1034 // TODO add const volatile overload
1035#endif // __cpp_lib_atomic_wait
1036
1037 template<typename _Tp>
1038 _GLIBCXX_ALWAYS_INLINE _Tp
1039 fetch_add(_Tp* __ptr, _Diff<_Tp> __i, memory_order __m) noexcept
1040 { return __atomic_fetch_add(__ptr, __i, int(__m)); }
1041
1042 template<typename _Tp>
1043 _GLIBCXX_ALWAYS_INLINE _Tp
1044 fetch_sub(_Tp* __ptr, _Diff<_Tp> __i, memory_order __m) noexcept
1045 { return __atomic_fetch_sub(__ptr, __i, int(__m)); }
1046
1047 template<typename _Tp>
1048 _GLIBCXX_ALWAYS_INLINE _Tp
1049 fetch_and(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
1050 { return __atomic_fetch_and(__ptr, __i, int(__m)); }
1051
1052 template<typename _Tp>
1053 _GLIBCXX_ALWAYS_INLINE _Tp
1054 fetch_or(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
1055 { return __atomic_fetch_or(__ptr, __i, int(__m)); }
1056
1057 template<typename _Tp>
1058 _GLIBCXX_ALWAYS_INLINE _Tp
1059 fetch_xor(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
1060 { return __atomic_fetch_xor(__ptr, __i, int(__m)); }
1061
1062 template<typename _Tp>
1063 _GLIBCXX_ALWAYS_INLINE _Tp
1064 __add_fetch(_Tp* __ptr, _Diff<_Tp> __i) noexcept
1065 { return __atomic_add_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
1066
1067 template<typename _Tp>
1068 _GLIBCXX_ALWAYS_INLINE _Tp
1069 __sub_fetch(_Tp* __ptr, _Diff<_Tp> __i) noexcept
1070 { return __atomic_sub_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
1071
1072 template<typename _Tp>
1073 _GLIBCXX_ALWAYS_INLINE _Tp
1074 __and_fetch(_Tp* __ptr, _Val<_Tp> __i) noexcept
1075 { return __atomic_and_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
1076
1077 template<typename _Tp>
1078 _GLIBCXX_ALWAYS_INLINE _Tp
1079 __or_fetch(_Tp* __ptr, _Val<_Tp> __i) noexcept
1080 { return __atomic_or_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
1081
1082 template<typename _Tp>
1083 _GLIBCXX_ALWAYS_INLINE _Tp
1084 __xor_fetch(_Tp* __ptr, _Val<_Tp> __i) noexcept
1085 { return __atomic_xor_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
1086
1087 template<typename _Tp>
1088 _Tp
1089 __fetch_add_flt(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
1090 {
1091 _Val<_Tp> __oldval = load(__ptr, memory_order_relaxed);
1092 _Val<_Tp> __newval = __oldval + __i;
1093 while (!compare_exchange_weak(__ptr, __oldval, __newval, __m,
1094 memory_order_relaxed))
1095 __newval = __oldval + __i;
1096 return __oldval;
1097 }
1098
1099 template<typename _Tp>
1100 _Tp
1101 __fetch_sub_flt(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
1102 {
1103 _Val<_Tp> __oldval = load(__ptr, memory_order_relaxed);
1104 _Val<_Tp> __newval = __oldval - __i;
1105 while (!compare_exchange_weak(__ptr, __oldval, __newval, __m,
1106 memory_order_relaxed))
1107 __newval = __oldval - __i;
1108 return __oldval;
1109 }
1110
1111 template<typename _Tp>
1112 _Tp
1113 __add_fetch_flt(_Tp* __ptr, _Val<_Tp> __i) noexcept
1114 {
1115 _Val<_Tp> __oldval = load(__ptr, memory_order_relaxed);
1116 _Val<_Tp> __newval = __oldval + __i;
1117 while (!compare_exchange_weak(__ptr, __oldval, __newval,
1118 memory_order_seq_cst,
1119 memory_order_relaxed))
1120 __newval = __oldval + __i;
1121 return __newval;
1122 }
1123
1124 template<typename _Tp>
1125 _Tp
1126 __sub_fetch_flt(_Tp* __ptr, _Val<_Tp> __i) noexcept
1127 {
1128 _Val<_Tp> __oldval = load(__ptr, memory_order_relaxed);
1129 _Val<_Tp> __newval = __oldval - __i;
1130 while (!compare_exchange_weak(__ptr, __oldval, __newval,
1131 memory_order_seq_cst,
1132 memory_order_relaxed))
1133 __newval = __oldval - __i;
1134 return __newval;
1135 }
1136 } // namespace __atomic_impl
1137
1138 // base class for atomic<floating-point-type>
1139 template<typename _Fp>
1140 struct __atomic_float
1141 {
1142 static_assert(is_floating_point_v<_Fp>);
1143
1144 static constexpr size_t _S_alignment = __alignof__(_Fp);
1145
1146 public:
1147 using value_type = _Fp;
1148 using difference_type = value_type;
1149
1150 static constexpr bool is_always_lock_free
1151 = __atomic_always_lock_free(sizeof(_Fp), 0);
1152
1153 __atomic_float() = default;
1154
1155 constexpr
1156 __atomic_float(_Fp __t) : _M_fp(__t)
1157 { }
1158
1159 __atomic_float(const __atomic_float&) = delete;
1160 __atomic_float& operator=(const __atomic_float&) = delete;
1161 __atomic_float& operator=(const __atomic_float&) volatile = delete;
1162
1163 _Fp
1164 operator=(_Fp __t) volatile noexcept
1165 {
1166 this->store(__t);
1167 return __t;
1168 }
1169
1170 _Fp
1171 operator=(_Fp __t) noexcept
1172 {
1173 this->store(__t);
1174 return __t;
1175 }
1176
1177 bool
1178 is_lock_free() const volatile noexcept
1179 { return __atomic_impl::is_lock_free<sizeof(_Fp), _S_alignment>(); }
1180
1181 bool
1182 is_lock_free() const noexcept
1183 { return __atomic_impl::is_lock_free<sizeof(_Fp), _S_alignment>(); }
1184
1185 void
1186 store(_Fp __t, memory_order __m = memory_order_seq_cst) volatile noexcept
1187 { __atomic_impl::store(&_M_fp, __t, __m); }
1188
1189 void
1190 store(_Fp __t, memory_order __m = memory_order_seq_cst) noexcept
1191 { __atomic_impl::store(&_M_fp, __t, __m); }
1192
1193 _Fp
1194 load(memory_order __m = memory_order_seq_cst) const volatile noexcept
1195 { return __atomic_impl::load(&_M_fp, __m); }
1196
1197 _Fp
1198 load(memory_order __m = memory_order_seq_cst) const noexcept
1199 { return __atomic_impl::load(&_M_fp, __m); }
1200
1201 operator _Fp() const volatile noexcept { return this->load(); }
1202 operator _Fp() const noexcept { return this->load(); }
1203
1204 _Fp
1205 exchange(_Fp __desired,
1206 memory_order __m = memory_order_seq_cst) volatile noexcept
1207 { return __atomic_impl::exchange(&_M_fp, __desired, __m); }
1208
1209 _Fp
1210 exchange(_Fp __desired,
1211 memory_order __m = memory_order_seq_cst) noexcept
1212 { return __atomic_impl::exchange(&_M_fp, __desired, __m); }
1213
1214 bool
1215 compare_exchange_weak(_Fp& __expected, _Fp __desired,
1216 memory_order __success,
1217 memory_order __failure) noexcept
1218 {
1219 return __atomic_impl::compare_exchange_weak(&_M_fp,
1220 __expected, __desired,
1221 __success, __failure);
1222 }
1223
1224 bool
1225 compare_exchange_weak(_Fp& __expected, _Fp __desired,
1226 memory_order __success,
1227 memory_order __failure) volatile noexcept
1228 {
1229 return __atomic_impl::compare_exchange_weak(&_M_fp,
1230 __expected, __desired,
1231 __success, __failure);
1232 }
1233
1234 bool
1235 compare_exchange_strong(_Fp& __expected, _Fp __desired,
1236 memory_order __success,
1237 memory_order __failure) noexcept
1238 {
1239 return __atomic_impl::compare_exchange_strong(&_M_fp,
1240 __expected, __desired,
1241 __success, __failure);
1242 }
1243
1244 bool
1245 compare_exchange_strong(_Fp& __expected, _Fp __desired,
1246 memory_order __success,
1247 memory_order __failure) volatile noexcept
1248 {
1249 return __atomic_impl::compare_exchange_strong(&_M_fp,
1250 __expected, __desired,
1251 __success, __failure);
1252 }
1253
1254 bool
1255 compare_exchange_weak(_Fp& __expected, _Fp __desired,
1256 memory_order __order = memory_order_seq_cst)
1257 noexcept
1258 {
1259 return compare_exchange_weak(__expected, __desired, __order,
1260 __cmpexch_failure_order(__order));
1261 }
1262
1263 bool
1264 compare_exchange_weak(_Fp& __expected, _Fp __desired,
1265 memory_order __order = memory_order_seq_cst)
1266 volatile noexcept
1267 {
1268 return compare_exchange_weak(__expected, __desired, __order,
1269 __cmpexch_failure_order(__order));
1270 }
1271
1272 bool
1273 compare_exchange_strong(_Fp& __expected, _Fp __desired,
1274 memory_order __order = memory_order_seq_cst)
1275 noexcept
1276 {
1277 return compare_exchange_strong(__expected, __desired, __order,
1278 __cmpexch_failure_order(__order));
1279 }
1280
1281 bool
1282 compare_exchange_strong(_Fp& __expected, _Fp __desired,
1283 memory_order __order = memory_order_seq_cst)
1284 volatile noexcept
1285 {
1286 return compare_exchange_strong(__expected, __desired, __order,
1287 __cmpexch_failure_order(__order));
1288 }
1289
1290#if __cpp_lib_atomic_wait
1291 _GLIBCXX_ALWAYS_INLINE void
1292 wait(_Fp __old, memory_order __m = memory_order_seq_cst) const noexcept
1293 { __atomic_impl::wait(&_M_fp, __old, __m); }
1294
1295 // TODO add const volatile overload
1296
1297 _GLIBCXX_ALWAYS_INLINE void
1298 notify_one() const noexcept
1299 { __atomic_impl::notify_one(&_M_fp); }
1300
1301 // TODO add const volatile overload
1302
1303 _GLIBCXX_ALWAYS_INLINE void
1304 notify_all() const noexcept
1305 { __atomic_impl::notify_all(&_M_fp); }
1306
1307 // TODO add const volatile overload
1308#endif // __cpp_lib_atomic_wait
1309
1310 value_type
1311 fetch_add(value_type __i,
1312 memory_order __m = memory_order_seq_cst) noexcept
1313 { return __atomic_impl::__fetch_add_flt(&_M_fp, __i, __m); }
1314
1315 value_type
1316 fetch_add(value_type __i,
1317 memory_order __m = memory_order_seq_cst) volatile noexcept
1318 { return __atomic_impl::__fetch_add_flt(&_M_fp, __i, __m); }
1319
1320 value_type
1321 fetch_sub(value_type __i,
1322 memory_order __m = memory_order_seq_cst) noexcept
1323 { return __atomic_impl::__fetch_sub_flt(&_M_fp, __i, __m); }
1324
1325 value_type
1326 fetch_sub(value_type __i,
1327 memory_order __m = memory_order_seq_cst) volatile noexcept
1328 { return __atomic_impl::__fetch_sub_flt(&_M_fp, __i, __m); }
1329
1330 value_type
1331 operator+=(value_type __i) noexcept
1332 { return __atomic_impl::__add_fetch_flt(&_M_fp, __i); }
1333
1334 value_type
1335 operator+=(value_type __i) volatile noexcept
1336 { return __atomic_impl::__add_fetch_flt(&_M_fp, __i); }
1337
1338 value_type
1339 operator-=(value_type __i) noexcept
1340 { return __atomic_impl::__sub_fetch_flt(&_M_fp, __i); }
1341
1342 value_type
1343 operator-=(value_type __i) volatile noexcept
1344 { return __atomic_impl::__sub_fetch_flt(&_M_fp, __i); }
1345
1346 private:
1347 alignas(_S_alignment) _Fp _M_fp _GLIBCXX20_INIT(0);
1348 };
1349#undef _GLIBCXX20_INIT
1350
1351 template<typename _Tp,
1352 bool = is_integral_v<_Tp>, bool = is_floating_point_v<_Tp>>
1353 struct __atomic_ref;
1354
1355 // base class for non-integral, non-floating-point, non-pointer types
1356 template<typename _Tp>
1357 struct __atomic_ref<_Tp, false, false>
1358 {
1359 static_assert(is_trivially_copyable_v<_Tp>);
1360
1361 // 1/2/4/8/16-byte types must be aligned to at least their size.
1362 static constexpr int _S_min_alignment
1363 = (sizeof(_Tp) & (sizeof(_Tp) - 1)) || sizeof(_Tp) > 16
1364 ? 0 : sizeof(_Tp);
1365
1366 public:
1367 using value_type = _Tp;
1368
1369 static constexpr bool is_always_lock_free
1370 = __atomic_always_lock_free(sizeof(_Tp), 0);
1371
1372 static constexpr size_t required_alignment
1373 = _S_min_alignment > alignof(_Tp) ? _S_min_alignment : alignof(_Tp);
1374
1375 __atomic_ref& operator=(const __atomic_ref&) = delete;
1376
1377 explicit
1378 __atomic_ref(_Tp& __t) : _M_ptr(std::__addressof(__t))
1379 { __glibcxx_assert(((uintptr_t)_M_ptr % required_alignment) == 0); }
1380
1381 __atomic_ref(const __atomic_ref&) noexcept = default;
1382
1383 _Tp
1384 operator=(_Tp __t) const noexcept
1385 {
1386 this->store(__t);
1387 return __t;
1388 }
1389
1390 operator _Tp() const noexcept { return this->load(); }
1391
1392 bool
1393 is_lock_free() const noexcept
1394 { return __atomic_impl::is_lock_free<sizeof(_Tp), required_alignment>(); }
1395
1396 void
1397 store(_Tp __t, memory_order __m = memory_order_seq_cst) const noexcept
1398 { __atomic_impl::store(_M_ptr, __t, __m); }
1399
1400 _Tp
1401 load(memory_order __m = memory_order_seq_cst) const noexcept
1402 { return __atomic_impl::load(_M_ptr, __m); }
1403
1404 _Tp
1405 exchange(_Tp __desired, memory_order __m = memory_order_seq_cst)
1406 const noexcept
1407 { return __atomic_impl::exchange(_M_ptr, __desired, __m); }
1408
1409 bool
1410 compare_exchange_weak(_Tp& __expected, _Tp __desired,
1411 memory_order __success,
1412 memory_order __failure) const noexcept
1413 {
1414 return __atomic_impl::compare_exchange_weak(_M_ptr,
1415 __expected, __desired,
1416 __success, __failure);
1417 }
1418
1419 bool
1420 compare_exchange_strong(_Tp& __expected, _Tp __desired,
1421 memory_order __success,
1422 memory_order __failure) const noexcept
1423 {
1424 return __atomic_impl::compare_exchange_strong(_M_ptr,
1425 __expected, __desired,
1426 __success, __failure);
1427 }
1428
1429 bool
1430 compare_exchange_weak(_Tp& __expected, _Tp __desired,
1431 memory_order __order = memory_order_seq_cst)
1432 const noexcept
1433 {
1434 return compare_exchange_weak(__expected, __desired, __order,
1435 __cmpexch_failure_order(__order));
1436 }
1437
1438 bool
1439 compare_exchange_strong(_Tp& __expected, _Tp __desired,
1440 memory_order __order = memory_order_seq_cst)
1441 const noexcept
1442 {
1443 return compare_exchange_strong(__expected, __desired, __order,
1444 __cmpexch_failure_order(__order));
1445 }
1446
1447#if __cpp_lib_atomic_wait
1448 _GLIBCXX_ALWAYS_INLINE void
1449 wait(_Tp __old, memory_order __m = memory_order_seq_cst) const noexcept
1450 { __atomic_impl::wait(_M_ptr, __old, __m); }
1451
1452 // TODO add const volatile overload
1453
1454 _GLIBCXX_ALWAYS_INLINE void
1455 notify_one() const noexcept
1456 { __atomic_impl::notify_one(_M_ptr); }
1457
1458 // TODO add const volatile overload
1459
1460 _GLIBCXX_ALWAYS_INLINE void
1461 notify_all() const noexcept
1462 { __atomic_impl::notify_all(_M_ptr); }
1463
1464 // TODO add const volatile overload
1465#endif // __cpp_lib_atomic_wait
1466
1467 private:
1468 _Tp* _M_ptr;
1469 };
1470
1471 // base class for atomic_ref<integral-type>
1472 template<typename _Tp>
1473 struct __atomic_ref<_Tp, true, false>
1474 {
1475 static_assert(is_integral_v<_Tp>);
1476
1477 public:
1478 using value_type = _Tp;
1479 using difference_type = value_type;
1480
1481 static constexpr bool is_always_lock_free
1482 = __atomic_always_lock_free(sizeof(_Tp), 0);
1483
1484 static constexpr size_t required_alignment
1485 = sizeof(_Tp) > alignof(_Tp) ? sizeof(_Tp) : alignof(_Tp);
1486
1487 __atomic_ref() = delete;
1488 __atomic_ref& operator=(const __atomic_ref&) = delete;
1489
1490 explicit
1491 __atomic_ref(_Tp& __t) : _M_ptr(&__t)
1492 { __glibcxx_assert(((uintptr_t)_M_ptr % required_alignment) == 0); }
1493
1494 __atomic_ref(const __atomic_ref&) noexcept = default;
1495
1496 _Tp
1497 operator=(_Tp __t) const noexcept
1498 {
1499 this->store(__t);
1500 return __t;
1501 }
1502
1503 operator _Tp() const noexcept { return this->load(); }
1504
1505 bool
1506 is_lock_free() const noexcept
1507 {
1508 return __atomic_impl::is_lock_free<sizeof(_Tp), required_alignment>();
1509 }
1510
1511 void
1512 store(_Tp __t, memory_order __m = memory_order_seq_cst) const noexcept
1513 { __atomic_impl::store(_M_ptr, __t, __m); }
1514
1515 _Tp
1516 load(memory_order __m = memory_order_seq_cst) const noexcept
1517 { return __atomic_impl::load(_M_ptr, __m); }
1518
1519 _Tp
1520 exchange(_Tp __desired,
1521 memory_order __m = memory_order_seq_cst) const noexcept
1522 { return __atomic_impl::exchange(_M_ptr, __desired, __m); }
1523
1524 bool
1525 compare_exchange_weak(_Tp& __expected, _Tp __desired,
1526 memory_order __success,
1527 memory_order __failure) const noexcept
1528 {
1529 return __atomic_impl::compare_exchange_weak(_M_ptr,
1530 __expected, __desired,
1531 __success, __failure);
1532 }
1533
1534 bool
1535 compare_exchange_strong(_Tp& __expected, _Tp __desired,
1536 memory_order __success,
1537 memory_order __failure) const noexcept
1538 {
1539 return __atomic_impl::compare_exchange_strong(_M_ptr,
1540 __expected, __desired,
1541 __success, __failure);
1542 }
1543
1544 bool
1545 compare_exchange_weak(_Tp& __expected, _Tp __desired,
1546 memory_order __order = memory_order_seq_cst)
1547 const noexcept
1548 {
1549 return compare_exchange_weak(__expected, __desired, __order,
1550 __cmpexch_failure_order(__order));
1551 }
1552
1553 bool
1554 compare_exchange_strong(_Tp& __expected, _Tp __desired,
1555 memory_order __order = memory_order_seq_cst)
1556 const noexcept
1557 {
1558 return compare_exchange_strong(__expected, __desired, __order,
1559 __cmpexch_failure_order(__order));
1560 }
1561
1562#if __cpp_lib_atomic_wait
1563 _GLIBCXX_ALWAYS_INLINE void
1564 wait(_Tp __old, memory_order __m = memory_order_seq_cst) const noexcept
1565 { __atomic_impl::wait(_M_ptr, __old, __m); }
1566
1567 // TODO add const volatile overload
1568
1569 _GLIBCXX_ALWAYS_INLINE void
1570 notify_one() const noexcept
1571 { __atomic_impl::notify_one(_M_ptr); }
1572
1573 // TODO add const volatile overload
1574
1575 _GLIBCXX_ALWAYS_INLINE void
1576 notify_all() const noexcept
1577 { __atomic_impl::notify_all(_M_ptr); }
1578
1579 // TODO add const volatile overload
1580#endif // __cpp_lib_atomic_wait
1581
1582 value_type
1583 fetch_add(value_type __i,
1584 memory_order __m = memory_order_seq_cst) const noexcept
1585 { return __atomic_impl::fetch_add(_M_ptr, __i, __m); }
1586
1587 value_type
1588 fetch_sub(value_type __i,
1589 memory_order __m = memory_order_seq_cst) const noexcept
1590 { return __atomic_impl::fetch_sub(_M_ptr, __i, __m); }
1591
1592 value_type
1593 fetch_and(value_type __i,
1594 memory_order __m = memory_order_seq_cst) const noexcept
1595 { return __atomic_impl::fetch_and(_M_ptr, __i, __m); }
1596
1597 value_type
1598 fetch_or(value_type __i,
1599 memory_order __m = memory_order_seq_cst) const noexcept
1600 { return __atomic_impl::fetch_or(_M_ptr, __i, __m); }
1601
1602 value_type
1603 fetch_xor(value_type __i,
1604 memory_order __m = memory_order_seq_cst) const noexcept
1605 { return __atomic_impl::fetch_xor(_M_ptr, __i, __m); }
1606
1607 _GLIBCXX_ALWAYS_INLINE value_type
1608 operator++(int) const noexcept
1609 { return fetch_add(1); }
1610
1611 _GLIBCXX_ALWAYS_INLINE value_type
1612 operator--(int) const noexcept
1613 { return fetch_sub(1); }
1614
1615 value_type
1616 operator++() const noexcept
1617 { return __atomic_impl::__add_fetch(_M_ptr, value_type(1)); }
1618
1619 value_type
1620 operator--() const noexcept
1621 { return __atomic_impl::__sub_fetch(_M_ptr, value_type(1)); }
1622
1623 value_type
1624 operator+=(value_type __i) const noexcept
1625 { return __atomic_impl::__add_fetch(_M_ptr, __i); }
1626
1627 value_type
1628 operator-=(value_type __i) const noexcept
1629 { return __atomic_impl::__sub_fetch(_M_ptr, __i); }
1630
1631 value_type
1632 operator&=(value_type __i) const noexcept
1633 { return __atomic_impl::__and_fetch(_M_ptr, __i); }
1634
1635 value_type
1636 operator|=(value_type __i) const noexcept
1637 { return __atomic_impl::__or_fetch(_M_ptr, __i); }
1638
1639 value_type
1640 operator^=(value_type __i) const noexcept
1641 { return __atomic_impl::__xor_fetch(_M_ptr, __i); }
1642
1643 private:
1644 _Tp* _M_ptr;
1645 };
1646
1647 // base class for atomic_ref<floating-point-type>
1648 template<typename _Fp>
1649 struct __atomic_ref<_Fp, false, true>
1650 {
1651 static_assert(is_floating_point_v<_Fp>);
1652
1653 public:
1654 using value_type = _Fp;
1655 using difference_type = value_type;
1656
1657 static constexpr bool is_always_lock_free
1658 = __atomic_always_lock_free(sizeof(_Fp), 0);
1659
1660 static constexpr size_t required_alignment = __alignof__(_Fp);
1661
1662 __atomic_ref() = delete;
1663 __atomic_ref& operator=(const __atomic_ref&) = delete;
1664
1665 explicit
1666 __atomic_ref(_Fp& __t) : _M_ptr(&__t)
1667 { __glibcxx_assert(((uintptr_t)_M_ptr % required_alignment) == 0); }
1668
1669 __atomic_ref(const __atomic_ref&) noexcept = default;
1670
1671 _Fp
1672 operator=(_Fp __t) const noexcept
1673 {
1674 this->store(__t);
1675 return __t;
1676 }
1677
1678 operator _Fp() const noexcept { return this->load(); }
1679
1680 bool
1681 is_lock_free() const noexcept
1682 {
1683 return __atomic_impl::is_lock_free<sizeof(_Fp), required_alignment>();
1684 }
1685
1686 void
1687 store(_Fp __t, memory_order __m = memory_order_seq_cst) const noexcept
1688 { __atomic_impl::store(_M_ptr, __t, __m); }
1689
1690 _Fp
1691 load(memory_order __m = memory_order_seq_cst) const noexcept
1692 { return __atomic_impl::load(_M_ptr, __m); }
1693
1694 _Fp
1695 exchange(_Fp __desired,
1696 memory_order __m = memory_order_seq_cst) const noexcept
1697 { return __atomic_impl::exchange(_M_ptr, __desired, __m); }
1698
1699 bool
1700 compare_exchange_weak(_Fp& __expected, _Fp __desired,
1701 memory_order __success,
1702 memory_order __failure) const noexcept
1703 {
1704 return __atomic_impl::compare_exchange_weak(_M_ptr,
1705 __expected, __desired,
1706 __success, __failure);
1707 }
1708
1709 bool
1710 compare_exchange_strong(_Fp& __expected, _Fp __desired,
1711 memory_order __success,
1712 memory_order __failure) const noexcept
1713 {
1714 return __atomic_impl::compare_exchange_strong(_M_ptr,
1715 __expected, __desired,
1716 __success, __failure);
1717 }
1718
1719 bool
1720 compare_exchange_weak(_Fp& __expected, _Fp __desired,
1721 memory_order __order = memory_order_seq_cst)
1722 const noexcept
1723 {
1724 return compare_exchange_weak(__expected, __desired, __order,
1725 __cmpexch_failure_order(__order));
1726 }
1727
1728 bool
1729 compare_exchange_strong(_Fp& __expected, _Fp __desired,
1730 memory_order __order = memory_order_seq_cst)
1731 const noexcept
1732 {
1733 return compare_exchange_strong(__expected, __desired, __order,
1734 __cmpexch_failure_order(__order));
1735 }
1736
1737#if __cpp_lib_atomic_wait
1738 _GLIBCXX_ALWAYS_INLINE void
1739 wait(_Fp __old, memory_order __m = memory_order_seq_cst) const noexcept
1740 { __atomic_impl::wait(_M_ptr, __old, __m); }
1741
1742 // TODO add const volatile overload
1743
1744 _GLIBCXX_ALWAYS_INLINE void
1745 notify_one() const noexcept
1746 { __atomic_impl::notify_one(_M_ptr); }
1747
1748 // TODO add const volatile overload
1749
1750 _GLIBCXX_ALWAYS_INLINE void
1751 notify_all() const noexcept
1752 { __atomic_impl::notify_all(_M_ptr); }
1753
1754 // TODO add const volatile overload
1755#endif // __cpp_lib_atomic_wait
1756
1757 value_type
1758 fetch_add(value_type __i,
1759 memory_order __m = memory_order_seq_cst) const noexcept
1760 { return __atomic_impl::__fetch_add_flt(_M_ptr, __i, __m); }
1761
1762 value_type
1763 fetch_sub(value_type __i,
1764 memory_order __m = memory_order_seq_cst) const noexcept
1765 { return __atomic_impl::__fetch_sub_flt(_M_ptr, __i, __m); }
1766
1767 value_type
1768 operator+=(value_type __i) const noexcept
1769 { return __atomic_impl::__add_fetch_flt(_M_ptr, __i); }
1770
1771 value_type
1772 operator-=(value_type __i) const noexcept
1773 { return __atomic_impl::__sub_fetch_flt(_M_ptr, __i); }
1774
1775 private:
1776 _Fp* _M_ptr;
1777 };
1778
1779 // base class for atomic_ref<pointer-type>
1780 template<typename _Tp>
1781 struct __atomic_ref<_Tp*, false, false>
1782 {
1783 public:
1784 using value_type = _Tp*;
1785 using difference_type = ptrdiff_t;
1786
1787 static constexpr bool is_always_lock_free = ATOMIC_POINTER_LOCK_FREE == 2;
1788
1789 static constexpr size_t required_alignment = __alignof__(_Tp*);
1790
1791 __atomic_ref() = delete;
1792 __atomic_ref& operator=(const __atomic_ref&) = delete;
1793
1794 explicit
1795 __atomic_ref(_Tp*& __t) : _M_ptr(std::__addressof(__t))
1796 { __glibcxx_assert(((uintptr_t)_M_ptr % required_alignment) == 0); }
1797
1798 __atomic_ref(const __atomic_ref&) noexcept = default;
1799
1800 _Tp*
1801 operator=(_Tp* __t) const noexcept
1802 {
1803 this->store(__t);
1804 return __t;
1805 }
1806
1807 operator _Tp*() const noexcept { return this->load(); }
1808
1809 bool
1810 is_lock_free() const noexcept
1811 {
1812 return __atomic_impl::is_lock_free<sizeof(_Tp*), required_alignment>();
1813 }
1814
1815 void
1816 store(_Tp* __t, memory_order __m = memory_order_seq_cst) const noexcept
1817 { __atomic_impl::store(_M_ptr, __t, __m); }
1818
1819 _Tp*
1820 load(memory_order __m = memory_order_seq_cst) const noexcept
1821 { return __atomic_impl::load(_M_ptr, __m); }
1822
1823 _Tp*
1824 exchange(_Tp* __desired,
1825 memory_order __m = memory_order_seq_cst) const noexcept
1826 { return __atomic_impl::exchange(_M_ptr, __desired, __m); }
1827
1828 bool
1829 compare_exchange_weak(_Tp*& __expected, _Tp* __desired,
1830 memory_order __success,
1831 memory_order __failure) const noexcept
1832 {
1833 return __atomic_impl::compare_exchange_weak(_M_ptr,
1834 __expected, __desired,
1835 __success, __failure);
1836 }
1837
1838 bool
1839 compare_exchange_strong(_Tp*& __expected, _Tp* __desired,
1840 memory_order __success,
1841 memory_order __failure) const noexcept
1842 {
1843 return __atomic_impl::compare_exchange_strong(_M_ptr,
1844 __expected, __desired,
1845 __success, __failure);
1846 }
1847
1848 bool
1849 compare_exchange_weak(_Tp*& __expected, _Tp* __desired,
1850 memory_order __order = memory_order_seq_cst)
1851 const noexcept
1852 {
1853 return compare_exchange_weak(__expected, __desired, __order,
1854 __cmpexch_failure_order(__order));
1855 }
1856
1857 bool
1858 compare_exchange_strong(_Tp*& __expected, _Tp* __desired,
1859 memory_order __order = memory_order_seq_cst)
1860 const noexcept
1861 {
1862 return compare_exchange_strong(__expected, __desired, __order,
1863 __cmpexch_failure_order(__order));
1864 }
1865
1866#if __cpp_lib_atomic_wait
1867 _GLIBCXX_ALWAYS_INLINE void
1868 wait(_Tp* __old, memory_order __m = memory_order_seq_cst) const noexcept
1869 { __atomic_impl::wait(_M_ptr, __old, __m); }
1870
1871 // TODO add const volatile overload
1872
1873 _GLIBCXX_ALWAYS_INLINE void
1874 notify_one() const noexcept
1875 { __atomic_impl::notify_one(_M_ptr); }
1876
1877 // TODO add const volatile overload
1878
1879 _GLIBCXX_ALWAYS_INLINE void
1880 notify_all() const noexcept
1881 { __atomic_impl::notify_all(_M_ptr); }
1882
1883 // TODO add const volatile overload
1884#endif // __cpp_lib_atomic_wait
1885
1886 _GLIBCXX_ALWAYS_INLINE value_type
1887 fetch_add(difference_type __d,
1888 memory_order __m = memory_order_seq_cst) const noexcept
1889 { return __atomic_impl::fetch_add(_M_ptr, _S_type_size(__d), __m); }
1890
1891 _GLIBCXX_ALWAYS_INLINE value_type
1892 fetch_sub(difference_type __d,
1893 memory_order __m = memory_order_seq_cst) const noexcept
1894 { return __atomic_impl::fetch_sub(_M_ptr, _S_type_size(__d), __m); }
1895
1896 value_type
1897 operator++(int) const noexcept
1898 { return fetch_add(1); }
1899
1900 value_type
1901 operator--(int) const noexcept
1902 { return fetch_sub(1); }
1903
1904 value_type
1905 operator++() const noexcept
1906 {
1907 return __atomic_impl::__add_fetch(_M_ptr, _S_type_size(1));
1908 }
1909
1910 value_type
1911 operator--() const noexcept
1912 {
1913 return __atomic_impl::__sub_fetch(_M_ptr, _S_type_size(1));
1914 }
1915
1916 value_type
1917 operator+=(difference_type __d) const noexcept
1918 {
1919 return __atomic_impl::__add_fetch(_M_ptr, _S_type_size(__d));
1920 }
1921
1922 value_type
1923 operator-=(difference_type __d) const noexcept
1924 {
1925 return __atomic_impl::__sub_fetch(_M_ptr, _S_type_size(__d));
1926 }
1927
1928 private:
1929 static constexpr ptrdiff_t
1930 _S_type_size(ptrdiff_t __d) noexcept
1931 {
1932 static_assert(is_object_v<_Tp>);
1933 return __d * sizeof(_Tp);
1934 }
1935
1936 _Tp** _M_ptr;
1937 };
1938
1939#endif // C++2a
1940
1941 /// @} group atomics
1942
1943_GLIBCXX_END_NAMESPACE_VERSION
1944} // namespace std
1945
1946#endif
1947