1 | /* |
2 | * Simple interface for 128-bit atomic operations. |
3 | * |
4 | * Copyright (C) 2018 Linaro, Ltd. |
5 | * |
6 | * This work is licensed under the terms of the GNU GPL, version 2 or later. |
7 | * See the COPYING file in the top-level directory. |
8 | * |
9 | * See docs/devel/atomics.txt for discussion about the guarantees each |
10 | * atomic primitive is meant to provide. |
11 | */ |
12 | |
13 | #ifndef QEMU_ATOMIC128_H |
14 | #define QEMU_ATOMIC128_H |
15 | |
16 | #include "qemu/int128.h" |
17 | |
18 | /* |
19 | * GCC is a house divided about supporting large atomic operations. |
20 | * |
21 | * For hosts that only have large compare-and-swap, a legalistic reading |
22 | * of the C++ standard means that one cannot implement __atomic_read on |
23 | * read-only memory, and thus all atomic operations must synchronize |
24 | * through libatomic. |
25 | * |
26 | * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80878 |
27 | * |
28 | * This interpretation is not especially helpful for QEMU. |
29 | * For softmmu, all RAM is always read/write from the hypervisor. |
30 | * For user-only, if the guest doesn't implement such an __atomic_read |
31 | * then the host need not worry about it either. |
32 | * |
33 | * Moreover, using libatomic is not an option, because its interface is |
34 | * built for std::atomic<T>, and requires that *all* accesses to such an |
35 | * object go through the library. In our case we do not have an object |
36 | * in the C/C++ sense, but a view of memory as seen by the guest. |
37 | * The guest may issue a large atomic operation and then access those |
38 | * pieces using word-sized accesses. From the hypervisor, we have no |
39 | * way to connect those two actions. |
40 | * |
41 | * Therefore, special case each platform. |
42 | */ |
43 | |
44 | #if defined(CONFIG_ATOMIC128) |
45 | static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) |
46 | { |
47 | return atomic_cmpxchg__nocheck(ptr, cmp, new); |
48 | } |
49 | # define HAVE_CMPXCHG128 1 |
50 | #elif defined(CONFIG_CMPXCHG128) |
51 | static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) |
52 | { |
53 | return __sync_val_compare_and_swap_16(ptr, cmp, new); |
54 | } |
55 | # define HAVE_CMPXCHG128 1 |
56 | #elif defined(__aarch64__) |
57 | /* Through gcc 8, aarch64 has no support for 128-bit at all. */ |
58 | static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new) |
59 | { |
60 | uint64_t cmpl = int128_getlo(cmp), cmph = int128_gethi(cmp); |
61 | uint64_t newl = int128_getlo(new), newh = int128_gethi(new); |
62 | uint64_t oldl, oldh; |
63 | uint32_t tmp; |
64 | |
65 | asm("0: ldaxp %[oldl], %[oldh], %[mem]\n\t" |
66 | "cmp %[oldl], %[cmpl]\n\t" |
67 | "ccmp %[oldh], %[cmph], #0, eq\n\t" |
68 | "b.ne 1f\n\t" |
69 | "stlxp %w[tmp], %[newl], %[newh], %[mem]\n\t" |
70 | "cbnz %w[tmp], 0b\n" |
71 | "1:" |
72 | : [mem] "+m" (*ptr), [tmp] "=&r" (tmp), |
73 | [oldl] "=&r" (oldl), [oldh] "=&r" (oldh) |
74 | : [cmpl] "r" (cmpl), [cmph] "r" (cmph), |
75 | [newl] "r" (newl), [newh] "r" (newh) |
76 | : "memory" , "cc" ); |
77 | |
78 | return int128_make128(oldl, oldh); |
79 | } |
80 | # define HAVE_CMPXCHG128 1 |
81 | #else |
82 | /* Fallback definition that must be optimized away, or error. */ |
83 | Int128 QEMU_ERROR("unsupported atomic" ) |
84 | atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new); |
85 | # define HAVE_CMPXCHG128 0 |
86 | #endif /* Some definition for HAVE_CMPXCHG128 */ |
87 | |
88 | |
89 | #if defined(CONFIG_ATOMIC128) |
90 | static inline Int128 atomic16_read(Int128 *ptr) |
91 | { |
92 | return atomic_read__nocheck(ptr); |
93 | } |
94 | |
95 | static inline void atomic16_set(Int128 *ptr, Int128 val) |
96 | { |
97 | atomic_set__nocheck(ptr, val); |
98 | } |
99 | |
100 | # define HAVE_ATOMIC128 1 |
101 | #elif !defined(CONFIG_USER_ONLY) && defined(__aarch64__) |
102 | /* We can do better than cmpxchg for AArch64. */ |
103 | static inline Int128 atomic16_read(Int128 *ptr) |
104 | { |
105 | uint64_t l, h; |
106 | uint32_t tmp; |
107 | |
108 | /* The load must be paired with the store to guarantee not tearing. */ |
109 | asm("0: ldxp %[l], %[h], %[mem]\n\t" |
110 | "stxp %w[tmp], %[l], %[h], %[mem]\n\t" |
111 | "cbnz %w[tmp], 0b" |
112 | : [mem] "+m" (*ptr), [tmp] "=r" (tmp), [l] "=r" (l), [h] "=r" (h)); |
113 | |
114 | return int128_make128(l, h); |
115 | } |
116 | |
117 | static inline void atomic16_set(Int128 *ptr, Int128 val) |
118 | { |
119 | uint64_t l = int128_getlo(val), h = int128_gethi(val); |
120 | uint64_t t1, t2; |
121 | |
122 | /* Load into temporaries to acquire the exclusive access lock. */ |
123 | asm("0: ldxp %[t1], %[t2], %[mem]\n\t" |
124 | "stxp %w[t1], %[l], %[h], %[mem]\n\t" |
125 | "cbnz %w[t1], 0b" |
126 | : [mem] "+m" (*ptr), [t1] "=&r" (t1), [t2] "=&r" (t2) |
127 | : [l] "r" (l), [h] "r" (h)); |
128 | } |
129 | |
130 | # define HAVE_ATOMIC128 1 |
131 | #elif !defined(CONFIG_USER_ONLY) && HAVE_CMPXCHG128 |
132 | static inline Int128 atomic16_read(Int128 *ptr) |
133 | { |
134 | /* Maybe replace 0 with 0, returning the old value. */ |
135 | return atomic16_cmpxchg(ptr, 0, 0); |
136 | } |
137 | |
138 | static inline void atomic16_set(Int128 *ptr, Int128 val) |
139 | { |
140 | Int128 old = *ptr, cmp; |
141 | do { |
142 | cmp = old; |
143 | old = atomic16_cmpxchg(ptr, cmp, val); |
144 | } while (old != cmp); |
145 | } |
146 | |
147 | # define HAVE_ATOMIC128 1 |
148 | #else |
149 | /* Fallback definitions that must be optimized away, or error. */ |
150 | Int128 QEMU_ERROR("unsupported atomic" ) atomic16_read(Int128 *ptr); |
151 | void QEMU_ERROR("unsupported atomic" ) atomic16_set(Int128 *ptr, Int128 val); |
152 | # define HAVE_ATOMIC128 0 |
153 | #endif /* Some definition for HAVE_ATOMIC128 */ |
154 | |
155 | #endif /* QEMU_ATOMIC128_H */ |
156 | |