1 | /* |
2 | * Atomic helper templates |
3 | * Included from tcg-runtime.c and cputlb.c. |
4 | * |
5 | * Copyright (c) 2016 Red Hat, Inc |
6 | * |
7 | * This library is free software; you can redistribute it and/or |
8 | * modify it under the terms of the GNU Lesser General Public |
9 | * License as published by the Free Software Foundation; either |
10 | * version 2.1 of the License, or (at your option) any later version. |
11 | * |
12 | * This library is distributed in the hope that it will be useful, |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
15 | * Lesser General Public License for more details. |
16 | * |
17 | * You should have received a copy of the GNU Lesser General Public |
18 | * License along with this library; if not, see <http://www.gnu.org/licenses/>. |
19 | */ |
20 | |
21 | #include "trace/mem.h" |
22 | |
23 | #if DATA_SIZE == 16 |
24 | # define SUFFIX o |
25 | # define DATA_TYPE Int128 |
26 | # define BSWAP bswap128 |
27 | # define SHIFT 4 |
28 | #elif DATA_SIZE == 8 |
29 | # define SUFFIX q |
30 | # define DATA_TYPE uint64_t |
31 | # define SDATA_TYPE int64_t |
32 | # define BSWAP bswap64 |
33 | # define SHIFT 3 |
34 | #elif DATA_SIZE == 4 |
35 | # define SUFFIX l |
36 | # define DATA_TYPE uint32_t |
37 | # define SDATA_TYPE int32_t |
38 | # define BSWAP bswap32 |
39 | # define SHIFT 2 |
40 | #elif DATA_SIZE == 2 |
41 | # define SUFFIX w |
42 | # define DATA_TYPE uint16_t |
43 | # define SDATA_TYPE int16_t |
44 | # define BSWAP bswap16 |
45 | # define SHIFT 1 |
46 | #elif DATA_SIZE == 1 |
47 | # define SUFFIX b |
48 | # define DATA_TYPE uint8_t |
49 | # define SDATA_TYPE int8_t |
50 | # define BSWAP |
51 | # define SHIFT 0 |
52 | #else |
53 | # error unsupported data size |
54 | #endif |
55 | |
56 | #if DATA_SIZE >= 4 |
57 | # define ABI_TYPE DATA_TYPE |
58 | #else |
59 | # define ABI_TYPE uint32_t |
60 | #endif |
61 | |
62 | #define ATOMIC_TRACE_RMW do { \ |
63 | uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \ |
64 | \ |
65 | trace_guest_mem_before_exec(env_cpu(env), addr, info); \ |
66 | trace_guest_mem_before_exec(env_cpu(env), addr, \ |
67 | info | TRACE_MEM_ST); \ |
68 | } while (0) |
69 | |
70 | #define ATOMIC_TRACE_LD do { \ |
71 | uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \ |
72 | \ |
73 | trace_guest_mem_before_exec(env_cpu(env), addr, info); \ |
74 | } while (0) |
75 | |
76 | # define ATOMIC_TRACE_ST do { \ |
77 | uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, true); \ |
78 | \ |
79 | trace_guest_mem_before_exec(env_cpu(env), addr, info); \ |
80 | } while (0) |
81 | |
82 | /* Define host-endian atomic operations. Note that END is used within |
83 | the ATOMIC_NAME macro, and redefined below. */ |
84 | #if DATA_SIZE == 1 |
85 | # define END |
86 | # define MEND _be /* either le or be would be fine */ |
87 | #elif defined(HOST_WORDS_BIGENDIAN) |
88 | # define END _be |
89 | # define MEND _be |
90 | #else |
91 | # define END _le |
92 | # define MEND _le |
93 | #endif |
94 | |
95 | ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, |
96 | ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) |
97 | { |
98 | ATOMIC_MMU_DECLS; |
99 | DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; |
100 | DATA_TYPE ret; |
101 | |
102 | ATOMIC_TRACE_RMW; |
103 | #if DATA_SIZE == 16 |
104 | ret = atomic16_cmpxchg(haddr, cmpv, newv); |
105 | #else |
106 | ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv); |
107 | #endif |
108 | ATOMIC_MMU_CLEANUP; |
109 | return ret; |
110 | } |
111 | |
112 | #if DATA_SIZE >= 16 |
113 | #if HAVE_ATOMIC128 |
114 | ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) |
115 | { |
116 | ATOMIC_MMU_DECLS; |
117 | DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; |
118 | |
119 | ATOMIC_TRACE_LD; |
120 | val = atomic16_read(haddr); |
121 | ATOMIC_MMU_CLEANUP; |
122 | return val; |
123 | } |
124 | |
125 | void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, |
126 | ABI_TYPE val EXTRA_ARGS) |
127 | { |
128 | ATOMIC_MMU_DECLS; |
129 | DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; |
130 | |
131 | ATOMIC_TRACE_ST; |
132 | atomic16_set(haddr, val); |
133 | ATOMIC_MMU_CLEANUP; |
134 | } |
135 | #endif |
136 | #else |
137 | ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, |
138 | ABI_TYPE val EXTRA_ARGS) |
139 | { |
140 | ATOMIC_MMU_DECLS; |
141 | DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; |
142 | DATA_TYPE ret; |
143 | |
144 | ATOMIC_TRACE_RMW; |
145 | ret = atomic_xchg__nocheck(haddr, val); |
146 | ATOMIC_MMU_CLEANUP; |
147 | return ret; |
148 | } |
149 | |
150 | #define GEN_ATOMIC_HELPER(X) \ |
151 | ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ |
152 | ABI_TYPE val EXTRA_ARGS) \ |
153 | { \ |
154 | ATOMIC_MMU_DECLS; \ |
155 | DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ |
156 | DATA_TYPE ret; \ |
157 | \ |
158 | ATOMIC_TRACE_RMW; \ |
159 | ret = atomic_##X(haddr, val); \ |
160 | ATOMIC_MMU_CLEANUP; \ |
161 | return ret; \ |
162 | } |
163 | |
164 | GEN_ATOMIC_HELPER(fetch_add) |
165 | GEN_ATOMIC_HELPER(fetch_and) |
166 | GEN_ATOMIC_HELPER(fetch_or) |
167 | GEN_ATOMIC_HELPER(fetch_xor) |
168 | GEN_ATOMIC_HELPER(add_fetch) |
169 | GEN_ATOMIC_HELPER(and_fetch) |
170 | GEN_ATOMIC_HELPER(or_fetch) |
171 | GEN_ATOMIC_HELPER(xor_fetch) |
172 | |
173 | #undef GEN_ATOMIC_HELPER |
174 | |
175 | /* These helpers are, as a whole, full barriers. Within the helper, |
176 | * the leading barrier is explicit and the trailing barrier is within |
177 | * cmpxchg primitive. |
178 | * |
179 | * Trace this load + RMW loop as a single RMW op. This way, regardless |
180 | * of CF_PARALLEL's value, we'll trace just a read and a write. |
181 | */ |
182 | #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ |
183 | ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ |
184 | ABI_TYPE xval EXTRA_ARGS) \ |
185 | { \ |
186 | ATOMIC_MMU_DECLS; \ |
187 | XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ |
188 | XDATA_TYPE cmp, old, new, val = xval; \ |
189 | \ |
190 | ATOMIC_TRACE_RMW; \ |
191 | smp_mb(); \ |
192 | cmp = atomic_read__nocheck(haddr); \ |
193 | do { \ |
194 | old = cmp; new = FN(old, val); \ |
195 | cmp = atomic_cmpxchg__nocheck(haddr, old, new); \ |
196 | } while (cmp != old); \ |
197 | ATOMIC_MMU_CLEANUP; \ |
198 | return RET; \ |
199 | } |
200 | |
201 | GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) |
202 | GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) |
203 | GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) |
204 | GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) |
205 | |
206 | GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) |
207 | GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) |
208 | GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) |
209 | GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) |
210 | |
211 | #undef GEN_ATOMIC_HELPER_FN |
212 | #endif /* DATA SIZE >= 16 */ |
213 | |
214 | #undef END |
215 | #undef MEND |
216 | |
217 | #if DATA_SIZE > 1 |
218 | |
219 | /* Define reverse-host-endian atomic operations. Note that END is used |
220 | within the ATOMIC_NAME macro. */ |
221 | #ifdef HOST_WORDS_BIGENDIAN |
222 | # define END _le |
223 | # define MEND _le |
224 | #else |
225 | # define END _be |
226 | # define MEND _be |
227 | #endif |
228 | |
229 | ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, |
230 | ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) |
231 | { |
232 | ATOMIC_MMU_DECLS; |
233 | DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; |
234 | DATA_TYPE ret; |
235 | |
236 | ATOMIC_TRACE_RMW; |
237 | #if DATA_SIZE == 16 |
238 | ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv)); |
239 | #else |
240 | ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)); |
241 | #endif |
242 | ATOMIC_MMU_CLEANUP; |
243 | return BSWAP(ret); |
244 | } |
245 | |
246 | #if DATA_SIZE >= 16 |
247 | #if HAVE_ATOMIC128 |
248 | ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) |
249 | { |
250 | ATOMIC_MMU_DECLS; |
251 | DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; |
252 | |
253 | ATOMIC_TRACE_LD; |
254 | val = atomic16_read(haddr); |
255 | ATOMIC_MMU_CLEANUP; |
256 | return BSWAP(val); |
257 | } |
258 | |
259 | void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, |
260 | ABI_TYPE val EXTRA_ARGS) |
261 | { |
262 | ATOMIC_MMU_DECLS; |
263 | DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; |
264 | |
265 | ATOMIC_TRACE_ST; |
266 | val = BSWAP(val); |
267 | atomic16_set(haddr, val); |
268 | ATOMIC_MMU_CLEANUP; |
269 | } |
270 | #endif |
271 | #else |
272 | ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, |
273 | ABI_TYPE val EXTRA_ARGS) |
274 | { |
275 | ATOMIC_MMU_DECLS; |
276 | DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; |
277 | ABI_TYPE ret; |
278 | |
279 | ATOMIC_TRACE_RMW; |
280 | ret = atomic_xchg__nocheck(haddr, BSWAP(val)); |
281 | ATOMIC_MMU_CLEANUP; |
282 | return BSWAP(ret); |
283 | } |
284 | |
285 | #define GEN_ATOMIC_HELPER(X) \ |
286 | ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ |
287 | ABI_TYPE val EXTRA_ARGS) \ |
288 | { \ |
289 | ATOMIC_MMU_DECLS; \ |
290 | DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ |
291 | DATA_TYPE ret; \ |
292 | \ |
293 | ATOMIC_TRACE_RMW; \ |
294 | ret = atomic_##X(haddr, BSWAP(val)); \ |
295 | ATOMIC_MMU_CLEANUP; \ |
296 | return BSWAP(ret); \ |
297 | } |
298 | |
299 | GEN_ATOMIC_HELPER(fetch_and) |
300 | GEN_ATOMIC_HELPER(fetch_or) |
301 | GEN_ATOMIC_HELPER(fetch_xor) |
302 | GEN_ATOMIC_HELPER(and_fetch) |
303 | GEN_ATOMIC_HELPER(or_fetch) |
304 | GEN_ATOMIC_HELPER(xor_fetch) |
305 | |
306 | #undef GEN_ATOMIC_HELPER |
307 | |
308 | /* These helpers are, as a whole, full barriers. Within the helper, |
309 | * the leading barrier is explicit and the trailing barrier is within |
310 | * cmpxchg primitive. |
311 | * |
312 | * Trace this load + RMW loop as a single RMW op. This way, regardless |
313 | * of CF_PARALLEL's value, we'll trace just a read and a write. |
314 | */ |
315 | #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ |
316 | ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ |
317 | ABI_TYPE xval EXTRA_ARGS) \ |
318 | { \ |
319 | ATOMIC_MMU_DECLS; \ |
320 | XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ |
321 | XDATA_TYPE ldo, ldn, old, new, val = xval; \ |
322 | \ |
323 | ATOMIC_TRACE_RMW; \ |
324 | smp_mb(); \ |
325 | ldn = atomic_read__nocheck(haddr); \ |
326 | do { \ |
327 | ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \ |
328 | ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \ |
329 | } while (ldo != ldn); \ |
330 | ATOMIC_MMU_CLEANUP; \ |
331 | return RET; \ |
332 | } |
333 | |
334 | GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old) |
335 | GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old) |
336 | GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old) |
337 | GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old) |
338 | |
339 | GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new) |
340 | GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new) |
341 | GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new) |
342 | GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) |
343 | |
344 | /* Note that for addition, we need to use a separate cmpxchg loop instead |
345 | of bswaps for the reverse-host-endian helpers. */ |
346 | #define ADD(X, Y) (X + Y) |
347 | GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old) |
348 | GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new) |
349 | #undef ADD |
350 | |
351 | #undef GEN_ATOMIC_HELPER_FN |
352 | #endif /* DATA_SIZE >= 16 */ |
353 | |
354 | #undef END |
355 | #undef MEND |
356 | #endif /* DATA_SIZE > 1 */ |
357 | |
358 | #undef ATOMIC_TRACE_ST |
359 | #undef ATOMIC_TRACE_LD |
360 | #undef ATOMIC_TRACE_RMW |
361 | |
362 | #undef BSWAP |
363 | #undef ABI_TYPE |
364 | #undef DATA_TYPE |
365 | #undef SDATA_TYPE |
366 | #undef SUFFIX |
367 | #undef DATA_SIZE |
368 | #undef SHIFT |
369 | |