1/*
2 * Atomic helper templates
3 * Included from tcg-runtime.c and cputlb.c.
4 *
5 * Copyright (c) 2016 Red Hat, Inc
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 */
20
21#include "trace/mem.h"
22
23#if DATA_SIZE == 16
24# define SUFFIX o
25# define DATA_TYPE Int128
26# define BSWAP bswap128
27# define SHIFT 4
28#elif DATA_SIZE == 8
29# define SUFFIX q
30# define DATA_TYPE uint64_t
31# define SDATA_TYPE int64_t
32# define BSWAP bswap64
33# define SHIFT 3
34#elif DATA_SIZE == 4
35# define SUFFIX l
36# define DATA_TYPE uint32_t
37# define SDATA_TYPE int32_t
38# define BSWAP bswap32
39# define SHIFT 2
40#elif DATA_SIZE == 2
41# define SUFFIX w
42# define DATA_TYPE uint16_t
43# define SDATA_TYPE int16_t
44# define BSWAP bswap16
45# define SHIFT 1
46#elif DATA_SIZE == 1
47# define SUFFIX b
48# define DATA_TYPE uint8_t
49# define SDATA_TYPE int8_t
50# define BSWAP
51# define SHIFT 0
52#else
53# error unsupported data size
54#endif
55
56#if DATA_SIZE >= 4
57# define ABI_TYPE DATA_TYPE
58#else
59# define ABI_TYPE uint32_t
60#endif
61
62#define ATOMIC_TRACE_RMW do { \
63 uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
64 \
65 trace_guest_mem_before_exec(env_cpu(env), addr, info); \
66 trace_guest_mem_before_exec(env_cpu(env), addr, \
67 info | TRACE_MEM_ST); \
68 } while (0)
69
70#define ATOMIC_TRACE_LD do { \
71 uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
72 \
73 trace_guest_mem_before_exec(env_cpu(env), addr, info); \
74 } while (0)
75
76# define ATOMIC_TRACE_ST do { \
77 uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, true); \
78 \
79 trace_guest_mem_before_exec(env_cpu(env), addr, info); \
80 } while (0)
81
82/* Define host-endian atomic operations. Note that END is used within
83 the ATOMIC_NAME macro, and redefined below. */
84#if DATA_SIZE == 1
85# define END
86# define MEND _be /* either le or be would be fine */
87#elif defined(HOST_WORDS_BIGENDIAN)
88# define END _be
89# define MEND _be
90#else
91# define END _le
92# define MEND _le
93#endif
94
95ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
96 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
97{
98 ATOMIC_MMU_DECLS;
99 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
100 DATA_TYPE ret;
101
102 ATOMIC_TRACE_RMW;
103#if DATA_SIZE == 16
104 ret = atomic16_cmpxchg(haddr, cmpv, newv);
105#else
106 ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
107#endif
108 ATOMIC_MMU_CLEANUP;
109 return ret;
110}
111
112#if DATA_SIZE >= 16
113#if HAVE_ATOMIC128
114ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
115{
116 ATOMIC_MMU_DECLS;
117 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
118
119 ATOMIC_TRACE_LD;
120 val = atomic16_read(haddr);
121 ATOMIC_MMU_CLEANUP;
122 return val;
123}
124
125void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
126 ABI_TYPE val EXTRA_ARGS)
127{
128 ATOMIC_MMU_DECLS;
129 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
130
131 ATOMIC_TRACE_ST;
132 atomic16_set(haddr, val);
133 ATOMIC_MMU_CLEANUP;
134}
135#endif
136#else
137ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
138 ABI_TYPE val EXTRA_ARGS)
139{
140 ATOMIC_MMU_DECLS;
141 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
142 DATA_TYPE ret;
143
144 ATOMIC_TRACE_RMW;
145 ret = atomic_xchg__nocheck(haddr, val);
146 ATOMIC_MMU_CLEANUP;
147 return ret;
148}
149
150#define GEN_ATOMIC_HELPER(X) \
151ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
152 ABI_TYPE val EXTRA_ARGS) \
153{ \
154 ATOMIC_MMU_DECLS; \
155 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
156 DATA_TYPE ret; \
157 \
158 ATOMIC_TRACE_RMW; \
159 ret = atomic_##X(haddr, val); \
160 ATOMIC_MMU_CLEANUP; \
161 return ret; \
162}
163
164GEN_ATOMIC_HELPER(fetch_add)
165GEN_ATOMIC_HELPER(fetch_and)
166GEN_ATOMIC_HELPER(fetch_or)
167GEN_ATOMIC_HELPER(fetch_xor)
168GEN_ATOMIC_HELPER(add_fetch)
169GEN_ATOMIC_HELPER(and_fetch)
170GEN_ATOMIC_HELPER(or_fetch)
171GEN_ATOMIC_HELPER(xor_fetch)
172
173#undef GEN_ATOMIC_HELPER
174
175/* These helpers are, as a whole, full barriers. Within the helper,
176 * the leading barrier is explicit and the trailing barrier is within
177 * cmpxchg primitive.
178 *
179 * Trace this load + RMW loop as a single RMW op. This way, regardless
180 * of CF_PARALLEL's value, we'll trace just a read and a write.
181 */
182#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
183ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
184 ABI_TYPE xval EXTRA_ARGS) \
185{ \
186 ATOMIC_MMU_DECLS; \
187 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
188 XDATA_TYPE cmp, old, new, val = xval; \
189 \
190 ATOMIC_TRACE_RMW; \
191 smp_mb(); \
192 cmp = atomic_read__nocheck(haddr); \
193 do { \
194 old = cmp; new = FN(old, val); \
195 cmp = atomic_cmpxchg__nocheck(haddr, old, new); \
196 } while (cmp != old); \
197 ATOMIC_MMU_CLEANUP; \
198 return RET; \
199}
200
201GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
202GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
203GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
204GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
205
206GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
207GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
208GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
209GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
210
211#undef GEN_ATOMIC_HELPER_FN
212#endif /* DATA SIZE >= 16 */
213
214#undef END
215#undef MEND
216
217#if DATA_SIZE > 1
218
219/* Define reverse-host-endian atomic operations. Note that END is used
220 within the ATOMIC_NAME macro. */
221#ifdef HOST_WORDS_BIGENDIAN
222# define END _le
223# define MEND _le
224#else
225# define END _be
226# define MEND _be
227#endif
228
229ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
230 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
231{
232 ATOMIC_MMU_DECLS;
233 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
234 DATA_TYPE ret;
235
236 ATOMIC_TRACE_RMW;
237#if DATA_SIZE == 16
238 ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
239#else
240 ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
241#endif
242 ATOMIC_MMU_CLEANUP;
243 return BSWAP(ret);
244}
245
246#if DATA_SIZE >= 16
247#if HAVE_ATOMIC128
248ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
249{
250 ATOMIC_MMU_DECLS;
251 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
252
253 ATOMIC_TRACE_LD;
254 val = atomic16_read(haddr);
255 ATOMIC_MMU_CLEANUP;
256 return BSWAP(val);
257}
258
259void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
260 ABI_TYPE val EXTRA_ARGS)
261{
262 ATOMIC_MMU_DECLS;
263 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
264
265 ATOMIC_TRACE_ST;
266 val = BSWAP(val);
267 atomic16_set(haddr, val);
268 ATOMIC_MMU_CLEANUP;
269}
270#endif
271#else
272ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
273 ABI_TYPE val EXTRA_ARGS)
274{
275 ATOMIC_MMU_DECLS;
276 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
277 ABI_TYPE ret;
278
279 ATOMIC_TRACE_RMW;
280 ret = atomic_xchg__nocheck(haddr, BSWAP(val));
281 ATOMIC_MMU_CLEANUP;
282 return BSWAP(ret);
283}
284
285#define GEN_ATOMIC_HELPER(X) \
286ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
287 ABI_TYPE val EXTRA_ARGS) \
288{ \
289 ATOMIC_MMU_DECLS; \
290 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
291 DATA_TYPE ret; \
292 \
293 ATOMIC_TRACE_RMW; \
294 ret = atomic_##X(haddr, BSWAP(val)); \
295 ATOMIC_MMU_CLEANUP; \
296 return BSWAP(ret); \
297}
298
299GEN_ATOMIC_HELPER(fetch_and)
300GEN_ATOMIC_HELPER(fetch_or)
301GEN_ATOMIC_HELPER(fetch_xor)
302GEN_ATOMIC_HELPER(and_fetch)
303GEN_ATOMIC_HELPER(or_fetch)
304GEN_ATOMIC_HELPER(xor_fetch)
305
306#undef GEN_ATOMIC_HELPER
307
308/* These helpers are, as a whole, full barriers. Within the helper,
309 * the leading barrier is explicit and the trailing barrier is within
310 * cmpxchg primitive.
311 *
312 * Trace this load + RMW loop as a single RMW op. This way, regardless
313 * of CF_PARALLEL's value, we'll trace just a read and a write.
314 */
315#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
316ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
317 ABI_TYPE xval EXTRA_ARGS) \
318{ \
319 ATOMIC_MMU_DECLS; \
320 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
321 XDATA_TYPE ldo, ldn, old, new, val = xval; \
322 \
323 ATOMIC_TRACE_RMW; \
324 smp_mb(); \
325 ldn = atomic_read__nocheck(haddr); \
326 do { \
327 ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \
328 ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \
329 } while (ldo != ldn); \
330 ATOMIC_MMU_CLEANUP; \
331 return RET; \
332}
333
334GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
335GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
336GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
337GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
338
339GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
340GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
341GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
342GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
343
344/* Note that for addition, we need to use a separate cmpxchg loop instead
345 of bswaps for the reverse-host-endian helpers. */
346#define ADD(X, Y) (X + Y)
347GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
348GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
349#undef ADD
350
351#undef GEN_ATOMIC_HELPER_FN
352#endif /* DATA_SIZE >= 16 */
353
354#undef END
355#undef MEND
356#endif /* DATA_SIZE > 1 */
357
358#undef ATOMIC_TRACE_ST
359#undef ATOMIC_TRACE_LD
360#undef ATOMIC_TRACE_RMW
361
362#undef BSWAP
363#undef ABI_TYPE
364#undef DATA_TYPE
365#undef SDATA_TYPE
366#undef SUFFIX
367#undef DATA_SIZE
368#undef SHIFT
369