1/*
2 * Generic vector operation expansion
3 *
4 * Copyright (c) 2018 Linaro
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20#ifndef TCG_TCG_OP_GVEC_H
21#define TCG_TCG_OP_GVEC_H
22
23/*
24 * "Generic" vectors. All operands are given as offsets from ENV,
25 * and therefore cannot also be allocated via tcg_global_mem_new_*.
26 * OPRSZ is the byte size of the vector upon which the operation is performed.
27 * MAXSZ is the byte size of the full vector; bytes beyond OPSZ are cleared.
28 *
29 * All sizes must be 8 or any multiple of 16.
30 * When OPRSZ is 8, the alignment may be 8, otherwise must be 16.
31 * Operands may completely, but not partially, overlap.
32 */
33
34/* Expand a call to a gvec-style helper, with pointers to two vector
35 operands, and a descriptor (see tcg-gvec-desc.h). */
36typedef void gen_helper_gvec_2(TCGv_ptr, TCGv_ptr, TCGv_i32);
37void tcg_gen_gvec_2_ool(uint32_t dofs, uint32_t aofs,
38 uint32_t oprsz, uint32_t maxsz, int32_t data,
39 gen_helper_gvec_2 *fn);
40
41/* Similarly, passing an extra data value. */
42typedef void gen_helper_gvec_2i(TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_i32);
43void tcg_gen_gvec_2i_ool(uint32_t dofs, uint32_t aofs, TCGv_i64 c,
44 uint32_t oprsz, uint32_t maxsz, int32_t data,
45 gen_helper_gvec_2i *fn);
46
47/* Similarly, passing an extra pointer (e.g. env or float_status). */
48typedef void gen_helper_gvec_2_ptr(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i32);
49void tcg_gen_gvec_2_ptr(uint32_t dofs, uint32_t aofs,
50 TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz,
51 int32_t data, gen_helper_gvec_2_ptr *fn);
52
53/* Similarly, with three vector operands. */
54typedef void gen_helper_gvec_3(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i32);
55void tcg_gen_gvec_3_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
56 uint32_t oprsz, uint32_t maxsz, int32_t data,
57 gen_helper_gvec_3 *fn);
58
59/* Similarly, with four vector operands. */
60typedef void gen_helper_gvec_4(TCGv_ptr, TCGv_ptr, TCGv_ptr,
61 TCGv_ptr, TCGv_i32);
62void tcg_gen_gvec_4_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
63 uint32_t cofs, uint32_t oprsz, uint32_t maxsz,
64 int32_t data, gen_helper_gvec_4 *fn);
65
66/* Similarly, with five vector operands. */
67typedef void gen_helper_gvec_5(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_ptr,
68 TCGv_ptr, TCGv_i32);
69void tcg_gen_gvec_5_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
70 uint32_t cofs, uint32_t xofs, uint32_t oprsz,
71 uint32_t maxsz, int32_t data, gen_helper_gvec_5 *fn);
72
73typedef void gen_helper_gvec_3_ptr(TCGv_ptr, TCGv_ptr, TCGv_ptr,
74 TCGv_ptr, TCGv_i32);
75void tcg_gen_gvec_3_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs,
76 TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz,
77 int32_t data, gen_helper_gvec_3_ptr *fn);
78
79typedef void gen_helper_gvec_4_ptr(TCGv_ptr, TCGv_ptr, TCGv_ptr,
80 TCGv_ptr, TCGv_ptr, TCGv_i32);
81void tcg_gen_gvec_4_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs,
82 uint32_t cofs, TCGv_ptr ptr, uint32_t oprsz,
83 uint32_t maxsz, int32_t data,
84 gen_helper_gvec_4_ptr *fn);
85
86/* Expand a gvec operation. Either inline or out-of-line depending on
87 the actual vector size and the operations supported by the host. */
88typedef struct {
89 /* Expand inline as a 64-bit or 32-bit integer.
90 Only one of these will be non-NULL. */
91 void (*fni8)(TCGv_i64, TCGv_i64);
92 void (*fni4)(TCGv_i32, TCGv_i32);
93 /* Expand inline with a host vector type. */
94 void (*fniv)(unsigned, TCGv_vec, TCGv_vec);
95 /* Expand out-of-line helper w/descriptor. */
96 gen_helper_gvec_2 *fno;
97 /* The optional opcodes, if any, utilized by .fniv. */
98 const TCGOpcode *opt_opc;
99 /* The data argument to the out-of-line helper. */
100 int32_t data;
101 /* The vector element size, if applicable. */
102 uint8_t vece;
103 /* Prefer i64 to v64. */
104 bool prefer_i64;
105} GVecGen2;
106
107typedef struct {
108 /* Expand inline as a 64-bit or 32-bit integer.
109 Only one of these will be non-NULL. */
110 void (*fni8)(TCGv_i64, TCGv_i64, int64_t);
111 void (*fni4)(TCGv_i32, TCGv_i32, int32_t);
112 /* Expand inline with a host vector type. */
113 void (*fniv)(unsigned, TCGv_vec, TCGv_vec, int64_t);
114 /* Expand out-of-line helper w/descriptor, data in descriptor. */
115 gen_helper_gvec_2 *fno;
116 /* Expand out-of-line helper w/descriptor, data as argument. */
117 gen_helper_gvec_2i *fnoi;
118 /* The optional opcodes, if any, utilized by .fniv. */
119 const TCGOpcode *opt_opc;
120 /* The vector element size, if applicable. */
121 uint8_t vece;
122 /* Prefer i64 to v64. */
123 bool prefer_i64;
124 /* Load dest as a 3rd source operand. */
125 bool load_dest;
126} GVecGen2i;
127
128typedef struct {
129 /* Expand inline as a 64-bit or 32-bit integer.
130 Only one of these will be non-NULL. */
131 void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64);
132 void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32);
133 /* Expand inline with a host vector type. */
134 void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec);
135 /* Expand out-of-line helper w/descriptor. */
136 gen_helper_gvec_2i *fno;
137 /* The optional opcodes, if any, utilized by .fniv. */
138 const TCGOpcode *opt_opc;
139 /* The data argument to the out-of-line helper. */
140 uint32_t data;
141 /* The vector element size, if applicable. */
142 uint8_t vece;
143 /* Prefer i64 to v64. */
144 bool prefer_i64;
145 /* Load scalar as 1st source operand. */
146 bool scalar_first;
147} GVecGen2s;
148
149typedef struct {
150 /* Expand inline as a 64-bit or 32-bit integer.
151 Only one of these will be non-NULL. */
152 void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64);
153 void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32);
154 /* Expand inline with a host vector type. */
155 void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec);
156 /* Expand out-of-line helper w/descriptor. */
157 gen_helper_gvec_3 *fno;
158 /* The optional opcodes, if any, utilized by .fniv. */
159 const TCGOpcode *opt_opc;
160 /* The data argument to the out-of-line helper. */
161 int32_t data;
162 /* The vector element size, if applicable. */
163 uint8_t vece;
164 /* Prefer i64 to v64. */
165 bool prefer_i64;
166 /* Load dest as a 3rd source operand. */
167 bool load_dest;
168} GVecGen3;
169
170typedef struct {
171 /*
172 * Expand inline as a 64-bit or 32-bit integer. Only one of these will be
173 * non-NULL.
174 */
175 void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64, int64_t);
176 void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32, int32_t);
177 /* Expand inline with a host vector type. */
178 void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec, int64_t);
179 /* Expand out-of-line helper w/descriptor, data in descriptor. */
180 gen_helper_gvec_3 *fno;
181 /* The optional opcodes, if any, utilized by .fniv. */
182 const TCGOpcode *opt_opc;
183 /* The vector element size, if applicable. */
184 uint8_t vece;
185 /* Prefer i64 to v64. */
186 bool prefer_i64;
187 /* Load dest as a 3rd source operand. */
188 bool load_dest;
189} GVecGen3i;
190
191typedef struct {
192 /* Expand inline as a 64-bit or 32-bit integer.
193 Only one of these will be non-NULL. */
194 void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64);
195 void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_i32);
196 /* Expand inline with a host vector type. */
197 void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec, TCGv_vec);
198 /* Expand out-of-line helper w/descriptor. */
199 gen_helper_gvec_4 *fno;
200 /* The optional opcodes, if any, utilized by .fniv. */
201 const TCGOpcode *opt_opc;
202 /* The data argument to the out-of-line helper. */
203 int32_t data;
204 /* The vector element size, if applicable. */
205 uint8_t vece;
206 /* Prefer i64 to v64. */
207 bool prefer_i64;
208 /* Write aofs as a 2nd dest operand. */
209 bool write_aofs;
210} GVecGen4;
211
212void tcg_gen_gvec_2(uint32_t dofs, uint32_t aofs,
213 uint32_t oprsz, uint32_t maxsz, const GVecGen2 *);
214void tcg_gen_gvec_2i(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
215 uint32_t maxsz, int64_t c, const GVecGen2i *);
216void tcg_gen_gvec_2s(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
217 uint32_t maxsz, TCGv_i64 c, const GVecGen2s *);
218void tcg_gen_gvec_3(uint32_t dofs, uint32_t aofs, uint32_t bofs,
219 uint32_t oprsz, uint32_t maxsz, const GVecGen3 *);
220void tcg_gen_gvec_3i(uint32_t dofs, uint32_t aofs, uint32_t bofs,
221 uint32_t oprsz, uint32_t maxsz, int64_t c,
222 const GVecGen3i *);
223void tcg_gen_gvec_4(uint32_t dofs, uint32_t aofs, uint32_t bofs, uint32_t cofs,
224 uint32_t oprsz, uint32_t maxsz, const GVecGen4 *);
225
226/* Expand a specific vector operation. */
227
228void tcg_gen_gvec_mov(unsigned vece, uint32_t dofs, uint32_t aofs,
229 uint32_t oprsz, uint32_t maxsz);
230void tcg_gen_gvec_not(unsigned vece, uint32_t dofs, uint32_t aofs,
231 uint32_t oprsz, uint32_t maxsz);
232void tcg_gen_gvec_neg(unsigned vece, uint32_t dofs, uint32_t aofs,
233 uint32_t oprsz, uint32_t maxsz);
234void tcg_gen_gvec_abs(unsigned vece, uint32_t dofs, uint32_t aofs,
235 uint32_t oprsz, uint32_t maxsz);
236
237void tcg_gen_gvec_add(unsigned vece, uint32_t dofs, uint32_t aofs,
238 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
239void tcg_gen_gvec_sub(unsigned vece, uint32_t dofs, uint32_t aofs,
240 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
241void tcg_gen_gvec_mul(unsigned vece, uint32_t dofs, uint32_t aofs,
242 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
243
244void tcg_gen_gvec_addi(unsigned vece, uint32_t dofs, uint32_t aofs,
245 int64_t c, uint32_t oprsz, uint32_t maxsz);
246void tcg_gen_gvec_muli(unsigned vece, uint32_t dofs, uint32_t aofs,
247 int64_t c, uint32_t oprsz, uint32_t maxsz);
248
249void tcg_gen_gvec_adds(unsigned vece, uint32_t dofs, uint32_t aofs,
250 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz);
251void tcg_gen_gvec_subs(unsigned vece, uint32_t dofs, uint32_t aofs,
252 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz);
253void tcg_gen_gvec_muls(unsigned vece, uint32_t dofs, uint32_t aofs,
254 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz);
255
256/* Saturated arithmetic. */
257void tcg_gen_gvec_ssadd(unsigned vece, uint32_t dofs, uint32_t aofs,
258 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
259void tcg_gen_gvec_sssub(unsigned vece, uint32_t dofs, uint32_t aofs,
260 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
261void tcg_gen_gvec_usadd(unsigned vece, uint32_t dofs, uint32_t aofs,
262 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
263void tcg_gen_gvec_ussub(unsigned vece, uint32_t dofs, uint32_t aofs,
264 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
265
266/* Min/max. */
267void tcg_gen_gvec_smin(unsigned vece, uint32_t dofs, uint32_t aofs,
268 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
269void tcg_gen_gvec_umin(unsigned vece, uint32_t dofs, uint32_t aofs,
270 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
271void tcg_gen_gvec_smax(unsigned vece, uint32_t dofs, uint32_t aofs,
272 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
273void tcg_gen_gvec_umax(unsigned vece, uint32_t dofs, uint32_t aofs,
274 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
275
276void tcg_gen_gvec_and(unsigned vece, uint32_t dofs, uint32_t aofs,
277 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
278void tcg_gen_gvec_or(unsigned vece, uint32_t dofs, uint32_t aofs,
279 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
280void tcg_gen_gvec_xor(unsigned vece, uint32_t dofs, uint32_t aofs,
281 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
282void tcg_gen_gvec_andc(unsigned vece, uint32_t dofs, uint32_t aofs,
283 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
284void tcg_gen_gvec_orc(unsigned vece, uint32_t dofs, uint32_t aofs,
285 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
286void tcg_gen_gvec_nand(unsigned vece, uint32_t dofs, uint32_t aofs,
287 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
288void tcg_gen_gvec_nor(unsigned vece, uint32_t dofs, uint32_t aofs,
289 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
290void tcg_gen_gvec_eqv(unsigned vece, uint32_t dofs, uint32_t aofs,
291 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
292
293void tcg_gen_gvec_andi(unsigned vece, uint32_t dofs, uint32_t aofs,
294 int64_t c, uint32_t oprsz, uint32_t maxsz);
295void tcg_gen_gvec_xori(unsigned vece, uint32_t dofs, uint32_t aofs,
296 int64_t c, uint32_t oprsz, uint32_t maxsz);
297void tcg_gen_gvec_ori(unsigned vece, uint32_t dofs, uint32_t aofs,
298 int64_t c, uint32_t oprsz, uint32_t maxsz);
299
300void tcg_gen_gvec_ands(unsigned vece, uint32_t dofs, uint32_t aofs,
301 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz);
302void tcg_gen_gvec_xors(unsigned vece, uint32_t dofs, uint32_t aofs,
303 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz);
304void tcg_gen_gvec_ors(unsigned vece, uint32_t dofs, uint32_t aofs,
305 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz);
306
307void tcg_gen_gvec_dup_mem(unsigned vece, uint32_t dofs, uint32_t aofs,
308 uint32_t s, uint32_t m);
309void tcg_gen_gvec_dup_i32(unsigned vece, uint32_t dofs, uint32_t s,
310 uint32_t m, TCGv_i32);
311void tcg_gen_gvec_dup_i64(unsigned vece, uint32_t dofs, uint32_t s,
312 uint32_t m, TCGv_i64);
313
314void tcg_gen_gvec_dup8i(uint32_t dofs, uint32_t s, uint32_t m, uint8_t x);
315void tcg_gen_gvec_dup16i(uint32_t dofs, uint32_t s, uint32_t m, uint16_t x);
316void tcg_gen_gvec_dup32i(uint32_t dofs, uint32_t s, uint32_t m, uint32_t x);
317void tcg_gen_gvec_dup64i(uint32_t dofs, uint32_t s, uint32_t m, uint64_t x);
318
319void tcg_gen_gvec_shli(unsigned vece, uint32_t dofs, uint32_t aofs,
320 int64_t shift, uint32_t oprsz, uint32_t maxsz);
321void tcg_gen_gvec_shri(unsigned vece, uint32_t dofs, uint32_t aofs,
322 int64_t shift, uint32_t oprsz, uint32_t maxsz);
323void tcg_gen_gvec_sari(unsigned vece, uint32_t dofs, uint32_t aofs,
324 int64_t shift, uint32_t oprsz, uint32_t maxsz);
325
326void tcg_gen_gvec_shls(unsigned vece, uint32_t dofs, uint32_t aofs,
327 TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz);
328void tcg_gen_gvec_shrs(unsigned vece, uint32_t dofs, uint32_t aofs,
329 TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz);
330void tcg_gen_gvec_sars(unsigned vece, uint32_t dofs, uint32_t aofs,
331 TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz);
332
333/*
334 * Perform vector shift by vector element, modulo the element size.
335 * E.g. D[i] = A[i] << (B[i] % (8 << vece)).
336 */
337void tcg_gen_gvec_shlv(unsigned vece, uint32_t dofs, uint32_t aofs,
338 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
339void tcg_gen_gvec_shrv(unsigned vece, uint32_t dofs, uint32_t aofs,
340 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
341void tcg_gen_gvec_sarv(unsigned vece, uint32_t dofs, uint32_t aofs,
342 uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
343
344void tcg_gen_gvec_cmp(TCGCond cond, unsigned vece, uint32_t dofs,
345 uint32_t aofs, uint32_t bofs,
346 uint32_t oprsz, uint32_t maxsz);
347
348/*
349 * Perform vector bit select: d = (b & a) | (c & ~a).
350 */
351void tcg_gen_gvec_bitsel(unsigned vece, uint32_t dofs, uint32_t aofs,
352 uint32_t bofs, uint32_t cofs,
353 uint32_t oprsz, uint32_t maxsz);
354
355/*
356 * 64-bit vector operations. Use these when the register has been allocated
357 * with tcg_global_mem_new_i64, and so we cannot also address it via pointer.
358 * OPRSZ = MAXSZ = 8.
359 */
360
361void tcg_gen_vec_neg8_i64(TCGv_i64 d, TCGv_i64 a);
362void tcg_gen_vec_neg16_i64(TCGv_i64 d, TCGv_i64 a);
363void tcg_gen_vec_neg32_i64(TCGv_i64 d, TCGv_i64 a);
364
365void tcg_gen_vec_add8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b);
366void tcg_gen_vec_add16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b);
367void tcg_gen_vec_add32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b);
368
369void tcg_gen_vec_sub8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b);
370void tcg_gen_vec_sub16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b);
371void tcg_gen_vec_sub32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b);
372
373void tcg_gen_vec_shl8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t);
374void tcg_gen_vec_shl16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t);
375void tcg_gen_vec_shr8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t);
376void tcg_gen_vec_shr16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t);
377void tcg_gen_vec_sar8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t);
378void tcg_gen_vec_sar16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t);
379
380#endif
381