1/*
2 * QEMU TCG support -- s390x vector floating point instruction support
3 *
4 * Copyright (C) 2019 Red Hat Inc
5 *
6 * Authors:
7 * David Hildenbrand <david@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2 or later.
10 * See the COPYING file in the top-level directory.
11 */
12#include "qemu/osdep.h"
13#include "qemu-common.h"
14#include "cpu.h"
15#include "internal.h"
16#include "vec.h"
17#include "tcg_s390x.h"
18#include "tcg/tcg-gvec-desc.h"
19#include "exec/exec-all.h"
20#include "exec/helper-proto.h"
21#include "fpu/softfloat.h"
22
23#define VIC_INVALID 0x1
24#define VIC_DIVBYZERO 0x2
25#define VIC_OVERFLOW 0x3
26#define VIC_UNDERFLOW 0x4
27#define VIC_INEXACT 0x5
28
29/* returns the VEX. If the VEX is 0, there is no trap */
30static uint8_t check_ieee_exc(CPUS390XState *env, uint8_t enr, bool XxC,
31 uint8_t *vec_exc)
32{
33 uint8_t vece_exc = 0, trap_exc;
34 unsigned qemu_exc;
35
36 /* Retrieve and clear the softfloat exceptions */
37 qemu_exc = env->fpu_status.float_exception_flags;
38 if (qemu_exc == 0) {
39 return 0;
40 }
41 env->fpu_status.float_exception_flags = 0;
42
43 vece_exc = s390_softfloat_exc_to_ieee(qemu_exc);
44
45 /* Add them to the vector-wide s390x exception bits */
46 *vec_exc |= vece_exc;
47
48 /* Check for traps and construct the VXC */
49 trap_exc = vece_exc & env->fpc >> 24;
50 if (trap_exc) {
51 if (trap_exc & S390_IEEE_MASK_INVALID) {
52 return enr << 4 | VIC_INVALID;
53 } else if (trap_exc & S390_IEEE_MASK_DIVBYZERO) {
54 return enr << 4 | VIC_DIVBYZERO;
55 } else if (trap_exc & S390_IEEE_MASK_OVERFLOW) {
56 return enr << 4 | VIC_OVERFLOW;
57 } else if (trap_exc & S390_IEEE_MASK_UNDERFLOW) {
58 return enr << 4 | VIC_UNDERFLOW;
59 } else if (!XxC) {
60 g_assert(trap_exc & S390_IEEE_MASK_INEXACT);
61 /* inexact has lowest priority on traps */
62 return enr << 4 | VIC_INEXACT;
63 }
64 }
65 return 0;
66}
67
68static void handle_ieee_exc(CPUS390XState *env, uint8_t vxc, uint8_t vec_exc,
69 uintptr_t retaddr)
70{
71 if (vxc) {
72 /* on traps, the fpc flags are not updated, instruction is suppressed */
73 tcg_s390_vector_exception(env, vxc, retaddr);
74 }
75 if (vec_exc) {
76 /* indicate exceptions for all elements combined */
77 env->fpc |= vec_exc << 16;
78 }
79}
80
81typedef uint64_t (*vop64_2_fn)(uint64_t a, float_status *s);
82static void vop64_2(S390Vector *v1, const S390Vector *v2, CPUS390XState *env,
83 bool s, bool XxC, uint8_t erm, vop64_2_fn fn,
84 uintptr_t retaddr)
85{
86 uint8_t vxc, vec_exc = 0;
87 S390Vector tmp = {};
88 int i, old_mode;
89
90 old_mode = s390_swap_bfp_rounding_mode(env, erm);
91 for (i = 0; i < 2; i++) {
92 const uint64_t a = s390_vec_read_element64(v2, i);
93
94 s390_vec_write_element64(&tmp, i, fn(a, &env->fpu_status));
95 vxc = check_ieee_exc(env, i, XxC, &vec_exc);
96 if (s || vxc) {
97 break;
98 }
99 }
100 s390_restore_bfp_rounding_mode(env, old_mode);
101 handle_ieee_exc(env, vxc, vec_exc, retaddr);
102 *v1 = tmp;
103}
104
105typedef uint64_t (*vop64_3_fn)(uint64_t a, uint64_t b, float_status *s);
106static void vop64_3(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
107 CPUS390XState *env, bool s, vop64_3_fn fn,
108 uintptr_t retaddr)
109{
110 uint8_t vxc, vec_exc = 0;
111 S390Vector tmp = {};
112 int i;
113
114 for (i = 0; i < 2; i++) {
115 const uint64_t a = s390_vec_read_element64(v2, i);
116 const uint64_t b = s390_vec_read_element64(v3, i);
117
118 s390_vec_write_element64(&tmp, i, fn(a, b, &env->fpu_status));
119 vxc = check_ieee_exc(env, i, false, &vec_exc);
120 if (s || vxc) {
121 break;
122 }
123 }
124 handle_ieee_exc(env, vxc, vec_exc, retaddr);
125 *v1 = tmp;
126}
127
128static uint64_t vfa64(uint64_t a, uint64_t b, float_status *s)
129{
130 return float64_add(a, b, s);
131}
132
133void HELPER(gvec_vfa64)(void *v1, const void *v2, const void *v3,
134 CPUS390XState *env, uint32_t desc)
135{
136 vop64_3(v1, v2, v3, env, false, vfa64, GETPC());
137}
138
139void HELPER(gvec_vfa64s)(void *v1, const void *v2, const void *v3,
140 CPUS390XState *env, uint32_t desc)
141{
142 vop64_3(v1, v2, v3, env, true, vfa64, GETPC());
143}
144
145static int wfc64(const S390Vector *v1, const S390Vector *v2,
146 CPUS390XState *env, bool signal, uintptr_t retaddr)
147{
148 /* only the zero-indexed elements are compared */
149 const float64 a = s390_vec_read_element64(v1, 0);
150 const float64 b = s390_vec_read_element64(v2, 0);
151 uint8_t vxc, vec_exc = 0;
152 int cmp;
153
154 if (signal) {
155 cmp = float64_compare(a, b, &env->fpu_status);
156 } else {
157 cmp = float64_compare_quiet(a, b, &env->fpu_status);
158 }
159 vxc = check_ieee_exc(env, 0, false, &vec_exc);
160 handle_ieee_exc(env, vxc, vec_exc, retaddr);
161
162 return float_comp_to_cc(env, cmp);
163}
164
165void HELPER(gvec_wfc64)(const void *v1, const void *v2, CPUS390XState *env,
166 uint32_t desc)
167{
168 env->cc_op = wfc64(v1, v2, env, false, GETPC());
169}
170
171void HELPER(gvec_wfk64)(const void *v1, const void *v2, CPUS390XState *env,
172 uint32_t desc)
173{
174 env->cc_op = wfc64(v1, v2, env, true, GETPC());
175}
176
177typedef int (*vfc64_fn)(float64 a, float64 b, float_status *status);
178static int vfc64(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
179 CPUS390XState *env, bool s, vfc64_fn fn, uintptr_t retaddr)
180{
181 uint8_t vxc, vec_exc = 0;
182 S390Vector tmp = {};
183 int match = 0;
184 int i;
185
186 for (i = 0; i < 2; i++) {
187 const float64 a = s390_vec_read_element64(v2, i);
188 const float64 b = s390_vec_read_element64(v3, i);
189
190 /* swap the order of the parameters, so we can use existing functions */
191 if (fn(b, a, &env->fpu_status)) {
192 match++;
193 s390_vec_write_element64(&tmp, i, -1ull);
194 }
195 vxc = check_ieee_exc(env, i, false, &vec_exc);
196 if (s || vxc) {
197 break;
198 }
199 }
200
201 handle_ieee_exc(env, vxc, vec_exc, retaddr);
202 *v1 = tmp;
203 if (match) {
204 return s || match == 2 ? 0 : 1;
205 }
206 return 3;
207}
208
209void HELPER(gvec_vfce64)(void *v1, const void *v2, const void *v3,
210 CPUS390XState *env, uint32_t desc)
211{
212 vfc64(v1, v2, v3, env, false, float64_eq_quiet, GETPC());
213}
214
215void HELPER(gvec_vfce64s)(void *v1, const void *v2, const void *v3,
216 CPUS390XState *env, uint32_t desc)
217{
218 vfc64(v1, v2, v3, env, true, float64_eq_quiet, GETPC());
219}
220
221void HELPER(gvec_vfce64_cc)(void *v1, const void *v2, const void *v3,
222 CPUS390XState *env, uint32_t desc)
223{
224 env->cc_op = vfc64(v1, v2, v3, env, false, float64_eq_quiet, GETPC());
225}
226
227void HELPER(gvec_vfce64s_cc)(void *v1, const void *v2, const void *v3,
228 CPUS390XState *env, uint32_t desc)
229{
230 env->cc_op = vfc64(v1, v2, v3, env, true, float64_eq_quiet, GETPC());
231}
232
233void HELPER(gvec_vfch64)(void *v1, const void *v2, const void *v3,
234 CPUS390XState *env, uint32_t desc)
235{
236 vfc64(v1, v2, v3, env, false, float64_lt_quiet, GETPC());
237}
238
239void HELPER(gvec_vfch64s)(void *v1, const void *v2, const void *v3,
240 CPUS390XState *env, uint32_t desc)
241{
242 vfc64(v1, v2, v3, env, true, float64_lt_quiet, GETPC());
243}
244
245void HELPER(gvec_vfch64_cc)(void *v1, const void *v2, const void *v3,
246 CPUS390XState *env, uint32_t desc)
247{
248 env->cc_op = vfc64(v1, v2, v3, env, false, float64_lt_quiet, GETPC());
249}
250
251void HELPER(gvec_vfch64s_cc)(void *v1, const void *v2, const void *v3,
252 CPUS390XState *env, uint32_t desc)
253{
254 env->cc_op = vfc64(v1, v2, v3, env, true, float64_lt_quiet, GETPC());
255}
256
257void HELPER(gvec_vfche64)(void *v1, const void *v2, const void *v3,
258 CPUS390XState *env, uint32_t desc)
259{
260 vfc64(v1, v2, v3, env, false, float64_le_quiet, GETPC());
261}
262
263void HELPER(gvec_vfche64s)(void *v1, const void *v2, const void *v3,
264 CPUS390XState *env, uint32_t desc)
265{
266 vfc64(v1, v2, v3, env, true, float64_le_quiet, GETPC());
267}
268
269void HELPER(gvec_vfche64_cc)(void *v1, const void *v2, const void *v3,
270 CPUS390XState *env, uint32_t desc)
271{
272 env->cc_op = vfc64(v1, v2, v3, env, false, float64_le_quiet, GETPC());
273}
274
275void HELPER(gvec_vfche64s_cc)(void *v1, const void *v2, const void *v3,
276 CPUS390XState *env, uint32_t desc)
277{
278 env->cc_op = vfc64(v1, v2, v3, env, true, float64_le_quiet, GETPC());
279}
280
281static uint64_t vcdg64(uint64_t a, float_status *s)
282{
283 return int64_to_float64(a, s);
284}
285
286void HELPER(gvec_vcdg64)(void *v1, const void *v2, CPUS390XState *env,
287 uint32_t desc)
288{
289 const uint8_t erm = extract32(simd_data(desc), 4, 4);
290 const bool XxC = extract32(simd_data(desc), 2, 1);
291
292 vop64_2(v1, v2, env, false, XxC, erm, vcdg64, GETPC());
293}
294
295void HELPER(gvec_vcdg64s)(void *v1, const void *v2, CPUS390XState *env,
296 uint32_t desc)
297{
298 const uint8_t erm = extract32(simd_data(desc), 4, 4);
299 const bool XxC = extract32(simd_data(desc), 2, 1);
300
301 vop64_2(v1, v2, env, true, XxC, erm, vcdg64, GETPC());
302}
303
304static uint64_t vcdlg64(uint64_t a, float_status *s)
305{
306 return uint64_to_float64(a, s);
307}
308
309void HELPER(gvec_vcdlg64)(void *v1, const void *v2, CPUS390XState *env,
310 uint32_t desc)
311{
312 const uint8_t erm = extract32(simd_data(desc), 4, 4);
313 const bool XxC = extract32(simd_data(desc), 2, 1);
314
315 vop64_2(v1, v2, env, false, XxC, erm, vcdlg64, GETPC());
316}
317
318void HELPER(gvec_vcdlg64s)(void *v1, const void *v2, CPUS390XState *env,
319 uint32_t desc)
320{
321 const uint8_t erm = extract32(simd_data(desc), 4, 4);
322 const bool XxC = extract32(simd_data(desc), 2, 1);
323
324 vop64_2(v1, v2, env, true, XxC, erm, vcdlg64, GETPC());
325}
326
327static uint64_t vcgd64(uint64_t a, float_status *s)
328{
329 return float64_to_int64(a, s);
330}
331
332void HELPER(gvec_vcgd64)(void *v1, const void *v2, CPUS390XState *env,
333 uint32_t desc)
334{
335 const uint8_t erm = extract32(simd_data(desc), 4, 4);
336 const bool XxC = extract32(simd_data(desc), 2, 1);
337
338 vop64_2(v1, v2, env, false, XxC, erm, vcgd64, GETPC());
339}
340
341void HELPER(gvec_vcgd64s)(void *v1, const void *v2, CPUS390XState *env,
342 uint32_t desc)
343{
344 const uint8_t erm = extract32(simd_data(desc), 4, 4);
345 const bool XxC = extract32(simd_data(desc), 2, 1);
346
347 vop64_2(v1, v2, env, true, XxC, erm, vcgd64, GETPC());
348}
349
350static uint64_t vclgd64(uint64_t a, float_status *s)
351{
352 return float64_to_uint64(a, s);
353}
354
355void HELPER(gvec_vclgd64)(void *v1, const void *v2, CPUS390XState *env,
356 uint32_t desc)
357{
358 const uint8_t erm = extract32(simd_data(desc), 4, 4);
359 const bool XxC = extract32(simd_data(desc), 2, 1);
360
361 vop64_2(v1, v2, env, false, XxC, erm, vclgd64, GETPC());
362}
363
364void HELPER(gvec_vclgd64s)(void *v1, const void *v2, CPUS390XState *env,
365 uint32_t desc)
366{
367 const uint8_t erm = extract32(simd_data(desc), 4, 4);
368 const bool XxC = extract32(simd_data(desc), 2, 1);
369
370 vop64_2(v1, v2, env, true, XxC, erm, vclgd64, GETPC());
371}
372
373static uint64_t vfd64(uint64_t a, uint64_t b, float_status *s)
374{
375 return float64_div(a, b, s);
376}
377
378void HELPER(gvec_vfd64)(void *v1, const void *v2, const void *v3,
379 CPUS390XState *env, uint32_t desc)
380{
381 vop64_3(v1, v2, v3, env, false, vfd64, GETPC());
382}
383
384void HELPER(gvec_vfd64s)(void *v1, const void *v2, const void *v3,
385 CPUS390XState *env, uint32_t desc)
386{
387 vop64_3(v1, v2, v3, env, true, vfd64, GETPC());
388}
389
390static uint64_t vfi64(uint64_t a, float_status *s)
391{
392 return float64_round_to_int(a, s);
393}
394
395void HELPER(gvec_vfi64)(void *v1, const void *v2, CPUS390XState *env,
396 uint32_t desc)
397{
398 const uint8_t erm = extract32(simd_data(desc), 4, 4);
399 const bool XxC = extract32(simd_data(desc), 2, 1);
400
401 vop64_2(v1, v2, env, false, XxC, erm, vfi64, GETPC());
402}
403
404void HELPER(gvec_vfi64s)(void *v1, const void *v2, CPUS390XState *env,
405 uint32_t desc)
406{
407 const uint8_t erm = extract32(simd_data(desc), 4, 4);
408 const bool XxC = extract32(simd_data(desc), 2, 1);
409
410 vop64_2(v1, v2, env, true, XxC, erm, vfi64, GETPC());
411}
412
413static void vfll32(S390Vector *v1, const S390Vector *v2, CPUS390XState *env,
414 bool s, uintptr_t retaddr)
415{
416 uint8_t vxc, vec_exc = 0;
417 S390Vector tmp = {};
418 int i;
419
420 for (i = 0; i < 2; i++) {
421 /* load from even element */
422 const float32 a = s390_vec_read_element32(v2, i * 2);
423 const uint64_t ret = float32_to_float64(a, &env->fpu_status);
424
425 s390_vec_write_element64(&tmp, i, ret);
426 /* indicate the source element */
427 vxc = check_ieee_exc(env, i * 2, false, &vec_exc);
428 if (s || vxc) {
429 break;
430 }
431 }
432 handle_ieee_exc(env, vxc, vec_exc, retaddr);
433 *v1 = tmp;
434}
435
436void HELPER(gvec_vfll32)(void *v1, const void *v2, CPUS390XState *env,
437 uint32_t desc)
438{
439 vfll32(v1, v2, env, false, GETPC());
440}
441
442void HELPER(gvec_vfll32s)(void *v1, const void *v2, CPUS390XState *env,
443 uint32_t desc)
444{
445 vfll32(v1, v2, env, true, GETPC());
446}
447
448static void vflr64(S390Vector *v1, const S390Vector *v2, CPUS390XState *env,
449 bool s, bool XxC, uint8_t erm, uintptr_t retaddr)
450{
451 uint8_t vxc, vec_exc = 0;
452 S390Vector tmp = {};
453 int i, old_mode;
454
455 old_mode = s390_swap_bfp_rounding_mode(env, erm);
456 for (i = 0; i < 2; i++) {
457 float64 a = s390_vec_read_element64(v2, i);
458 uint32_t ret = float64_to_float32(a, &env->fpu_status);
459
460 /* place at even element */
461 s390_vec_write_element32(&tmp, i * 2, ret);
462 /* indicate the source element */
463 vxc = check_ieee_exc(env, i, XxC, &vec_exc);
464 if (s || vxc) {
465 break;
466 }
467 }
468 s390_restore_bfp_rounding_mode(env, old_mode);
469 handle_ieee_exc(env, vxc, vec_exc, retaddr);
470 *v1 = tmp;
471}
472
473void HELPER(gvec_vflr64)(void *v1, const void *v2, CPUS390XState *env,
474 uint32_t desc)
475{
476 const uint8_t erm = extract32(simd_data(desc), 4, 4);
477 const bool XxC = extract32(simd_data(desc), 2, 1);
478
479 vflr64(v1, v2, env, false, XxC, erm, GETPC());
480}
481
482void HELPER(gvec_vflr64s)(void *v1, const void *v2, CPUS390XState *env,
483 uint32_t desc)
484{
485 const uint8_t erm = extract32(simd_data(desc), 4, 4);
486 const bool XxC = extract32(simd_data(desc), 2, 1);
487
488 vflr64(v1, v2, env, true, XxC, erm, GETPC());
489}
490
491static uint64_t vfm64(uint64_t a, uint64_t b, float_status *s)
492{
493 return float64_mul(a, b, s);
494}
495
496void HELPER(gvec_vfm64)(void *v1, const void *v2, const void *v3,
497 CPUS390XState *env, uint32_t desc)
498{
499 vop64_3(v1, v2, v3, env, false, vfm64, GETPC());
500}
501
502void HELPER(gvec_vfm64s)(void *v1, const void *v2, const void *v3,
503 CPUS390XState *env, uint32_t desc)
504{
505 vop64_3(v1, v2, v3, env, true, vfm64, GETPC());
506}
507
508static void vfma64(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
509 const S390Vector *v4, CPUS390XState *env, bool s, int flags,
510 uintptr_t retaddr)
511{
512 uint8_t vxc, vec_exc = 0;
513 S390Vector tmp = {};
514 int i;
515
516 for (i = 0; i < 2; i++) {
517 const uint64_t a = s390_vec_read_element64(v2, i);
518 const uint64_t b = s390_vec_read_element64(v3, i);
519 const uint64_t c = s390_vec_read_element64(v4, i);
520 uint64_t ret = float64_muladd(a, b, c, flags, &env->fpu_status);
521
522 s390_vec_write_element64(&tmp, i, ret);
523 vxc = check_ieee_exc(env, i, false, &vec_exc);
524 if (s || vxc) {
525 break;
526 }
527 }
528 handle_ieee_exc(env, vxc, vec_exc, retaddr);
529 *v1 = tmp;
530}
531
532void HELPER(gvec_vfma64)(void *v1, const void *v2, const void *v3,
533 const void *v4, CPUS390XState *env, uint32_t desc)
534{
535 vfma64(v1, v2, v3, v4, env, false, 0, GETPC());
536}
537
538void HELPER(gvec_vfma64s)(void *v1, const void *v2, const void *v3,
539 const void *v4, CPUS390XState *env, uint32_t desc)
540{
541 vfma64(v1, v2, v3, v4, env, true, 0, GETPC());
542}
543
544void HELPER(gvec_vfms64)(void *v1, const void *v2, const void *v3,
545 const void *v4, CPUS390XState *env, uint32_t desc)
546{
547 vfma64(v1, v2, v3, v4, env, false, float_muladd_negate_c, GETPC());
548}
549
550void HELPER(gvec_vfms64s)(void *v1, const void *v2, const void *v3,
551 const void *v4, CPUS390XState *env, uint32_t desc)
552{
553 vfma64(v1, v2, v3, v4, env, true, float_muladd_negate_c, GETPC());
554}
555
556static uint64_t vfsq64(uint64_t a, float_status *s)
557{
558 return float64_sqrt(a, s);
559}
560
561void HELPER(gvec_vfsq64)(void *v1, const void *v2, CPUS390XState *env,
562 uint32_t desc)
563{
564 vop64_2(v1, v2, env, false, false, 0, vfsq64, GETPC());
565}
566
567void HELPER(gvec_vfsq64s)(void *v1, const void *v2, CPUS390XState *env,
568 uint32_t desc)
569{
570 vop64_2(v1, v2, env, true, false, 0, vfsq64, GETPC());
571}
572
573static uint64_t vfs64(uint64_t a, uint64_t b, float_status *s)
574{
575 return float64_sub(a, b, s);
576}
577
578void HELPER(gvec_vfs64)(void *v1, const void *v2, const void *v3,
579 CPUS390XState *env, uint32_t desc)
580{
581 vop64_3(v1, v2, v3, env, false, vfs64, GETPC());
582}
583
584void HELPER(gvec_vfs64s)(void *v1, const void *v2, const void *v3,
585 CPUS390XState *env, uint32_t desc)
586{
587 vop64_3(v1, v2, v3, env, true, vfs64, GETPC());
588}
589
590static int vftci64(S390Vector *v1, const S390Vector *v2, CPUS390XState *env,
591 bool s, uint16_t i3)
592{
593 int i, match = 0;
594
595 for (i = 0; i < 2; i++) {
596 float64 a = s390_vec_read_element64(v2, i);
597
598 if (float64_dcmask(env, a) & i3) {
599 match++;
600 s390_vec_write_element64(v1, i, -1ull);
601 } else {
602 s390_vec_write_element64(v1, i, 0);
603 }
604 if (s) {
605 break;
606 }
607 }
608
609 if (match) {
610 return s || match == 2 ? 0 : 1;
611 }
612 return 3;
613}
614
615void HELPER(gvec_vftci64)(void *v1, const void *v2, CPUS390XState *env,
616 uint32_t desc)
617{
618 env->cc_op = vftci64(v1, v2, env, false, simd_data(desc));
619}
620
621void HELPER(gvec_vftci64s)(void *v1, const void *v2, CPUS390XState *env,
622 uint32_t desc)
623{
624 env->cc_op = vftci64(v1, v2, env, true, simd_data(desc));
625}
626