1/*
2 * Copyright 2019 Google Inc.
3 *
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
6 */
7
8#ifndef SKVX_DEFINED
9#define SKVX_DEFINED
10
11// skvx::Vec<N,T> are SIMD vectors of N T's, a v1.5 successor to SkNx<N,T>.
12//
13// This time we're leaning a bit less on platform-specific intrinsics and a bit
14// more on Clang/GCC vector extensions, but still keeping the option open to
15// drop in platform-specific intrinsics, actually more easily than before.
16//
17// We've also fixed a few of the caveats that used to make SkNx awkward to work
18// with across translation units. skvx::Vec<N,T> always has N*sizeof(T) size
19// and alignment[1][2] and is safe to use across translation units freely.
20//
21// [1] Ideally we'd only align to T, but that tanks ARMv7 NEON codegen.
22// [2] Some compilers barf if we try to use N*sizeof(T), so instead we leave them at T.
23
24// Please try to keep this file independent of Skia headers.
25#include <algorithm> // std::min, std::max
26#include <cmath> // std::ceil, std::floor, std::trunc, std::round, std::sqrt, etc.
27#include <cstdint> // intXX_t
28#include <cstring> // memcpy()
29#include <initializer_list> // std::initializer_list
30
31#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__)
32 #include <immintrin.h>
33#elif defined(__ARM_NEON)
34 #include <arm_neon.h>
35#endif
36
37#if !defined(__clang__) && defined(__GNUC__) && defined(__mips64)
38 // GCC 7 hits an internal compiler error when targeting MIPS64.
39 #define SKVX_ALIGNMENT
40#elif !defined(__clang__) && defined(_MSC_VER) && defined(_M_IX86)
41 // Our SkVx unit tests fail when built by MSVC for 32-bit x86.
42 #define SKVX_ALIGNMENT
43#else
44 #define SKVX_ALIGNMENT alignas(N * sizeof(T))
45#endif
46
47#if defined(__GNUC__) && !defined(__clang__) && defined(__SSE__)
48 // GCC warns about ABI changes when returning >= 32 byte vectors when -mavx is not enabled.
49 // This only happens for types like VExt whose ABI we don't care about, not for Vec itself.
50 #pragma GCC diagnostic ignored "-Wpsabi"
51#endif
52
53// To avoid ODR violations, all methods must be force-inlined,
54// and all standalone functions must be static, perhaps using these helpers.
55#if defined(_MSC_VER)
56 #define SKVX_ALWAYS_INLINE __forceinline
57#else
58 #define SKVX_ALWAYS_INLINE __attribute__((always_inline))
59#endif
60
61#define SIT template < typename T> static inline
62#define SINT template <int N, typename T> static inline
63#define SINTU template <int N, typename T, typename U, \
64 typename=typename std::enable_if<std::is_convertible<U,T>::value>::type> \
65 static inline
66
67namespace skvx {
68
69// All Vec have the same simple memory layout, the same as `T vec[N]`.
70template <int N, typename T>
71struct SKVX_ALIGNMENT Vec {
72 static_assert((N & (N-1)) == 0, "N must be a power of 2.");
73 static_assert(sizeof(T) >= alignof(T), "What kind of crazy T is this?");
74
75 Vec<N/2,T> lo, hi;
76
77 // Methods belong here in the class declaration of Vec only if:
78 // - they must be here, like constructors or operator[];
79 // - they'll definitely never want a specialized implementation.
80 // Other operations on Vec should be defined outside the type.
81
82 SKVX_ALWAYS_INLINE Vec() = default;
83
84 template <typename U,
85 typename=typename std::enable_if<std::is_convertible<U,T>::value>::type>
86 SKVX_ALWAYS_INLINE
87 Vec(U x) : lo(x), hi(x) {}
88
89 SKVX_ALWAYS_INLINE Vec(std::initializer_list<T> xs) {
90 T vals[N] = {0};
91 memcpy(vals, xs.begin(), std::min(xs.size(), (size_t)N)*sizeof(T));
92
93 lo = Vec<N/2,T>::Load(vals + 0);
94 hi = Vec<N/2,T>::Load(vals + N/2);
95 }
96
97 SKVX_ALWAYS_INLINE T operator[](int i) const { return i < N/2 ? lo[i] : hi[i-N/2]; }
98 SKVX_ALWAYS_INLINE T& operator[](int i) { return i < N/2 ? lo[i] : hi[i-N/2]; }
99
100 SKVX_ALWAYS_INLINE static Vec Load(const void* ptr) {
101 Vec v;
102 memcpy(&v, ptr, sizeof(Vec));
103 return v;
104 }
105 SKVX_ALWAYS_INLINE void store(void* ptr) const {
106 memcpy(ptr, this, sizeof(Vec));
107 }
108};
109
110template <typename T>
111struct Vec<1,T> {
112 T val;
113
114 SKVX_ALWAYS_INLINE Vec() = default;
115
116 template <typename U,
117 typename=typename std::enable_if<std::is_convertible<U,T>::value>::type>
118 SKVX_ALWAYS_INLINE
119 Vec(U x) : val(x) {}
120
121 SKVX_ALWAYS_INLINE Vec(std::initializer_list<T> xs) : val(xs.size() ? *xs.begin() : 0) {}
122
123 SKVX_ALWAYS_INLINE T operator[](int) const { return val; }
124 SKVX_ALWAYS_INLINE T& operator[](int) { return val; }
125
126 SKVX_ALWAYS_INLINE static Vec Load(const void* ptr) {
127 Vec v;
128 memcpy(&v, ptr, sizeof(Vec));
129 return v;
130 }
131 SKVX_ALWAYS_INLINE void store(void* ptr) const {
132 memcpy(ptr, this, sizeof(Vec));
133 }
134};
135
136template <typename D, typename S>
137static inline D bit_pun(const S& s) {
138 static_assert(sizeof(D) == sizeof(S), "");
139 D d;
140 memcpy(&d, &s, sizeof(D));
141 return d;
142}
143
144// Translate from a value type T to its corresponding Mask, the result of a comparison.
145template <typename T> struct Mask { using type = T; };
146template <> struct Mask<float > { using type = int32_t; };
147template <> struct Mask<double> { using type = int64_t; };
148template <typename T> using M = typename Mask<T>::type;
149
150// Join two Vec<N,T> into one Vec<2N,T>.
151SINT Vec<2*N,T> join(const Vec<N,T>& lo, const Vec<N,T>& hi) {
152 Vec<2*N,T> v;
153 v.lo = lo;
154 v.hi = hi;
155 return v;
156}
157
158// We have two default strategies for implementing most operations:
159// 1) lean on Clang/GCC vector extensions when available;
160// 2) recurse to scalar portable implementations when not.
161// At the end we can drop in platform-specific implementations that override either default.
162
163#if !defined(SKNX_NO_SIMD) && (defined(__clang__) || defined(__GNUC__))
164
165 // VExt<N,T> types have the same size as Vec<N,T> and support most operations directly.
166 // N.B. VExt<N,T> alignment is N*alignof(T), stricter than Vec<N,T>'s alignof(T).
167 #if defined(__clang__)
168 template <int N, typename T>
169 using VExt = T __attribute__((ext_vector_type(N)));
170
171 #elif defined(__GNUC__)
172 template <int N, typename T>
173 struct VExtHelper {
174 typedef T __attribute__((vector_size(N*sizeof(T)))) type;
175 };
176
177 template <int N, typename T>
178 using VExt = typename VExtHelper<N,T>::type;
179
180 // For some reason some (new!) versions of GCC cannot seem to deduce N in the generic
181 // to_vec<N,T>() below for N=4 and T=float. This workaround seems to help...
182 static inline Vec<4,float> to_vec(VExt<4,float> v) { return bit_pun<Vec<4,float>>(v); }
183 #endif
184
185 SINT VExt<N,T> to_vext(const Vec<N,T>& v) { return bit_pun<VExt<N,T>>(v); }
186 SINT Vec <N,T> to_vec(const VExt<N,T>& v) { return bit_pun<Vec <N,T>>(v); }
187
188 SINT Vec<N,T> operator+(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) + to_vext(y)); }
189 SINT Vec<N,T> operator-(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) - to_vext(y)); }
190 SINT Vec<N,T> operator*(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) * to_vext(y)); }
191 SINT Vec<N,T> operator/(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) / to_vext(y)); }
192
193 SINT Vec<N,T> operator^(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) ^ to_vext(y)); }
194 SINT Vec<N,T> operator&(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) & to_vext(y)); }
195 SINT Vec<N,T> operator|(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) | to_vext(y)); }
196
197 SINT Vec<N,T> operator!(const Vec<N,T>& x) { return to_vec<N,T>(!to_vext(x)); }
198 SINT Vec<N,T> operator-(const Vec<N,T>& x) { return to_vec<N,T>(-to_vext(x)); }
199 SINT Vec<N,T> operator~(const Vec<N,T>& x) { return to_vec<N,T>(~to_vext(x)); }
200
201 SINT Vec<N,T> operator<<(const Vec<N,T>& x, int bits) { return to_vec<N,T>(to_vext(x) << bits); }
202 SINT Vec<N,T> operator>>(const Vec<N,T>& x, int bits) { return to_vec<N,T>(to_vext(x) >> bits); }
203
204 SINT Vec<N,M<T>> operator==(const Vec<N,T>& x, const Vec<N,T>& y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) == to_vext(y)); }
205 SINT Vec<N,M<T>> operator!=(const Vec<N,T>& x, const Vec<N,T>& y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) != to_vext(y)); }
206 SINT Vec<N,M<T>> operator<=(const Vec<N,T>& x, const Vec<N,T>& y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) <= to_vext(y)); }
207 SINT Vec<N,M<T>> operator>=(const Vec<N,T>& x, const Vec<N,T>& y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) >= to_vext(y)); }
208 SINT Vec<N,M<T>> operator< (const Vec<N,T>& x, const Vec<N,T>& y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) < to_vext(y)); }
209 SINT Vec<N,M<T>> operator> (const Vec<N,T>& x, const Vec<N,T>& y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) > to_vext(y)); }
210
211#else
212
213 // Either SKNX_NO_SIMD is defined, or Clang/GCC vector extensions are not available.
214 // We'll implement things portably, in a way that should be easily autovectorizable.
215
216 // N == 1 scalar implementations.
217 SIT Vec<1,T> operator+(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val + y.val; }
218 SIT Vec<1,T> operator-(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val - y.val; }
219 SIT Vec<1,T> operator*(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val * y.val; }
220 SIT Vec<1,T> operator/(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val / y.val; }
221
222 SIT Vec<1,T> operator^(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val ^ y.val; }
223 SIT Vec<1,T> operator&(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val & y.val; }
224 SIT Vec<1,T> operator|(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val | y.val; }
225
226 SIT Vec<1,T> operator!(const Vec<1,T>& x) { return !x.val; }
227 SIT Vec<1,T> operator-(const Vec<1,T>& x) { return -x.val; }
228 SIT Vec<1,T> operator~(const Vec<1,T>& x) { return ~x.val; }
229
230 SIT Vec<1,T> operator<<(const Vec<1,T>& x, int bits) { return x.val << bits; }
231 SIT Vec<1,T> operator>>(const Vec<1,T>& x, int bits) { return x.val >> bits; }
232
233 SIT Vec<1,M<T>> operator==(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val == y.val ? ~0 : 0; }
234 SIT Vec<1,M<T>> operator!=(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val != y.val ? ~0 : 0; }
235 SIT Vec<1,M<T>> operator<=(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val <= y.val ? ~0 : 0; }
236 SIT Vec<1,M<T>> operator>=(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val >= y.val ? ~0 : 0; }
237 SIT Vec<1,M<T>> operator< (const Vec<1,T>& x, const Vec<1,T>& y) { return x.val < y.val ? ~0 : 0; }
238 SIT Vec<1,M<T>> operator> (const Vec<1,T>& x, const Vec<1,T>& y) { return x.val > y.val ? ~0 : 0; }
239
240 // All default N != 1 implementations just recurse on lo and hi halves.
241 SINT Vec<N,T> operator+(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo + y.lo, x.hi + y.hi); }
242 SINT Vec<N,T> operator-(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo - y.lo, x.hi - y.hi); }
243 SINT Vec<N,T> operator*(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo * y.lo, x.hi * y.hi); }
244 SINT Vec<N,T> operator/(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo / y.lo, x.hi / y.hi); }
245
246 SINT Vec<N,T> operator^(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo ^ y.lo, x.hi ^ y.hi); }
247 SINT Vec<N,T> operator&(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo & y.lo, x.hi & y.hi); }
248 SINT Vec<N,T> operator|(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo | y.lo, x.hi | y.hi); }
249
250 SINT Vec<N,T> operator!(const Vec<N,T>& x) { return join(!x.lo, !x.hi); }
251 SINT Vec<N,T> operator-(const Vec<N,T>& x) { return join(-x.lo, -x.hi); }
252 SINT Vec<N,T> operator~(const Vec<N,T>& x) { return join(~x.lo, ~x.hi); }
253
254 SINT Vec<N,T> operator<<(const Vec<N,T>& x, int bits) { return join(x.lo << bits, x.hi << bits); }
255 SINT Vec<N,T> operator>>(const Vec<N,T>& x, int bits) { return join(x.lo >> bits, x.hi >> bits); }
256
257 SINT Vec<N,M<T>> operator==(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo == y.lo, x.hi == y.hi); }
258 SINT Vec<N,M<T>> operator!=(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo != y.lo, x.hi != y.hi); }
259 SINT Vec<N,M<T>> operator<=(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo <= y.lo, x.hi <= y.hi); }
260 SINT Vec<N,M<T>> operator>=(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo >= y.lo, x.hi >= y.hi); }
261 SINT Vec<N,M<T>> operator< (const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo < y.lo, x.hi < y.hi); }
262 SINT Vec<N,M<T>> operator> (const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo > y.lo, x.hi > y.hi); }
263#endif
264
265// Some operations we want are not expressible with Clang/GCC vector
266// extensions, so we implement them using the recursive approach.
267
268// N == 1 scalar implementations.
269SIT Vec<1,T> if_then_else(const Vec<1,M<T>>& cond, const Vec<1,T>& t, const Vec<1,T>& e) {
270 auto t_bits = bit_pun<M<T>>(t),
271 e_bits = bit_pun<M<T>>(e);
272 return bit_pun<T>( (cond.val & t_bits) | (~cond.val & e_bits) );
273}
274
275SIT bool any(const Vec<1,T>& x) { return x.val != 0; }
276SIT bool all(const Vec<1,T>& x) { return x.val != 0; }
277
278SIT T min(const Vec<1,T>& x) { return x.val; }
279SIT T max(const Vec<1,T>& x) { return x.val; }
280
281SIT Vec<1,T> min(const Vec<1,T>& x, const Vec<1,T>& y) { return std::min(x.val, y.val); }
282SIT Vec<1,T> max(const Vec<1,T>& x, const Vec<1,T>& y) { return std::max(x.val, y.val); }
283
284SIT Vec<1,T> ceil(const Vec<1,T>& x) { return std:: ceil(x.val); }
285SIT Vec<1,T> floor(const Vec<1,T>& x) { return std::floor(x.val); }
286SIT Vec<1,T> trunc(const Vec<1,T>& x) { return std::trunc(x.val); }
287SIT Vec<1,T> round(const Vec<1,T>& x) { return std::round(x.val); }
288SIT Vec<1,T> sqrt(const Vec<1,T>& x) { return std:: sqrt(x.val); }
289SIT Vec<1,T> abs(const Vec<1,T>& x) { return std:: abs(x.val); }
290
291SIT Vec<1,int> lrint(const Vec<1,T>& x) { return (int)std::lrint(x.val); }
292
293SIT Vec<1,T> rcp(const Vec<1,T>& x) { return 1 / x.val; }
294SIT Vec<1,T> rsqrt(const Vec<1,T>& x) { return rcp(sqrt(x)); }
295SIT Vec<1,T> mad(const Vec<1,T>& f,
296 const Vec<1,T>& m,
297 const Vec<1,T>& a) { return f*m+a; }
298
299// All default N != 1 implementations just recurse on lo and hi halves.
300SINT Vec<N,T> if_then_else(const Vec<N,M<T>>& cond, const Vec<N,T>& t, const Vec<N,T>& e) {
301 return join(if_then_else(cond.lo, t.lo, e.lo),
302 if_then_else(cond.hi, t.hi, e.hi));
303}
304
305SINT bool any(const Vec<N,T>& x) { return any(x.lo) || any(x.hi); }
306SINT bool all(const Vec<N,T>& x) { return all(x.lo) && all(x.hi); }
307
308SINT T min(const Vec<N,T>& x) { return std::min(min(x.lo), min(x.hi)); }
309SINT T max(const Vec<N,T>& x) { return std::max(max(x.lo), max(x.hi)); }
310
311SINT Vec<N,T> min(const Vec<N,T>& x, const Vec<N,T>& y) { return join(min(x.lo, y.lo), min(x.hi, y.hi)); }
312SINT Vec<N,T> max(const Vec<N,T>& x, const Vec<N,T>& y) { return join(max(x.lo, y.lo), max(x.hi, y.hi)); }
313
314SINT Vec<N,T> ceil(const Vec<N,T>& x) { return join( ceil(x.lo), ceil(x.hi)); }
315SINT Vec<N,T> floor(const Vec<N,T>& x) { return join(floor(x.lo), floor(x.hi)); }
316SINT Vec<N,T> trunc(const Vec<N,T>& x) { return join(trunc(x.lo), trunc(x.hi)); }
317SINT Vec<N,T> round(const Vec<N,T>& x) { return join(round(x.lo), round(x.hi)); }
318SINT Vec<N,T> sqrt(const Vec<N,T>& x) { return join( sqrt(x.lo), sqrt(x.hi)); }
319SINT Vec<N,T> abs(const Vec<N,T>& x) { return join( abs(x.lo), abs(x.hi)); }
320
321SINT Vec<N,int> lrint(const Vec<N,T>& x) { return join(lrint(x.lo), lrint(x.hi)); }
322
323SINT Vec<N,T> rcp(const Vec<N,T>& x) { return join( rcp(x.lo), rcp(x.hi)); }
324SINT Vec<N,T> rsqrt(const Vec<N,T>& x) { return join(rsqrt(x.lo), rsqrt(x.hi)); }
325SINT Vec<N,T> mad(const Vec<N,T>& f,
326 const Vec<N,T>& m,
327 const Vec<N,T>& a) { return join(mad(f.lo, m.lo, a.lo), mad(f.hi, m.hi, a.hi)); }
328
329
330// Scalar/vector operations just splat the scalar to a vector...
331SINTU Vec<N,T> operator+ (U x, const Vec<N,T>& y) { return Vec<N,T>(x) + y; }
332SINTU Vec<N,T> operator- (U x, const Vec<N,T>& y) { return Vec<N,T>(x) - y; }
333SINTU Vec<N,T> operator* (U x, const Vec<N,T>& y) { return Vec<N,T>(x) * y; }
334SINTU Vec<N,T> operator/ (U x, const Vec<N,T>& y) { return Vec<N,T>(x) / y; }
335SINTU Vec<N,T> operator^ (U x, const Vec<N,T>& y) { return Vec<N,T>(x) ^ y; }
336SINTU Vec<N,T> operator& (U x, const Vec<N,T>& y) { return Vec<N,T>(x) & y; }
337SINTU Vec<N,T> operator| (U x, const Vec<N,T>& y) { return Vec<N,T>(x) | y; }
338SINTU Vec<N,M<T>> operator==(U x, const Vec<N,T>& y) { return Vec<N,T>(x) == y; }
339SINTU Vec<N,M<T>> operator!=(U x, const Vec<N,T>& y) { return Vec<N,T>(x) != y; }
340SINTU Vec<N,M<T>> operator<=(U x, const Vec<N,T>& y) { return Vec<N,T>(x) <= y; }
341SINTU Vec<N,M<T>> operator>=(U x, const Vec<N,T>& y) { return Vec<N,T>(x) >= y; }
342SINTU Vec<N,M<T>> operator< (U x, const Vec<N,T>& y) { return Vec<N,T>(x) < y; }
343SINTU Vec<N,M<T>> operator> (U x, const Vec<N,T>& y) { return Vec<N,T>(x) > y; }
344SINTU Vec<N,T> min(U x, const Vec<N,T>& y) { return min(Vec<N,T>(x), y); }
345SINTU Vec<N,T> max(U x, const Vec<N,T>& y) { return max(Vec<N,T>(x), y); }
346
347// ... and same deal for vector/scalar operations.
348SINTU Vec<N,T> operator+ (const Vec<N,T>& x, U y) { return x + Vec<N,T>(y); }
349SINTU Vec<N,T> operator- (const Vec<N,T>& x, U y) { return x - Vec<N,T>(y); }
350SINTU Vec<N,T> operator* (const Vec<N,T>& x, U y) { return x * Vec<N,T>(y); }
351SINTU Vec<N,T> operator/ (const Vec<N,T>& x, U y) { return x / Vec<N,T>(y); }
352SINTU Vec<N,T> operator^ (const Vec<N,T>& x, U y) { return x ^ Vec<N,T>(y); }
353SINTU Vec<N,T> operator& (const Vec<N,T>& x, U y) { return x & Vec<N,T>(y); }
354SINTU Vec<N,T> operator| (const Vec<N,T>& x, U y) { return x | Vec<N,T>(y); }
355SINTU Vec<N,M<T>> operator==(const Vec<N,T>& x, U y) { return x == Vec<N,T>(y); }
356SINTU Vec<N,M<T>> operator!=(const Vec<N,T>& x, U y) { return x != Vec<N,T>(y); }
357SINTU Vec<N,M<T>> operator<=(const Vec<N,T>& x, U y) { return x <= Vec<N,T>(y); }
358SINTU Vec<N,M<T>> operator>=(const Vec<N,T>& x, U y) { return x >= Vec<N,T>(y); }
359SINTU Vec<N,M<T>> operator< (const Vec<N,T>& x, U y) { return x < Vec<N,T>(y); }
360SINTU Vec<N,M<T>> operator> (const Vec<N,T>& x, U y) { return x > Vec<N,T>(y); }
361SINTU Vec<N,T> min(const Vec<N,T>& x, U y) { return min(x, Vec<N,T>(y)); }
362SINTU Vec<N,T> max(const Vec<N,T>& x, U y) { return max(x, Vec<N,T>(y)); }
363
364// All vector/scalar combinations for mad() with at least one vector.
365SINTU Vec<N,T> mad(U f, const Vec<N,T>& m, const Vec<N,T>& a) { return Vec<N,T>(f)*m + a; }
366SINTU Vec<N,T> mad(const Vec<N,T>& f, U m, const Vec<N,T>& a) { return f*Vec<N,T>(m) + a; }
367SINTU Vec<N,T> mad(const Vec<N,T>& f, const Vec<N,T>& m, U a) { return f*m + Vec<N,T>(a); }
368SINTU Vec<N,T> mad(const Vec<N,T>& f, U m, U a) { return f*Vec<N,T>(m) + Vec<N,T>(a); }
369SINTU Vec<N,T> mad(U f, const Vec<N,T>& m, U a) { return Vec<N,T>(f)*m + Vec<N,T>(a); }
370SINTU Vec<N,T> mad(U f, U m, const Vec<N,T>& a) { return Vec<N,T>(f)*Vec<N,T>(m) + a; }
371
372// The various op= operators, for vectors...
373SINT Vec<N,T>& operator+=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x + y); }
374SINT Vec<N,T>& operator-=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x - y); }
375SINT Vec<N,T>& operator*=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x * y); }
376SINT Vec<N,T>& operator/=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x / y); }
377SINT Vec<N,T>& operator^=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x ^ y); }
378SINT Vec<N,T>& operator&=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x & y); }
379SINT Vec<N,T>& operator|=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x | y); }
380
381// ... for scalars...
382SINTU Vec<N,T>& operator+=(Vec<N,T>& x, U y) { return (x = x + Vec<N,T>(y)); }
383SINTU Vec<N,T>& operator-=(Vec<N,T>& x, U y) { return (x = x - Vec<N,T>(y)); }
384SINTU Vec<N,T>& operator*=(Vec<N,T>& x, U y) { return (x = x * Vec<N,T>(y)); }
385SINTU Vec<N,T>& operator/=(Vec<N,T>& x, U y) { return (x = x / Vec<N,T>(y)); }
386SINTU Vec<N,T>& operator^=(Vec<N,T>& x, U y) { return (x = x ^ Vec<N,T>(y)); }
387SINTU Vec<N,T>& operator&=(Vec<N,T>& x, U y) { return (x = x & Vec<N,T>(y)); }
388SINTU Vec<N,T>& operator|=(Vec<N,T>& x, U y) { return (x = x | Vec<N,T>(y)); }
389
390// ... and for shifts.
391SINT Vec<N,T>& operator<<=(Vec<N,T>& x, int bits) { return (x = x << bits); }
392SINT Vec<N,T>& operator>>=(Vec<N,T>& x, int bits) { return (x = x >> bits); }
393
394// cast() Vec<N,S> to Vec<N,D>, as if applying a C-cast to each lane.
395template <typename D, typename S>
396static inline Vec<1,D> cast(const Vec<1,S>& src) { return (D)src.val; }
397
398template <typename D, int N, typename S>
399static inline Vec<N,D> cast(const Vec<N,S>& src) {
400#if !defined(SKNX_NO_SIMD) && defined(__clang__)
401 return to_vec(__builtin_convertvector(to_vext(src), VExt<N,D>));
402#else
403 return join(cast<D>(src.lo), cast<D>(src.hi));
404#endif
405}
406
407// Shuffle values from a vector pretty arbitrarily:
408// skvx::Vec<4,float> rgba = {R,G,B,A};
409// shuffle<2,1,0,3> (rgba) ~> {B,G,R,A}
410// shuffle<2,1> (rgba) ~> {B,G}
411// shuffle<2,1,2,1,2,1,2,1>(rgba) ~> {B,G,B,G,B,G,B,G}
412// shuffle<3,3,3,3> (rgba) ~> {A,A,A,A}
413// The only real restriction is that the output also be a legal N=power-of-two sknx::Vec.
414template <int... Ix, int N, typename T>
415static inline Vec<sizeof...(Ix),T> shuffle(const Vec<N,T>& x) {
416#if !defined(SKNX_NO_SIMD) && defined(__clang__)
417 return to_vec<sizeof...(Ix),T>(__builtin_shufflevector(to_vext(x), to_vext(x), Ix...));
418#else
419 return { x[Ix]... };
420#endif
421}
422
423// fma() delivers a fused mul-add, even if that's really expensive. Call it when you know it's not.
424static inline Vec<1,float> fma(const Vec<1,float>& x,
425 const Vec<1,float>& y,
426 const Vec<1,float>& z) {
427 return std::fma(x.val, y.val, z.val);
428}
429template <int N>
430static inline Vec<N,float> fma(const Vec<N,float>& x,
431 const Vec<N,float>& y,
432 const Vec<N,float>& z) {
433 return join(fma(x.lo, y.lo, z.lo),
434 fma(x.hi, y.hi, z.hi));
435}
436
437// div255(x) = (x + 127) / 255 is a bit-exact rounding divide-by-255, packing down to 8-bit.
438template <int N>
439static inline Vec<N,uint8_t> div255(const Vec<N,uint16_t>& x) {
440 return cast<uint8_t>( (x+127)/255 );
441}
442
443// approx_scale(x,y) approximates div255(cast<uint16_t>(x)*cast<uint16_t>(y)) within a bit,
444// and is always perfect when x or y is 0 or 255.
445template <int N>
446static inline Vec<N,uint8_t> approx_scale(const Vec<N,uint8_t>& x, const Vec<N,uint8_t>& y) {
447 // All of (x*y+x)/256, (x*y+y)/256, and (x*y+255)/256 meet the criteria above.
448 // We happen to have historically picked (x*y+x)/256.
449 auto X = cast<uint16_t>(x),
450 Y = cast<uint16_t>(y);
451 return cast<uint8_t>( (X*Y+X)/256 );
452}
453
454#if !defined(SKNX_NO_SIMD) && defined(__ARM_NEON)
455 // With NEON we can do eight u8*u8 -> u16 in one instruction, vmull_u8 (read, mul-long).
456 static inline Vec<8,uint16_t> mull(const Vec<8,uint8_t>& x,
457 const Vec<8,uint8_t>& y) {
458 return to_vec<8,uint16_t>(vmull_u8(to_vext(x),
459 to_vext(y)));
460 }
461
462 template <int N>
463 static inline typename std::enable_if<(N < 8),
464 Vec<N,uint16_t>>::type mull(const Vec<N,uint8_t>& x,
465 const Vec<N,uint8_t>& y) {
466 // N < 8 --> double up data until N == 8, returning the part we need.
467 return mull(join(x,x),
468 join(y,y)).lo;
469 }
470
471 template <int N>
472 static inline typename std::enable_if<(N > 8),
473 Vec<N,uint16_t>>::type mull(const Vec<N,uint8_t>& x,
474 const Vec<N,uint8_t>& y) {
475 // N > 8 --> usual join(lo,hi) strategy to recurse down to N == 8.
476 return join(mull(x.lo, y.lo),
477 mull(x.hi, y.hi));
478 }
479#else
480 // Nothing special when we don't have NEON... just cast up to 16-bit and multiply.
481 template <int N>
482 static inline Vec<N,uint16_t> mull(const Vec<N,uint8_t>& x,
483 const Vec<N,uint8_t>& y) {
484 return cast<uint16_t>(x)
485 * cast<uint16_t>(y);
486 }
487#endif
488
489#if !defined(SKNX_NO_SIMD)
490
491 // Platform-specific specializations and overloads can now drop in here.
492
493 #if defined(__AVX__)
494 static inline Vec<8,float> sqrt(const Vec<8,float>& x) {
495 return bit_pun<Vec<8,float>>(_mm256_sqrt_ps(bit_pun<__m256>(x)));
496 }
497 static inline Vec<8,float> rsqrt(const Vec<8,float>& x) {
498 return bit_pun<Vec<8,float>>(_mm256_rsqrt_ps(bit_pun<__m256>(x)));
499 }
500 static inline Vec<8,float> rcp(const Vec<8,float>& x) {
501 return bit_pun<Vec<8,float>>(_mm256_rcp_ps(bit_pun<__m256>(x)));
502 }
503 static inline Vec<8,int> lrint(const Vec<8,float>& x) {
504 return bit_pun<Vec<8,int>>(_mm256_cvtps_epi32(bit_pun<__m256>(x)));
505 }
506 #endif
507
508 #if defined(__SSE__)
509 static inline Vec<4,float> sqrt(const Vec<4,float>& x) {
510 return bit_pun<Vec<4,float>>(_mm_sqrt_ps(bit_pun<__m128>(x)));
511 }
512 static inline Vec<4,float> rsqrt(const Vec<4,float>& x) {
513 return bit_pun<Vec<4,float>>(_mm_rsqrt_ps(bit_pun<__m128>(x)));
514 }
515 static inline Vec<4,float> rcp(const Vec<4,float>& x) {
516 return bit_pun<Vec<4,float>>(_mm_rcp_ps(bit_pun<__m128>(x)));
517 }
518 static inline Vec<4,int> lrint(const Vec<4,float>& x) {
519 return bit_pun<Vec<4,int>>(_mm_cvtps_epi32(bit_pun<__m128>(x)));
520 }
521
522 static inline Vec<2,float> sqrt(const Vec<2,float>& x) {
523 return shuffle<0,1>( sqrt(shuffle<0,1,0,1>(x)));
524 }
525 static inline Vec<2,float> rsqrt(const Vec<2,float>& x) {
526 return shuffle<0,1>(rsqrt(shuffle<0,1,0,1>(x)));
527 }
528 static inline Vec<2,float> rcp(const Vec<2,float>& x) {
529 return shuffle<0,1>( rcp(shuffle<0,1,0,1>(x)));
530 }
531 static inline Vec<2,int> lrint(const Vec<2,float>& x) {
532 return shuffle<0,1>(lrint(shuffle<0,1,0,1>(x)));
533 }
534 #endif
535
536 #if defined(__SSE4_1__)
537 static inline Vec<4,float> if_then_else(const Vec<4,int >& c,
538 const Vec<4,float>& t,
539 const Vec<4,float>& e) {
540 return bit_pun<Vec<4,float>>(_mm_blendv_ps(bit_pun<__m128>(e),
541 bit_pun<__m128>(t),
542 bit_pun<__m128>(c)));
543 }
544 #elif defined(__SSE__)
545 static inline Vec<4,float> if_then_else(const Vec<4,int >& c,
546 const Vec<4,float>& t,
547 const Vec<4,float>& e) {
548 return bit_pun<Vec<4,float>>(_mm_or_ps(_mm_and_ps (bit_pun<__m128>(c),
549 bit_pun<__m128>(t)),
550 _mm_andnot_ps(bit_pun<__m128>(c),
551 bit_pun<__m128>(e))));
552 }
553 #elif defined(__ARM_NEON)
554 static inline Vec<4,float> if_then_else(const Vec<4,int >& c,
555 const Vec<4,float>& t,
556 const Vec<4,float>& e) {
557 return bit_pun<Vec<4,float>>(vbslq_f32(bit_pun<uint32x4_t> (c),
558 bit_pun<float32x4_t>(t),
559 bit_pun<float32x4_t>(e)));
560 }
561 #endif
562
563 #if defined(__AVX2__)
564 static inline Vec<4,float> fma(const Vec<4,float>& x,
565 const Vec<4,float>& y,
566 const Vec<4,float>& z) {
567 return bit_pun<Vec<4,float>>(_mm_fmadd_ps(bit_pun<__m128>(x),
568 bit_pun<__m128>(y),
569 bit_pun<__m128>(z)));
570 }
571
572 static inline Vec<8,float> fma(const Vec<8,float>& x,
573 const Vec<8,float>& y,
574 const Vec<8,float>& z) {
575 return bit_pun<Vec<8,float>>(_mm256_fmadd_ps(bit_pun<__m256>(x),
576 bit_pun<__m256>(y),
577 bit_pun<__m256>(z)));
578 }
579 #elif defined(__aarch64__)
580 static inline Vec<4,float> fma(const Vec<4,float>& x,
581 const Vec<4,float>& y,
582 const Vec<4,float>& z) {
583 // These instructions tend to work like z += xy, so the order here is z,x,y.
584 return bit_pun<Vec<4,float>>(vfmaq_f32(bit_pun<float32x4_t>(z),
585 bit_pun<float32x4_t>(x),
586 bit_pun<float32x4_t>(y)));
587 }
588 #endif
589
590#endif // !defined(SKNX_NO_SIMD)
591
592} // namespace skvx
593
594#undef SINTU
595#undef SINT
596#undef SIT
597#undef SKVX_ALIGNMENT
598
599#endif//SKVX_DEFINED
600