1// Copyright 2009-2021 Intel Corporation
2// SPDX-License-Identifier: Apache-2.0
3
4#pragma once
5
6#include "../../common/sys/platform.h"
7#include "../../common/sys/sysinfo.h"
8
9namespace embree
10{
11#define DEFINE_SYMBOL2(type,name) \
12 typedef type (*name##Func)(); \
13 name##Func name;
14
15#define DECLARE_SYMBOL2(type,name) \
16 namespace sse2 { extern type name(); } \
17 namespace sse42 { extern type name(); } \
18 namespace avx { extern type name(); } \
19 namespace avx2 { extern type name(); } \
20 namespace avx512 { extern type name(); } \
21 void name##_error2() { throw_RTCError(RTC_ERROR_UNKNOWN,"internal error in ISA selection for " TOSTRING(name)); } \
22 type name##_error() { return type(name##_error2); } \
23 type name##_zero() { return type(nullptr); }
24
25#define DECLARE_ISA_FUNCTION(type,symbol,args) \
26 namespace sse2 { extern type symbol(args); } \
27 namespace sse42 { extern type symbol(args); } \
28 namespace avx { extern type symbol(args); } \
29 namespace avx2 { extern type symbol(args); } \
30 namespace avx512 { extern type symbol(args); } \
31 inline type symbol##_error(args) { throw_RTCError(RTC_ERROR_UNSUPPORTED_CPU,"function " TOSTRING(symbol) " not supported by your CPU"); } \
32 typedef type (*symbol##Ty)(args); \
33
34#define DEFINE_ISA_FUNCTION(type,symbol,args) \
35 typedef type (*symbol##Func)(args); \
36 symbol##Func symbol;
37
38#define ZERO_SYMBOL(features,intersector) \
39 intersector = intersector##_zero;
40
41#define INIT_SYMBOL(features,intersector) \
42 intersector = decltype(intersector)(intersector##_error);
43
44#define SELECT_SYMBOL_DEFAULT(features,intersector) \
45 intersector = isa::intersector;
46
47#if defined(__SSE__) || defined(__ARM_NEON)
48#if !defined(EMBREE_TARGET_SIMD4)
49#define EMBREE_TARGET_SIMD4
50#endif
51#endif
52
53#if defined(EMBREE_TARGET_SSE42)
54#define SELECT_SYMBOL_SSE42(features,intersector) \
55 if ((features & SSE42) == SSE42) intersector = sse42::intersector;
56#else
57#define SELECT_SYMBOL_SSE42(features,intersector)
58#endif
59
60#if defined(EMBREE_TARGET_AVX) || defined(__AVX__)
61#if !defined(EMBREE_TARGET_SIMD8)
62#define EMBREE_TARGET_SIMD8
63#endif
64#if defined(__AVX__) // if default ISA is >= AVX we treat AVX target as default target
65#define SELECT_SYMBOL_AVX(features,intersector) \
66 if ((features & ISA) == ISA) intersector = isa::intersector;
67#else
68#define SELECT_SYMBOL_AVX(features,intersector) \
69 if ((features & AVX) == AVX) intersector = avx::intersector;
70#endif
71#else
72#define SELECT_SYMBOL_AVX(features,intersector)
73#endif
74
75#if defined(EMBREE_TARGET_AVX2)
76#if !defined(EMBREE_TARGET_SIMD8)
77#define EMBREE_TARGET_SIMD8
78#endif
79#define SELECT_SYMBOL_AVX2(features,intersector) \
80 if ((features & AVX2) == AVX2) intersector = avx2::intersector;
81#else
82#define SELECT_SYMBOL_AVX2(features,intersector)
83#endif
84
85#if defined(EMBREE_TARGET_AVX512)
86#if !defined(EMBREE_TARGET_SIMD16)
87#define EMBREE_TARGET_SIMD16
88#endif
89#define SELECT_SYMBOL_AVX512(features,intersector) \
90 if ((features & AVX512) == AVX512) intersector = avx512::intersector;
91#else
92#define SELECT_SYMBOL_AVX512(features,intersector)
93#endif
94
95#define SELECT_SYMBOL_DEFAULT_SSE42(features,intersector) \
96 SELECT_SYMBOL_DEFAULT(features,intersector); \
97 SELECT_SYMBOL_SSE42(features,intersector);
98
99#define SELECT_SYMBOL_DEFAULT_SSE42_AVX(features,intersector) \
100 SELECT_SYMBOL_DEFAULT(features,intersector); \
101 SELECT_SYMBOL_SSE42(features,intersector); \
102 SELECT_SYMBOL_AVX(features,intersector);
103
104#define SELECT_SYMBOL_DEFAULT_SSE42_AVX_AVX2(features,intersector) \
105 SELECT_SYMBOL_DEFAULT(features,intersector); \
106 SELECT_SYMBOL_SSE42(features,intersector); \
107 SELECT_SYMBOL_AVX(features,intersector); \
108 SELECT_SYMBOL_AVX2(features,intersector);
109
110#define SELECT_SYMBOL_DEFAULT_SSE42_AVX_AVX512(features,intersector) \
111 SELECT_SYMBOL_DEFAULT(features,intersector); \
112 SELECT_SYMBOL_SSE42(features,intersector); \
113 SELECT_SYMBOL_AVX(features,intersector); \
114 SELECT_SYMBOL_AVX512(features,intersector);
115
116#define SELECT_SYMBOL_DEFAULT_AVX_AVX2_AVX512(features,intersector) \
117 SELECT_SYMBOL_DEFAULT(features,intersector); \
118 SELECT_SYMBOL_AVX(features,intersector); \
119 SELECT_SYMBOL_AVX2(features,intersector); \
120 SELECT_SYMBOL_AVX512(features,intersector);
121
122#define SELECT_SYMBOL_DEFAULT_AVX_AVX2_AVX512(features,intersector) \
123 SELECT_SYMBOL_DEFAULT(features,intersector); \
124 SELECT_SYMBOL_AVX(features,intersector); \
125 SELECT_SYMBOL_AVX2(features,intersector); \
126 SELECT_SYMBOL_AVX512(features,intersector);
127
128#define SELECT_SYMBOL_DEFAULT_SSE42_AVX_AVX2_AVX512(features,intersector) \
129 SELECT_SYMBOL_DEFAULT(features,intersector); \
130 SELECT_SYMBOL_SSE42(features,intersector); \
131 SELECT_SYMBOL_AVX(features,intersector); \
132 SELECT_SYMBOL_AVX2(features,intersector); \
133 SELECT_SYMBOL_AVX512(features,intersector);
134
135#define SELECT_SYMBOL_DEFAULT_SSE42_AVX_AVX2_AVX512(features,intersector) \
136 SELECT_SYMBOL_DEFAULT(features,intersector); \
137 SELECT_SYMBOL_SSE42(features,intersector); \
138 SELECT_SYMBOL_AVX(features,intersector); \
139 SELECT_SYMBOL_AVX2(features,intersector); \
140 SELECT_SYMBOL_AVX512(features,intersector);
141
142#define SELECT_SYMBOL_DEFAULT_AVX(features,intersector) \
143 SELECT_SYMBOL_DEFAULT(features,intersector); \
144 SELECT_SYMBOL_AVX(features,intersector);
145
146#define SELECT_SYMBOL_DEFAULT_AVX_AVX2(features,intersector) \
147 SELECT_SYMBOL_DEFAULT(features,intersector); \
148 SELECT_SYMBOL_AVX(features,intersector); \
149 SELECT_SYMBOL_AVX2(features,intersector);
150
151#define SELECT_SYMBOL_DEFAULT_AVX(features,intersector) \
152 SELECT_SYMBOL_DEFAULT(features,intersector); \
153 SELECT_SYMBOL_AVX(features,intersector);
154
155#define SELECT_SYMBOL_DEFAULT_AVX_AVX512(features,intersector) \
156 SELECT_SYMBOL_DEFAULT(features,intersector); \
157 SELECT_SYMBOL_AVX(features,intersector); \
158 SELECT_SYMBOL_AVX512(features,intersector);
159
160#define SELECT_SYMBOL_DEFAULT_AVX_AVX512(features,intersector) \
161 SELECT_SYMBOL_DEFAULT(features,intersector); \
162 SELECT_SYMBOL_AVX(features,intersector); \
163 SELECT_SYMBOL_AVX512(features,intersector);
164
165#define SELECT_SYMBOL_INIT_AVX(features,intersector) \
166 INIT_SYMBOL(features,intersector); \
167 SELECT_SYMBOL_AVX(features,intersector);
168
169#define SELECT_SYMBOL_INIT_AVX_AVX2(features,intersector) \
170 INIT_SYMBOL(features,intersector); \
171 SELECT_SYMBOL_AVX(features,intersector); \
172 SELECT_SYMBOL_AVX2(features,intersector);
173
174#define SELECT_SYMBOL_INIT_AVX_AVX2_AVX512(features,intersector) \
175 INIT_SYMBOL(features,intersector); \
176 SELECT_SYMBOL_AVX(features,intersector); \
177 SELECT_SYMBOL_AVX2(features,intersector); \
178 SELECT_SYMBOL_AVX512(features,intersector);
179
180#define SELECT_SYMBOL_INIT_SSE42_AVX_AVX2(features,intersector) \
181 INIT_SYMBOL(features,intersector); \
182 SELECT_SYMBOL_SSE42(features,intersector); \
183 SELECT_SYMBOL_AVX(features,intersector); \
184 SELECT_SYMBOL_AVX2(features,intersector);
185
186#define SELECT_SYMBOL_INIT_AVX(features,intersector) \
187 INIT_SYMBOL(features,intersector); \
188 SELECT_SYMBOL_AVX(features,intersector);
189
190#define SELECT_SYMBOL_INIT_AVX_AVX512(features,intersector) \
191 INIT_SYMBOL(features,intersector); \
192 SELECT_SYMBOL_AVX(features,intersector); \
193 SELECT_SYMBOL_AVX512(features,intersector);
194
195#define SELECT_SYMBOL_INIT_AVX_AVX2(features,intersector) \
196 INIT_SYMBOL(features,intersector); \
197 SELECT_SYMBOL_AVX(features,intersector); \
198 SELECT_SYMBOL_AVX2(features,intersector);
199
200#define SELECT_SYMBOL_INIT_AVX_AVX2_AVX512(features,intersector) \
201 INIT_SYMBOL(features,intersector); \
202 SELECT_SYMBOL_AVX(features,intersector); \
203 SELECT_SYMBOL_AVX2(features,intersector); \
204 SELECT_SYMBOL_AVX512(features,intersector);
205
206#define SELECT_SYMBOL_INIT_SSE42_AVX_AVX2_AVX512(features,intersector) \
207 INIT_SYMBOL(features,intersector); \
208 SELECT_SYMBOL_SSE42(features,intersector); \
209 SELECT_SYMBOL_AVX(features,intersector); \
210 SELECT_SYMBOL_AVX2(features,intersector); \
211 SELECT_SYMBOL_AVX512(features,intersector);
212
213#define SELECT_SYMBOL_ZERO_SSE42_AVX_AVX2_AVX512(features,intersector) \
214 ZERO_SYMBOL(features,intersector); \
215 SELECT_SYMBOL_SSE42(features,intersector); \
216 SELECT_SYMBOL_AVX(features,intersector); \
217 SELECT_SYMBOL_AVX2(features,intersector); \
218 SELECT_SYMBOL_AVX512(features,intersector);
219
220#define SELECT_SYMBOL_DEFAULT_AVX_AVX2_AVX512(features,intersector) \
221 SELECT_SYMBOL_DEFAULT(features,intersector); \
222 SELECT_SYMBOL_AVX(features,intersector); \
223 SELECT_SYMBOL_AVX2(features,intersector); \
224 SELECT_SYMBOL_AVX512(features,intersector);
225
226#define SELECT_SYMBOL_INIT_AVX512(features,intersector) \
227 INIT_SYMBOL(features,intersector); \
228 SELECT_SYMBOL_AVX512(features,intersector);
229
230#define SELECT_SYMBOL_SSE42_AVX_AVX2(features,intersector) \
231 SELECT_SYMBOL_SSE42(features,intersector); \
232 SELECT_SYMBOL_AVX(features,intersector); \
233 SELECT_SYMBOL_AVX2(features,intersector);
234
235 struct VerifyMultiTargetLinking {
236 static __noinline int getISA(int depth = 5) {
237 if (depth == 0) return ISA;
238 else return getISA(depth-1);
239 }
240 };
241 namespace sse2 { int getISA(); };
242 namespace sse42 { int getISA(); };
243 namespace avx { int getISA(); };
244 namespace avx2 { int getISA(); };
245 namespace avx512 { int getISA(); };
246}
247