1 | // Copyright 2009-2021 Intel Corporation |
2 | // SPDX-License-Identifier: Apache-2.0 |
3 | |
4 | #pragma once |
5 | |
6 | #include "../../common/sys/platform.h" |
7 | #include "../../common/sys/sysinfo.h" |
8 | |
9 | namespace embree |
10 | { |
11 | #define DEFINE_SYMBOL2(type,name) \ |
12 | typedef type (*name##Func)(); \ |
13 | name##Func name; |
14 | |
15 | #define DECLARE_SYMBOL2(type,name) \ |
16 | namespace sse2 { extern type name(); } \ |
17 | namespace sse42 { extern type name(); } \ |
18 | namespace avx { extern type name(); } \ |
19 | namespace avx2 { extern type name(); } \ |
20 | namespace avx512 { extern type name(); } \ |
21 | void name##_error2() { throw_RTCError(RTC_ERROR_UNKNOWN,"internal error in ISA selection for " TOSTRING(name)); } \ |
22 | type name##_error() { return type(name##_error2); } \ |
23 | type name##_zero() { return type(nullptr); } |
24 | |
25 | #define DECLARE_ISA_FUNCTION(type,symbol,args) \ |
26 | namespace sse2 { extern type symbol(args); } \ |
27 | namespace sse42 { extern type symbol(args); } \ |
28 | namespace avx { extern type symbol(args); } \ |
29 | namespace avx2 { extern type symbol(args); } \ |
30 | namespace avx512 { extern type symbol(args); } \ |
31 | inline type symbol##_error(args) { throw_RTCError(RTC_ERROR_UNSUPPORTED_CPU,"function " TOSTRING(symbol) " not supported by your CPU"); } \ |
32 | typedef type (*symbol##Ty)(args); \ |
33 | |
34 | #define DEFINE_ISA_FUNCTION(type,symbol,args) \ |
35 | typedef type (*symbol##Func)(args); \ |
36 | symbol##Func symbol; |
37 | |
38 | #define ZERO_SYMBOL(features,intersector) \ |
39 | intersector = intersector##_zero; |
40 | |
41 | #define INIT_SYMBOL(features,intersector) \ |
42 | intersector = decltype(intersector)(intersector##_error); |
43 | |
44 | #define SELECT_SYMBOL_DEFAULT(features,intersector) \ |
45 | intersector = isa::intersector; |
46 | |
47 | #if defined(__SSE__) || defined(__ARM_NEON) |
48 | #if !defined(EMBREE_TARGET_SIMD4) |
49 | #define EMBREE_TARGET_SIMD4 |
50 | #endif |
51 | #endif |
52 | |
53 | #if defined(EMBREE_TARGET_SSE42) |
54 | #define SELECT_SYMBOL_SSE42(features,intersector) \ |
55 | if ((features & SSE42) == SSE42) intersector = sse42::intersector; |
56 | #else |
57 | #define SELECT_SYMBOL_SSE42(features,intersector) |
58 | #endif |
59 | |
60 | #if defined(EMBREE_TARGET_AVX) || defined(__AVX__) |
61 | #if !defined(EMBREE_TARGET_SIMD8) |
62 | #define EMBREE_TARGET_SIMD8 |
63 | #endif |
64 | #if defined(__AVX__) // if default ISA is >= AVX we treat AVX target as default target |
65 | #define SELECT_SYMBOL_AVX(features,intersector) \ |
66 | if ((features & ISA) == ISA) intersector = isa::intersector; |
67 | #else |
68 | #define SELECT_SYMBOL_AVX(features,intersector) \ |
69 | if ((features & AVX) == AVX) intersector = avx::intersector; |
70 | #endif |
71 | #else |
72 | #define SELECT_SYMBOL_AVX(features,intersector) |
73 | #endif |
74 | |
75 | #if defined(EMBREE_TARGET_AVX2) |
76 | #if !defined(EMBREE_TARGET_SIMD8) |
77 | #define EMBREE_TARGET_SIMD8 |
78 | #endif |
79 | #define SELECT_SYMBOL_AVX2(features,intersector) \ |
80 | if ((features & AVX2) == AVX2) intersector = avx2::intersector; |
81 | #else |
82 | #define SELECT_SYMBOL_AVX2(features,intersector) |
83 | #endif |
84 | |
85 | #if defined(EMBREE_TARGET_AVX512) |
86 | #if !defined(EMBREE_TARGET_SIMD16) |
87 | #define EMBREE_TARGET_SIMD16 |
88 | #endif |
89 | #define SELECT_SYMBOL_AVX512(features,intersector) \ |
90 | if ((features & AVX512) == AVX512) intersector = avx512::intersector; |
91 | #else |
92 | #define SELECT_SYMBOL_AVX512(features,intersector) |
93 | #endif |
94 | |
95 | #define SELECT_SYMBOL_DEFAULT_SSE42(features,intersector) \ |
96 | SELECT_SYMBOL_DEFAULT(features,intersector); \ |
97 | SELECT_SYMBOL_SSE42(features,intersector); |
98 | |
99 | #define SELECT_SYMBOL_DEFAULT_SSE42_AVX(features,intersector) \ |
100 | SELECT_SYMBOL_DEFAULT(features,intersector); \ |
101 | SELECT_SYMBOL_SSE42(features,intersector); \ |
102 | SELECT_SYMBOL_AVX(features,intersector); |
103 | |
104 | #define SELECT_SYMBOL_DEFAULT_SSE42_AVX_AVX2(features,intersector) \ |
105 | SELECT_SYMBOL_DEFAULT(features,intersector); \ |
106 | SELECT_SYMBOL_SSE42(features,intersector); \ |
107 | SELECT_SYMBOL_AVX(features,intersector); \ |
108 | SELECT_SYMBOL_AVX2(features,intersector); |
109 | |
110 | #define SELECT_SYMBOL_DEFAULT_SSE42_AVX_AVX512(features,intersector) \ |
111 | SELECT_SYMBOL_DEFAULT(features,intersector); \ |
112 | SELECT_SYMBOL_SSE42(features,intersector); \ |
113 | SELECT_SYMBOL_AVX(features,intersector); \ |
114 | SELECT_SYMBOL_AVX512(features,intersector); |
115 | |
116 | #define SELECT_SYMBOL_DEFAULT_AVX_AVX2_AVX512(features,intersector) \ |
117 | SELECT_SYMBOL_DEFAULT(features,intersector); \ |
118 | SELECT_SYMBOL_AVX(features,intersector); \ |
119 | SELECT_SYMBOL_AVX2(features,intersector); \ |
120 | SELECT_SYMBOL_AVX512(features,intersector); |
121 | |
122 | #define SELECT_SYMBOL_DEFAULT_AVX_AVX2_AVX512(features,intersector) \ |
123 | SELECT_SYMBOL_DEFAULT(features,intersector); \ |
124 | SELECT_SYMBOL_AVX(features,intersector); \ |
125 | SELECT_SYMBOL_AVX2(features,intersector); \ |
126 | SELECT_SYMBOL_AVX512(features,intersector); |
127 | |
128 | #define SELECT_SYMBOL_DEFAULT_SSE42_AVX_AVX2_AVX512(features,intersector) \ |
129 | SELECT_SYMBOL_DEFAULT(features,intersector); \ |
130 | SELECT_SYMBOL_SSE42(features,intersector); \ |
131 | SELECT_SYMBOL_AVX(features,intersector); \ |
132 | SELECT_SYMBOL_AVX2(features,intersector); \ |
133 | SELECT_SYMBOL_AVX512(features,intersector); |
134 | |
135 | #define SELECT_SYMBOL_DEFAULT_SSE42_AVX_AVX2_AVX512(features,intersector) \ |
136 | SELECT_SYMBOL_DEFAULT(features,intersector); \ |
137 | SELECT_SYMBOL_SSE42(features,intersector); \ |
138 | SELECT_SYMBOL_AVX(features,intersector); \ |
139 | SELECT_SYMBOL_AVX2(features,intersector); \ |
140 | SELECT_SYMBOL_AVX512(features,intersector); |
141 | |
142 | #define SELECT_SYMBOL_DEFAULT_AVX(features,intersector) \ |
143 | SELECT_SYMBOL_DEFAULT(features,intersector); \ |
144 | SELECT_SYMBOL_AVX(features,intersector); |
145 | |
146 | #define SELECT_SYMBOL_DEFAULT_AVX_AVX2(features,intersector) \ |
147 | SELECT_SYMBOL_DEFAULT(features,intersector); \ |
148 | SELECT_SYMBOL_AVX(features,intersector); \ |
149 | SELECT_SYMBOL_AVX2(features,intersector); |
150 | |
151 | #define SELECT_SYMBOL_DEFAULT_AVX(features,intersector) \ |
152 | SELECT_SYMBOL_DEFAULT(features,intersector); \ |
153 | SELECT_SYMBOL_AVX(features,intersector); |
154 | |
155 | #define SELECT_SYMBOL_DEFAULT_AVX_AVX512(features,intersector) \ |
156 | SELECT_SYMBOL_DEFAULT(features,intersector); \ |
157 | SELECT_SYMBOL_AVX(features,intersector); \ |
158 | SELECT_SYMBOL_AVX512(features,intersector); |
159 | |
160 | #define SELECT_SYMBOL_DEFAULT_AVX_AVX512(features,intersector) \ |
161 | SELECT_SYMBOL_DEFAULT(features,intersector); \ |
162 | SELECT_SYMBOL_AVX(features,intersector); \ |
163 | SELECT_SYMBOL_AVX512(features,intersector); |
164 | |
165 | #define SELECT_SYMBOL_INIT_AVX(features,intersector) \ |
166 | INIT_SYMBOL(features,intersector); \ |
167 | SELECT_SYMBOL_AVX(features,intersector); |
168 | |
169 | #define SELECT_SYMBOL_INIT_AVX_AVX2(features,intersector) \ |
170 | INIT_SYMBOL(features,intersector); \ |
171 | SELECT_SYMBOL_AVX(features,intersector); \ |
172 | SELECT_SYMBOL_AVX2(features,intersector); |
173 | |
174 | #define SELECT_SYMBOL_INIT_AVX_AVX2_AVX512(features,intersector) \ |
175 | INIT_SYMBOL(features,intersector); \ |
176 | SELECT_SYMBOL_AVX(features,intersector); \ |
177 | SELECT_SYMBOL_AVX2(features,intersector); \ |
178 | SELECT_SYMBOL_AVX512(features,intersector); |
179 | |
180 | #define SELECT_SYMBOL_INIT_SSE42_AVX_AVX2(features,intersector) \ |
181 | INIT_SYMBOL(features,intersector); \ |
182 | SELECT_SYMBOL_SSE42(features,intersector); \ |
183 | SELECT_SYMBOL_AVX(features,intersector); \ |
184 | SELECT_SYMBOL_AVX2(features,intersector); |
185 | |
186 | #define SELECT_SYMBOL_INIT_AVX(features,intersector) \ |
187 | INIT_SYMBOL(features,intersector); \ |
188 | SELECT_SYMBOL_AVX(features,intersector); |
189 | |
190 | #define SELECT_SYMBOL_INIT_AVX_AVX512(features,intersector) \ |
191 | INIT_SYMBOL(features,intersector); \ |
192 | SELECT_SYMBOL_AVX(features,intersector); \ |
193 | SELECT_SYMBOL_AVX512(features,intersector); |
194 | |
195 | #define SELECT_SYMBOL_INIT_AVX_AVX2(features,intersector) \ |
196 | INIT_SYMBOL(features,intersector); \ |
197 | SELECT_SYMBOL_AVX(features,intersector); \ |
198 | SELECT_SYMBOL_AVX2(features,intersector); |
199 | |
200 | #define SELECT_SYMBOL_INIT_AVX_AVX2_AVX512(features,intersector) \ |
201 | INIT_SYMBOL(features,intersector); \ |
202 | SELECT_SYMBOL_AVX(features,intersector); \ |
203 | SELECT_SYMBOL_AVX2(features,intersector); \ |
204 | SELECT_SYMBOL_AVX512(features,intersector); |
205 | |
206 | #define SELECT_SYMBOL_INIT_SSE42_AVX_AVX2_AVX512(features,intersector) \ |
207 | INIT_SYMBOL(features,intersector); \ |
208 | SELECT_SYMBOL_SSE42(features,intersector); \ |
209 | SELECT_SYMBOL_AVX(features,intersector); \ |
210 | SELECT_SYMBOL_AVX2(features,intersector); \ |
211 | SELECT_SYMBOL_AVX512(features,intersector); |
212 | |
213 | #define SELECT_SYMBOL_ZERO_SSE42_AVX_AVX2_AVX512(features,intersector) \ |
214 | ZERO_SYMBOL(features,intersector); \ |
215 | SELECT_SYMBOL_SSE42(features,intersector); \ |
216 | SELECT_SYMBOL_AVX(features,intersector); \ |
217 | SELECT_SYMBOL_AVX2(features,intersector); \ |
218 | SELECT_SYMBOL_AVX512(features,intersector); |
219 | |
220 | #define SELECT_SYMBOL_DEFAULT_AVX_AVX2_AVX512(features,intersector) \ |
221 | SELECT_SYMBOL_DEFAULT(features,intersector); \ |
222 | SELECT_SYMBOL_AVX(features,intersector); \ |
223 | SELECT_SYMBOL_AVX2(features,intersector); \ |
224 | SELECT_SYMBOL_AVX512(features,intersector); |
225 | |
226 | #define SELECT_SYMBOL_INIT_AVX512(features,intersector) \ |
227 | INIT_SYMBOL(features,intersector); \ |
228 | SELECT_SYMBOL_AVX512(features,intersector); |
229 | |
230 | #define SELECT_SYMBOL_SSE42_AVX_AVX2(features,intersector) \ |
231 | SELECT_SYMBOL_SSE42(features,intersector); \ |
232 | SELECT_SYMBOL_AVX(features,intersector); \ |
233 | SELECT_SYMBOL_AVX2(features,intersector); |
234 | |
235 | struct VerifyMultiTargetLinking { |
236 | static __noinline int getISA(int depth = 5) { |
237 | if (depth == 0) return ISA; |
238 | else return getISA(depth-1); |
239 | } |
240 | }; |
241 | namespace sse2 { int getISA(); }; |
242 | namespace sse42 { int getISA(); }; |
243 | namespace avx { int getISA(); }; |
244 | namespace avx2 { int getISA(); }; |
245 | namespace avx512 { int getISA(); }; |
246 | } |
247 | |