1// Licensed to the .NET Foundation under one or more agreements.
2// The .NET Foundation licenses this file to you under the MIT license.
3// See the LICENSE file in the project root for more information.
4
5#include "jitpch.h"
6#include "hwintrinsic.h"
7
8#ifdef FEATURE_HW_INTRINSICS
9
10namespace IsaFlag
11{
12enum Flag
13{
14#define HARDWARE_INTRINSIC_CLASS(flag, isa) isa = 1ULL << InstructionSet_##isa,
15#include "hwintrinsiclistArm64.h"
16 None = 0,
17 Base = 1ULL << InstructionSet_Base,
18 EveryISA = ~0ULL
19};
20
21Flag operator|(Flag a, Flag b)
22{
23 return Flag(uint64_t(a) | uint64_t(b));
24}
25
26Flag flag(InstructionSet isa)
27{
28 return Flag(1ULL << isa);
29}
30}
31
32// clang-format off
33static const HWIntrinsicInfo hwIntrinsicInfoArray[] = {
34 // Add lookupHWIntrinsic special cases see lookupHWIntrinsic() below
35 // NI_ARM64_IsSupported_True is used to expand get_IsSupported to const true
36 // NI_ARM64_IsSupported_False is used to expand get_IsSupported to const false
37 // NI_ARM64_PlatformNotSupported to throw PlatformNotSupported exception for every intrinsic not supported on the running platform
38 {NI_ARM64_IsSupported_True, "get_IsSupported", IsaFlag::EveryISA, HWIntrinsicInfo::IsSupported, HWIntrinsicInfo::None, {}},
39 {NI_ARM64_IsSupported_False, "::NI_ARM64_IsSupported_False", IsaFlag::EveryISA, HWIntrinsicInfo::IsSupported, HWIntrinsicInfo::None, {}},
40 {NI_ARM64_PlatformNotSupported, "::NI_ARM64_PlatformNotSupported", IsaFlag::EveryISA, HWIntrinsicInfo::Unsupported, HWIntrinsicInfo::None, {}},
41#define HARDWARE_INTRINSIC(id, isa, name, form, i0, i1, i2, flags) \
42 {id, #name, IsaFlag::isa, HWIntrinsicInfo::form, HWIntrinsicInfo::flags, { i0, i1, i2 }},
43#include "hwintrinsiclistArm64.h"
44};
45// clang-format on
46
47//------------------------------------------------------------------------
48// lookup: Gets the HWIntrinsicInfo associated with a given NamedIntrinsic
49//
50// Arguments:
51// id -- The NamedIntrinsic associated with the HWIntrinsic to lookup
52//
53// Return Value:
54// The HWIntrinsicInfo associated with id
55const HWIntrinsicInfo& HWIntrinsicInfo::lookup(NamedIntrinsic id)
56{
57 assert(id != NI_Illegal);
58
59 assert(id > NI_HW_INTRINSIC_START);
60 assert(id < NI_HW_INTRINSIC_END);
61
62 return hwIntrinsicInfoArray[id - NI_HW_INTRINSIC_START - 1];
63}
64
65//------------------------------------------------------------------------
66// lookupHWIntrinsicISA: map class name to InstructionSet value
67//
68// Arguments:
69// className -- class name in System.Runtime.Intrinsics.Arm.Arm64
70//
71// Return Value:
72// Id for the ISA class if enabled.
73//
74InstructionSet Compiler::lookupHWIntrinsicISA(const char* className)
75{
76 if (className != nullptr)
77 {
78 if (strcmp(className, "Base") == 0)
79 return InstructionSet_Base;
80#define HARDWARE_INTRINSIC_CLASS(flag, isa) \
81 if (strcmp(className, #isa) == 0) \
82 return InstructionSet_##isa;
83#include "hwintrinsiclistArm64.h"
84 }
85
86 return InstructionSet_NONE;
87}
88
89//------------------------------------------------------------------------
90// lookupHWIntrinsic: map intrinsic name to named intrinsic value
91//
92// Arguments:
93// methodName -- name of the intrinsic function.
94// isa -- instruction set of the intrinsic.
95//
96// Return Value:
97// Id for the hardware intrinsic.
98//
99// TODO-Throughput: replace sequential search by hash lookup
100NamedIntrinsic Compiler::lookupHWIntrinsic(const char* className, const char* methodName)
101{
102 InstructionSet isa = lookupHWIntrinsicISA(className);
103 NamedIntrinsic result = NI_Illegal;
104 if (isa != InstructionSet_NONE)
105 {
106 IsaFlag::Flag isaFlag = IsaFlag::flag(isa);
107 for (int i = 0; i < NI_HW_INTRINSIC_END - NI_HW_INTRINSIC_START; i++)
108 {
109 if ((isaFlag & hwIntrinsicInfoArray[i].isaflags) && strcmp(methodName, hwIntrinsicInfoArray[i].name) == 0)
110 {
111 if (compSupportsHWIntrinsic(isa))
112 {
113 // Intrinsic is supported on platform
114 result = hwIntrinsicInfoArray[i].id;
115 }
116 else
117 {
118 // When the intrinsic class is not supported
119 // Return NI_ARM64_PlatformNotSupported for all intrinsics
120 // Return NI_ARM64_IsSupported_False for the IsSupported property
121 result = (hwIntrinsicInfoArray[i].id != NI_ARM64_IsSupported_True) ? NI_ARM64_PlatformNotSupported
122 : NI_ARM64_IsSupported_False;
123 }
124 break;
125 }
126 }
127 }
128 return result;
129}
130
131//------------------------------------------------------------------------
132// impCheckImmediate: check if immediate is const and in range for inlining
133//
134bool Compiler::impCheckImmediate(GenTree* immediateOp, unsigned int max)
135{
136 return immediateOp->IsCnsIntOrI() && (immediateOp->AsIntConCommon()->IconValue() < max);
137}
138
139//------------------------------------------------------------------------
140// isFullyImplementedIsa: Gets a value that indicates whether the InstructionSet is fully implemented
141//
142// Arguments:
143// isa - The InstructionSet to check
144//
145// Return Value:
146// true if isa is supported; otherwise, false
147//
148// Notes:
149// This currently returns true for all partially-implemented ISAs.
150// TODO-Bug: Set this to return the correct values as GH 20427 is resolved.
151//
152bool HWIntrinsicInfo::isFullyImplementedIsa(InstructionSet isa)
153{
154 switch (isa)
155 {
156 case InstructionSet_Base:
157 case InstructionSet_Crc32:
158 case InstructionSet_Aes:
159 case InstructionSet_Simd:
160 case InstructionSet_Sha1:
161 case InstructionSet_Sha256:
162 return true;
163
164 default:
165 assert(!"Unexpected Arm64 HW intrinsics ISA");
166 return false;
167 }
168}
169
170//------------------------------------------------------------------------
171// isScalarIsa: Gets a value that indicates whether the InstructionSet is scalar
172//
173// Arguments:
174// isa - The InstructionSet to check
175//
176// Return Value:
177// true if isa is scalar; otherwise, false
178bool HWIntrinsicInfo::isScalarIsa(InstructionSet isa)
179{
180 switch (isa)
181 {
182 case InstructionSet_Base:
183 case InstructionSet_Crc32:
184 return true;
185
186 case InstructionSet_Aes:
187 case InstructionSet_Simd:
188 case InstructionSet_Sha1:
189 case InstructionSet_Sha256:
190 return false;
191
192 default:
193 assert(!"Unexpected Arm64 HW intrinsics ISA");
194 return true;
195 }
196}
197
198//------------------------------------------------------------------------
199// compSupportsHWIntrinsic: compiler support of hardware intrinsics
200//
201// Arguments:
202// isa - Instruction set
203// Return Value:
204// true if
205// - isa is a scalar ISA
206// - isa is a SIMD ISA and featureSIMD=true
207// - isa is fully implemented or EnableIncompleteISAClass=true
208bool Compiler::compSupportsHWIntrinsic(InstructionSet isa)
209{
210 return (featureSIMD || HWIntrinsicInfo::isScalarIsa(isa)) && (
211#ifdef DEBUG
212 JitConfig.EnableIncompleteISAClass() ||
213#endif
214 HWIntrinsicInfo::isFullyImplementedIsa(isa));
215}
216
217//------------------------------------------------------------------------
218// lookupNumArgs: gets the number of arguments for the hardware intrinsic.
219// This attempts to do a table based lookup but will fallback to the number
220// of operands in 'node' if the table entry is -1.
221//
222// Arguments:
223// node -- GenTreeHWIntrinsic* node with nullptr default value
224//
225// Return Value:
226// number of arguments
227//
228int HWIntrinsicInfo::lookupNumArgs(const GenTreeHWIntrinsic* node)
229{
230 NamedIntrinsic intrinsic = node->gtHWIntrinsicId;
231
232 assert(intrinsic != NI_Illegal);
233 assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
234
235 GenTree* op1 = node->gtGetOp1();
236 GenTree* op2 = node->gtGetOp2();
237 int numArgs = 0;
238
239 if (op1 == nullptr)
240 {
241 return 0;
242 }
243
244 if (op1->OperIsList())
245 {
246 numArgs = 0;
247 GenTreeArgList* list = op1->AsArgList();
248
249 while (list != nullptr)
250 {
251 numArgs++;
252 list = list->Rest();
253 }
254
255 // We should only use a list if we have 3 operands.
256 assert(numArgs >= 3);
257 return numArgs;
258 }
259
260 if (op2 == nullptr)
261 {
262 return 1;
263 }
264
265 return 2;
266}
267
268//------------------------------------------------------------------------
269// impHWIntrinsic: dispatch hardware intrinsics to their own implementation
270// function
271//
272// Arguments:
273// intrinsic -- id of the intrinsic function.
274// method -- method handle of the intrinsic function.
275// sig -- signature of the intrinsic call
276//
277// Return Value:
278// the expanded intrinsic.
279//
280GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic,
281 CORINFO_METHOD_HANDLE method,
282 CORINFO_SIG_INFO* sig,
283 bool mustExpand)
284{
285 GenTree* retNode = nullptr;
286 GenTree* op1 = nullptr;
287 GenTree* op2 = nullptr;
288 GenTree* op3 = nullptr;
289 CORINFO_CLASS_HANDLE simdClass = nullptr;
290 var_types simdType = TYP_UNKNOWN;
291 var_types simdBaseType = TYP_UNKNOWN;
292 unsigned simdSizeBytes = 0;
293
294 switch (HWIntrinsicInfo::lookup(intrinsic).form)
295 {
296 case HWIntrinsicInfo::SimdBinaryOp:
297 case HWIntrinsicInfo::SimdInsertOp:
298 case HWIntrinsicInfo::SimdSelectOp:
299 case HWIntrinsicInfo::SimdSetAllOp:
300 case HWIntrinsicInfo::SimdUnaryOp:
301 case HWIntrinsicInfo::SimdBinaryRMWOp:
302 case HWIntrinsicInfo::SimdTernaryRMWOp:
303 case HWIntrinsicInfo::Sha1HashOp:
304 simdClass = sig->retTypeClass;
305 break;
306 case HWIntrinsicInfo::SimdExtractOp:
307 info.compCompHnd->getArgType(sig, sig->args, &simdClass);
308 break;
309 default:
310 break;
311 }
312
313 // Simd instantiation type check
314 if (simdClass != nullptr)
315 {
316 if (featureSIMD)
317 {
318 compFloatingPointUsed = true;
319
320 simdBaseType = getBaseTypeAndSizeOfSIMDType(simdClass, &simdSizeBytes);
321 }
322
323 if (simdBaseType == TYP_UNKNOWN)
324 {
325 return impUnsupportedHWIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand);
326 }
327 simdType = getSIMDTypeForSize(simdSizeBytes);
328 }
329
330 switch (HWIntrinsicInfo::lookup(intrinsic).form)
331 {
332 case HWIntrinsicInfo::IsSupported:
333 return gtNewIconNode((intrinsic == NI_ARM64_IsSupported_True) ? 1 : 0);
334
335 case HWIntrinsicInfo::Unsupported:
336 return impUnsupportedHWIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand);
337
338 case HWIntrinsicInfo::UnaryOp:
339 op1 = impPopStack().val;
340
341 return gtNewScalarHWIntrinsicNode(JITtype2varType(sig->retType), op1, intrinsic);
342
343 case HWIntrinsicInfo::SimdBinaryOp:
344 case HWIntrinsicInfo::SimdBinaryRMWOp:
345 // op1 is the first operand
346 // op2 is the second operand
347 op2 = impSIMDPopStack(simdType);
348 op1 = impSIMDPopStack(simdType);
349
350 return gtNewSimdHWIntrinsicNode(simdType, op1, op2, intrinsic, simdBaseType, simdSizeBytes);
351
352 case HWIntrinsicInfo::SimdTernaryRMWOp:
353 case HWIntrinsicInfo::SimdSelectOp:
354 // op1 is the first operand
355 // op2 is the second operand
356 // op3 is the third operand
357 op3 = impSIMDPopStack(simdType);
358 op2 = impSIMDPopStack(simdType);
359 op1 = impSIMDPopStack(simdType);
360
361 return gtNewSimdHWIntrinsicNode(simdType, op1, op2, op3, intrinsic, simdBaseType, simdSizeBytes);
362
363 case HWIntrinsicInfo::SimdSetAllOp:
364 op1 = impPopStack().val;
365
366 return gtNewSimdHWIntrinsicNode(simdType, op1, intrinsic, simdBaseType, simdSizeBytes);
367
368 case HWIntrinsicInfo::SimdUnaryOp:
369 op1 = impSIMDPopStack(simdType);
370
371 return gtNewSimdHWIntrinsicNode(simdType, op1, intrinsic, simdBaseType, simdSizeBytes);
372
373 case HWIntrinsicInfo::SimdExtractOp:
374 if (!mustExpand && !impCheckImmediate(impStackTop(0).val, getSIMDVectorLength(simdSizeBytes, simdBaseType)))
375 {
376 // Immediate lane not constant or out of range
377 return nullptr;
378 }
379 op2 = impPopStack().val;
380 op1 = impSIMDPopStack(simdType);
381
382 return gtNewScalarHWIntrinsicNode(JITtype2varType(sig->retType), op1, op2, intrinsic);
383
384 case HWIntrinsicInfo::SimdInsertOp:
385 if (!mustExpand && !impCheckImmediate(impStackTop(1).val, getSIMDVectorLength(simdSizeBytes, simdBaseType)))
386 {
387 // Immediate lane not constant or out of range
388 return nullptr;
389 }
390 op3 = impPopStack().val;
391 op2 = impPopStack().val;
392 op1 = impSIMDPopStack(simdType);
393
394 return gtNewSimdHWIntrinsicNode(simdType, op1, op2, op3, intrinsic, simdBaseType, simdSizeBytes);
395
396 case HWIntrinsicInfo::Sha1HashOp:
397 op3 = impSIMDPopStack(simdType);
398 op2 = impPopStack().val;
399 op1 = impSIMDPopStack(simdType);
400
401 return gtNewSimdHWIntrinsicNode(simdType, op1, op2, op3, intrinsic, simdBaseType, simdSizeBytes);
402
403 case HWIntrinsicInfo::Sha1RotateOp:
404 assert(sig->numArgs == 1);
405 compFloatingPointUsed = true;
406 return gtNewScalarHWIntrinsicNode(TYP_UINT, impPopStack().val, NI_ARM64_Sha1FixedRotate);
407
408 default:
409 JITDUMP("Not implemented hardware intrinsic form");
410 assert(!"Unimplemented SIMD Intrinsic form");
411
412 break;
413 }
414 return retNode;
415}
416
417#endif // FEATURE_HW_INTRINSICS
418