1// Licensed to the .NET Foundation under one or more agreements.
2// The .NET Foundation licenses this file to you under the MIT license.
3// See the LICENSE file in the project root for more information.
4
5#include "jitpch.h"
6#include "hwintrinsic.h"
7
8#ifdef FEATURE_HW_INTRINSICS
9
10static const HWIntrinsicInfo hwIntrinsicInfoArray[] = {
11// clang-format off
12#define HARDWARE_INTRINSIC(id, name, isa, ival, size, numarg, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, category, flag) \
13 {NI_##id, name, InstructionSet_##isa, ival, size, numarg, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, category, static_cast<HWIntrinsicFlag>(flag)},
14// clang-format on
15#include "hwintrinsiclistxarch.h"
16};
17
18//------------------------------------------------------------------------
19// lookup: Gets the HWIntrinsicInfo associated with a given NamedIntrinsic
20//
21// Arguments:
22// id -- The NamedIntrinsic associated with the HWIntrinsic to lookup
23//
24// Return Value:
25// The HWIntrinsicInfo associated with id
26const HWIntrinsicInfo& HWIntrinsicInfo::lookup(NamedIntrinsic id)
27{
28 assert(id != NI_Illegal);
29
30 assert(id > NI_HW_INTRINSIC_START);
31 assert(id < NI_HW_INTRINSIC_END);
32
33 return hwIntrinsicInfoArray[id - NI_HW_INTRINSIC_START - 1];
34}
35
36//------------------------------------------------------------------------
37// lookupId: Gets the NamedIntrinsic for a given method name and InstructionSet
38//
39// Arguments:
40// className -- The name of the class associated with the HWIntrinsic to lookup
41// methodName -- The name of the method associated with the HWIntrinsic to lookup
42// enclosingClassName -- The name of the enclosing class of X64 classes
43//
44// Return Value:
45// The NamedIntrinsic associated with methodName and isa
46NamedIntrinsic HWIntrinsicInfo::lookupId(const char* className, const char* methodName, const char* enclosingClassName)
47{
48 // TODO-Throughput: replace sequential search by binary search
49
50 InstructionSet isa = lookupIsa(className, enclosingClassName);
51 assert(isa != InstructionSet_ILLEGAL);
52
53 assert(methodName != nullptr);
54
55 for (int i = 0; i < (NI_HW_INTRINSIC_END - NI_HW_INTRINSIC_START - 1); i++)
56 {
57 if (isa != hwIntrinsicInfoArray[i].isa)
58 {
59 continue;
60 }
61
62 if (strcmp(methodName, hwIntrinsicInfoArray[i].name) == 0)
63 {
64 return hwIntrinsicInfoArray[i].id;
65 }
66 }
67
68 // There are several helper intrinsics that are implemented in managed code
69 // Those intrinsics will hit this code path and need to return NI_Illegal
70 return NI_Illegal;
71}
72
73//------------------------------------------------------------------------
74// X64VersionOfIsa: Gets the corresponding 64-bit only InstructionSet for a given InstructionSet
75//
76// Arguments:
77// isa -- The InstructionSet ID
78//
79// Return Value:
80// The 64-bit only InstructionSet associated with isa
81static InstructionSet X64VersionOfIsa(InstructionSet isa)
82{
83 switch (isa)
84 {
85 case InstructionSet_SSE:
86 return InstructionSet_SSE_X64;
87 case InstructionSet_SSE2:
88 return InstructionSet_SSE2_X64;
89 case InstructionSet_SSE41:
90 return InstructionSet_SSE41_X64;
91 case InstructionSet_SSE42:
92 return InstructionSet_SSE42_X64;
93 case InstructionSet_BMI1:
94 return InstructionSet_BMI1_X64;
95 case InstructionSet_BMI2:
96 return InstructionSet_BMI2_X64;
97 case InstructionSet_LZCNT:
98 return InstructionSet_LZCNT_X64;
99 case InstructionSet_POPCNT:
100 return InstructionSet_POPCNT_X64;
101 default:
102 unreached();
103 return InstructionSet_ILLEGAL;
104 }
105}
106
107//------------------------------------------------------------------------
108// lookupInstructionSet: Gets the InstructionSet for a given class name
109//
110// Arguments:
111// className -- The name of the class associated with the InstructionSet to lookup
112//
113// Return Value:
114// The InstructionSet associated with className
115static InstructionSet lookupInstructionSet(const char* className)
116{
117 assert(className != nullptr);
118 if (className[0] == 'A')
119 {
120 if (strcmp(className, "Aes") == 0)
121 {
122 return InstructionSet_AES;
123 }
124 if (strcmp(className, "Avx") == 0)
125 {
126 return InstructionSet_AVX;
127 }
128 if (strcmp(className, "Avx2") == 0)
129 {
130 return InstructionSet_AVX2;
131 }
132 }
133 else if (className[0] == 'S')
134 {
135 if (strcmp(className, "Sse") == 0)
136 {
137 return InstructionSet_SSE;
138 }
139 if (strcmp(className, "Sse2") == 0)
140 {
141 return InstructionSet_SSE2;
142 }
143 if (strcmp(className, "Sse3") == 0)
144 {
145 return InstructionSet_SSE3;
146 }
147 if (strcmp(className, "Ssse3") == 0)
148 {
149 return InstructionSet_SSSE3;
150 }
151 if (strcmp(className, "Sse41") == 0)
152 {
153 return InstructionSet_SSE41;
154 }
155 if (strcmp(className, "Sse42") == 0)
156 {
157 return InstructionSet_SSE42;
158 }
159 }
160 else if (className[0] == 'B')
161 {
162 if (strcmp(className, "Bmi1") == 0)
163 {
164 return InstructionSet_BMI1;
165 }
166 if (strcmp(className, "Bmi2") == 0)
167 {
168 return InstructionSet_BMI2;
169 }
170 }
171 else if (className[0] == 'P')
172 {
173 if (strcmp(className, "Pclmulqdq") == 0)
174 {
175 return InstructionSet_PCLMULQDQ;
176 }
177 if (strcmp(className, "Popcnt") == 0)
178 {
179 return InstructionSet_POPCNT;
180 }
181 }
182 else if (strcmp(className, "Fma") == 0)
183 {
184 return InstructionSet_FMA;
185 }
186 else if (strcmp(className, "Lzcnt") == 0)
187 {
188 return InstructionSet_LZCNT;
189 }
190
191 unreached();
192 return InstructionSet_ILLEGAL;
193}
194
195//------------------------------------------------------------------------
196// lookupIsa: Gets the InstructionSet for a given class name and enclsoing class name
197//
198// Arguments:
199// className -- The name of the class associated with the InstructionSet to lookup
200// enclosingClassName -- The name of the enclosing class of X64 classes
201//
202// Return Value:
203// The InstructionSet associated with className and enclosingClassName
204InstructionSet HWIntrinsicInfo::lookupIsa(const char* className, const char* enclosingClassName)
205{
206 assert(className != nullptr);
207
208 if (strcmp(className, "X64") == 0)
209 {
210 assert(enclosingClassName != nullptr);
211 return X64VersionOfIsa(lookupInstructionSet(enclosingClassName));
212 }
213 else
214 {
215 return lookupInstructionSet(className);
216 }
217}
218
219//------------------------------------------------------------------------
220// lookupSimdSize: Gets the SimdSize for a given HWIntrinsic and signature
221//
222// Arguments:
223// id -- The ID associated with the HWIntrinsic to lookup
224// sig -- The signature of the HWIntrinsic to lookup
225//
226// Return Value:
227// The SIMD size for the HWIntrinsic associated with id and sig
228//
229// Remarks:
230// This function is only used by the importer. After importation, we can
231// get the SIMD size from the GenTreeHWIntrinsic node.
232unsigned HWIntrinsicInfo::lookupSimdSize(Compiler* comp, NamedIntrinsic id, CORINFO_SIG_INFO* sig)
233{
234 if (HWIntrinsicInfo::HasFixedSimdSize(id))
235 {
236 return lookupSimdSize(id);
237 }
238
239 CORINFO_CLASS_HANDLE typeHnd = nullptr;
240
241 if (JITtype2varType(sig->retType) == TYP_STRUCT)
242 {
243 typeHnd = sig->retTypeSigClass;
244 }
245 else if (HWIntrinsicInfo::BaseTypeFromFirstArg(id))
246 {
247 typeHnd = comp->info.compCompHnd->getArgClass(sig, sig->args);
248 }
249 else
250 {
251 assert(HWIntrinsicInfo::BaseTypeFromSecondArg(id));
252 CORINFO_ARG_LIST_HANDLE secondArg = comp->info.compCompHnd->getArgNext(sig->args);
253 typeHnd = comp->info.compCompHnd->getArgClass(sig, secondArg);
254 }
255
256 unsigned simdSize = 0;
257 var_types baseType = comp->getBaseTypeAndSizeOfSIMDType(typeHnd, &simdSize);
258 assert((simdSize > 0) && (baseType != TYP_UNKNOWN));
259 return simdSize;
260}
261
262//------------------------------------------------------------------------
263// lookupNumArgs: Gets the number of args for a given HWIntrinsic node
264//
265// Arguments:
266// node -- The HWIntrinsic node to get the number of args for
267//
268// Return Value:
269// The number of args for the HWIntrinsic associated with node
270int HWIntrinsicInfo::lookupNumArgs(const GenTreeHWIntrinsic* node)
271{
272 assert(node != nullptr);
273
274 NamedIntrinsic id = node->gtHWIntrinsicId;
275 int numArgs = lookupNumArgs(id);
276
277 if (numArgs >= 0)
278 {
279 return numArgs;
280 }
281
282 assert(numArgs == -1);
283
284 GenTree* op1 = node->gtGetOp1();
285
286 if (op1 == nullptr)
287 {
288 return 0;
289 }
290
291 if (op1->OperIsList())
292 {
293 GenTreeArgList* list = op1->AsArgList();
294 numArgs = 0;
295
296 do
297 {
298 numArgs++;
299 list = list->Rest();
300 } while (list != nullptr);
301
302 return numArgs;
303 }
304
305 GenTree* op2 = node->gtGetOp2();
306
307 return (op2 == nullptr) ? 1 : 2;
308}
309
310//------------------------------------------------------------------------
311// lookupLastOp: Gets the last operand for a given HWIntrinsic node
312//
313// Arguments:
314// node -- The HWIntrinsic node to get the last operand for
315//
316// Return Value:
317// The last operand for node
318GenTree* HWIntrinsicInfo::lookupLastOp(const GenTreeHWIntrinsic* node)
319{
320 int numArgs = lookupNumArgs(node);
321
322 switch (numArgs)
323 {
324 case 0:
325 {
326 assert(node->gtGetOp1() == nullptr);
327 assert(node->gtGetOp2() == nullptr);
328 return nullptr;
329 }
330
331 case 1:
332 {
333 assert(node->gtGetOp1() != nullptr);
334 assert(!node->gtGetOp1()->OperIsList());
335 assert(node->gtGetOp2() == nullptr);
336
337 return node->gtGetOp1();
338 }
339
340 case 2:
341 {
342 assert(node->gtGetOp1() != nullptr);
343 assert(!node->gtGetOp1()->OperIsList());
344 assert(node->gtGetOp2() != nullptr);
345
346 return node->gtGetOp2();
347 }
348
349 case 3:
350 {
351 assert(node->gtGetOp1() != nullptr);
352 assert(node->gtGetOp1()->OperIsList());
353 assert(node->gtGetOp2() == nullptr);
354 assert(node->gtGetOp1()->AsArgList()->Rest()->Rest()->Current() != nullptr);
355 assert(node->gtGetOp1()->AsArgList()->Rest()->Rest()->Rest() == nullptr);
356
357 return node->gtGetOp1()->AsArgList()->Rest()->Rest()->Current();
358 }
359
360 case 5:
361 {
362 assert(node->gtGetOp1() != nullptr);
363 assert(node->gtGetOp1()->OperIsList());
364 assert(node->gtGetOp2() == nullptr);
365 assert(node->gtGetOp1()->AsArgList()->Rest()->Rest()->Rest()->Rest()->Current() != nullptr);
366 assert(node->gtGetOp1()->AsArgList()->Rest()->Rest()->Rest()->Rest()->Rest() == nullptr);
367
368 return node->gtGetOp1()->AsArgList()->Rest()->Rest()->Rest()->Rest()->Current();
369 }
370
371 default:
372 {
373 unreached();
374 return nullptr;
375 }
376 }
377}
378
379//------------------------------------------------------------------------
380// isImmOp: Gets a value that indicates whether the HWIntrinsic node has an imm operand
381//
382// Arguments:
383// id -- The NamedIntrinsic associated with the HWIntrinsic to lookup
384// op -- The operand to check
385//
386// Return Value:
387// true if the node has an imm operand; otherwise, false
388bool HWIntrinsicInfo::isImmOp(NamedIntrinsic id, const GenTree* op)
389{
390 if (HWIntrinsicInfo::lookupCategory(id) != HW_Category_IMM)
391 {
392 return false;
393 }
394
395 if (!HWIntrinsicInfo::MaybeImm(id))
396 {
397 return true;
398 }
399
400 if (genActualType(op->TypeGet()) != TYP_INT)
401 {
402 return false;
403 }
404
405 return true;
406}
407
408//------------------------------------------------------------------------
409// lookupImmUpperBound: Gets the upper bound for the imm-value of a given NamedIntrinsic
410//
411// Arguments:
412// id -- The NamedIntrinsic associated with the HWIntrinsic to lookup
413//
414// Return Value:
415// The upper bound for the imm-value of the intrinsic associated with id
416//
417int HWIntrinsicInfo::lookupImmUpperBound(NamedIntrinsic id)
418{
419 assert(HWIntrinsicInfo::lookupCategory(id) == HW_Category_IMM);
420
421 switch (id)
422 {
423 case NI_AVX_Compare:
424 case NI_AVX_CompareScalar:
425 {
426 assert(!HWIntrinsicInfo::HasFullRangeImm(id));
427 return 31; // enum FloatComparisonMode has 32 values
428 }
429
430 case NI_AVX2_GatherVector128:
431 case NI_AVX2_GatherVector256:
432 case NI_AVX2_GatherMaskVector128:
433 case NI_AVX2_GatherMaskVector256:
434 return 8;
435
436 default:
437 {
438 assert(HWIntrinsicInfo::HasFullRangeImm(id));
439 return 255;
440 }
441 }
442}
443
444//------------------------------------------------------------------------
445// isInImmRange: Check if ival is valid for the intrinsic
446//
447// Arguments:
448// id -- The NamedIntrinsic associated with the HWIntrinsic to lookup
449// ival -- the imm value to be checked
450//
451// Return Value:
452// true if ival is valid for the intrinsic
453//
454bool HWIntrinsicInfo::isInImmRange(NamedIntrinsic id, int ival)
455{
456 assert(HWIntrinsicInfo::lookupCategory(id) == HW_Category_IMM);
457
458 if (isAVX2GatherIntrinsic(id))
459 {
460 return ival == 1 || ival == 2 || ival == 4 || ival == 8;
461 }
462 else
463 {
464 return ival <= lookupImmUpperBound(id) && ival >= 0;
465 }
466}
467
468//------------------------------------------------------------------------
469// isAVX2GatherIntrinsic: Check if the intrinsic is AVX Gather*
470//
471// Arguments:
472// id -- The NamedIntrinsic associated with the HWIntrinsic to lookup
473//
474// Return Value:
475// true if id is AVX Gather* intrinsic
476//
477bool HWIntrinsicInfo::isAVX2GatherIntrinsic(NamedIntrinsic id)
478{
479 switch (id)
480 {
481 case NI_AVX2_GatherVector128:
482 case NI_AVX2_GatherVector256:
483 case NI_AVX2_GatherMaskVector128:
484 case NI_AVX2_GatherMaskVector256:
485 return true;
486 default:
487 return false;
488 }
489}
490
491//------------------------------------------------------------------------
492// isFullyImplementedIsa: Gets a value that indicates whether the InstructionSet is fully implemented
493//
494// Arguments:
495// isa - The InstructionSet to check
496//
497// Return Value:
498// true if isa is supported; otherwise, false
499bool HWIntrinsicInfo::isFullyImplementedIsa(InstructionSet isa)
500{
501 switch (isa)
502 {
503 // These ISAs are fully implemented
504 case InstructionSet_AES:
505 case InstructionSet_AVX:
506 case InstructionSet_AVX2:
507 case InstructionSet_BMI1:
508 case InstructionSet_BMI2:
509 case InstructionSet_BMI1_X64:
510 case InstructionSet_BMI2_X64:
511 case InstructionSet_FMA:
512 case InstructionSet_LZCNT:
513 case InstructionSet_LZCNT_X64:
514 case InstructionSet_PCLMULQDQ:
515 case InstructionSet_POPCNT:
516 case InstructionSet_POPCNT_X64:
517 case InstructionSet_SSE:
518 case InstructionSet_SSE_X64:
519 case InstructionSet_SSE2:
520 case InstructionSet_SSE2_X64:
521 case InstructionSet_SSE3:
522 case InstructionSet_SSSE3:
523 case InstructionSet_SSE41:
524 case InstructionSet_SSE41_X64:
525 case InstructionSet_SSE42:
526 case InstructionSet_SSE42_X64:
527 {
528 return true;
529 }
530
531 default:
532 {
533 unreached();
534 }
535 }
536}
537
538//------------------------------------------------------------------------
539// isScalarIsa: Gets a value that indicates whether the InstructionSet is scalar
540//
541// Arguments:
542// isa - The InstructionSet to check
543//
544// Return Value:
545// true if isa is scalar; otherwise, false
546bool HWIntrinsicInfo::isScalarIsa(InstructionSet isa)
547{
548 switch (isa)
549 {
550 case InstructionSet_BMI1:
551 case InstructionSet_BMI2:
552 case InstructionSet_BMI1_X64:
553 case InstructionSet_BMI2_X64:
554 case InstructionSet_LZCNT:
555 case InstructionSet_LZCNT_X64:
556 case InstructionSet_POPCNT:
557 case InstructionSet_POPCNT_X64:
558 {
559 return true;
560 }
561
562 default:
563 {
564 return false;
565 }
566 }
567}
568
569//------------------------------------------------------------------------
570// getArgForHWIntrinsic: get the argument from the stack and match the signature
571//
572// Arguments:
573// argType -- the required type of argument
574// argClass -- the class handle of argType
575//
576// Return Value:
577// get the argument at the given index from the stack and match the signature
578//
579GenTree* Compiler::getArgForHWIntrinsic(var_types argType, CORINFO_CLASS_HANDLE argClass)
580{
581 GenTree* arg = nullptr;
582 if (argType == TYP_STRUCT)
583 {
584 unsigned int argSizeBytes;
585 var_types base = getBaseTypeAndSizeOfSIMDType(argClass, &argSizeBytes);
586 argType = getSIMDTypeForSize(argSizeBytes);
587 assert((argType == TYP_SIMD32) || (argType == TYP_SIMD16));
588 arg = impSIMDPopStack(argType);
589 assert((arg->TypeGet() == TYP_SIMD16) || (arg->TypeGet() == TYP_SIMD32));
590 }
591 else
592 {
593 assert(varTypeIsArithmetic(argType));
594 arg = impPopStack().val;
595 assert(varTypeIsArithmetic(arg->TypeGet()));
596 assert(genActualType(arg->gtType) == genActualType(argType));
597 }
598 return arg;
599}
600
601//------------------------------------------------------------------------
602// impNonConstFallback: convert certain SSE2/AVX2 shift intrinsic to its semantic alternative when the imm-arg is
603// not a compile-time constant
604//
605// Arguments:
606// intrinsic -- intrinsic ID
607// simdType -- Vector type
608// baseType -- base type of the Vector128/256<T>
609//
610// Return Value:
611// return the IR of semantic alternative on non-const imm-arg
612//
613GenTree* Compiler::impNonConstFallback(NamedIntrinsic intrinsic, var_types simdType, var_types baseType)
614{
615 assert(HWIntrinsicInfo::NoJmpTableImm(intrinsic));
616 switch (intrinsic)
617 {
618 case NI_SSE2_ShiftLeftLogical:
619 case NI_SSE2_ShiftRightArithmetic:
620 case NI_SSE2_ShiftRightLogical:
621 case NI_AVX2_ShiftLeftLogical:
622 case NI_AVX2_ShiftRightArithmetic:
623 case NI_AVX2_ShiftRightLogical:
624 {
625 GenTree* op2 = impPopStack().val;
626 GenTree* op1 = impSIMDPopStack(simdType);
627 GenTree* tmpOp =
628 gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, NI_SSE2_ConvertScalarToVector128Int32, TYP_INT, 16);
629 return gtNewSimdHWIntrinsicNode(simdType, op1, tmpOp, intrinsic, baseType, genTypeSize(simdType));
630 }
631
632 default:
633 unreached();
634 return nullptr;
635 }
636}
637
638//------------------------------------------------------------------------
639// addRangeCheckIfNeeded: add a GT_HW_INTRINSIC_CHK node for non-full-range imm-intrinsic
640//
641// Arguments:
642// intrinsic -- intrinsic ID
643// lastOp -- the last operand of the intrinsic that points to the imm-arg
644// mustExpand -- true if the compiler is compiling the fallback(GT_CALL) of this intrinsics
645//
646// Return Value:
647// add a GT_HW_INTRINSIC_CHK node for non-full-range imm-intrinsic, which would throw ArgumentOutOfRangeException
648// when the imm-argument is not in the valid range
649//
650GenTree* Compiler::addRangeCheckIfNeeded(NamedIntrinsic intrinsic, GenTree* lastOp, bool mustExpand)
651{
652 assert(lastOp != nullptr);
653 // Full-range imm-intrinsics do not need the range-check
654 // because the imm-parameter of the intrinsic method is a byte.
655 // AVX2 Gather intrinsics no not need the range-check
656 // because their imm-parameter have discrete valid values that are handle by managed code
657 if (mustExpand && !HWIntrinsicInfo::HasFullRangeImm(intrinsic) && HWIntrinsicInfo::isImmOp(intrinsic, lastOp) &&
658 !HWIntrinsicInfo::isAVX2GatherIntrinsic(intrinsic))
659 {
660 assert(!lastOp->IsCnsIntOrI());
661 GenTree* upperBoundNode =
662 new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, HWIntrinsicInfo::lookupImmUpperBound(intrinsic));
663 GenTree* index = nullptr;
664 if ((lastOp->gtFlags & GTF_SIDE_EFFECT) != 0)
665 {
666 index = fgInsertCommaFormTemp(&lastOp);
667 }
668 else
669 {
670 index = gtCloneExpr(lastOp);
671 }
672 GenTreeBoundsChk* hwIntrinsicChk = new (this, GT_HW_INTRINSIC_CHK)
673 GenTreeBoundsChk(GT_HW_INTRINSIC_CHK, TYP_VOID, index, upperBoundNode, SCK_RNGCHK_FAIL);
674 hwIntrinsicChk->gtThrowKind = SCK_ARG_RNG_EXCPN;
675 return gtNewOperNode(GT_COMMA, lastOp->TypeGet(), hwIntrinsicChk, lastOp);
676 }
677 else
678 {
679 return lastOp;
680 }
681}
682
683//------------------------------------------------------------------------
684// compSupportsHWIntrinsic: compiler support of hardware intrinsics
685//
686// Arguments:
687// isa - Instruction set
688// Return Value:
689// true if
690// - isa is a scalar ISA
691// - isa is a SIMD ISA and featureSIMD=true
692// - isa is fully implemented or EnableIncompleteISAClass=true
693bool Compiler::compSupportsHWIntrinsic(InstructionSet isa)
694{
695 return (featureSIMD || HWIntrinsicInfo::isScalarIsa(isa)) && (
696#ifdef DEBUG
697 JitConfig.EnableIncompleteISAClass() ||
698#endif
699 HWIntrinsicInfo::isFullyImplementedIsa(isa));
700}
701
702//------------------------------------------------------------------------
703// impIsTableDrivenHWIntrinsic:
704//
705// Arguments:
706// category - category of a HW intrinsic
707//
708// Return Value:
709// returns true if this category can be table-driven in the importer
710//
711static bool impIsTableDrivenHWIntrinsic(NamedIntrinsic intrinsicId, HWIntrinsicCategory category)
712{
713 // HW_Flag_NoCodeGen implies this intrinsic should be manually morphed in the importer.
714 return (category != HW_Category_Special) && (category != HW_Category_Scalar) &&
715 HWIntrinsicInfo::RequiresCodegen(intrinsicId) && !HWIntrinsicInfo::HasSpecialImport(intrinsicId);
716}
717
718//------------------------------------------------------------------------
719// impHWIntrinsic: dispatch hardware intrinsics to their own implementation
720//
721// Arguments:
722// intrinsic -- id of the intrinsic function.
723// method -- method handle of the intrinsic function.
724// sig -- signature of the intrinsic call
725//
726// Return Value:
727// the expanded intrinsic.
728//
729GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic,
730 CORINFO_METHOD_HANDLE method,
731 CORINFO_SIG_INFO* sig,
732 bool mustExpand)
733{
734 InstructionSet isa = HWIntrinsicInfo::lookupIsa(intrinsic);
735 HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(intrinsic);
736 int numArgs = sig->numArgs;
737 var_types retType = JITtype2varType(sig->retType);
738 var_types baseType = TYP_UNKNOWN;
739
740 if ((retType == TYP_STRUCT) && featureSIMD)
741 {
742 unsigned int sizeBytes;
743 baseType = getBaseTypeAndSizeOfSIMDType(sig->retTypeSigClass, &sizeBytes);
744 retType = getSIMDTypeForSize(sizeBytes);
745 assert(sizeBytes != 0);
746 }
747
748 // This intrinsic is supported if
749 // - the ISA is available on the underlying hardware (compSupports returns true)
750 // - the compiler supports this hardware intrinsics (compSupportsHWIntrinsic returns true)
751 bool issupported = compSupports(isa) && compSupportsHWIntrinsic(isa);
752
753 if (category == HW_Category_IsSupportedProperty)
754 {
755 return gtNewIconNode(issupported);
756 }
757 // - calling to unsupported intrinsics must throw PlatforNotSupportedException
758 else if (!issupported)
759 {
760 return impUnsupportedHWIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand);
761 }
762 // Avoid checking stacktop for 0-op intrinsics
763 if (sig->numArgs > 0 && HWIntrinsicInfo::isImmOp(intrinsic, impStackTop().val))
764 {
765 GenTree* lastOp = impStackTop().val;
766 // The imm-HWintrinsics that do not accept all imm8 values may throw
767 // ArgumentOutOfRangeException when the imm argument is not in the valid range
768 if (!HWIntrinsicInfo::HasFullRangeImm(intrinsic))
769 {
770 if (!mustExpand && lastOp->IsCnsIntOrI() &&
771 !HWIntrinsicInfo::isInImmRange(intrinsic, (int)lastOp->AsIntCon()->IconValue()))
772 {
773 return nullptr;
774 }
775 }
776
777 if (!lastOp->IsCnsIntOrI())
778 {
779 if (HWIntrinsicInfo::NoJmpTableImm(intrinsic))
780 {
781 return impNonConstFallback(intrinsic, retType, baseType);
782 }
783
784 if (!mustExpand)
785 {
786 // When the imm-argument is not a constant and we are not being forced to expand, we need to
787 // return nullptr so a GT_CALL to the intrinsic method is emitted instead. The
788 // intrinsic method is recursive and will be forced to expand, at which point
789 // we emit some less efficient fallback code.
790 return nullptr;
791 }
792 }
793 }
794
795 bool isTableDriven = impIsTableDrivenHWIntrinsic(intrinsic, category);
796
797 if (isTableDriven && ((category == HW_Category_MemoryStore) || HWIntrinsicInfo::BaseTypeFromFirstArg(intrinsic) ||
798 HWIntrinsicInfo::BaseTypeFromSecondArg(intrinsic)))
799 {
800 if (HWIntrinsicInfo::BaseTypeFromFirstArg(intrinsic))
801 {
802 baseType = getBaseTypeOfSIMDType(info.compCompHnd->getArgClass(sig, sig->args));
803 }
804 else
805 {
806 assert((category == HW_Category_MemoryStore) || HWIntrinsicInfo::BaseTypeFromSecondArg(intrinsic));
807 CORINFO_ARG_LIST_HANDLE secondArg = info.compCompHnd->getArgNext(sig->args);
808 CORINFO_CLASS_HANDLE secondArgClass = info.compCompHnd->getArgClass(sig, secondArg);
809 baseType = getBaseTypeOfSIMDType(secondArgClass);
810
811 if (baseType == TYP_UNKNOWN) // the second argument is not a vector
812 {
813 baseType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, secondArg, &secondArgClass)));
814 }
815 }
816 }
817
818 if (HWIntrinsicInfo::IsFloatingPointUsed(intrinsic))
819 {
820 // Set `compFloatingPointUsed` to cover the scenario where an intrinsic is being on SIMD fields, but
821 // where no SIMD local vars are in use. This is the same logic as is used for FEATURE_SIMD.
822 compFloatingPointUsed = true;
823 }
824
825 // table-driven importer of simple intrinsics
826 if (isTableDriven)
827 {
828 unsigned simdSize = HWIntrinsicInfo::lookupSimdSize(this, intrinsic, sig);
829 CORINFO_ARG_LIST_HANDLE argList = sig->args;
830 CORINFO_CLASS_HANDLE argClass;
831 var_types argType = TYP_UNKNOWN;
832
833 assert(numArgs >= 0);
834 assert(HWIntrinsicInfo::lookupIns(intrinsic, baseType) != INS_invalid);
835 assert(simdSize == 32 || simdSize == 16);
836
837 GenTreeHWIntrinsic* retNode = nullptr;
838 GenTree* op1 = nullptr;
839 GenTree* op2 = nullptr;
840
841 switch (numArgs)
842 {
843 case 0:
844 retNode = gtNewSimdHWIntrinsicNode(retType, intrinsic, baseType, simdSize);
845 break;
846 case 1:
847 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, argList, &argClass)));
848 op1 = getArgForHWIntrinsic(argType, argClass);
849 retNode = gtNewSimdHWIntrinsicNode(retType, op1, intrinsic, baseType, simdSize);
850 break;
851 case 2:
852 argType = JITtype2varType(
853 strip(info.compCompHnd->getArgType(sig, info.compCompHnd->getArgNext(argList), &argClass)));
854 op2 = getArgForHWIntrinsic(argType, argClass);
855
856 op2 = addRangeCheckIfNeeded(intrinsic, op2, mustExpand);
857
858 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, argList, &argClass)));
859 op1 = getArgForHWIntrinsic(argType, argClass);
860
861 retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2, intrinsic, baseType, simdSize);
862 break;
863
864 case 3:
865 {
866 CORINFO_ARG_LIST_HANDLE arg2 = info.compCompHnd->getArgNext(argList);
867 CORINFO_ARG_LIST_HANDLE arg3 = info.compCompHnd->getArgNext(arg2);
868
869 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg3, &argClass)));
870 GenTree* op3 = getArgForHWIntrinsic(argType, argClass);
871
872 op3 = addRangeCheckIfNeeded(intrinsic, op3, mustExpand);
873
874 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg2, &argClass)));
875 op2 = getArgForHWIntrinsic(argType, argClass);
876 var_types op2Type;
877 if (intrinsic == NI_AVX2_GatherVector128 || intrinsic == NI_AVX2_GatherVector256)
878 {
879 assert(varTypeIsSIMD(op2->TypeGet()));
880 op2Type = getBaseTypeOfSIMDType(argClass);
881 }
882
883 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, argList, &argClass)));
884 op1 = getArgForHWIntrinsic(argType, argClass);
885
886 retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2, op3, intrinsic, baseType, simdSize);
887
888 if (intrinsic == NI_AVX2_GatherVector128 || intrinsic == NI_AVX2_GatherVector256)
889 {
890 assert(varTypeIsSIMD(op2->TypeGet()));
891 retNode->AsHWIntrinsic()->gtIndexBaseType = op2Type;
892 }
893 break;
894 }
895
896 default:
897 unreached();
898 }
899
900 bool isMemoryStore = retNode->OperIsMemoryStore();
901 if (isMemoryStore || retNode->OperIsMemoryLoad())
902 {
903 if (isMemoryStore)
904 {
905 // A MemoryStore operation is an assignment
906 retNode->gtFlags |= GTF_ASG;
907 }
908
909 // This operation contains an implicit indirection
910 // it could point into the gloabal heap or
911 // it could throw a null reference exception.
912 //
913 retNode->gtFlags |= (GTF_GLOB_REF | GTF_EXCEPT);
914 }
915 return retNode;
916 }
917
918 // other intrinsics need special importation
919 switch (isa)
920 {
921 case InstructionSet_SSE:
922 return impSSEIntrinsic(intrinsic, method, sig, mustExpand);
923 case InstructionSet_SSE2:
924 return impSSE2Intrinsic(intrinsic, method, sig, mustExpand);
925 case InstructionSet_SSE42:
926 case InstructionSet_SSE42_X64:
927 return impSSE42Intrinsic(intrinsic, method, sig, mustExpand);
928 case InstructionSet_AVX:
929 case InstructionSet_AVX2:
930 return impAvxOrAvx2Intrinsic(intrinsic, method, sig, mustExpand);
931
932 case InstructionSet_AES:
933 return impAESIntrinsic(intrinsic, method, sig, mustExpand);
934 case InstructionSet_BMI1:
935 case InstructionSet_BMI1_X64:
936 case InstructionSet_BMI2:
937 case InstructionSet_BMI2_X64:
938 return impBMI1OrBMI2Intrinsic(intrinsic, method, sig, mustExpand);
939
940 case InstructionSet_FMA:
941 return impFMAIntrinsic(intrinsic, method, sig, mustExpand);
942 case InstructionSet_LZCNT:
943 case InstructionSet_LZCNT_X64:
944 return impLZCNTIntrinsic(intrinsic, method, sig, mustExpand);
945 case InstructionSet_PCLMULQDQ:
946 return impPCLMULQDQIntrinsic(intrinsic, method, sig, mustExpand);
947 case InstructionSet_POPCNT:
948 case InstructionSet_POPCNT_X64:
949 return impPOPCNTIntrinsic(intrinsic, method, sig, mustExpand);
950 default:
951 return nullptr;
952 }
953}
954
955GenTree* Compiler::impSSEIntrinsic(NamedIntrinsic intrinsic,
956 CORINFO_METHOD_HANDLE method,
957 CORINFO_SIG_INFO* sig,
958 bool mustExpand)
959{
960 GenTree* retNode = nullptr;
961 GenTree* op1 = nullptr;
962 GenTree* op2 = nullptr;
963 GenTree* op3 = nullptr;
964 GenTree* op4 = nullptr;
965 int simdSize = HWIntrinsicInfo::lookupSimdSize(this, intrinsic, sig);
966
967 // The Prefetch and StoreFence intrinsics don't take any SIMD operands
968 // and have a simdSize of 0
969 assert((simdSize == 16) || (simdSize == 0));
970
971 switch (intrinsic)
972 {
973 case NI_SSE_MoveMask:
974 assert(sig->numArgs == 1);
975 assert(JITtype2varType(sig->retType) == TYP_INT);
976 assert(getBaseTypeOfSIMDType(info.compCompHnd->getArgClass(sig, sig->args)) == TYP_FLOAT);
977 op1 = impSIMDPopStack(TYP_SIMD16);
978 retNode = gtNewSimdHWIntrinsicNode(TYP_INT, op1, intrinsic, TYP_FLOAT, simdSize);
979 break;
980
981 case NI_SSE_Prefetch0:
982 case NI_SSE_Prefetch1:
983 case NI_SSE_Prefetch2:
984 case NI_SSE_PrefetchNonTemporal:
985 {
986 assert(sig->numArgs == 1);
987 assert(JITtype2varType(sig->retType) == TYP_VOID);
988 op1 = impPopStack().val;
989 retNode = gtNewSimdHWIntrinsicNode(TYP_VOID, op1, intrinsic, TYP_UBYTE, 0);
990 break;
991 }
992
993 case NI_SSE_StoreFence:
994 assert(sig->numArgs == 0);
995 assert(JITtype2varType(sig->retType) == TYP_VOID);
996 retNode = gtNewSimdHWIntrinsicNode(TYP_VOID, intrinsic, TYP_VOID, 0);
997 break;
998
999 default:
1000 JITDUMP("Not implemented hardware intrinsic");
1001 break;
1002 }
1003 return retNode;
1004}
1005
1006GenTree* Compiler::impSSE2Intrinsic(NamedIntrinsic intrinsic,
1007 CORINFO_METHOD_HANDLE method,
1008 CORINFO_SIG_INFO* sig,
1009 bool mustExpand)
1010{
1011 GenTree* retNode = nullptr;
1012 GenTree* op1 = nullptr;
1013 GenTree* op2 = nullptr;
1014 int ival = -1;
1015 int simdSize = HWIntrinsicInfo::lookupSimdSize(this, intrinsic, sig);
1016 var_types baseType = TYP_UNKNOWN;
1017 var_types retType = TYP_UNKNOWN;
1018
1019 // The fencing intrinsics don't take any operands and simdSize is 0
1020 assert((simdSize == 16) || (simdSize == 0));
1021
1022 CORINFO_ARG_LIST_HANDLE argList = sig->args;
1023 var_types argType = TYP_UNKNOWN;
1024
1025 switch (intrinsic)
1026 {
1027 case NI_SSE2_CompareLessThan:
1028 {
1029 assert(sig->numArgs == 2);
1030 op2 = impSIMDPopStack(TYP_SIMD16);
1031 op1 = impSIMDPopStack(TYP_SIMD16);
1032 baseType = getBaseTypeOfSIMDType(sig->retTypeSigClass);
1033 if (baseType == TYP_DOUBLE)
1034 {
1035 retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, intrinsic, baseType, simdSize);
1036 }
1037 else
1038 {
1039 retNode =
1040 gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, op1, NI_SSE2_CompareGreaterThan, baseType, simdSize);
1041 }
1042 break;
1043 }
1044
1045 case NI_SSE2_LoadFence:
1046 case NI_SSE2_MemoryFence:
1047 {
1048 assert(sig->numArgs == 0);
1049 assert(JITtype2varType(sig->retType) == TYP_VOID);
1050 assert(simdSize == 0);
1051
1052 retNode = gtNewSimdHWIntrinsicNode(TYP_VOID, intrinsic, TYP_VOID, simdSize);
1053 break;
1054 }
1055
1056 case NI_SSE2_MoveMask:
1057 {
1058 assert(sig->numArgs == 1);
1059 retType = JITtype2varType(sig->retType);
1060 assert(retType == TYP_INT);
1061 op1 = impSIMDPopStack(TYP_SIMD16);
1062 baseType = getBaseTypeOfSIMDType(info.compCompHnd->getArgClass(sig, sig->args));
1063 retNode = gtNewSimdHWIntrinsicNode(retType, op1, intrinsic, baseType, simdSize);
1064 break;
1065 }
1066
1067 case NI_SSE2_StoreNonTemporal:
1068 {
1069 assert(sig->numArgs == 2);
1070 assert(JITtype2varType(sig->retType) == TYP_VOID);
1071 op2 = impPopStack().val;
1072 op1 = impPopStack().val;
1073 retNode = gtNewSimdHWIntrinsicNode(TYP_VOID, op1, op2, NI_SSE2_StoreNonTemporal, op2->TypeGet(), 0);
1074 break;
1075 }
1076
1077 default:
1078 JITDUMP("Not implemented hardware intrinsic");
1079 break;
1080 }
1081 return retNode;
1082}
1083
1084GenTree* Compiler::impSSE42Intrinsic(NamedIntrinsic intrinsic,
1085 CORINFO_METHOD_HANDLE method,
1086 CORINFO_SIG_INFO* sig,
1087 bool mustExpand)
1088{
1089 GenTree* retNode = nullptr;
1090 GenTree* op1 = nullptr;
1091 GenTree* op2 = nullptr;
1092 var_types callType = JITtype2varType(sig->retType);
1093
1094 CORINFO_ARG_LIST_HANDLE argList = sig->args;
1095 CORINFO_CLASS_HANDLE argClass;
1096 CorInfoType corType;
1097 switch (intrinsic)
1098 {
1099 case NI_SSE42_Crc32:
1100 case NI_SSE42_X64_Crc32:
1101 assert(sig->numArgs == 2);
1102 op2 = impPopStack().val;
1103 op1 = impPopStack().val;
1104 argList = info.compCompHnd->getArgNext(argList); // the second argument
1105 corType = strip(info.compCompHnd->getArgType(sig, argList, &argClass)); // type of the second argument
1106
1107 retNode = gtNewScalarHWIntrinsicNode(callType, op1, op2, intrinsic);
1108
1109 // TODO - currently we use the BaseType to bring the type of the second argument
1110 // to the code generator. May encode the overload info in other way.
1111 retNode->gtHWIntrinsic.gtSIMDBaseType = JITtype2varType(corType);
1112 break;
1113
1114 default:
1115 JITDUMP("Not implemented hardware intrinsic");
1116 break;
1117 }
1118 return retNode;
1119}
1120
1121GenTree* Compiler::impAvxOrAvx2Intrinsic(NamedIntrinsic intrinsic,
1122 CORINFO_METHOD_HANDLE method,
1123 CORINFO_SIG_INFO* sig,
1124 bool mustExpand)
1125{
1126 GenTree* retNode = nullptr;
1127 GenTree* op1 = nullptr;
1128 GenTree* op2 = nullptr;
1129 var_types baseType = TYP_UNKNOWN;
1130 int simdSize = HWIntrinsicInfo::lookupSimdSize(this, intrinsic, sig);
1131
1132 switch (intrinsic)
1133 {
1134 case NI_AVX_ExtractVector128:
1135 case NI_AVX2_ExtractVector128:
1136 {
1137 GenTree* lastOp = impPopStack().val;
1138 assert(lastOp->IsCnsIntOrI() || mustExpand);
1139 GenTree* vectorOp = impSIMDPopStack(TYP_SIMD32);
1140 if (sig->numArgs == 2)
1141 {
1142 baseType = getBaseTypeOfSIMDType(sig->retTypeSigClass);
1143 retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, vectorOp, lastOp, intrinsic, baseType, 32);
1144 }
1145 else
1146 {
1147 assert(sig->numArgs == 3);
1148 op1 = impPopStack().val;
1149 CORINFO_ARG_LIST_HANDLE secondArg = info.compCompHnd->getArgNext(sig->args);
1150 CORINFO_CLASS_HANDLE secondArgClass = info.compCompHnd->getArgClass(sig, secondArg);
1151 baseType = getBaseTypeOfSIMDType(secondArgClass);
1152 retNode = gtNewSimdHWIntrinsicNode(TYP_VOID, op1, vectorOp, lastOp, intrinsic, baseType, 32);
1153 }
1154 break;
1155 }
1156
1157 case NI_AVX2_PermuteVar8x32:
1158 {
1159 baseType = getBaseTypeOfSIMDType(sig->retTypeSigClass);
1160 // swap the two operands
1161 GenTree* indexVector = impSIMDPopStack(TYP_SIMD32);
1162 GenTree* sourceVector = impSIMDPopStack(TYP_SIMD32);
1163 retNode =
1164 gtNewSimdHWIntrinsicNode(TYP_SIMD32, indexVector, sourceVector, NI_AVX2_PermuteVar8x32, baseType, 32);
1165 break;
1166 }
1167
1168 case NI_AVX2_GatherMaskVector128:
1169 case NI_AVX2_GatherMaskVector256:
1170 {
1171 CORINFO_ARG_LIST_HANDLE argList = sig->args;
1172 CORINFO_CLASS_HANDLE argClass;
1173 var_types argType = TYP_UNKNOWN;
1174 unsigned int sizeBytes;
1175 baseType = getBaseTypeAndSizeOfSIMDType(sig->retTypeSigClass, &sizeBytes);
1176 var_types retType = getSIMDTypeForSize(sizeBytes);
1177
1178 assert(sig->numArgs == 5);
1179 CORINFO_ARG_LIST_HANDLE arg2 = info.compCompHnd->getArgNext(argList);
1180 CORINFO_ARG_LIST_HANDLE arg3 = info.compCompHnd->getArgNext(arg2);
1181 CORINFO_ARG_LIST_HANDLE arg4 = info.compCompHnd->getArgNext(arg3);
1182 CORINFO_ARG_LIST_HANDLE arg5 = info.compCompHnd->getArgNext(arg4);
1183
1184 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg5, &argClass)));
1185 GenTree* op5 = getArgForHWIntrinsic(argType, argClass);
1186 SetOpLclRelatedToSIMDIntrinsic(op5);
1187
1188 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg4, &argClass)));
1189 GenTree* op4 = getArgForHWIntrinsic(argType, argClass);
1190 SetOpLclRelatedToSIMDIntrinsic(op4);
1191
1192 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg3, &argClass)));
1193 var_types indexbaseType = getBaseTypeOfSIMDType(argClass);
1194 GenTree* op3 = getArgForHWIntrinsic(argType, argClass);
1195 SetOpLclRelatedToSIMDIntrinsic(op3);
1196
1197 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg2, &argClass)));
1198 op2 = getArgForHWIntrinsic(argType, argClass);
1199 SetOpLclRelatedToSIMDIntrinsic(op2);
1200
1201 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, argList, &argClass)));
1202 op1 = getArgForHWIntrinsic(argType, argClass);
1203 SetOpLclRelatedToSIMDIntrinsic(op1);
1204
1205 GenTree* opList = new (this, GT_LIST) GenTreeArgList(op1, gtNewArgList(op2, op3, op4, op5));
1206 retNode = new (this, GT_HWIntrinsic) GenTreeHWIntrinsic(retType, opList, intrinsic, baseType, simdSize);
1207 retNode->AsHWIntrinsic()->gtIndexBaseType = indexbaseType;
1208 break;
1209 }
1210
1211 default:
1212 JITDUMP("Not implemented hardware intrinsic");
1213 break;
1214 }
1215 return retNode;
1216}
1217
1218GenTree* Compiler::impAESIntrinsic(NamedIntrinsic intrinsic,
1219 CORINFO_METHOD_HANDLE method,
1220 CORINFO_SIG_INFO* sig,
1221 bool mustExpand)
1222{
1223 return nullptr;
1224}
1225
1226GenTree* Compiler::impBMI1OrBMI2Intrinsic(NamedIntrinsic intrinsic,
1227 CORINFO_METHOD_HANDLE method,
1228 CORINFO_SIG_INFO* sig,
1229 bool mustExpand)
1230{
1231 var_types callType = JITtype2varType(sig->retType);
1232
1233 switch (intrinsic)
1234 {
1235 case NI_BMI1_AndNot:
1236 case NI_BMI1_X64_AndNot:
1237 case NI_BMI2_ParallelBitDeposit:
1238 case NI_BMI2_ParallelBitExtract:
1239 case NI_BMI2_X64_ParallelBitDeposit:
1240 case NI_BMI2_X64_ParallelBitExtract:
1241 {
1242 assert(sig->numArgs == 2);
1243
1244 GenTree* op2 = impPopStack().val;
1245 GenTree* op1 = impPopStack().val;
1246
1247 return gtNewScalarHWIntrinsicNode(callType, op1, op2, intrinsic);
1248 }
1249
1250 case NI_BMI2_ZeroHighBits:
1251 case NI_BMI2_X64_ZeroHighBits:
1252 {
1253 assert(sig->numArgs == 2);
1254
1255 GenTree* op2 = impPopStack().val;
1256 GenTree* op1 = impPopStack().val;
1257 // Instruction BZHI requires to encode op2 (3rd register) in VEX.vvvv and op1 maybe memory operand,
1258 // so swap op1 and op2 to unify the backend code.
1259 return gtNewScalarHWIntrinsicNode(callType, op2, op1, intrinsic);
1260 }
1261
1262 case NI_BMI1_ExtractLowestSetBit:
1263 case NI_BMI1_GetMaskUpToLowestSetBit:
1264 case NI_BMI1_ResetLowestSetBit:
1265 case NI_BMI1_TrailingZeroCount:
1266 case NI_BMI1_X64_ExtractLowestSetBit:
1267 case NI_BMI1_X64_GetMaskUpToLowestSetBit:
1268 case NI_BMI1_X64_ResetLowestSetBit:
1269 case NI_BMI1_X64_TrailingZeroCount:
1270 {
1271 assert(sig->numArgs == 1);
1272 GenTree* op1 = impPopStack().val;
1273 return gtNewScalarHWIntrinsicNode(callType, op1, intrinsic);
1274 }
1275
1276 case NI_BMI1_BitFieldExtract:
1277 case NI_BMI1_X64_BitFieldExtract:
1278 {
1279 // The 3-arg version is implemented in managed code
1280 if (sig->numArgs == 3)
1281 {
1282 return nullptr;
1283 }
1284 assert(sig->numArgs == 2);
1285
1286 GenTree* op2 = impPopStack().val;
1287 GenTree* op1 = impPopStack().val;
1288 // Instruction BEXTR requires to encode op2 (3rd register) in VEX.vvvv and op1 maybe memory operand,
1289 // so swap op1 and op2 to unify the backend code.
1290 return gtNewScalarHWIntrinsicNode(callType, op2, op1, intrinsic);
1291 }
1292
1293 case NI_BMI2_MultiplyNoFlags:
1294 case NI_BMI2_X64_MultiplyNoFlags:
1295 {
1296 assert(sig->numArgs == 2 || sig->numArgs == 3);
1297 GenTree* op3 = nullptr;
1298 if (sig->numArgs == 3)
1299 {
1300 op3 = impPopStack().val;
1301 }
1302
1303 GenTree* op2 = impPopStack().val;
1304 GenTree* op1 = impPopStack().val;
1305
1306 if (sig->numArgs == 3)
1307 {
1308 return gtNewScalarHWIntrinsicNode(callType, op1, op2, op3, intrinsic);
1309 }
1310 else
1311 {
1312 return gtNewScalarHWIntrinsicNode(callType, op1, op2, intrinsic);
1313 }
1314 }
1315
1316 default:
1317 {
1318 unreached();
1319 return nullptr;
1320 }
1321 }
1322}
1323
1324GenTree* Compiler::impFMAIntrinsic(NamedIntrinsic intrinsic,
1325 CORINFO_METHOD_HANDLE method,
1326 CORINFO_SIG_INFO* sig,
1327 bool mustExpand)
1328{
1329 return nullptr;
1330}
1331
1332GenTree* Compiler::impLZCNTIntrinsic(NamedIntrinsic intrinsic,
1333 CORINFO_METHOD_HANDLE method,
1334 CORINFO_SIG_INFO* sig,
1335 bool mustExpand)
1336{
1337 assert(sig->numArgs == 1);
1338 var_types callType = JITtype2varType(sig->retType);
1339 return gtNewScalarHWIntrinsicNode(callType, impPopStack().val, intrinsic);
1340}
1341
1342GenTree* Compiler::impPCLMULQDQIntrinsic(NamedIntrinsic intrinsic,
1343 CORINFO_METHOD_HANDLE method,
1344 CORINFO_SIG_INFO* sig,
1345 bool mustExpand)
1346{
1347 return nullptr;
1348}
1349
1350GenTree* Compiler::impPOPCNTIntrinsic(NamedIntrinsic intrinsic,
1351 CORINFO_METHOD_HANDLE method,
1352 CORINFO_SIG_INFO* sig,
1353 bool mustExpand)
1354{
1355 assert(sig->numArgs == 1);
1356 var_types callType = JITtype2varType(sig->retType);
1357 return gtNewScalarHWIntrinsicNode(callType, impPopStack().val, intrinsic);
1358}
1359
1360#endif // FEATURE_HW_INTRINSICS
1361