1// Licensed to the .NET Foundation under one or more agreements.
2// The .NET Foundation licenses this file to you under the MIT license.
3// See the LICENSE file in the project root for more information.
4
5//
6// SIMD Support
7//
8// IMPORTANT NOTES AND CAVEATS:
9//
10// This implementation is preliminary, and may change dramatically.
11//
12// New JIT types, TYP_SIMDxx, are introduced, and the SIMD intrinsics are created as GT_SIMD nodes.
13// Nodes of SIMD types will be typed as TYP_SIMD* (e.g. TYP_SIMD8, TYP_SIMD16, etc.).
14//
15// Note that currently the "reference implementation" is the same as the runtime dll. As such, it is currently
16// providing implementations for those methods not currently supported by the JIT as intrinsics.
17//
18// These are currently recognized using string compares, in order to provide an implementation in the JIT
19// without taking a dependency on the VM.
20// Furthermore, in the CTP, in order to limit the impact of doing these string compares
21// against assembly names, we only look for the SIMDVector assembly if we are compiling a class constructor. This
22// makes it somewhat more "pay for play" but is a significant usability compromise.
23// This has been addressed for RTM by doing the assembly recognition in the VM.
24// --------------------------------------------------------------------------------------
25
26#include "jitpch.h"
27#include "simd.h"
28
29#ifdef _MSC_VER
30#pragma hdrstop
31#endif
32
33#ifdef FEATURE_SIMD
34
35// Intrinsic Id to intrinsic info map
36const SIMDIntrinsicInfo simdIntrinsicInfoArray[] = {
37#define SIMD_INTRINSIC(mname, inst, id, name, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, \
38 t10) \
39 {SIMDIntrinsic##id, mname, inst, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10},
40#include "simdintrinsiclist.h"
41};
42
43//------------------------------------------------------------------------
44// getSIMDVectorLength: Get the length (number of elements of base type) of
45// SIMD Vector given its size and base (element) type.
46//
47// Arguments:
48// simdSize - size of the SIMD vector
49// baseType - type of the elements of the SIMD vector
50//
51// static
52int Compiler::getSIMDVectorLength(unsigned simdSize, var_types baseType)
53{
54 return simdSize / genTypeSize(baseType);
55}
56
57//------------------------------------------------------------------------
58// Get the length (number of elements of base type) of SIMD Vector given by typeHnd.
59//
60// Arguments:
61// typeHnd - type handle of the SIMD vector
62//
63int Compiler::getSIMDVectorLength(CORINFO_CLASS_HANDLE typeHnd)
64{
65 unsigned sizeBytes = 0;
66 var_types baseType = getBaseTypeAndSizeOfSIMDType(typeHnd, &sizeBytes);
67 return getSIMDVectorLength(sizeBytes, baseType);
68}
69
70//------------------------------------------------------------------------
71// Get the preferred alignment of SIMD vector type for better performance.
72//
73// Arguments:
74// typeHnd - type handle of the SIMD vector
75//
76int Compiler::getSIMDTypeAlignment(var_types simdType)
77{
78#ifdef _TARGET_XARCH_
79 // Fixed length vectors have the following alignment preference
80 // Vector2 = 8 byte alignment
81 // Vector3/4 = 16-byte alignment
82 unsigned size = genTypeSize(simdType);
83
84 // preferred alignment for SSE2 128-bit vectors is 16-bytes
85 if (size == 8)
86 {
87 return 8;
88 }
89 else if (size <= 16)
90 {
91 assert((size == 12) || (size == 16));
92 return 16;
93 }
94 else
95 {
96 assert(size == 32);
97 return 32;
98 }
99#elif defined(_TARGET_ARM64_)
100 return 16;
101#else
102 assert(!"getSIMDTypeAlignment() unimplemented on target arch");
103 unreached();
104#endif
105}
106
107//----------------------------------------------------------------------------------
108// Return the base type and size of SIMD vector type given its type handle.
109//
110// Arguments:
111// typeHnd - The handle of the type we're interested in.
112// sizeBytes - out param
113//
114// Return Value:
115// base type of SIMD vector.
116// sizeBytes if non-null is set to size in bytes.
117//
118// TODO-Throughput: current implementation parses class name to find base type. Change
119// this when we implement SIMD intrinsic identification for the final
120// product.
121//
122var_types Compiler::getBaseTypeAndSizeOfSIMDType(CORINFO_CLASS_HANDLE typeHnd, unsigned* sizeBytes /*= nullptr */)
123{
124 assert(featureSIMD);
125
126 if (m_simdHandleCache == nullptr)
127 {
128 if (impInlineInfo == nullptr)
129 {
130 m_simdHandleCache = new (this, CMK_Generic) SIMDHandlesCache();
131 }
132 else
133 {
134 // Steal the inliner compiler's cache (create it if not available).
135
136 if (impInlineInfo->InlineRoot->m_simdHandleCache == nullptr)
137 {
138 impInlineInfo->InlineRoot->m_simdHandleCache = new (this, CMK_Generic) SIMDHandlesCache();
139 }
140
141 m_simdHandleCache = impInlineInfo->InlineRoot->m_simdHandleCache;
142 }
143 }
144
145 if (typeHnd == nullptr)
146 {
147 return TYP_UNKNOWN;
148 }
149
150 // fast path search using cached type handles of important types
151 var_types simdBaseType = TYP_UNKNOWN;
152 unsigned size = 0;
153
154 // TODO - Optimize SIMD type recognition by IntrinsicAttribute
155 if (isSIMDClass(typeHnd))
156 {
157 // The most likely to be used type handles are looked up first followed by
158 // less likely to be used type handles
159 if (typeHnd == m_simdHandleCache->SIMDFloatHandle)
160 {
161 simdBaseType = TYP_FLOAT;
162 JITDUMP(" Known type SIMD Vector<Float>\n");
163 }
164 else if (typeHnd == m_simdHandleCache->SIMDIntHandle)
165 {
166 simdBaseType = TYP_INT;
167 JITDUMP(" Known type SIMD Vector<Int>\n");
168 }
169 else if (typeHnd == m_simdHandleCache->SIMDVector2Handle)
170 {
171 simdBaseType = TYP_FLOAT;
172 size = 2 * genTypeSize(TYP_FLOAT);
173 assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
174 JITDUMP(" Known type Vector2\n");
175 }
176 else if (typeHnd == m_simdHandleCache->SIMDVector3Handle)
177 {
178 simdBaseType = TYP_FLOAT;
179 size = 3 * genTypeSize(TYP_FLOAT);
180 assert(size == info.compCompHnd->getClassSize(typeHnd));
181 JITDUMP(" Known type Vector3\n");
182 }
183 else if (typeHnd == m_simdHandleCache->SIMDVector4Handle)
184 {
185 simdBaseType = TYP_FLOAT;
186 size = 4 * genTypeSize(TYP_FLOAT);
187 assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
188 JITDUMP(" Known type Vector4\n");
189 }
190 else if (typeHnd == m_simdHandleCache->SIMDVectorHandle)
191 {
192 JITDUMP(" Known type Vector\n");
193 }
194 else if (typeHnd == m_simdHandleCache->SIMDUShortHandle)
195 {
196 simdBaseType = TYP_USHORT;
197 JITDUMP(" Known type SIMD Vector<ushort>\n");
198 }
199 else if (typeHnd == m_simdHandleCache->SIMDUByteHandle)
200 {
201 simdBaseType = TYP_UBYTE;
202 JITDUMP(" Known type SIMD Vector<ubyte>\n");
203 }
204 else if (typeHnd == m_simdHandleCache->SIMDDoubleHandle)
205 {
206 simdBaseType = TYP_DOUBLE;
207 JITDUMP(" Known type SIMD Vector<Double>\n");
208 }
209 else if (typeHnd == m_simdHandleCache->SIMDLongHandle)
210 {
211 simdBaseType = TYP_LONG;
212 JITDUMP(" Known type SIMD Vector<Long>\n");
213 }
214 else if (typeHnd == m_simdHandleCache->SIMDShortHandle)
215 {
216 simdBaseType = TYP_SHORT;
217 JITDUMP(" Known type SIMD Vector<short>\n");
218 }
219 else if (typeHnd == m_simdHandleCache->SIMDByteHandle)
220 {
221 simdBaseType = TYP_BYTE;
222 JITDUMP(" Known type SIMD Vector<byte>\n");
223 }
224 else if (typeHnd == m_simdHandleCache->SIMDUIntHandle)
225 {
226 simdBaseType = TYP_UINT;
227 JITDUMP(" Known type SIMD Vector<uint>\n");
228 }
229 else if (typeHnd == m_simdHandleCache->SIMDULongHandle)
230 {
231 simdBaseType = TYP_ULONG;
232 JITDUMP(" Known type SIMD Vector<ulong>\n");
233 }
234
235 // slow path search
236 if (simdBaseType == TYP_UNKNOWN)
237 {
238 // Doesn't match with any of the cached type handles.
239 // Obtain base type by parsing fully qualified class name.
240 //
241 // TODO-Throughput: implement product shipping solution to query base type.
242 WCHAR className[256] = {0};
243 WCHAR* pbuf = &className[0];
244 int len = _countof(className);
245 info.compCompHnd->appendClassName(&pbuf, &len, typeHnd, TRUE, FALSE, FALSE);
246 noway_assert(pbuf < &className[256]);
247 JITDUMP("SIMD Candidate Type %S\n", className);
248
249 if (wcsncmp(className, W("System.Numerics."), 16) == 0)
250 {
251 if (wcsncmp(&(className[16]), W("Vector`1["), 9) == 0)
252 {
253 if (wcsncmp(&(className[25]), W("System.Single"), 13) == 0)
254 {
255 m_simdHandleCache->SIMDFloatHandle = typeHnd;
256 simdBaseType = TYP_FLOAT;
257 JITDUMP(" Found type SIMD Vector<Float>\n");
258 }
259 else if (wcsncmp(&(className[25]), W("System.Int32"), 12) == 0)
260 {
261 m_simdHandleCache->SIMDIntHandle = typeHnd;
262 simdBaseType = TYP_INT;
263 JITDUMP(" Found type SIMD Vector<Int>\n");
264 }
265 else if (wcsncmp(&(className[25]), W("System.UInt16"), 13) == 0)
266 {
267 m_simdHandleCache->SIMDUShortHandle = typeHnd;
268 simdBaseType = TYP_USHORT;
269 JITDUMP(" Found type SIMD Vector<ushort>\n");
270 }
271 else if (wcsncmp(&(className[25]), W("System.Byte"), 11) == 0)
272 {
273 m_simdHandleCache->SIMDUByteHandle = typeHnd;
274 simdBaseType = TYP_UBYTE;
275 JITDUMP(" Found type SIMD Vector<ubyte>\n");
276 }
277 else if (wcsncmp(&(className[25]), W("System.Double"), 13) == 0)
278 {
279 m_simdHandleCache->SIMDDoubleHandle = typeHnd;
280 simdBaseType = TYP_DOUBLE;
281 JITDUMP(" Found type SIMD Vector<Double>\n");
282 }
283 else if (wcsncmp(&(className[25]), W("System.Int64"), 12) == 0)
284 {
285 m_simdHandleCache->SIMDLongHandle = typeHnd;
286 simdBaseType = TYP_LONG;
287 JITDUMP(" Found type SIMD Vector<Long>\n");
288 }
289 else if (wcsncmp(&(className[25]), W("System.Int16"), 12) == 0)
290 {
291 m_simdHandleCache->SIMDShortHandle = typeHnd;
292 simdBaseType = TYP_SHORT;
293 JITDUMP(" Found type SIMD Vector<short>\n");
294 }
295 else if (wcsncmp(&(className[25]), W("System.SByte"), 12) == 0)
296 {
297 m_simdHandleCache->SIMDByteHandle = typeHnd;
298 simdBaseType = TYP_BYTE;
299 JITDUMP(" Found type SIMD Vector<byte>\n");
300 }
301 else if (wcsncmp(&(className[25]), W("System.UInt32"), 13) == 0)
302 {
303 m_simdHandleCache->SIMDUIntHandle = typeHnd;
304 simdBaseType = TYP_UINT;
305 JITDUMP(" Found type SIMD Vector<uint>\n");
306 }
307 else if (wcsncmp(&(className[25]), W("System.UInt64"), 13) == 0)
308 {
309 m_simdHandleCache->SIMDULongHandle = typeHnd;
310 simdBaseType = TYP_ULONG;
311 JITDUMP(" Found type SIMD Vector<ulong>\n");
312 }
313 else
314 {
315 JITDUMP(" Unknown SIMD Vector<T>\n");
316 }
317 }
318 else if (wcsncmp(&(className[16]), W("Vector2"), 8) == 0)
319 {
320 m_simdHandleCache->SIMDVector2Handle = typeHnd;
321
322 simdBaseType = TYP_FLOAT;
323 size = 2 * genTypeSize(TYP_FLOAT);
324 assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
325 JITDUMP(" Found Vector2\n");
326 }
327 else if (wcsncmp(&(className[16]), W("Vector3"), 8) == 0)
328 {
329 m_simdHandleCache->SIMDVector3Handle = typeHnd;
330
331 simdBaseType = TYP_FLOAT;
332 size = 3 * genTypeSize(TYP_FLOAT);
333 assert(size == info.compCompHnd->getClassSize(typeHnd));
334 JITDUMP(" Found Vector3\n");
335 }
336 else if (wcsncmp(&(className[16]), W("Vector4"), 8) == 0)
337 {
338 m_simdHandleCache->SIMDVector4Handle = typeHnd;
339
340 simdBaseType = TYP_FLOAT;
341 size = 4 * genTypeSize(TYP_FLOAT);
342 assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
343 JITDUMP(" Found Vector4\n");
344 }
345 else if (wcsncmp(&(className[16]), W("Vector"), 6) == 0)
346 {
347 m_simdHandleCache->SIMDVectorHandle = typeHnd;
348 JITDUMP(" Found type Vector\n");
349 }
350 else
351 {
352 JITDUMP(" Unknown SIMD Type\n");
353 }
354 }
355 }
356 if (simdBaseType != TYP_UNKNOWN && sizeBytes != nullptr)
357 {
358 // If not a fixed size vector then its size is same as SIMD vector
359 // register length in bytes
360 if (size == 0)
361 {
362 size = getSIMDVectorRegisterByteLength();
363 }
364
365 *sizeBytes = size;
366 setUsesSIMDTypes(true);
367 }
368 }
369#ifdef FEATURE_HW_INTRINSICS
370 else if (isIntrinsicType(typeHnd))
371 {
372 const size_t Vector64SizeBytes = 64 / 8;
373 const size_t Vector128SizeBytes = 128 / 8;
374 const size_t Vector256SizeBytes = 256 / 8;
375
376#if defined(_TARGET_XARCH_)
377 static_assert_no_msg(YMM_REGSIZE_BYTES == Vector256SizeBytes);
378 static_assert_no_msg(XMM_REGSIZE_BYTES == Vector128SizeBytes);
379
380 if (typeHnd == m_simdHandleCache->Vector256FloatHandle)
381 {
382 simdBaseType = TYP_FLOAT;
383 size = Vector256SizeBytes;
384 JITDUMP(" Known type Vector256<float>\n");
385 }
386 else if (typeHnd == m_simdHandleCache->Vector256DoubleHandle)
387 {
388 simdBaseType = TYP_DOUBLE;
389 size = Vector256SizeBytes;
390 JITDUMP(" Known type Vector256<double>\n");
391 }
392 else if (typeHnd == m_simdHandleCache->Vector256IntHandle)
393 {
394 simdBaseType = TYP_INT;
395 size = Vector256SizeBytes;
396 JITDUMP(" Known type Vector256<int>\n");
397 }
398 else if (typeHnd == m_simdHandleCache->Vector256UIntHandle)
399 {
400 simdBaseType = TYP_UINT;
401 size = Vector256SizeBytes;
402 JITDUMP(" Known type Vector256<uint>\n");
403 }
404 else if (typeHnd == m_simdHandleCache->Vector256ShortHandle)
405 {
406 simdBaseType = TYP_SHORT;
407 size = Vector256SizeBytes;
408 JITDUMP(" Known type Vector256<short>\n");
409 }
410 else if (typeHnd == m_simdHandleCache->Vector256UShortHandle)
411 {
412 simdBaseType = TYP_USHORT;
413 size = Vector256SizeBytes;
414 JITDUMP(" Known type Vector256<ushort>\n");
415 }
416 else if (typeHnd == m_simdHandleCache->Vector256ByteHandle)
417 {
418 simdBaseType = TYP_BYTE;
419 size = Vector256SizeBytes;
420 JITDUMP(" Known type Vector256<sbyte>\n");
421 }
422 else if (typeHnd == m_simdHandleCache->Vector256UByteHandle)
423 {
424 simdBaseType = TYP_UBYTE;
425 size = Vector256SizeBytes;
426 JITDUMP(" Known type Vector256<byte>\n");
427 }
428 else if (typeHnd == m_simdHandleCache->Vector256LongHandle)
429 {
430 simdBaseType = TYP_LONG;
431 size = Vector256SizeBytes;
432 JITDUMP(" Known type Vector256<long>\n");
433 }
434 else if (typeHnd == m_simdHandleCache->Vector256ULongHandle)
435 {
436 simdBaseType = TYP_ULONG;
437 size = Vector256SizeBytes;
438 JITDUMP(" Known type Vector256<ulong>\n");
439 }
440 else
441#endif // defined(_TARGET_XARCH)
442 if (typeHnd == m_simdHandleCache->Vector128FloatHandle)
443 {
444 simdBaseType = TYP_FLOAT;
445 size = Vector128SizeBytes;
446 JITDUMP(" Known type Vector128<float>\n");
447 }
448 else if (typeHnd == m_simdHandleCache->Vector128DoubleHandle)
449 {
450 simdBaseType = TYP_DOUBLE;
451 size = Vector128SizeBytes;
452 JITDUMP(" Known type Vector128<double>\n");
453 }
454 else if (typeHnd == m_simdHandleCache->Vector128IntHandle)
455 {
456 simdBaseType = TYP_INT;
457 size = Vector128SizeBytes;
458 JITDUMP(" Known type Vector128<int>\n");
459 }
460 else if (typeHnd == m_simdHandleCache->Vector128UIntHandle)
461 {
462 simdBaseType = TYP_UINT;
463 size = Vector128SizeBytes;
464 JITDUMP(" Known type Vector128<uint>\n");
465 }
466 else if (typeHnd == m_simdHandleCache->Vector128ShortHandle)
467 {
468 simdBaseType = TYP_SHORT;
469 size = Vector128SizeBytes;
470 JITDUMP(" Known type Vector128<short>\n");
471 }
472 else if (typeHnd == m_simdHandleCache->Vector128UShortHandle)
473 {
474 simdBaseType = TYP_USHORT;
475 size = Vector128SizeBytes;
476 JITDUMP(" Known type Vector128<ushort>\n");
477 }
478 else if (typeHnd == m_simdHandleCache->Vector128ByteHandle)
479 {
480 simdBaseType = TYP_BYTE;
481 size = Vector128SizeBytes;
482 JITDUMP(" Known type Vector128<sbyte>\n");
483 }
484 else if (typeHnd == m_simdHandleCache->Vector128UByteHandle)
485 {
486 simdBaseType = TYP_UBYTE;
487 size = Vector128SizeBytes;
488 JITDUMP(" Known type Vector128<byte>\n");
489 }
490 else if (typeHnd == m_simdHandleCache->Vector128LongHandle)
491 {
492 simdBaseType = TYP_LONG;
493 size = Vector128SizeBytes;
494 JITDUMP(" Known type Vector128<long>\n");
495 }
496 else if (typeHnd == m_simdHandleCache->Vector128ULongHandle)
497 {
498 simdBaseType = TYP_ULONG;
499 size = Vector128SizeBytes;
500 JITDUMP(" Known type Vector128<ulong>\n");
501 }
502 else
503#if defined(_TARGET_ARM64_)
504 if (typeHnd == m_simdHandleCache->Vector64FloatHandle)
505 {
506 simdBaseType = TYP_FLOAT;
507 size = Vector64SizeBytes;
508 JITDUMP(" Known type Vector64<float>\n");
509 }
510 else if (typeHnd == m_simdHandleCache->Vector64IntHandle)
511 {
512 simdBaseType = TYP_INT;
513 size = Vector64SizeBytes;
514 JITDUMP(" Known type Vector64<int>\n");
515 }
516 else if (typeHnd == m_simdHandleCache->Vector64UIntHandle)
517 {
518 simdBaseType = TYP_UINT;
519 size = Vector64SizeBytes;
520 JITDUMP(" Known type Vector64<uint>\n");
521 }
522 else if (typeHnd == m_simdHandleCache->Vector64ShortHandle)
523 {
524 simdBaseType = TYP_SHORT;
525 size = Vector64SizeBytes;
526 JITDUMP(" Known type Vector64<short>\n");
527 }
528 else if (typeHnd == m_simdHandleCache->Vector64UShortHandle)
529 {
530 simdBaseType = TYP_USHORT;
531 size = Vector64SizeBytes;
532 JITDUMP(" Known type Vector64<ushort>\n");
533 }
534 else if (typeHnd == m_simdHandleCache->Vector64ByteHandle)
535 {
536 simdBaseType = TYP_BYTE;
537 size = Vector64SizeBytes;
538 JITDUMP(" Known type Vector64<sbyte>\n");
539 }
540 else if (typeHnd == m_simdHandleCache->Vector64UByteHandle)
541 {
542 simdBaseType = TYP_UBYTE;
543 size = Vector64SizeBytes;
544 JITDUMP(" Known type Vector64<byte>\n");
545 }
546#endif // defined(_TARGET_ARM64_)
547
548 // slow path search
549 if (simdBaseType == TYP_UNKNOWN)
550 {
551 // Doesn't match with any of the cached type handles.
552 const char* className = getClassNameFromMetadata(typeHnd, nullptr);
553 CORINFO_CLASS_HANDLE baseTypeHnd = getTypeInstantiationArgument(typeHnd, 0);
554
555 if (baseTypeHnd != nullptr)
556 {
557 CorInfoType type = info.compCompHnd->getTypeForPrimitiveNumericClass(baseTypeHnd);
558
559 JITDUMP("HW Intrinsic SIMD Candidate Type %s with Base Type %s\n", className,
560 getClassNameFromMetadata(baseTypeHnd, nullptr));
561
562#if defined(_TARGET_XARCH_)
563 if (strcmp(className, "Vector256`1") == 0)
564 {
565 size = Vector256SizeBytes;
566 switch (type)
567 {
568 case CORINFO_TYPE_FLOAT:
569 m_simdHandleCache->Vector256FloatHandle = typeHnd;
570 simdBaseType = TYP_FLOAT;
571 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<float>\n");
572 break;
573 case CORINFO_TYPE_DOUBLE:
574 m_simdHandleCache->Vector256DoubleHandle = typeHnd;
575 simdBaseType = TYP_DOUBLE;
576 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<double>\n");
577 break;
578 case CORINFO_TYPE_INT:
579 m_simdHandleCache->Vector256IntHandle = typeHnd;
580 simdBaseType = TYP_INT;
581 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<int>\n");
582 break;
583 case CORINFO_TYPE_UINT:
584 m_simdHandleCache->Vector256UIntHandle = typeHnd;
585 simdBaseType = TYP_UINT;
586 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<uint>\n");
587 break;
588 case CORINFO_TYPE_SHORT:
589 m_simdHandleCache->Vector256ShortHandle = typeHnd;
590 simdBaseType = TYP_SHORT;
591 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<short>\n");
592 break;
593 case CORINFO_TYPE_USHORT:
594 m_simdHandleCache->Vector256UShortHandle = typeHnd;
595 simdBaseType = TYP_USHORT;
596 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<ushort>\n");
597 break;
598 case CORINFO_TYPE_LONG:
599 m_simdHandleCache->Vector256LongHandle = typeHnd;
600 simdBaseType = TYP_LONG;
601 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<long>\n");
602 break;
603 case CORINFO_TYPE_ULONG:
604 m_simdHandleCache->Vector256ULongHandle = typeHnd;
605 simdBaseType = TYP_ULONG;
606 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<ulong>\n");
607 break;
608 case CORINFO_TYPE_UBYTE:
609 m_simdHandleCache->Vector256UByteHandle = typeHnd;
610 simdBaseType = TYP_UBYTE;
611 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<byte>\n");
612 break;
613 case CORINFO_TYPE_BYTE:
614 m_simdHandleCache->Vector256ByteHandle = typeHnd;
615 simdBaseType = TYP_BYTE;
616 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<sbyte>\n");
617 break;
618
619 default:
620 JITDUMP(" Unknown Hardware Intrinsic SIMD Type Vector256<T>\n");
621 }
622 }
623 else
624#endif // defined(_TARGET_XARCH_)
625 if (strcmp(className, "Vector128`1") == 0)
626 {
627 size = Vector128SizeBytes;
628 switch (type)
629 {
630 case CORINFO_TYPE_FLOAT:
631 m_simdHandleCache->Vector128FloatHandle = typeHnd;
632 simdBaseType = TYP_FLOAT;
633 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<float>\n");
634 break;
635 case CORINFO_TYPE_DOUBLE:
636 m_simdHandleCache->Vector128DoubleHandle = typeHnd;
637 simdBaseType = TYP_DOUBLE;
638 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<double>\n");
639 break;
640 case CORINFO_TYPE_INT:
641 m_simdHandleCache->Vector128IntHandle = typeHnd;
642 simdBaseType = TYP_INT;
643 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<int>\n");
644 break;
645 case CORINFO_TYPE_UINT:
646 m_simdHandleCache->Vector128UIntHandle = typeHnd;
647 simdBaseType = TYP_UINT;
648 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<uint>\n");
649 break;
650 case CORINFO_TYPE_SHORT:
651 m_simdHandleCache->Vector128ShortHandle = typeHnd;
652 simdBaseType = TYP_SHORT;
653 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<short>\n");
654 break;
655 case CORINFO_TYPE_USHORT:
656 m_simdHandleCache->Vector128UShortHandle = typeHnd;
657 simdBaseType = TYP_USHORT;
658 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<ushort>\n");
659 break;
660 case CORINFO_TYPE_LONG:
661 m_simdHandleCache->Vector128LongHandle = typeHnd;
662 simdBaseType = TYP_LONG;
663 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<long>\n");
664 break;
665 case CORINFO_TYPE_ULONG:
666 m_simdHandleCache->Vector128ULongHandle = typeHnd;
667 simdBaseType = TYP_ULONG;
668 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<ulong>\n");
669 break;
670 case CORINFO_TYPE_UBYTE:
671 m_simdHandleCache->Vector128UByteHandle = typeHnd;
672 simdBaseType = TYP_UBYTE;
673 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<byte>\n");
674 break;
675 case CORINFO_TYPE_BYTE:
676 m_simdHandleCache->Vector128ByteHandle = typeHnd;
677 simdBaseType = TYP_BYTE;
678 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<sbyte>\n");
679 break;
680
681 default:
682 JITDUMP(" Unknown Hardware Intrinsic SIMD Type Vector128<T>\n");
683 }
684 }
685#if defined(_TARGET_ARM64_)
686 else if (strcmp(className, "Vector64`1") == 0)
687 {
688 size = Vector64SizeBytes;
689 switch (type)
690 {
691 case CORINFO_TYPE_FLOAT:
692 m_simdHandleCache->Vector64FloatHandle = typeHnd;
693 simdBaseType = TYP_FLOAT;
694 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<float>\n");
695 break;
696 case CORINFO_TYPE_INT:
697 m_simdHandleCache->Vector64IntHandle = typeHnd;
698 simdBaseType = TYP_INT;
699 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<int>\n");
700 break;
701 case CORINFO_TYPE_UINT:
702 m_simdHandleCache->Vector64UIntHandle = typeHnd;
703 simdBaseType = TYP_UINT;
704 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<uint>\n");
705 break;
706 case CORINFO_TYPE_SHORT:
707 m_simdHandleCache->Vector64ShortHandle = typeHnd;
708 simdBaseType = TYP_SHORT;
709 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<short>\n");
710 break;
711 case CORINFO_TYPE_USHORT:
712 m_simdHandleCache->Vector64UShortHandle = typeHnd;
713 simdBaseType = TYP_USHORT;
714 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<ushort>\n");
715 break;
716 case CORINFO_TYPE_UBYTE:
717 m_simdHandleCache->Vector64UByteHandle = typeHnd;
718 simdBaseType = TYP_UBYTE;
719 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<byte>\n");
720 break;
721 case CORINFO_TYPE_BYTE:
722 m_simdHandleCache->Vector64ByteHandle = typeHnd;
723 simdBaseType = TYP_BYTE;
724 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<sbyte>\n");
725 break;
726
727 default:
728 JITDUMP(" Unknown Hardware Intrinsic SIMD Type Vector64<T>\n");
729 }
730 }
731#endif // defined(_TARGET_ARM64_)
732 }
733 }
734
735 if (sizeBytes != nullptr)
736 {
737 *sizeBytes = size;
738 }
739
740 if (simdBaseType != TYP_UNKNOWN)
741 {
742 setUsesSIMDTypes(true);
743 }
744 }
745#endif // FEATURE_HW_INTRINSICS
746
747 return simdBaseType;
748}
749
750//--------------------------------------------------------------------------------------
751// getSIMDIntrinsicInfo: get SIMD intrinsic info given the method handle.
752//
753// Arguments:
754// inOutTypeHnd - The handle of the type on which the method is invoked. This is an in-out param.
755// methodHnd - The handle of the method we're interested in.
756// sig - method signature info
757// isNewObj - whether this call represents a newboj constructor call
758// argCount - argument count - out pram
759// baseType - base type of the intrinsic - out param
760// sizeBytes - size of SIMD vector type on which the method is invoked - out param
761//
762// Return Value:
763// SIMDIntrinsicInfo struct initialized corresponding to methodHnd.
764// Sets SIMDIntrinsicInfo.id to SIMDIntrinsicInvalid if methodHnd doesn't correspond
765// to any SIMD intrinsic. Also, sets the out params inOutTypeHnd, argCount, baseType and
766// sizeBytes.
767//
768// Note that VectorMath class doesn't have a base type and first argument of the method
769// determines the SIMD vector type on which intrinsic is invoked. In such a case inOutTypeHnd
770// is modified by this routine.
771//
772// TODO-Throughput: The current implementation is based on method name string parsing.
773// Although we now have type identification from the VM, the parsing of intrinsic names
774// could be made more efficient.
775//
776const SIMDIntrinsicInfo* Compiler::getSIMDIntrinsicInfo(CORINFO_CLASS_HANDLE* inOutTypeHnd,
777 CORINFO_METHOD_HANDLE methodHnd,
778 CORINFO_SIG_INFO* sig,
779 bool isNewObj,
780 unsigned* argCount,
781 var_types* baseType,
782 unsigned* sizeBytes)
783{
784 assert(featureSIMD);
785 assert(baseType != nullptr);
786 assert(sizeBytes != nullptr);
787
788 // get baseType and size of the type
789 CORINFO_CLASS_HANDLE typeHnd = *inOutTypeHnd;
790 *baseType = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
791
792 if (typeHnd == m_simdHandleCache->SIMDVectorHandle)
793 {
794 // All of the supported intrinsics on this static class take a first argument that's a vector,
795 // which determines the baseType.
796 // The exception is the IsHardwareAccelerated property, which is handled as a special case.
797 assert(*baseType == TYP_UNKNOWN);
798 if (sig->numArgs == 0)
799 {
800 const SIMDIntrinsicInfo* hwAccelIntrinsicInfo = &(simdIntrinsicInfoArray[SIMDIntrinsicHWAccel]);
801 if ((strcmp(eeGetMethodName(methodHnd, nullptr), hwAccelIntrinsicInfo->methodName) == 0) &&
802 JITtype2varType(sig->retType) == hwAccelIntrinsicInfo->retType)
803 {
804 // Sanity check
805 assert(hwAccelIntrinsicInfo->argCount == 0 && hwAccelIntrinsicInfo->isInstMethod == false);
806 return hwAccelIntrinsicInfo;
807 }
808 return nullptr;
809 }
810 else
811 {
812 typeHnd = info.compCompHnd->getArgClass(sig, sig->args);
813 *inOutTypeHnd = typeHnd;
814 *baseType = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
815 }
816 }
817
818 if (*baseType == TYP_UNKNOWN)
819 {
820 JITDUMP("NOT a SIMD Intrinsic: unsupported baseType\n");
821 return nullptr;
822 }
823
824 // account for implicit "this" arg
825 *argCount = sig->numArgs;
826 if (sig->hasThis())
827 {
828 *argCount += 1;
829 }
830
831 // Get the Intrinsic Id by parsing method name.
832 //
833 // TODO-Throughput: replace sequential search by binary search by arranging entries
834 // sorted by method name.
835 SIMDIntrinsicID intrinsicId = SIMDIntrinsicInvalid;
836 const char* methodName = eeGetMethodName(methodHnd, nullptr);
837 for (int i = SIMDIntrinsicNone + 1; i < SIMDIntrinsicInvalid; ++i)
838 {
839 if (strcmp(methodName, simdIntrinsicInfoArray[i].methodName) == 0)
840 {
841 // Found an entry for the method; further check whether it is one of
842 // the supported base types.
843 bool found = false;
844 for (int j = 0; j < SIMD_INTRINSIC_MAX_BASETYPE_COUNT; ++j)
845 {
846 // Convention: if there are fewer base types supported than MAX_BASETYPE_COUNT,
847 // the end of the list is marked by TYP_UNDEF.
848 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == TYP_UNDEF)
849 {
850 break;
851 }
852
853 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == *baseType)
854 {
855 found = true;
856 break;
857 }
858 }
859
860 if (!found)
861 {
862 continue;
863 }
864
865 // Now, check the arguments.
866 unsigned int fixedArgCnt = simdIntrinsicInfoArray[i].argCount;
867 unsigned int expectedArgCnt = fixedArgCnt;
868
869 // First handle SIMDIntrinsicInitN, where the arg count depends on the type.
870 // The listed arg types include the vector and the first two init values, which is the expected number
871 // for Vector2. For other cases, we'll check their types here.
872 if (*argCount > expectedArgCnt)
873 {
874 if (i == SIMDIntrinsicInitN)
875 {
876 if (*argCount == 3 && typeHnd == m_simdHandleCache->SIMDVector2Handle)
877 {
878 expectedArgCnt = 3;
879 }
880 else if (*argCount == 4 && typeHnd == m_simdHandleCache->SIMDVector3Handle)
881 {
882 expectedArgCnt = 4;
883 }
884 else if (*argCount == 5 && typeHnd == m_simdHandleCache->SIMDVector4Handle)
885 {
886 expectedArgCnt = 5;
887 }
888 }
889 else if (i == SIMDIntrinsicInitFixed)
890 {
891 if (*argCount == 4 && typeHnd == m_simdHandleCache->SIMDVector4Handle)
892 {
893 expectedArgCnt = 4;
894 }
895 }
896 }
897 if (*argCount != expectedArgCnt)
898 {
899 continue;
900 }
901
902 // Validate the types of individual args passed are what is expected of.
903 // If any of the types don't match with what is expected, don't consider
904 // as an intrinsic. This will make an older JIT with SIMD capabilities
905 // resilient to breaking changes to SIMD managed API.
906 //
907 // Note that from IL type stack, args get popped in right to left order
908 // whereas args get listed in method signatures in left to right order.
909
910 int stackIndex = (expectedArgCnt - 1);
911
912 // Track the arguments from the signature - we currently only use this to distinguish
913 // integral and pointer types, both of which will by TYP_I_IMPL on the importer stack.
914 CORINFO_ARG_LIST_HANDLE argLst = sig->args;
915
916 CORINFO_CLASS_HANDLE argClass;
917 for (unsigned int argIndex = 0; found == true && argIndex < expectedArgCnt; argIndex++)
918 {
919 bool isThisPtr = ((argIndex == 0) && sig->hasThis());
920
921 // In case of "newobj SIMDVector<T>(T val)", thisPtr won't be present on type stack.
922 // We don't check anything in that case.
923 if (!isThisPtr || !isNewObj)
924 {
925 GenTree* arg = impStackTop(stackIndex).val;
926 var_types argType = arg->TypeGet();
927
928 var_types expectedArgType;
929 if (argIndex < fixedArgCnt)
930 {
931 // Convention:
932 // - intrinsicInfo.argType[i] == TYP_UNDEF - intrinsic doesn't have a valid arg at position i
933 // - intrinsicInfo.argType[i] == TYP_UNKNOWN - arg type should be same as basetype
934 // Note that we pop the args off in reverse order.
935 expectedArgType = simdIntrinsicInfoArray[i].argType[argIndex];
936 assert(expectedArgType != TYP_UNDEF);
937 if (expectedArgType == TYP_UNKNOWN)
938 {
939 // The type of the argument will be genActualType(*baseType).
940 expectedArgType = genActualType(*baseType);
941 argType = genActualType(argType);
942 }
943 }
944 else
945 {
946 expectedArgType = *baseType;
947 }
948
949 if (!isThisPtr && argType == TYP_I_IMPL)
950 {
951 // The reference implementation has a constructor that takes a pointer.
952 // We don't want to recognize that one. This requires us to look at the CorInfoType
953 // in order to distinguish a signature with a pointer argument from one with an
954 // integer argument of pointer size, both of which will be TYP_I_IMPL on the stack.
955 // TODO-Review: This seems quite fragile. We should consider beefing up the checking
956 // here.
957 CorInfoType corType = strip(info.compCompHnd->getArgType(sig, argLst, &argClass));
958 if (corType == CORINFO_TYPE_PTR)
959 {
960 found = false;
961 }
962 }
963
964 if (varTypeIsSIMD(argType))
965 {
966 argType = TYP_STRUCT;
967 }
968 if (argType != expectedArgType)
969 {
970 found = false;
971 }
972 }
973 if (argIndex != 0 || !sig->hasThis())
974 {
975 argLst = info.compCompHnd->getArgNext(argLst);
976 }
977 stackIndex--;
978 }
979
980 // Cross check return type and static vs. instance is what we are expecting.
981 // If not, don't consider it as an intrinsic.
982 // Note that ret type of TYP_UNKNOWN means that it is not known apriori and must be same as baseType
983 if (found)
984 {
985 var_types expectedRetType = simdIntrinsicInfoArray[i].retType;
986 if (expectedRetType == TYP_UNKNOWN)
987 {
988 // JIT maps uint/ulong type vars to TYP_INT/TYP_LONG.
989 expectedRetType =
990 (*baseType == TYP_UINT || *baseType == TYP_ULONG) ? genActualType(*baseType) : *baseType;
991 }
992
993 if (JITtype2varType(sig->retType) != expectedRetType ||
994 sig->hasThis() != simdIntrinsicInfoArray[i].isInstMethod)
995 {
996 found = false;
997 }
998 }
999
1000 if (found)
1001 {
1002 intrinsicId = (SIMDIntrinsicID)i;
1003 break;
1004 }
1005 }
1006 }
1007
1008 if (intrinsicId != SIMDIntrinsicInvalid)
1009 {
1010 JITDUMP("Method %s maps to SIMD intrinsic %s\n", methodName, simdIntrinsicNames[intrinsicId]);
1011 return &simdIntrinsicInfoArray[intrinsicId];
1012 }
1013 else
1014 {
1015 JITDUMP("Method %s is NOT a SIMD intrinsic\n", methodName);
1016 }
1017
1018 return nullptr;
1019}
1020
1021// Pops and returns GenTree node from importer's type stack.
1022// Normalizes TYP_STRUCT value in case of GT_CALL, GT_RET_EXPR and arg nodes.
1023//
1024// Arguments:
1025// type - the type of value that the caller expects to be popped off the stack.
1026// expectAddr - if true indicates we are expecting type stack entry to be a TYP_BYREF.
1027// structType - the class handle to use when normalizing if it is not the same as the stack entry class handle;
1028// this can happen for certain scenarios, such as folding away a static cast, where we want the
1029// value popped to have the type that would have been returned.
1030//
1031// Notes:
1032// If the popped value is a struct, and the expected type is a simd type, it will be set
1033// to that type, otherwise it will assert if the type being popped is not the expected type.
1034
1035GenTree* Compiler::impSIMDPopStack(var_types type, bool expectAddr, CORINFO_CLASS_HANDLE structType)
1036{
1037 StackEntry se = impPopStack();
1038 typeInfo ti = se.seTypeInfo;
1039 GenTree* tree = se.val;
1040
1041 // If expectAddr is true implies what we have on stack is address and we need
1042 // SIMD type struct that it points to.
1043 if (expectAddr)
1044 {
1045 assert(tree->TypeGet() == TYP_BYREF);
1046 if (tree->OperGet() == GT_ADDR)
1047 {
1048 tree = tree->gtGetOp1();
1049 }
1050 else
1051 {
1052 tree = gtNewOperNode(GT_IND, type, tree);
1053 }
1054 }
1055
1056 bool isParam = false;
1057
1058 // If we have a ldobj of a SIMD local we need to transform it.
1059 if (tree->OperGet() == GT_OBJ)
1060 {
1061 GenTree* addr = tree->gtOp.gtOp1;
1062 if ((addr->OperGet() == GT_ADDR) && isSIMDTypeLocal(addr->gtOp.gtOp1))
1063 {
1064 tree = addr->gtOp.gtOp1;
1065 }
1066 }
1067
1068 if (tree->OperGet() == GT_LCL_VAR)
1069 {
1070 unsigned lclNum = tree->AsLclVarCommon()->GetLclNum();
1071 LclVarDsc* lclVarDsc = &lvaTable[lclNum];
1072 isParam = lclVarDsc->lvIsParam;
1073 }
1074
1075 // normalize TYP_STRUCT value
1076 if (varTypeIsStruct(tree) && ((tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) || isParam))
1077 {
1078 assert(ti.IsType(TI_STRUCT));
1079
1080 if (structType == nullptr)
1081 {
1082 structType = ti.GetClassHandleForValueClass();
1083 }
1084
1085 tree = impNormStructVal(tree, structType, (unsigned)CHECK_SPILL_ALL);
1086 }
1087
1088 // Now set the type of the tree to the specialized SIMD struct type, if applicable.
1089 if (genActualType(tree->gtType) != genActualType(type))
1090 {
1091 assert(tree->gtType == TYP_STRUCT);
1092 tree->gtType = type;
1093 }
1094 else if (tree->gtType == TYP_BYREF)
1095 {
1096 assert(tree->IsLocal() || (tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) ||
1097 ((tree->gtOper == GT_ADDR) && varTypeIsSIMD(tree->gtGetOp1())));
1098 }
1099
1100 return tree;
1101}
1102
1103// impSIMDGetFixed: Create a GT_SIMD tree for a Get property of SIMD vector with a fixed index.
1104//
1105// Arguments:
1106// baseType - The base (element) type of the SIMD vector.
1107// simdSize - The total size in bytes of the SIMD vector.
1108// index - The index of the field to get.
1109//
1110// Return Value:
1111// Returns a GT_SIMD node with the SIMDIntrinsicGetItem intrinsic id.
1112//
1113GenTreeSIMD* Compiler::impSIMDGetFixed(var_types simdType, var_types baseType, unsigned simdSize, int index)
1114{
1115 assert(simdSize >= ((index + 1) * genTypeSize(baseType)));
1116
1117 // op1 is a SIMD source.
1118 GenTree* op1 = impSIMDPopStack(simdType, true);
1119
1120 GenTree* op2 = gtNewIconNode(index);
1121 GenTreeSIMD* simdTree = gtNewSIMDNode(baseType, op1, op2, SIMDIntrinsicGetItem, baseType, simdSize);
1122 return simdTree;
1123}
1124
1125#ifdef _TARGET_XARCH_
1126// impSIMDLongRelOpEqual: transforms operands and returns the SIMD intrinsic to be applied on
1127// transformed operands to obtain == comparison result.
1128//
1129// Arguments:
1130// typeHnd - type handle of SIMD vector
1131// size - SIMD vector size
1132// op1 - in-out parameter; first operand
1133// op2 - in-out parameter; second operand
1134//
1135// Return Value:
1136// Modifies in-out params op1, op2 and returns intrinsic ID to be applied to modified operands
1137//
1138SIMDIntrinsicID Compiler::impSIMDLongRelOpEqual(CORINFO_CLASS_HANDLE typeHnd,
1139 unsigned size,
1140 GenTree** pOp1,
1141 GenTree** pOp2)
1142{
1143 var_types simdType = (*pOp1)->TypeGet();
1144 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1145
1146 // There is no direct SSE2 support for comparing TYP_LONG vectors.
1147 // These have to be implemented in terms of TYP_INT vector comparison operations.
1148 //
1149 // Equality(v1, v2):
1150 // tmp = (v1 == v2) i.e. compare for equality as if v1 and v2 are vector<int>
1151 // result = BitwiseAnd(t, shuffle(t, (2, 3, 0, 1)))
1152 // Shuffle is meant to swap the comparison results of low-32-bits and high 32-bits of respective long elements.
1153
1154 // Compare vector<long> as if they were vector<int> and assign the result to a temp
1155 GenTree* compResult = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, TYP_INT, size);
1156 unsigned lclNum = lvaGrabTemp(true DEBUGARG("SIMD Long =="));
1157 lvaSetStruct(lclNum, typeHnd, false);
1158 GenTree* tmp = gtNewLclvNode(lclNum, simdType);
1159 GenTree* asg = gtNewTempAssign(lclNum, compResult);
1160
1161 // op1 = GT_COMMA(tmp=compResult, tmp)
1162 // op2 = Shuffle(tmp, 0xB1)
1163 // IntrinsicId = BitwiseAnd
1164 *pOp1 = gtNewOperNode(GT_COMMA, simdType, asg, tmp);
1165 *pOp2 = gtNewSIMDNode(simdType, gtNewLclvNode(lclNum, simdType), gtNewIconNode(SHUFFLE_ZWXY, TYP_INT),
1166 SIMDIntrinsicShuffleSSE2, TYP_INT, size);
1167 return SIMDIntrinsicBitwiseAnd;
1168}
1169
1170// impSIMDLongRelOpGreaterThan: transforms operands and returns the SIMD intrinsic to be applied on
1171// transformed operands to obtain > comparison result.
1172//
1173// Arguments:
1174// typeHnd - type handle of SIMD vector
1175// size - SIMD vector size
1176// pOp1 - in-out parameter; first operand
1177// pOp2 - in-out parameter; second operand
1178//
1179// Return Value:
1180// Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1181//
1182SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThan(CORINFO_CLASS_HANDLE typeHnd,
1183 unsigned size,
1184 GenTree** pOp1,
1185 GenTree** pOp2)
1186{
1187 var_types simdType = (*pOp1)->TypeGet();
1188 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1189
1190 // GreaterThan(v1, v2) where v1 and v2 are vector long.
1191 // Let us consider the case of single long element comparison.
1192 // say L1 = (x1, y1) and L2 = (x2, y2) where x1, y1, x2, and y2 are 32-bit integers that comprise the longs L1 and
1193 // L2.
1194 //
1195 // GreaterThan(L1, L2) can be expressed in terms of > relationship between 32-bit integers that comprise L1 and L2
1196 // as
1197 // = (x1, y1) > (x2, y2)
1198 // = (x1 > x2) || [(x1 == x2) && (y1 > y2)] - eq (1)
1199 //
1200 // t = (v1 > v2) 32-bit signed comparison
1201 // u = (v1 == v2) 32-bit sized element equality
1202 // v = (v1 > v2) 32-bit unsigned comparison
1203 //
1204 // z = shuffle(t, (3, 3, 1, 1)) - This corresponds to (x1 > x2) in eq(1) above
1205 // t1 = Shuffle(v, (2, 2, 0, 0)) - This corresponds to (y1 > y2) in eq(1) above
1206 // u1 = Shuffle(u, (3, 3, 1, 1)) - This corresponds to (x1 == x2) in eq(1) above
1207 // w = And(t1, u1) - This corresponds to [(x1 == x2) && (y1 > y2)] in eq(1) above
1208 // Result = BitwiseOr(z, w)
1209
1210 // Since op1 and op2 gets used multiple times, make sure side effects are computed.
1211 GenTree* dupOp1 = nullptr;
1212 GenTree* dupOp2 = nullptr;
1213 GenTree* dupDupOp1 = nullptr;
1214 GenTree* dupDupOp2 = nullptr;
1215
1216 if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1217 {
1218 dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1219 dupDupOp1 = gtNewLclvNode(dupOp1->AsLclVarCommon()->GetLclNum(), simdType);
1220 }
1221 else
1222 {
1223 dupOp1 = gtCloneExpr(*pOp1);
1224 dupDupOp1 = gtCloneExpr(*pOp1);
1225 }
1226
1227 if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1228 {
1229 dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1230 dupDupOp2 = gtNewLclvNode(dupOp2->AsLclVarCommon()->GetLclNum(), simdType);
1231 }
1232 else
1233 {
1234 dupOp2 = gtCloneExpr(*pOp2);
1235 dupDupOp2 = gtCloneExpr(*pOp2);
1236 }
1237
1238 assert(dupDupOp1 != nullptr && dupDupOp2 != nullptr);
1239 assert(dupOp1 != nullptr && dupOp2 != nullptr);
1240 assert(*pOp1 != nullptr && *pOp2 != nullptr);
1241
1242 // v1GreaterThanv2Signed - signed 32-bit comparison
1243 GenTree* v1GreaterThanv2Signed = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicGreaterThan, TYP_INT, size);
1244
1245 // v1Equalsv2 - 32-bit equality
1246 GenTree* v1Equalsv2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicEqual, TYP_INT, size);
1247
1248 // v1GreaterThanv2Unsigned - unsigned 32-bit comparison
1249 var_types tempBaseType = TYP_UINT;
1250 SIMDIntrinsicID sid = impSIMDRelOp(SIMDIntrinsicGreaterThan, typeHnd, size, &tempBaseType, &dupDupOp1, &dupDupOp2);
1251 GenTree* v1GreaterThanv2Unsigned = gtNewSIMDNode(simdType, dupDupOp1, dupDupOp2, sid, tempBaseType, size);
1252
1253 GenTree* z = gtNewSIMDNode(simdType, v1GreaterThanv2Signed, gtNewIconNode(SHUFFLE_WWYY, TYP_INT),
1254 SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1255 GenTree* t1 = gtNewSIMDNode(simdType, v1GreaterThanv2Unsigned, gtNewIconNode(SHUFFLE_ZZXX, TYP_INT),
1256 SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1257 GenTree* u1 = gtNewSIMDNode(simdType, v1Equalsv2, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), SIMDIntrinsicShuffleSSE2,
1258 TYP_FLOAT, size);
1259 GenTree* w = gtNewSIMDNode(simdType, u1, t1, SIMDIntrinsicBitwiseAnd, TYP_INT, size);
1260
1261 *pOp1 = z;
1262 *pOp2 = w;
1263 return SIMDIntrinsicBitwiseOr;
1264}
1265
1266// impSIMDLongRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1267// transformed operands to obtain >= comparison result.
1268//
1269// Arguments:
1270// typeHnd - type handle of SIMD vector
1271// size - SIMD vector size
1272// pOp1 - in-out parameter; first operand
1273// pOp2 - in-out parameter; second operand
1274//
1275// Return Value:
1276// Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1277//
1278SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThanOrEqual(CORINFO_CLASS_HANDLE typeHnd,
1279 unsigned size,
1280 GenTree** pOp1,
1281 GenTree** pOp2)
1282{
1283 var_types simdType = (*pOp1)->TypeGet();
1284 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1285
1286 // expand this to (a == b) | (a > b)
1287 GenTree* dupOp1 = nullptr;
1288 GenTree* dupOp2 = nullptr;
1289
1290 if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1291 {
1292 dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1293 }
1294 else
1295 {
1296 dupOp1 = gtCloneExpr(*pOp1);
1297 }
1298
1299 if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1300 {
1301 dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1302 }
1303 else
1304 {
1305 dupOp2 = gtCloneExpr(*pOp2);
1306 }
1307
1308 assert(dupOp1 != nullptr && dupOp2 != nullptr);
1309 assert(*pOp1 != nullptr && *pOp2 != nullptr);
1310
1311 // (a==b)
1312 SIMDIntrinsicID id = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1313 *pOp1 = gtNewSIMDNode(simdType, *pOp1, *pOp2, id, TYP_LONG, size);
1314
1315 // (a > b)
1316 id = impSIMDLongRelOpGreaterThan(typeHnd, size, &dupOp1, &dupOp2);
1317 *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, id, TYP_LONG, size);
1318
1319 return SIMDIntrinsicBitwiseOr;
1320}
1321
1322// impSIMDInt32OrSmallIntRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1323// transformed operands to obtain >= comparison result in case of integer base type vectors
1324//
1325// Arguments:
1326// typeHnd - type handle of SIMD vector
1327// size - SIMD vector size
1328// baseType - base type of SIMD vector
1329// pOp1 - in-out parameter; first operand
1330// pOp2 - in-out parameter; second operand
1331//
1332// Return Value:
1333// Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1334//
1335SIMDIntrinsicID Compiler::impSIMDIntegralRelOpGreaterThanOrEqual(
1336 CORINFO_CLASS_HANDLE typeHnd, unsigned size, var_types baseType, GenTree** pOp1, GenTree** pOp2)
1337{
1338 var_types simdType = (*pOp1)->TypeGet();
1339 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1340
1341 // This routine should be used only for integer base type vectors
1342 assert(varTypeIsIntegral(baseType));
1343 if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && ((baseType == TYP_LONG) || baseType == TYP_UBYTE))
1344 {
1345 return impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1346 }
1347
1348 // expand this to (a == b) | (a > b)
1349 GenTree* dupOp1 = nullptr;
1350 GenTree* dupOp2 = nullptr;
1351
1352 if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1353 {
1354 dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1355 }
1356 else
1357 {
1358 dupOp1 = gtCloneExpr(*pOp1);
1359 }
1360
1361 if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1362 {
1363 dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1364 }
1365 else
1366 {
1367 dupOp2 = gtCloneExpr(*pOp2);
1368 }
1369
1370 assert(dupOp1 != nullptr && dupOp2 != nullptr);
1371 assert(*pOp1 != nullptr && *pOp2 != nullptr);
1372
1373 // (a==b)
1374 *pOp1 = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, baseType, size);
1375
1376 // (a > b)
1377 *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicGreaterThan, baseType, size);
1378
1379 return SIMDIntrinsicBitwiseOr;
1380}
1381#endif // _TARGET_XARCH_
1382
1383// Transforms operands and returns the SIMD intrinsic to be applied on
1384// transformed operands to obtain given relop result.
1385//
1386// Arguments:
1387// relOpIntrinsicId - Relational operator SIMD intrinsic
1388// typeHnd - type handle of SIMD vector
1389// size - SIMD vector size
1390// inOutBaseType - base type of SIMD vector
1391// pOp1 - in-out parameter; first operand
1392// pOp2 - in-out parameter; second operand
1393//
1394// Return Value:
1395// Modifies in-out params pOp1, pOp2, inOutBaseType and returns intrinsic ID to be applied to modified operands
1396//
1397SIMDIntrinsicID Compiler::impSIMDRelOp(SIMDIntrinsicID relOpIntrinsicId,
1398 CORINFO_CLASS_HANDLE typeHnd,
1399 unsigned size,
1400 var_types* inOutBaseType,
1401 GenTree** pOp1,
1402 GenTree** pOp2)
1403{
1404 var_types simdType = (*pOp1)->TypeGet();
1405 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1406
1407 assert(isRelOpSIMDIntrinsic(relOpIntrinsicId));
1408
1409 SIMDIntrinsicID intrinsicID = relOpIntrinsicId;
1410#ifdef _TARGET_XARCH_
1411 var_types baseType = *inOutBaseType;
1412
1413 if (varTypeIsFloating(baseType))
1414 {
1415 // SSE2/AVX doesn't support > and >= on vector float/double.
1416 // Therefore, we need to use < and <= with swapped operands
1417 if (relOpIntrinsicId == SIMDIntrinsicGreaterThan || relOpIntrinsicId == SIMDIntrinsicGreaterThanOrEqual)
1418 {
1419 GenTree* tmp = *pOp1;
1420 *pOp1 = *pOp2;
1421 *pOp2 = tmp;
1422
1423 intrinsicID =
1424 (relOpIntrinsicId == SIMDIntrinsicGreaterThan) ? SIMDIntrinsicLessThan : SIMDIntrinsicLessThanOrEqual;
1425 }
1426 }
1427 else if (varTypeIsIntegral(baseType))
1428 {
1429 // SSE/AVX doesn't support < and <= on integer base type vectors.
1430 // Therefore, we need to use > and >= with swapped operands.
1431 if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1432 {
1433 GenTree* tmp = *pOp1;
1434 *pOp1 = *pOp2;
1435 *pOp2 = tmp;
1436
1437 intrinsicID = (relOpIntrinsicId == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan
1438 : SIMDIntrinsicGreaterThanOrEqual;
1439 }
1440
1441 if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && baseType == TYP_LONG)
1442 {
1443 // There is no direct SSE2 support for comparing TYP_LONG vectors.
1444 // These have to be implemented interms of TYP_INT vector comparison operations.
1445 if (intrinsicID == SIMDIntrinsicEqual)
1446 {
1447 intrinsicID = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1448 }
1449 else if (intrinsicID == SIMDIntrinsicGreaterThan)
1450 {
1451 intrinsicID = impSIMDLongRelOpGreaterThan(typeHnd, size, pOp1, pOp2);
1452 }
1453 else if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1454 {
1455 intrinsicID = impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1456 }
1457 else
1458 {
1459 unreached();
1460 }
1461 }
1462 // SSE2 and AVX direct support for signed comparison of int32, int16 and int8 types
1463 else if (!varTypeIsUnsigned(baseType))
1464 {
1465 if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1466 {
1467 intrinsicID = impSIMDIntegralRelOpGreaterThanOrEqual(typeHnd, size, baseType, pOp1, pOp2);
1468 }
1469 }
1470 else // unsigned
1471 {
1472 // Vector<byte>, Vector<ushort>, Vector<uint> and Vector<ulong>:
1473 // SSE2 supports > for signed comparison. Therefore, to use it for
1474 // comparing unsigned numbers, we subtract a constant from both the
1475 // operands such that the result fits within the corresponding signed
1476 // type. The resulting signed numbers are compared using SSE2 signed
1477 // comparison.
1478 //
1479 // Vector<byte>: constant to be subtracted is 2^7
1480 // Vector<ushort> constant to be subtracted is 2^15
1481 // Vector<uint> constant to be subtracted is 2^31
1482 // Vector<ulong> constant to be subtracted is 2^63
1483 //
1484 // We need to treat op1 and op2 as signed for comparison purpose after
1485 // the transformation.
1486 __int64 constVal = 0;
1487 switch (baseType)
1488 {
1489 case TYP_UBYTE:
1490 constVal = 0x80808080;
1491 *inOutBaseType = TYP_BYTE;
1492 break;
1493 case TYP_USHORT:
1494 constVal = 0x80008000;
1495 *inOutBaseType = TYP_SHORT;
1496 break;
1497 case TYP_UINT:
1498 constVal = 0x80000000;
1499 *inOutBaseType = TYP_INT;
1500 break;
1501 case TYP_ULONG:
1502 constVal = 0x8000000000000000LL;
1503 *inOutBaseType = TYP_LONG;
1504 break;
1505 default:
1506 unreached();
1507 break;
1508 }
1509 assert(constVal != 0);
1510
1511 // This transformation is not required for equality.
1512 if (intrinsicID != SIMDIntrinsicEqual)
1513 {
1514 // For constructing const vector use either long or int base type.
1515 var_types tempBaseType;
1516 GenTree* initVal;
1517 if (baseType == TYP_ULONG)
1518 {
1519 tempBaseType = TYP_LONG;
1520 initVal = gtNewLconNode(constVal);
1521 }
1522 else
1523 {
1524 tempBaseType = TYP_INT;
1525 initVal = gtNewIconNode((ssize_t)constVal);
1526 }
1527 initVal->gtType = tempBaseType;
1528 GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, tempBaseType, size);
1529
1530 // Assign constVector to a temp, since we intend to use it more than once
1531 // TODO-CQ: We have quite a few such constant vectors constructed during
1532 // the importation of SIMD intrinsics. Make sure that we have a single
1533 // temp per distinct constant per method.
1534 GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1535
1536 // op1 = op1 - constVector
1537 // op2 = op2 - constVector
1538 *pOp1 = gtNewSIMDNode(simdType, *pOp1, constVector, SIMDIntrinsicSub, baseType, size);
1539 *pOp2 = gtNewSIMDNode(simdType, *pOp2, tmp, SIMDIntrinsicSub, baseType, size);
1540 }
1541
1542 return impSIMDRelOp(intrinsicID, typeHnd, size, inOutBaseType, pOp1, pOp2);
1543 }
1544 }
1545#elif defined(_TARGET_ARM64_)
1546 // TODO-ARM64-CQ handle comparisons against zero
1547
1548 // _TARGET_ARM64_ doesn't support < and <= on register register comparisons
1549 // Therefore, we need to use > and >= with swapped operands.
1550 if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1551 {
1552 GenTree* tmp = *pOp1;
1553 *pOp1 = *pOp2;
1554 *pOp2 = tmp;
1555
1556 intrinsicID =
1557 (intrinsicID == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan : SIMDIntrinsicGreaterThanOrEqual;
1558 }
1559#else // !_TARGET_XARCH_
1560 assert(!"impSIMDRelOp() unimplemented on target arch");
1561 unreached();
1562#endif // !_TARGET_XARCH_
1563
1564 return intrinsicID;
1565}
1566
1567//-------------------------------------------------------------------------
1568// impSIMDAbs: creates GT_SIMD node to compute Abs value of a given vector.
1569//
1570// Arguments:
1571// typeHnd - type handle of SIMD vector
1572// baseType - base type of vector
1573// size - vector size in bytes
1574// op1 - operand of Abs intrinsic
1575//
1576GenTree* Compiler::impSIMDAbs(CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1)
1577{
1578 assert(varTypeIsSIMD(op1));
1579
1580 var_types simdType = op1->TypeGet();
1581 GenTree* retVal = nullptr;
1582
1583#ifdef _TARGET_XARCH_
1584 // When there is no direct support, Abs(v) could be computed
1585 // on integer vectors as follows:
1586 // BitVector = v < vector.Zero
1587 // result = ConditionalSelect(BitVector, vector.Zero - v, v)
1588
1589 bool useConditionalSelect = false;
1590 if (getSIMDSupportLevel() == SIMD_SSE2_Supported)
1591 {
1592 // SSE2 doesn't support abs on signed integer type vectors.
1593 if (baseType == TYP_LONG || baseType == TYP_INT || baseType == TYP_SHORT || baseType == TYP_BYTE)
1594 {
1595 useConditionalSelect = true;
1596 }
1597 }
1598 else
1599 {
1600 assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1601 if (baseType == TYP_LONG)
1602 {
1603 // SSE4/AVX2 don't support abs on long type vector.
1604 useConditionalSelect = true;
1605 }
1606 }
1607
1608 if (useConditionalSelect)
1609 {
1610 // This works only on integer vectors not on float/double vectors.
1611 assert(varTypeIsIntegral(baseType));
1612
1613 GenTree* op1Assign;
1614 unsigned op1LclNum;
1615
1616 if (op1->OperGet() == GT_LCL_VAR)
1617 {
1618 op1LclNum = op1->gtLclVarCommon.gtLclNum;
1619 op1Assign = nullptr;
1620 }
1621 else
1622 {
1623 op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs op1"));
1624 lvaSetStruct(op1LclNum, typeHnd, false);
1625 op1Assign = gtNewTempAssign(op1LclNum, op1);
1626 op1 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1627 }
1628
1629 // Assign Vector.Zero to a temp since it is needed more than once
1630 GenTree* vecZero = gtNewSIMDVectorZero(simdType, baseType, size);
1631 unsigned vecZeroLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs VecZero"));
1632 lvaSetStruct(vecZeroLclNum, typeHnd, false);
1633 GenTree* vecZeroAssign = gtNewTempAssign(vecZeroLclNum, vecZero);
1634
1635 // Construct BitVector = v < vector.Zero
1636 GenTree* bitVecOp1 = op1;
1637 GenTree* bitVecOp2 = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1638 var_types relOpBaseType = baseType;
1639 SIMDIntrinsicID relOpIntrinsic =
1640 impSIMDRelOp(SIMDIntrinsicLessThan, typeHnd, size, &relOpBaseType, &bitVecOp1, &bitVecOp2);
1641 GenTree* bitVec = gtNewSIMDNode(simdType, bitVecOp1, bitVecOp2, relOpIntrinsic, relOpBaseType, size);
1642 unsigned bitVecLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs bitVec"));
1643 lvaSetStruct(bitVecLclNum, typeHnd, false);
1644 GenTree* bitVecAssign = gtNewTempAssign(bitVecLclNum, bitVec);
1645 bitVec = gtNewLclvNode(bitVecLclNum, bitVec->TypeGet());
1646
1647 // Construct condSelectOp1 = vector.Zero - v
1648 GenTree* subOp1 = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1649 GenTree* subOp2 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1650 GenTree* negVec = gtNewSIMDNode(simdType, subOp1, subOp2, SIMDIntrinsicSub, baseType, size);
1651
1652 // Construct ConditionalSelect(bitVec, vector.Zero - v, v)
1653 GenTree* vec = gtNewLclvNode(op1LclNum, op1->TypeGet());
1654 retVal = impSIMDSelect(typeHnd, baseType, size, bitVec, negVec, vec);
1655
1656 // Prepend bitVec assignment to retVal.
1657 // retVal = (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1658 retVal = gtNewOperNode(GT_COMMA, simdType, bitVecAssign, retVal);
1659
1660 // Prepend vecZero assignment to retVal.
1661 // retVal = (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1662 retVal = gtNewOperNode(GT_COMMA, simdType, vecZeroAssign, retVal);
1663
1664 // If op1 was assigned to a temp, prepend that to retVal.
1665 if (op1Assign != nullptr)
1666 {
1667 // retVal = (v=op1), (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1668 retVal = gtNewOperNode(GT_COMMA, simdType, op1Assign, retVal);
1669 }
1670 }
1671 else if (varTypeIsFloating(baseType))
1672 {
1673 // Abs(vf) = vf & new SIMDVector<float>(0x7fffffff);
1674 // Abs(vd) = vf & new SIMDVector<double>(0x7fffffffffffffff);
1675 GenTree* bitMask = nullptr;
1676 if (baseType == TYP_FLOAT)
1677 {
1678 float f;
1679 static_assert_no_msg(sizeof(float) == sizeof(int));
1680 *((int*)&f) = 0x7fffffff;
1681 bitMask = gtNewDconNode(f);
1682 }
1683 else if (baseType == TYP_DOUBLE)
1684 {
1685 double d;
1686 static_assert_no_msg(sizeof(double) == sizeof(__int64));
1687 *((__int64*)&d) = 0x7fffffffffffffffLL;
1688 bitMask = gtNewDconNode(d);
1689 }
1690
1691 assert(bitMask != nullptr);
1692 bitMask->gtType = baseType;
1693 GenTree* bitMaskVector = gtNewSIMDNode(simdType, bitMask, SIMDIntrinsicInit, baseType, size);
1694 retVal = gtNewSIMDNode(simdType, op1, bitMaskVector, SIMDIntrinsicBitwiseAnd, baseType, size);
1695 }
1696 else if (baseType == TYP_USHORT || baseType == TYP_UBYTE || baseType == TYP_UINT || baseType == TYP_ULONG)
1697 {
1698 // Abs is a no-op on unsigned integer type vectors
1699 retVal = op1;
1700 }
1701 else
1702 {
1703 assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1704 assert(baseType != TYP_LONG);
1705
1706 retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1707 }
1708#elif defined(_TARGET_ARM64_)
1709 if (varTypeIsUnsigned(baseType))
1710 {
1711 // Abs is a no-op on unsigned integer type vectors
1712 retVal = op1;
1713 }
1714 else
1715 {
1716 retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1717 }
1718#else // !defined(_TARGET_XARCH)_ && !defined(_TARGET_ARM64_)
1719 assert(!"Abs intrinsic on non-xarch target not implemented");
1720#endif // !_TARGET_XARCH_
1721
1722 return retVal;
1723}
1724
1725// Creates a GT_SIMD tree for Select operation
1726//
1727// Arguments:
1728// typeHnd - type handle of SIMD vector
1729// baseType - base type of SIMD vector
1730// size - SIMD vector size
1731// op1 - first operand = Condition vector vc
1732// op2 - second operand = va
1733// op3 - third operand = vb
1734//
1735// Return Value:
1736// Returns GT_SIMD tree that computes Select(vc, va, vb)
1737//
1738GenTree* Compiler::impSIMDSelect(
1739 CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1, GenTree* op2, GenTree* op3)
1740{
1741 assert(varTypeIsSIMD(op1));
1742 var_types simdType = op1->TypeGet();
1743 assert(op2->TypeGet() == simdType);
1744 assert(op3->TypeGet() == simdType);
1745
1746 // TODO-ARM64-CQ Support generating select instruction for SIMD
1747
1748 // Select(BitVector vc, va, vb) = (va & vc) | (vb & !vc)
1749 // Select(op1, op2, op3) = (op2 & op1) | (op3 & !op1)
1750 // = SIMDIntrinsicBitwiseOr(SIMDIntrinsicBitwiseAnd(op2, op1),
1751 // SIMDIntrinsicBitwiseAndNot(op3, op1))
1752 //
1753 // If Op1 has side effect, create an assignment to a temp
1754 GenTree* tmp = op1;
1755 GenTree* asg = nullptr;
1756 if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1757 {
1758 unsigned lclNum = lvaGrabTemp(true DEBUGARG("SIMD Select"));
1759 lvaSetStruct(lclNum, typeHnd, false);
1760 tmp = gtNewLclvNode(lclNum, op1->TypeGet());
1761 asg = gtNewTempAssign(lclNum, op1);
1762 }
1763
1764 GenTree* andExpr = gtNewSIMDNode(simdType, op2, tmp, SIMDIntrinsicBitwiseAnd, baseType, size);
1765 GenTree* dupOp1 = gtCloneExpr(tmp);
1766 assert(dupOp1 != nullptr);
1767#ifdef _TARGET_ARM64_
1768 // ARM64 implements SIMDIntrinsicBitwiseAndNot as Left & ~Right
1769 GenTree* andNotExpr = gtNewSIMDNode(simdType, op3, dupOp1, SIMDIntrinsicBitwiseAndNot, baseType, size);
1770#else
1771 // XARCH implements SIMDIntrinsicBitwiseAndNot as ~Left & Right
1772 GenTree* andNotExpr = gtNewSIMDNode(simdType, dupOp1, op3, SIMDIntrinsicBitwiseAndNot, baseType, size);
1773#endif
1774 GenTree* simdTree = gtNewSIMDNode(simdType, andExpr, andNotExpr, SIMDIntrinsicBitwiseOr, baseType, size);
1775
1776 // If asg not null, create a GT_COMMA tree.
1777 if (asg != nullptr)
1778 {
1779 simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), asg, simdTree);
1780 }
1781
1782 return simdTree;
1783}
1784
1785// Creates a GT_SIMD tree for Min/Max operation
1786//
1787// Arguments:
1788// IntrinsicId - SIMD intrinsic Id, either Min or Max
1789// typeHnd - type handle of SIMD vector
1790// baseType - base type of SIMD vector
1791// size - SIMD vector size
1792// op1 - first operand = va
1793// op2 - second operand = vb
1794//
1795// Return Value:
1796// Returns GT_SIMD tree that computes Max(va, vb)
1797//
1798GenTree* Compiler::impSIMDMinMax(SIMDIntrinsicID intrinsicId,
1799 CORINFO_CLASS_HANDLE typeHnd,
1800 var_types baseType,
1801 unsigned size,
1802 GenTree* op1,
1803 GenTree* op2)
1804{
1805 assert(intrinsicId == SIMDIntrinsicMin || intrinsicId == SIMDIntrinsicMax);
1806 assert(varTypeIsSIMD(op1));
1807 var_types simdType = op1->TypeGet();
1808 assert(op2->TypeGet() == simdType);
1809
1810#if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
1811 GenTree* simdTree = nullptr;
1812
1813#ifdef _TARGET_XARCH_
1814 // SSE2 has direct support for float/double/signed word/unsigned byte.
1815 // SSE4.1 has direct support for int32/uint32/signed byte/unsigned word.
1816 // For other integer types we compute min/max as follows
1817 //
1818 // int32/uint32 (SSE2)
1819 // int64/uint64 (SSE2&SSE4):
1820 // compResult = (op1 < op2) in case of Min
1821 // (op1 > op2) in case of Max
1822 // Min/Max(op1, op2) = Select(compResult, op1, op2)
1823 //
1824 // unsigned word (SSE2):
1825 // op1 = op1 - 2^15 ; to make it fit within a signed word
1826 // op2 = op2 - 2^15 ; to make it fit within a signed word
1827 // result = SSE2 signed word Min/Max(op1, op2)
1828 // result = result + 2^15 ; readjust it back
1829 //
1830 // signed byte (SSE2):
1831 // op1 = op1 + 2^7 ; to make it unsigned
1832 // op1 = op1 + 2^7 ; to make it unsigned
1833 // result = SSE2 unsigned byte Min/Max(op1, op2)
1834 // result = result - 2^15 ; readjust it back
1835
1836 if (varTypeIsFloating(baseType) || baseType == TYP_SHORT || baseType == TYP_UBYTE ||
1837 (getSIMDSupportLevel() >= SIMD_SSE4_Supported &&
1838 (baseType == TYP_BYTE || baseType == TYP_INT || baseType == TYP_UINT || baseType == TYP_USHORT)))
1839 {
1840 // SSE2 or SSE4.1 has direct support
1841 simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1842 }
1843 else if (baseType == TYP_USHORT || baseType == TYP_BYTE)
1844 {
1845 assert(getSIMDSupportLevel() == SIMD_SSE2_Supported);
1846 int constVal;
1847 SIMDIntrinsicID operIntrinsic;
1848 SIMDIntrinsicID adjustIntrinsic;
1849 var_types minMaxOperBaseType;
1850 if (baseType == TYP_USHORT)
1851 {
1852 constVal = 0x80008000;
1853 operIntrinsic = SIMDIntrinsicSub;
1854 adjustIntrinsic = SIMDIntrinsicAdd;
1855 minMaxOperBaseType = TYP_SHORT;
1856 }
1857 else
1858 {
1859 assert(baseType == TYP_BYTE);
1860 constVal = 0x80808080;
1861 operIntrinsic = SIMDIntrinsicAdd;
1862 adjustIntrinsic = SIMDIntrinsicSub;
1863 minMaxOperBaseType = TYP_UBYTE;
1864 }
1865
1866 GenTree* initVal = gtNewIconNode(constVal);
1867 GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
1868
1869 // Assign constVector to a temp, since we intend to use it more than once
1870 // TODO-CQ: We have quite a few such constant vectors constructed during
1871 // the importation of SIMD intrinsics. Make sure that we have a single
1872 // temp per distinct constant per method.
1873 GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1874
1875 // op1 = op1 - constVector
1876 // op2 = op2 - constVector
1877 op1 = gtNewSIMDNode(simdType, op1, constVector, operIntrinsic, baseType, size);
1878 op2 = gtNewSIMDNode(simdType, op2, tmp, operIntrinsic, baseType, size);
1879
1880 // compute min/max of op1 and op2 considering them as if minMaxOperBaseType
1881 simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, minMaxOperBaseType, size);
1882
1883 // re-adjust the value by adding or subtracting constVector
1884 tmp = gtNewLclvNode(tmp->AsLclVarCommon()->GetLclNum(), tmp->TypeGet());
1885 simdTree = gtNewSIMDNode(simdType, simdTree, tmp, adjustIntrinsic, baseType, size);
1886 }
1887#elif defined(_TARGET_ARM64_)
1888 // Arm64 has direct support for all types except int64/uint64
1889 // For which we compute min/max as follows
1890 //
1891 // int64/uint64
1892 // compResult = (op1 < op2) in case of Min
1893 // (op1 > op2) in case of Max
1894 // Min/Max(op1, op2) = Select(compResult, op1, op2)
1895 if (baseType != TYP_ULONG && baseType != TYP_LONG)
1896 {
1897 simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1898 }
1899#endif
1900 else
1901 {
1902 GenTree* dupOp1 = nullptr;
1903 GenTree* dupOp2 = nullptr;
1904 GenTree* op1Assign = nullptr;
1905 GenTree* op2Assign = nullptr;
1906 unsigned op1LclNum;
1907 unsigned op2LclNum;
1908
1909 if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1910 {
1911 op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1912 dupOp1 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1913 lvaSetStruct(op1LclNum, typeHnd, false);
1914 op1Assign = gtNewTempAssign(op1LclNum, op1);
1915 op1 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1916 }
1917 else
1918 {
1919 dupOp1 = gtCloneExpr(op1);
1920 }
1921
1922 if ((op2->gtFlags & GTF_SIDE_EFFECT) != 0)
1923 {
1924 op2LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1925 dupOp2 = gtNewLclvNode(op2LclNum, op2->TypeGet());
1926 lvaSetStruct(op2LclNum, typeHnd, false);
1927 op2Assign = gtNewTempAssign(op2LclNum, op2);
1928 op2 = gtNewLclvNode(op2LclNum, op2->TypeGet());
1929 }
1930 else
1931 {
1932 dupOp2 = gtCloneExpr(op2);
1933 }
1934
1935 SIMDIntrinsicID relOpIntrinsic =
1936 (intrinsicId == SIMDIntrinsicMin) ? SIMDIntrinsicLessThan : SIMDIntrinsicGreaterThan;
1937 var_types relOpBaseType = baseType;
1938
1939 // compResult = op1 relOp op2
1940 // simdTree = Select(compResult, op1, op2);
1941 assert(dupOp1 != nullptr);
1942 assert(dupOp2 != nullptr);
1943 relOpIntrinsic = impSIMDRelOp(relOpIntrinsic, typeHnd, size, &relOpBaseType, &dupOp1, &dupOp2);
1944 GenTree* compResult = gtNewSIMDNode(simdType, dupOp1, dupOp2, relOpIntrinsic, relOpBaseType, size);
1945 unsigned compResultLclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1946 lvaSetStruct(compResultLclNum, typeHnd, false);
1947 GenTree* compResultAssign = gtNewTempAssign(compResultLclNum, compResult);
1948 compResult = gtNewLclvNode(compResultLclNum, compResult->TypeGet());
1949 simdTree = impSIMDSelect(typeHnd, baseType, size, compResult, op1, op2);
1950 simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), compResultAssign, simdTree);
1951
1952 // Now create comma trees if we have created assignments of op1/op2 to temps
1953 if (op2Assign != nullptr)
1954 {
1955 simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op2Assign, simdTree);
1956 }
1957
1958 if (op1Assign != nullptr)
1959 {
1960 simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op1Assign, simdTree);
1961 }
1962 }
1963
1964 assert(simdTree != nullptr);
1965 return simdTree;
1966#else // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1967 assert(!"impSIMDMinMax() unimplemented on target arch");
1968 unreached();
1969#endif // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1970}
1971
1972//------------------------------------------------------------------------
1973// getOp1ForConstructor: Get the op1 for a constructor call.
1974//
1975// Arguments:
1976// opcode - the opcode being handled (needed to identify the CEE_NEWOBJ case)
1977// newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
1978// clsHnd - The handle of the class of the method.
1979//
1980// Return Value:
1981// The tree node representing the object to be initialized with the constructor.
1982//
1983// Notes:
1984// This method handles the differences between the CEE_NEWOBJ and constructor cases.
1985//
1986GenTree* Compiler::getOp1ForConstructor(OPCODE opcode, GenTree* newobjThis, CORINFO_CLASS_HANDLE clsHnd)
1987{
1988 GenTree* op1;
1989 if (opcode == CEE_NEWOBJ)
1990 {
1991 op1 = newobjThis;
1992 assert(newobjThis->gtOper == GT_ADDR && newobjThis->gtOp.gtOp1->gtOper == GT_LCL_VAR);
1993
1994 // push newobj result on type stack
1995 unsigned tmp = op1->gtOp.gtOp1->gtLclVarCommon.gtLclNum;
1996 impPushOnStack(gtNewLclvNode(tmp, lvaGetRealType(tmp)), verMakeTypeInfo(clsHnd).NormaliseForStack());
1997 }
1998 else
1999 {
2000 op1 = impSIMDPopStack(TYP_BYREF);
2001 }
2002 assert(op1->TypeGet() == TYP_BYREF);
2003 return op1;
2004}
2005
2006//-------------------------------------------------------------------
2007// Set the flag that indicates that the lclVar referenced by this tree
2008// is used in a SIMD intrinsic.
2009// Arguments:
2010// tree - GenTree*
2011
2012void Compiler::setLclRelatedToSIMDIntrinsic(GenTree* tree)
2013{
2014 assert(tree->OperIsLocal());
2015 unsigned lclNum = tree->AsLclVarCommon()->GetLclNum();
2016 LclVarDsc* lclVarDsc = &lvaTable[lclNum];
2017 lclVarDsc->lvUsedInSIMDIntrinsic = true;
2018}
2019
2020//-------------------------------------------------------------
2021// Check if two field nodes reference at the same memory location.
2022// Notice that this check is just based on pattern matching.
2023// Arguments:
2024// op1 - GenTree*.
2025// op2 - GenTree*.
2026// Return Value:
2027// If op1's parents node and op2's parents node are at the same location, return true. Otherwise, return false
2028
2029bool areFieldsParentsLocatedSame(GenTree* op1, GenTree* op2)
2030{
2031 assert(op1->OperGet() == GT_FIELD);
2032 assert(op2->OperGet() == GT_FIELD);
2033
2034 GenTree* op1ObjRef = op1->gtField.gtFldObj;
2035 GenTree* op2ObjRef = op2->gtField.gtFldObj;
2036 while (op1ObjRef != nullptr && op2ObjRef != nullptr)
2037 {
2038
2039 if (op1ObjRef->OperGet() != op2ObjRef->OperGet())
2040 {
2041 break;
2042 }
2043 else if (op1ObjRef->OperGet() == GT_ADDR)
2044 {
2045 op1ObjRef = op1ObjRef->gtOp.gtOp1;
2046 op2ObjRef = op2ObjRef->gtOp.gtOp1;
2047 }
2048
2049 if (op1ObjRef->OperIsLocal() && op2ObjRef->OperIsLocal() &&
2050 op1ObjRef->AsLclVarCommon()->GetLclNum() == op2ObjRef->AsLclVarCommon()->GetLclNum())
2051 {
2052 return true;
2053 }
2054 else if (op1ObjRef->OperGet() == GT_FIELD && op2ObjRef->OperGet() == GT_FIELD &&
2055 op1ObjRef->gtField.gtFldHnd == op2ObjRef->gtField.gtFldHnd)
2056 {
2057 op1ObjRef = op1ObjRef->gtField.gtFldObj;
2058 op2ObjRef = op2ObjRef->gtField.gtFldObj;
2059 continue;
2060 }
2061 else
2062 {
2063 break;
2064 }
2065 }
2066
2067 return false;
2068}
2069
2070//----------------------------------------------------------------------
2071// Check whether two field are contiguous
2072// Arguments:
2073// first - GenTree*. The Type of the node should be TYP_FLOAT
2074// second - GenTree*. The Type of the node should be TYP_FLOAT
2075// Return Value:
2076// if the first field is located before second field, and they are located contiguously,
2077// then return true. Otherwise, return false.
2078
2079bool Compiler::areFieldsContiguous(GenTree* first, GenTree* second)
2080{
2081 assert(first->OperGet() == GT_FIELD);
2082 assert(second->OperGet() == GT_FIELD);
2083 assert(first->gtType == TYP_FLOAT);
2084 assert(second->gtType == TYP_FLOAT);
2085
2086 var_types firstFieldType = first->gtType;
2087 var_types secondFieldType = second->gtType;
2088
2089 unsigned firstFieldEndOffset = first->gtField.gtFldOffset + genTypeSize(firstFieldType);
2090 unsigned secondFieldOffset = second->gtField.gtFldOffset;
2091 if (firstFieldEndOffset == secondFieldOffset && firstFieldType == secondFieldType &&
2092 areFieldsParentsLocatedSame(first, second))
2093 {
2094 return true;
2095 }
2096
2097 return false;
2098}
2099
2100//-------------------------------------------------------------------------------
2101// Check whether two array element nodes are located contiguously or not.
2102// Arguments:
2103// op1 - GenTree*.
2104// op2 - GenTree*.
2105// Return Value:
2106// if the array element op1 is located before array element op2, and they are contiguous,
2107// then return true. Otherwise, return false.
2108// TODO-CQ:
2109// Right this can only check array element with const number as index. In future,
2110// we should consider to allow this function to check the index using expression.
2111
2112bool Compiler::areArrayElementsContiguous(GenTree* op1, GenTree* op2)
2113{
2114 noway_assert(op1->gtOper == GT_INDEX);
2115 noway_assert(op2->gtOper == GT_INDEX);
2116 GenTreeIndex* op1Index = op1->AsIndex();
2117 GenTreeIndex* op2Index = op2->AsIndex();
2118
2119 GenTree* op1ArrayRef = op1Index->Arr();
2120 GenTree* op2ArrayRef = op2Index->Arr();
2121 assert(op1ArrayRef->TypeGet() == TYP_REF);
2122 assert(op2ArrayRef->TypeGet() == TYP_REF);
2123
2124 GenTree* op1IndexNode = op1Index->Index();
2125 GenTree* op2IndexNode = op2Index->Index();
2126 if ((op1IndexNode->OperGet() == GT_CNS_INT && op2IndexNode->OperGet() == GT_CNS_INT) &&
2127 op1IndexNode->gtIntCon.gtIconVal + 1 == op2IndexNode->gtIntCon.gtIconVal)
2128 {
2129 if (op1ArrayRef->OperGet() == GT_FIELD && op2ArrayRef->OperGet() == GT_FIELD &&
2130 areFieldsParentsLocatedSame(op1ArrayRef, op2ArrayRef))
2131 {
2132 return true;
2133 }
2134 else if (op1ArrayRef->OperIsLocal() && op2ArrayRef->OperIsLocal() &&
2135 op1ArrayRef->AsLclVarCommon()->GetLclNum() == op2ArrayRef->AsLclVarCommon()->GetLclNum())
2136 {
2137 return true;
2138 }
2139 }
2140 return false;
2141}
2142
2143//-------------------------------------------------------------------------------
2144// Check whether two argument nodes are contiguous or not.
2145// Arguments:
2146// op1 - GenTree*.
2147// op2 - GenTree*.
2148// Return Value:
2149// if the argument node op1 is located before argument node op2, and they are located contiguously,
2150// then return true. Otherwise, return false.
2151// TODO-CQ:
2152// Right now this can only check field and array. In future we should add more cases.
2153//
2154
2155bool Compiler::areArgumentsContiguous(GenTree* op1, GenTree* op2)
2156{
2157 if (op1->OperGet() == GT_INDEX && op2->OperGet() == GT_INDEX)
2158 {
2159 return areArrayElementsContiguous(op1, op2);
2160 }
2161 else if (op1->OperGet() == GT_FIELD && op2->OperGet() == GT_FIELD)
2162 {
2163 return areFieldsContiguous(op1, op2);
2164 }
2165 return false;
2166}
2167
2168//--------------------------------------------------------------------------------------------------------
2169// createAddressNodeForSIMDInit: Generate the address node(GT_LEA) if we want to intialize vector2, vector3 or vector4
2170// from first argument's address.
2171//
2172// Arguments:
2173// tree - GenTree*. This the tree node which is used to get the address for indir.
2174// simdsize - unsigned. This the simd vector size.
2175// arrayElementsCount - unsigned. This is used for generating the boundary check for array.
2176//
2177// Return value:
2178// return the address node.
2179//
2180// TODO-CQ:
2181// 1. Currently just support for GT_FIELD and GT_INDEX, because we can only verify the GT_INDEX node or GT_Field
2182// are located contiguously or not. In future we should support more cases.
2183// 2. Though it happens to just work fine front-end phases are not aware of GT_LEA node. Therefore, convert these
2184// to use GT_ADDR.
2185GenTree* Compiler::createAddressNodeForSIMDInit(GenTree* tree, unsigned simdSize)
2186{
2187 assert(tree->OperGet() == GT_FIELD || tree->OperGet() == GT_INDEX);
2188 GenTree* byrefNode = nullptr;
2189 GenTree* startIndex = nullptr;
2190 unsigned offset = 0;
2191 var_types baseType = tree->gtType;
2192
2193 if (tree->OperGet() == GT_FIELD)
2194 {
2195 GenTree* objRef = tree->gtField.gtFldObj;
2196 if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2197 {
2198 GenTree* obj = objRef->gtOp.gtOp1;
2199
2200 // If the field is directly from a struct, then in this case,
2201 // we should set this struct's lvUsedInSIMDIntrinsic as true,
2202 // so that this sturct won't be promoted.
2203 // e.g. s.x x is a field, and s is a struct, then we should set the s's lvUsedInSIMDIntrinsic as true.
2204 // so that s won't be promoted.
2205 // Notice that if we have a case like s1.s2.x. s1 s2 are struct, and x is a field, then it is possible that
2206 // s1 can be promoted, so that s2 can be promoted. The reason for that is if we don't allow s1 to be
2207 // promoted, then this will affect the other optimizations which are depend on s1's struct promotion.
2208 // TODO-CQ:
2209 // In future, we should optimize this case so that if there is a nested field like s1.s2.x and s1.s2.x's
2210 // address is used for initializing the vector, then s1 can be promoted but s2 can't.
2211 if (varTypeIsSIMD(obj) && obj->OperIsLocal())
2212 {
2213 setLclRelatedToSIMDIntrinsic(obj);
2214 }
2215 }
2216
2217 byrefNode = gtCloneExpr(tree->gtField.gtFldObj);
2218 assert(byrefNode != nullptr);
2219 offset = tree->gtField.gtFldOffset;
2220 }
2221 else if (tree->OperGet() == GT_INDEX)
2222 {
2223
2224 GenTree* index = tree->AsIndex()->Index();
2225 assert(index->OperGet() == GT_CNS_INT);
2226
2227 GenTree* checkIndexExpr = nullptr;
2228 unsigned indexVal = (unsigned)(index->gtIntCon.gtIconVal);
2229 offset = indexVal * genTypeSize(tree->TypeGet());
2230 GenTree* arrayRef = tree->AsIndex()->Arr();
2231
2232 // Generate the boundary check exception.
2233 // The length for boundary check should be the maximum index number which should be
2234 // (first argument's index number) + (how many array arguments we have) - 1
2235 // = indexVal + arrayElementsCount - 1
2236 unsigned arrayElementsCount = simdSize / genTypeSize(baseType);
2237 checkIndexExpr = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, indexVal + arrayElementsCount - 1);
2238 GenTreeArrLen* arrLen = gtNewArrLen(TYP_INT, arrayRef, (int)OFFSETOF__CORINFO_Array__length);
2239 GenTreeBoundsChk* arrBndsChk = new (this, GT_ARR_BOUNDS_CHECK)
2240 GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, SCK_RNGCHK_FAIL);
2241
2242 offset += OFFSETOF__CORINFO_Array__data;
2243 byrefNode = gtNewOperNode(GT_COMMA, arrayRef->TypeGet(), arrBndsChk, gtCloneExpr(arrayRef));
2244 }
2245 else
2246 {
2247 unreached();
2248 }
2249 GenTree* address =
2250 new (this, GT_LEA) GenTreeAddrMode(TYP_BYREF, byrefNode, startIndex, genTypeSize(tree->TypeGet()), offset);
2251 return address;
2252}
2253
2254//-------------------------------------------------------------------------------
2255// impMarkContiguousSIMDFieldAssignments: Try to identify if there are contiguous
2256// assignments from SIMD field to memory. If there are, then mark the related
2257// lclvar so that it won't be promoted.
2258//
2259// Arguments:
2260// stmt - GenTree*. Input statement node.
2261
2262void Compiler::impMarkContiguousSIMDFieldAssignments(GenTree* stmt)
2263{
2264 if (!featureSIMD || opts.OptimizationDisabled())
2265 {
2266 return;
2267 }
2268 GenTree* expr = stmt->gtStmt.gtStmtExpr;
2269 if (expr->OperGet() == GT_ASG && expr->TypeGet() == TYP_FLOAT)
2270 {
2271 GenTree* curDst = expr->gtOp.gtOp1;
2272 GenTree* curSrc = expr->gtOp.gtOp2;
2273 unsigned index = 0;
2274 var_types baseType = TYP_UNKNOWN;
2275 unsigned simdSize = 0;
2276 GenTree* srcSimdStructNode = getSIMDStructFromField(curSrc, &baseType, &index, &simdSize, true);
2277 if (srcSimdStructNode == nullptr || baseType != TYP_FLOAT)
2278 {
2279 fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2280 }
2281 else if (index == 0 && isSIMDTypeLocal(srcSimdStructNode))
2282 {
2283 fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2284 }
2285 else if (fgPreviousCandidateSIMDFieldAsgStmt != nullptr)
2286 {
2287 assert(index > 0);
2288 GenTree* prevAsgExpr = fgPreviousCandidateSIMDFieldAsgStmt->gtStmt.gtStmtExpr;
2289 GenTree* prevDst = prevAsgExpr->gtOp.gtOp1;
2290 GenTree* prevSrc = prevAsgExpr->gtOp.gtOp2;
2291 if (!areArgumentsContiguous(prevDst, curDst) || !areArgumentsContiguous(prevSrc, curSrc))
2292 {
2293 fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2294 }
2295 else
2296 {
2297 if (index == (simdSize / genTypeSize(baseType) - 1))
2298 {
2299 // Successfully found the pattern, mark the lclvar as UsedInSIMDIntrinsic
2300 if (srcSimdStructNode->OperIsLocal())
2301 {
2302 setLclRelatedToSIMDIntrinsic(srcSimdStructNode);
2303 }
2304
2305 if (curDst->OperGet() == GT_FIELD)
2306 {
2307 GenTree* objRef = curDst->gtField.gtFldObj;
2308 if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2309 {
2310 GenTree* obj = objRef->gtOp.gtOp1;
2311 if (varTypeIsStruct(obj) && obj->OperIsLocal())
2312 {
2313 setLclRelatedToSIMDIntrinsic(obj);
2314 }
2315 }
2316 }
2317 }
2318 else
2319 {
2320 fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2321 }
2322 }
2323 }
2324 }
2325 else
2326 {
2327 fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2328 }
2329}
2330
2331//------------------------------------------------------------------------
2332// impSIMDIntrinsic: Check method to see if it is a SIMD method
2333//
2334// Arguments:
2335// opcode - the opcode being handled (needed to identify the CEE_NEWOBJ case)
2336// newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
2337// clsHnd - The handle of the class of the method.
2338// method - The handle of the method.
2339// sig - The call signature for the method.
2340// memberRef - The memberRef token for the method reference.
2341//
2342// Return Value:
2343// If clsHnd is a known SIMD type, and 'method' is one of the methods that are
2344// implemented as an intrinsic in the JIT, then return the tree that implements
2345// it.
2346//
2347GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode,
2348 GenTree* newobjThis,
2349 CORINFO_CLASS_HANDLE clsHnd,
2350 CORINFO_METHOD_HANDLE methodHnd,
2351 CORINFO_SIG_INFO* sig,
2352 unsigned methodFlags,
2353 int memberRef)
2354{
2355 assert(featureSIMD);
2356
2357 // Exit early if we are not in one of the SIMD types.
2358 if (!isSIMDClass(clsHnd))
2359 {
2360 return nullptr;
2361 }
2362
2363#ifdef FEATURE_CORECLR
2364 // For coreclr, we also exit early if the method is not a JIT Intrinsic (which requires the [Intrinsic] attribute).
2365 if ((methodFlags & CORINFO_FLG_JIT_INTRINSIC) == 0)
2366 {
2367 return nullptr;
2368 }
2369#endif // FEATURE_CORECLR
2370
2371 // Get base type and intrinsic Id
2372 var_types baseType = TYP_UNKNOWN;
2373 unsigned size = 0;
2374 unsigned argCount = 0;
2375 const SIMDIntrinsicInfo* intrinsicInfo =
2376 getSIMDIntrinsicInfo(&clsHnd, methodHnd, sig, (opcode == CEE_NEWOBJ), &argCount, &baseType, &size);
2377 if (intrinsicInfo == nullptr || intrinsicInfo->id == SIMDIntrinsicInvalid)
2378 {
2379 return nullptr;
2380 }
2381
2382 SIMDIntrinsicID simdIntrinsicID = intrinsicInfo->id;
2383 var_types simdType;
2384 if (baseType != TYP_UNKNOWN)
2385 {
2386 simdType = getSIMDTypeForSize(size);
2387 }
2388 else
2389 {
2390 assert(simdIntrinsicID == SIMDIntrinsicHWAccel);
2391 simdType = TYP_UNKNOWN;
2392 }
2393 bool instMethod = intrinsicInfo->isInstMethod;
2394 var_types callType = JITtype2varType(sig->retType);
2395 if (callType == TYP_STRUCT)
2396 {
2397 // Note that here we are assuming that, if the call returns a struct, that it is the same size as the
2398 // struct on which the method is declared. This is currently true for all methods on Vector types,
2399 // but if this ever changes, we will need to determine the callType from the signature.
2400 assert(info.compCompHnd->getClassSize(sig->retTypeClass) == genTypeSize(simdType));
2401 callType = simdType;
2402 }
2403
2404 GenTree* simdTree = nullptr;
2405 GenTree* op1 = nullptr;
2406 GenTree* op2 = nullptr;
2407 GenTree* op3 = nullptr;
2408 GenTree* retVal = nullptr;
2409 GenTree* copyBlkDst = nullptr;
2410 bool doCopyBlk = false;
2411
2412 switch (simdIntrinsicID)
2413 {
2414 case SIMDIntrinsicGetCount:
2415 {
2416 int length = getSIMDVectorLength(clsHnd);
2417 GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, length);
2418 retVal = intConstTree;
2419
2420 intConstTree->gtFlags |= GTF_ICON_SIMD_COUNT;
2421 }
2422 break;
2423
2424 case SIMDIntrinsicGetZero:
2425 retVal = gtNewSIMDVectorZero(simdType, baseType, size);
2426 break;
2427
2428 case SIMDIntrinsicGetOne:
2429 retVal = gtNewSIMDVectorOne(simdType, baseType, size);
2430 break;
2431
2432 case SIMDIntrinsicGetAllOnes:
2433 {
2434 // Equivalent to (Vector<T>) new Vector<int>(0xffffffff);
2435 GenTree* initVal = gtNewIconNode(0xffffffff, TYP_INT);
2436 simdTree = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
2437 if (baseType != TYP_INT)
2438 {
2439 // cast it to required baseType if different from TYP_INT
2440 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2441 }
2442 retVal = simdTree;
2443 }
2444 break;
2445
2446 case SIMDIntrinsicInit:
2447 case SIMDIntrinsicInitN:
2448 {
2449 // SIMDIntrinsicInit:
2450 // op2 - the initializer value
2451 // op1 - byref of vector
2452 //
2453 // SIMDIntrinsicInitN
2454 // op2 - list of initializer values stitched into a list
2455 // op1 - byref of vector
2456 bool initFromFirstArgIndir = false;
2457 if (simdIntrinsicID == SIMDIntrinsicInit)
2458 {
2459 op2 = impSIMDPopStack(baseType);
2460 }
2461 else
2462 {
2463 assert(simdIntrinsicID == SIMDIntrinsicInitN);
2464 assert(baseType == TYP_FLOAT);
2465
2466 unsigned initCount = argCount - 1;
2467 unsigned elementCount = getSIMDVectorLength(size, baseType);
2468 noway_assert(initCount == elementCount);
2469 GenTree* nextArg = op2;
2470
2471 // Build a GT_LIST with the N values.
2472 // We must maintain left-to-right order of the args, but we will pop
2473 // them off in reverse order (the Nth arg was pushed onto the stack last).
2474
2475 GenTree* list = nullptr;
2476 GenTree* firstArg = nullptr;
2477 GenTree* prevArg = nullptr;
2478 int offset = 0;
2479 bool areArgsContiguous = true;
2480 for (unsigned i = 0; i < initCount; i++)
2481 {
2482 GenTree* nextArg = impSIMDPopStack(baseType);
2483 if (areArgsContiguous)
2484 {
2485 GenTree* curArg = nextArg;
2486 firstArg = curArg;
2487
2488 if (prevArg != nullptr)
2489 {
2490 // Recall that we are popping the args off the stack in reverse order.
2491 areArgsContiguous = areArgumentsContiguous(curArg, prevArg);
2492 }
2493 prevArg = curArg;
2494 }
2495
2496 list = new (this, GT_LIST) GenTreeOp(GT_LIST, baseType, nextArg, list);
2497 }
2498
2499 if (areArgsContiguous && baseType == TYP_FLOAT)
2500 {
2501 // Since Vector2, Vector3 and Vector4's arguments type are only float,
2502 // we intialize the vector from first argument address, only when
2503 // the baseType is TYP_FLOAT and the arguments are located contiguously in memory
2504 initFromFirstArgIndir = true;
2505 GenTree* op2Address = createAddressNodeForSIMDInit(firstArg, size);
2506 var_types simdType = getSIMDTypeForSize(size);
2507 op2 = gtNewOperNode(GT_IND, simdType, op2Address);
2508 }
2509 else
2510 {
2511 op2 = list;
2512 }
2513 }
2514
2515 op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2516
2517 assert(op1->TypeGet() == TYP_BYREF);
2518 assert(genActualType(op2->TypeGet()) == genActualType(baseType) || initFromFirstArgIndir);
2519
2520 // For integral base types of size less than TYP_INT, expand the initializer
2521 // to fill size of TYP_INT bytes.
2522 if (varTypeIsSmallInt(baseType))
2523 {
2524 // This case should occur only for Init intrinsic.
2525 assert(simdIntrinsicID == SIMDIntrinsicInit);
2526
2527 unsigned baseSize = genTypeSize(baseType);
2528 int multiplier;
2529 if (baseSize == 1)
2530 {
2531 multiplier = 0x01010101;
2532 }
2533 else
2534 {
2535 assert(baseSize == 2);
2536 multiplier = 0x00010001;
2537 }
2538
2539 GenTree* t1 = nullptr;
2540 if (baseType == TYP_BYTE)
2541 {
2542 // What we have is a signed byte initializer,
2543 // which when loaded to a reg will get sign extended to TYP_INT.
2544 // But what we need is the initializer without sign extended or
2545 // rather zero extended to 32-bits.
2546 t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xff, TYP_INT));
2547 }
2548 else if (baseType == TYP_SHORT)
2549 {
2550 // What we have is a signed short initializer,
2551 // which when loaded to a reg will get sign extended to TYP_INT.
2552 // But what we need is the initializer without sign extended or
2553 // rather zero extended to 32-bits.
2554 t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xffff, TYP_INT));
2555 }
2556 else
2557 {
2558 assert(baseType == TYP_UBYTE || baseType == TYP_USHORT);
2559 t1 = gtNewCastNode(TYP_INT, op2, false, TYP_INT);
2560 }
2561
2562 assert(t1 != nullptr);
2563 GenTree* t2 = gtNewIconNode(multiplier, TYP_INT);
2564 op2 = gtNewOperNode(GT_MUL, TYP_INT, t1, t2);
2565
2566 // Construct a vector of TYP_INT with the new initializer and cast it back to vector of baseType
2567 simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, TYP_INT, size);
2568 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2569 }
2570 else
2571 {
2572
2573 if (initFromFirstArgIndir)
2574 {
2575 simdTree = op2;
2576 if (op1->gtOp.gtOp1->OperIsLocal())
2577 {
2578 // label the dst struct's lclvar is used for SIMD intrinsic,
2579 // so that this dst struct won't be promoted.
2580 setLclRelatedToSIMDIntrinsic(op1->gtOp.gtOp1);
2581 }
2582 }
2583 else
2584 {
2585 simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, baseType, size);
2586 }
2587 }
2588
2589 copyBlkDst = op1;
2590 doCopyBlk = true;
2591 }
2592 break;
2593
2594 case SIMDIntrinsicInitArray:
2595 case SIMDIntrinsicInitArrayX:
2596 case SIMDIntrinsicCopyToArray:
2597 case SIMDIntrinsicCopyToArrayX:
2598 {
2599 // op3 - index into array in case of SIMDIntrinsicCopyToArrayX and SIMDIntrinsicInitArrayX
2600 // op2 - array itself
2601 // op1 - byref to vector struct
2602
2603 unsigned int vectorLength = getSIMDVectorLength(size, baseType);
2604 // (This constructor takes only the zero-based arrays.)
2605 // We will add one or two bounds checks:
2606 // 1. If we have an index, we must do a check on that first.
2607 // We can't combine it with the index + vectorLength check because
2608 // a. It might be negative, and b. It may need to raise a different exception
2609 // (captured as SCK_ARG_RNG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init).
2610 // 2. We need to generate a check (SCK_ARG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init)
2611 // for the last array element we will access.
2612 // We'll either check against (vectorLength - 1) or (index + vectorLength - 1).
2613
2614 GenTree* checkIndexExpr = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength - 1);
2615
2616 // Get the index into the array. If it has been provided, it will be on the
2617 // top of the stack. Otherwise, it is null.
2618 if (argCount == 3)
2619 {
2620 op3 = impSIMDPopStack(TYP_INT);
2621 if (op3->IsIntegralConst(0))
2622 {
2623 op3 = nullptr;
2624 }
2625 }
2626 else
2627 {
2628 // TODO-CQ: Here, or elsewhere, check for the pattern where op2 is a newly constructed array, and
2629 // change this to the InitN form.
2630 // op3 = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 0);
2631 op3 = nullptr;
2632 }
2633
2634 // Clone the array for use in the bounds check.
2635 op2 = impSIMDPopStack(TYP_REF);
2636 assert(op2->TypeGet() == TYP_REF);
2637 GenTree* arrayRefForArgChk = op2;
2638 GenTree* argRngChk = nullptr;
2639 GenTree* asg = nullptr;
2640 if ((arrayRefForArgChk->gtFlags & GTF_SIDE_EFFECT) != 0)
2641 {
2642 op2 = fgInsertCommaFormTemp(&arrayRefForArgChk);
2643 }
2644 else
2645 {
2646 op2 = gtCloneExpr(arrayRefForArgChk);
2647 }
2648 assert(op2 != nullptr);
2649
2650 if (op3 != nullptr)
2651 {
2652 SpecialCodeKind op3CheckKind;
2653 if (simdIntrinsicID == SIMDIntrinsicInitArrayX)
2654 {
2655 op3CheckKind = SCK_RNGCHK_FAIL;
2656 }
2657 else
2658 {
2659 assert(simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2660 op3CheckKind = SCK_ARG_RNG_EXCPN;
2661 }
2662 // We need to use the original expression on this, which is the first check.
2663 GenTree* arrayRefForArgRngChk = arrayRefForArgChk;
2664 // Then we clone the clone we just made for the next check.
2665 arrayRefForArgChk = gtCloneExpr(op2);
2666 // We know we MUST have had a cloneable expression.
2667 assert(arrayRefForArgChk != nullptr);
2668 GenTree* index = op3;
2669 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2670 {
2671 op3 = fgInsertCommaFormTemp(&index);
2672 }
2673 else
2674 {
2675 op3 = gtCloneExpr(index);
2676 }
2677
2678 GenTreeArrLen* arrLen =
2679 gtNewArrLen(TYP_INT, arrayRefForArgRngChk, (int)OFFSETOF__CORINFO_Array__length);
2680 argRngChk = new (this, GT_ARR_BOUNDS_CHECK)
2681 GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, index, arrLen, op3CheckKind);
2682 // Now, clone op3 to create another node for the argChk
2683 GenTree* index2 = gtCloneExpr(op3);
2684 assert(index != nullptr);
2685 checkIndexExpr = gtNewOperNode(GT_ADD, TYP_INT, index2, checkIndexExpr);
2686 }
2687
2688 // Insert a bounds check for index + offset - 1.
2689 // This must be a "normal" array.
2690 SpecialCodeKind op2CheckKind;
2691 if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2692 {
2693 op2CheckKind = SCK_RNGCHK_FAIL;
2694 }
2695 else
2696 {
2697 op2CheckKind = SCK_ARG_EXCPN;
2698 }
2699 GenTreeArrLen* arrLen = gtNewArrLen(TYP_INT, arrayRefForArgChk, (int)OFFSETOF__CORINFO_Array__length);
2700 GenTreeBoundsChk* argChk = new (this, GT_ARR_BOUNDS_CHECK)
2701 GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, op2CheckKind);
2702
2703 // Create a GT_COMMA tree for the bounds check(s).
2704 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argChk, op2);
2705 if (argRngChk != nullptr)
2706 {
2707 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argRngChk, op2);
2708 }
2709
2710 if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2711 {
2712 op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2713 simdTree = gtNewSIMDNode(simdType, op2, op3, SIMDIntrinsicInitArray, baseType, size);
2714 copyBlkDst = op1;
2715 doCopyBlk = true;
2716 }
2717 else
2718 {
2719 assert(simdIntrinsicID == SIMDIntrinsicCopyToArray || simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2720 op1 = impSIMDPopStack(simdType, instMethod);
2721 assert(op1->TypeGet() == simdType);
2722
2723 // copy vector (op1) to array (op2) starting at index (op3)
2724 simdTree = op1;
2725
2726 // TODO-Cleanup: Though it happens to just work fine front-end phases are not aware of GT_LEA node.
2727 // Therefore, convert these to use GT_ADDR .
2728 copyBlkDst = new (this, GT_LEA)
2729 GenTreeAddrMode(TYP_BYREF, op2, op3, genTypeSize(baseType), OFFSETOF__CORINFO_Array__data);
2730 doCopyBlk = true;
2731 }
2732 }
2733 break;
2734
2735 case SIMDIntrinsicInitFixed:
2736 {
2737 // We are initializing a fixed-length vector VLarge with a smaller fixed-length vector VSmall, plus 1 or 2
2738 // additional floats.
2739 // op4 (optional) - float value for VLarge.W, if VLarge is Vector4, and VSmall is Vector2
2740 // op3 - float value for VLarge.Z or VLarge.W
2741 // op2 - VSmall
2742 // op1 - byref of VLarge
2743 assert(baseType == TYP_FLOAT);
2744 unsigned elementByteCount = 4;
2745
2746 GenTree* op4 = nullptr;
2747 if (argCount == 4)
2748 {
2749 op4 = impSIMDPopStack(TYP_FLOAT);
2750 assert(op4->TypeGet() == TYP_FLOAT);
2751 }
2752 op3 = impSIMDPopStack(TYP_FLOAT);
2753 assert(op3->TypeGet() == TYP_FLOAT);
2754 // The input vector will either be TYP_SIMD8 or TYP_SIMD12.
2755 var_types smallSIMDType = TYP_SIMD8;
2756 if ((op4 == nullptr) && (simdType == TYP_SIMD16))
2757 {
2758 smallSIMDType = TYP_SIMD12;
2759 }
2760 op2 = impSIMDPopStack(smallSIMDType);
2761 op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2762
2763 // We are going to redefine the operands so that:
2764 // - op3 is the value that's going into the Z position, or null if it's a Vector4 constructor with a single
2765 // operand, and
2766 // - op4 is the W position value, or null if this is a Vector3 constructor.
2767 if (size == 16 && argCount == 3)
2768 {
2769 op4 = op3;
2770 op3 = nullptr;
2771 }
2772
2773 simdTree = op2;
2774 if (op3 != nullptr)
2775 {
2776 simdTree = gtNewSIMDNode(simdType, simdTree, op3, SIMDIntrinsicSetZ, baseType, size);
2777 }
2778 if (op4 != nullptr)
2779 {
2780 simdTree = gtNewSIMDNode(simdType, simdTree, op4, SIMDIntrinsicSetW, baseType, size);
2781 }
2782
2783 copyBlkDst = op1;
2784 doCopyBlk = true;
2785 }
2786 break;
2787
2788 case SIMDIntrinsicOpEquality:
2789 case SIMDIntrinsicInstEquals:
2790 {
2791 op2 = impSIMDPopStack(simdType);
2792 op1 = impSIMDPopStack(simdType, instMethod);
2793
2794 assert(op1->TypeGet() == simdType);
2795 assert(op2->TypeGet() == simdType);
2796
2797 simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpEquality, baseType, size);
2798 if (simdType == TYP_SIMD12)
2799 {
2800 simdTree->gtFlags |= GTF_SIMD12_OP;
2801 }
2802 retVal = simdTree;
2803 }
2804 break;
2805
2806 case SIMDIntrinsicOpInEquality:
2807 {
2808 // op1 is the first operand
2809 // op2 is the second operand
2810 op2 = impSIMDPopStack(simdType);
2811 op1 = impSIMDPopStack(simdType, instMethod);
2812 simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpInEquality, baseType, size);
2813 if (simdType == TYP_SIMD12)
2814 {
2815 simdTree->gtFlags |= GTF_SIMD12_OP;
2816 }
2817 retVal = simdTree;
2818 }
2819 break;
2820
2821 case SIMDIntrinsicEqual:
2822 case SIMDIntrinsicLessThan:
2823 case SIMDIntrinsicLessThanOrEqual:
2824 case SIMDIntrinsicGreaterThan:
2825 case SIMDIntrinsicGreaterThanOrEqual:
2826 {
2827 op2 = impSIMDPopStack(simdType);
2828 op1 = impSIMDPopStack(simdType, instMethod);
2829
2830 SIMDIntrinsicID intrinsicID = impSIMDRelOp(simdIntrinsicID, clsHnd, size, &baseType, &op1, &op2);
2831 simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, intrinsicID, baseType, size);
2832 retVal = simdTree;
2833 }
2834 break;
2835
2836 case SIMDIntrinsicAdd:
2837 case SIMDIntrinsicSub:
2838 case SIMDIntrinsicMul:
2839 case SIMDIntrinsicDiv:
2840 case SIMDIntrinsicBitwiseAnd:
2841 case SIMDIntrinsicBitwiseAndNot:
2842 case SIMDIntrinsicBitwiseOr:
2843 case SIMDIntrinsicBitwiseXor:
2844 {
2845#if defined(DEBUG)
2846 // check for the cases where we don't support intrinsics.
2847 // This check should be done before we make modifications to type stack.
2848 // Note that this is more of a double safety check for robustness since
2849 // we expect getSIMDIntrinsicInfo() to have filtered out intrinsics on
2850 // unsupported base types. If getSIMdIntrinsicInfo() doesn't filter due
2851 // to some bug, assert in chk/dbg will fire.
2852 if (!varTypeIsFloating(baseType))
2853 {
2854 if (simdIntrinsicID == SIMDIntrinsicMul)
2855 {
2856#if defined(_TARGET_XARCH_)
2857 if ((baseType != TYP_INT) && (baseType != TYP_SHORT))
2858 {
2859 // TODO-CQ: implement mul on these integer vectors.
2860 // Note that SSE2 has no direct support for these vectors.
2861 assert(!"Mul not supported on long/ulong/uint/small int vectors\n");
2862 return nullptr;
2863 }
2864#endif // _TARGET_XARCH_
2865#if defined(_TARGET_ARM64_)
2866 if ((baseType == TYP_ULONG) && (baseType == TYP_LONG))
2867 {
2868 // TODO-CQ: implement mul on these integer vectors.
2869 // Note that ARM64 has no direct support for these vectors.
2870 assert(!"Mul not supported on long/ulong vectors\n");
2871 return nullptr;
2872 }
2873#endif // _TARGET_ARM64_
2874 }
2875#if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2876 // common to all integer type vectors
2877 if (simdIntrinsicID == SIMDIntrinsicDiv)
2878 {
2879 // SSE2 doesn't support div on non-floating point vectors.
2880 assert(!"Div not supported on integer type vectors\n");
2881 return nullptr;
2882 }
2883#endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2884 }
2885#endif // DEBUG
2886
2887 // op1 is the first operand; if instance method, op1 is "this" arg
2888 // op2 is the second operand
2889 op2 = impSIMDPopStack(simdType);
2890 op1 = impSIMDPopStack(simdType, instMethod);
2891
2892#ifdef _TARGET_XARCH_
2893 if (simdIntrinsicID == SIMDIntrinsicBitwiseAndNot)
2894 {
2895 // XARCH implements SIMDIntrinsicBitwiseAndNot as ~op1 & op2, while the
2896 // software implementation does op1 & ~op2, so we need to swap the operands
2897
2898 GenTree* tmp = op2;
2899 op2 = op1;
2900 op1 = tmp;
2901 }
2902#endif // _TARGET_XARCH_
2903
2904 simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
2905 retVal = simdTree;
2906 }
2907 break;
2908
2909 case SIMDIntrinsicSelect:
2910 {
2911 // op3 is a SIMD variable that is the second source
2912 // op2 is a SIMD variable that is the first source
2913 // op1 is a SIMD variable which is the bit mask.
2914 op3 = impSIMDPopStack(simdType);
2915 op2 = impSIMDPopStack(simdType);
2916 op1 = impSIMDPopStack(simdType);
2917
2918 retVal = impSIMDSelect(clsHnd, baseType, size, op1, op2, op3);
2919 }
2920 break;
2921
2922 case SIMDIntrinsicMin:
2923 case SIMDIntrinsicMax:
2924 {
2925 // op1 is the first operand; if instance method, op1 is "this" arg
2926 // op2 is the second operand
2927 op2 = impSIMDPopStack(simdType);
2928 op1 = impSIMDPopStack(simdType, instMethod);
2929
2930 retVal = impSIMDMinMax(simdIntrinsicID, clsHnd, baseType, size, op1, op2);
2931 }
2932 break;
2933
2934 case SIMDIntrinsicGetItem:
2935 {
2936 // op1 is a SIMD variable that is "this" arg
2937 // op2 is an index of TYP_INT
2938 op2 = impSIMDPopStack(TYP_INT);
2939 op1 = impSIMDPopStack(simdType, instMethod);
2940 int vectorLength = getSIMDVectorLength(size, baseType);
2941 if (!op2->IsCnsIntOrI() || op2->AsIntCon()->gtIconVal >= vectorLength || op2->AsIntCon()->gtIconVal < 0)
2942 {
2943 // We need to bounds-check the length of the vector.
2944 // For that purpose, we need to clone the index expression.
2945 GenTree* index = op2;
2946 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2947 {
2948 op2 = fgInsertCommaFormTemp(&index);
2949 }
2950 else
2951 {
2952 op2 = gtCloneExpr(index);
2953 }
2954
2955 GenTree* lengthNode = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength);
2956 GenTreeBoundsChk* simdChk =
2957 new (this, GT_SIMD_CHK) GenTreeBoundsChk(GT_SIMD_CHK, TYP_VOID, index, lengthNode, SCK_RNGCHK_FAIL);
2958
2959 // Create a GT_COMMA tree for the bounds check.
2960 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), simdChk, op2);
2961 }
2962
2963 assert(op1->TypeGet() == simdType);
2964 assert(op2->TypeGet() == TYP_INT);
2965
2966 simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, simdIntrinsicID, baseType, size);
2967 retVal = simdTree;
2968 }
2969 break;
2970
2971 case SIMDIntrinsicDotProduct:
2972 {
2973#if defined(_TARGET_XARCH_)
2974 // Right now dot product is supported only for float/double vectors and
2975 // int vectors on SSE4/AVX.
2976 if (!varTypeIsFloating(baseType) && !(baseType == TYP_INT && getSIMDSupportLevel() >= SIMD_SSE4_Supported))
2977 {
2978 return nullptr;
2979 }
2980#endif // _TARGET_XARCH_
2981
2982 // op1 is a SIMD variable that is the first source and also "this" arg.
2983 // op2 is a SIMD variable which is the second source.
2984 op2 = impSIMDPopStack(simdType);
2985 op1 = impSIMDPopStack(simdType, instMethod);
2986
2987 simdTree = gtNewSIMDNode(baseType, op1, op2, simdIntrinsicID, baseType, size);
2988 if (simdType == TYP_SIMD12)
2989 {
2990 simdTree->gtFlags |= GTF_SIMD12_OP;
2991 }
2992 retVal = simdTree;
2993 }
2994 break;
2995
2996 case SIMDIntrinsicSqrt:
2997 {
2998#if (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
2999 // SSE/AVX/ARM64 doesn't support sqrt on integer type vectors and hence
3000 // should never be seen as an intrinsic here. See SIMDIntrinsicList.h
3001 // for supported base types for this intrinsic.
3002 if (!varTypeIsFloating(baseType))
3003 {
3004 assert(!"Sqrt not supported on integer vectors\n");
3005 return nullptr;
3006 }
3007#endif // (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
3008
3009 op1 = impSIMDPopStack(simdType);
3010
3011 retVal = gtNewSIMDNode(genActualType(callType), op1, nullptr, simdIntrinsicID, baseType, size);
3012 }
3013 break;
3014
3015 case SIMDIntrinsicAbs:
3016 op1 = impSIMDPopStack(simdType);
3017 retVal = impSIMDAbs(clsHnd, baseType, size, op1);
3018 break;
3019
3020 case SIMDIntrinsicGetW:
3021 retVal = impSIMDGetFixed(simdType, baseType, size, 3);
3022 break;
3023
3024 case SIMDIntrinsicGetZ:
3025 retVal = impSIMDGetFixed(simdType, baseType, size, 2);
3026 break;
3027
3028 case SIMDIntrinsicGetY:
3029 retVal = impSIMDGetFixed(simdType, baseType, size, 1);
3030 break;
3031
3032 case SIMDIntrinsicGetX:
3033 retVal = impSIMDGetFixed(simdType, baseType, size, 0);
3034 break;
3035
3036 case SIMDIntrinsicSetW:
3037 case SIMDIntrinsicSetZ:
3038 case SIMDIntrinsicSetY:
3039 case SIMDIntrinsicSetX:
3040 {
3041 // op2 is the value to be set at indexTemp position
3042 // op1 is SIMD vector that is going to be modified, which is a byref
3043
3044 // If op1 has a side-effect, then don't make it an intrinsic.
3045 // It would be in-efficient to read the entire vector into xmm reg,
3046 // modify it and write back entire xmm reg.
3047 //
3048 // TODO-CQ: revisit this later.
3049 op1 = impStackTop(1).val;
3050 if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
3051 {
3052 return nullptr;
3053 }
3054
3055 op2 = impSIMDPopStack(baseType);
3056 op1 = impSIMDPopStack(simdType, instMethod);
3057
3058 GenTree* src = gtCloneExpr(op1);
3059 assert(src != nullptr);
3060 simdTree = gtNewSIMDNode(simdType, src, op2, simdIntrinsicID, baseType, size);
3061
3062 copyBlkDst = gtNewOperNode(GT_ADDR, TYP_BYREF, op1);
3063 doCopyBlk = true;
3064 }
3065 break;
3066
3067 // Unary operators that take and return a Vector.
3068 case SIMDIntrinsicCast:
3069 case SIMDIntrinsicConvertToSingle:
3070 case SIMDIntrinsicConvertToDouble:
3071 case SIMDIntrinsicConvertToInt32:
3072 {
3073 op1 = impSIMDPopStack(simdType, instMethod);
3074
3075 simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3076 retVal = simdTree;
3077 }
3078 break;
3079
3080 case SIMDIntrinsicConvertToInt64:
3081 {
3082#ifdef _TARGET_64BIT_
3083 op1 = impSIMDPopStack(simdType, instMethod);
3084
3085 simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3086 retVal = simdTree;
3087#else
3088 JITDUMP("SIMD Conversion to Int64 is not supported on this platform\n");
3089 return nullptr;
3090#endif
3091 }
3092 break;
3093
3094 case SIMDIntrinsicNarrow:
3095 {
3096 assert(!instMethod);
3097 op2 = impSIMDPopStack(simdType);
3098 op1 = impSIMDPopStack(simdType);
3099 // op1 and op2 are two input Vector<T>.
3100 simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
3101 retVal = simdTree;
3102 }
3103 break;
3104
3105 case SIMDIntrinsicWiden:
3106 {
3107 GenTree* dstAddrHi = impSIMDPopStack(TYP_BYREF);
3108 GenTree* dstAddrLo = impSIMDPopStack(TYP_BYREF);
3109 op1 = impSIMDPopStack(simdType);
3110 GenTree* dupOp1 = fgInsertCommaFormTemp(&op1, gtGetStructHandleForSIMD(simdType, baseType));
3111
3112 // Widen the lower half and assign it to dstAddrLo.
3113 simdTree = gtNewSIMDNode(simdType, op1, nullptr, SIMDIntrinsicWidenLo, baseType, size);
3114 GenTree* loDest =
3115 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrLo, getSIMDTypeSizeInBytes(clsHnd));
3116 GenTree* loAsg = gtNewBlkOpNode(loDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3117 false, // not volatile
3118 true); // copyBlock
3119 loAsg->gtFlags |= ((simdTree->gtFlags | dstAddrLo->gtFlags) & GTF_ALL_EFFECT);
3120
3121 // Widen the upper half and assign it to dstAddrHi.
3122 simdTree = gtNewSIMDNode(simdType, dupOp1, nullptr, SIMDIntrinsicWidenHi, baseType, size);
3123 GenTree* hiDest =
3124 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrHi, getSIMDTypeSizeInBytes(clsHnd));
3125 GenTree* hiAsg = gtNewBlkOpNode(hiDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3126 false, // not volatile
3127 true); // copyBlock
3128 hiAsg->gtFlags |= ((simdTree->gtFlags | dstAddrHi->gtFlags) & GTF_ALL_EFFECT);
3129
3130 retVal = gtNewOperNode(GT_COMMA, simdType, loAsg, hiAsg);
3131 }
3132 break;
3133
3134 case SIMDIntrinsicHWAccel:
3135 {
3136 GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 1);
3137 retVal = intConstTree;
3138 }
3139 break;
3140
3141 default:
3142 assert(!"Unimplemented SIMD Intrinsic");
3143 return nullptr;
3144 }
3145
3146#if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3147 // XArch/Arm64: also indicate that we use floating point registers.
3148 // The need for setting this here is that a method may not have SIMD
3149 // type lclvars, but might be exercising SIMD intrinsics on fields of
3150 // SIMD type.
3151 //
3152 // e.g. public Vector<float> ComplexVecFloat::sqabs() { return this.r * this.r + this.i * this.i; }
3153 compFloatingPointUsed = true;
3154#endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3155
3156 // At this point, we have a tree that we are going to store into a destination.
3157 // TODO-1stClassStructs: This should be a simple store or assignment, and should not require
3158 // GTF_ALL_EFFECT for the dest. This is currently emulating the previous behavior of
3159 // block ops.
3160 if (doCopyBlk)
3161 {
3162 GenTree* dest = new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, copyBlkDst, getSIMDTypeSizeInBytes(clsHnd));
3163 dest->gtFlags |= GTF_GLOB_REF;
3164 retVal = gtNewBlkOpNode(dest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3165 false, // not volatile
3166 true); // copyBlock
3167 retVal->gtFlags |= ((simdTree->gtFlags | copyBlkDst->gtFlags) & GTF_ALL_EFFECT);
3168 }
3169
3170 return retVal;
3171}
3172
3173#endif // FEATURE_SIMD
3174