1// Licensed to the .NET Foundation under one or more agreements.
2// The .NET Foundation licenses this file to you under the MIT license.
3// See the LICENSE file in the project root for more information.
4
5#include "jitpch.h"
6#ifdef _MSC_VER
7#pragma hdrstop
8#endif
9
10// The IndirectCallTransformer transforms indirect calls that involve fat function
11// pointers or guarded devirtualization candidates. These transformations introduce
12// control flow and so can't easily be done in the importer.
13//
14// A fat function pointer is a pointer with the second least significant bit
15// (aka FAT_POINTER_MASK) set. If the bit is set, the pointer (after clearing the bit)
16// actually points to a tuple <method pointer, instantiation argument pointer> where
17// instantiationArgument is a hidden first argument required by method pointer.
18//
19// Fat pointers are used in CoreRT as a replacement for instantiating stubs,
20// because CoreRT can't generate stubs in runtime.
21//
22// The JIT is responsible for emitting code to check the bit at runtime, branching
23// to one of two call sites.
24//
25// When the bit is not set, the code should execute the original indirect call.
26//
27// When the bit is set, the code should mask off the bit, use the resulting pointer
28// to load the real target address and the extra argument, and then call indirect
29// via the target, passing the extra argument.
30//
31// before:
32// current block
33// {
34// previous statements
35// transforming statement
36// {
37// call with GTF_CALL_M_FAT_POINTER_CHECK flag set in function ptr
38// }
39// subsequent statements
40// }
41//
42// after:
43// current block
44// {
45// previous statements
46// } BBJ_NONE check block
47// check block
48// {
49// jump to else if function ptr has the FAT_POINTER_MASK bit set.
50// } BBJ_COND then block, else block
51// then block
52// {
53// original statement
54// } BBJ_ALWAYS remainder block
55// else block
56// {
57// clear FAT_POINTER_MASK bit
58// load actual function pointer
59// load instantiation argument
60// create newArgList = (instantiation argument, original argList)
61// call (actual function pointer, newArgList)
62// } BBJ_NONE remainder block
63// remainder block
64// {
65// subsequent statements
66// }
67//
68class IndirectCallTransformer
69{
70public:
71 IndirectCallTransformer(Compiler* compiler) : compiler(compiler)
72 {
73 }
74
75 //------------------------------------------------------------------------
76 // Run: run transformation for each block.
77 //
78 // Returns:
79 // Count of calls transformed.
80 int Run()
81 {
82 int count = 0;
83
84 for (BasicBlock* block = compiler->fgFirstBB; block != nullptr; block = block->bbNext)
85 {
86 count += TransformBlock(block);
87 }
88
89 return count;
90 }
91
92private:
93 //------------------------------------------------------------------------
94 // TransformBlock: look through statements and transform statements with
95 // particular indirect calls
96 //
97 // Returns:
98 // Count of calls transformed.
99 //
100 int TransformBlock(BasicBlock* block)
101 {
102 int count = 0;
103
104 for (GenTreeStmt* stmt = block->firstStmt(); stmt != nullptr; stmt = stmt->gtNextStmt)
105 {
106 if (ContainsFatCalli(stmt))
107 {
108 FatPointerCallTransformer transformer(compiler, block, stmt);
109 transformer.Run();
110 count++;
111 }
112
113 if (ContainsGuardedDevirtualizationCandidate(stmt))
114 {
115 GuardedDevirtualizationTransformer transformer(compiler, block, stmt);
116 transformer.Run();
117 count++;
118 }
119 }
120
121 return count;
122 }
123
124 //------------------------------------------------------------------------
125 // ContainsFatCalli: check does this statement contain fat pointer call.
126 //
127 // Checks fatPointerCandidate in form of call() or lclVar = call().
128 //
129 // Return Value:
130 // true if contains, false otherwise.
131 //
132 bool ContainsFatCalli(GenTreeStmt* stmt)
133 {
134 GenTree* fatPointerCandidate = stmt->gtStmtExpr;
135 if (fatPointerCandidate->OperIs(GT_ASG))
136 {
137 fatPointerCandidate = fatPointerCandidate->gtGetOp2();
138 }
139 return fatPointerCandidate->IsCall() && fatPointerCandidate->AsCall()->IsFatPointerCandidate();
140 }
141
142 //------------------------------------------------------------------------
143 // ContainsGuardedDevirtualizationCandidate: check does this statement contain a virtual
144 // call that we'd like to guardedly devirtualize?
145 //
146 // Return Value:
147 // true if contains, false otherwise.
148 //
149 // Notes:
150 // calls are hoisted to top level ... (we hope)
151 bool ContainsGuardedDevirtualizationCandidate(GenTreeStmt* stmt)
152 {
153 GenTree* candidate = stmt->gtStmtExpr;
154 return candidate->IsCall() && candidate->AsCall()->IsGuardedDevirtualizationCandidate();
155 }
156
157 class Transformer
158 {
159 public:
160 Transformer(Compiler* compiler, BasicBlock* block, GenTreeStmt* stmt)
161 : compiler(compiler), currBlock(block), stmt(stmt)
162 {
163 remainderBlock = nullptr;
164 checkBlock = nullptr;
165 thenBlock = nullptr;
166 elseBlock = nullptr;
167 origCall = nullptr;
168 }
169
170 //------------------------------------------------------------------------
171 // Run: transform the statement as described above.
172 //
173 virtual void Run()
174 {
175 Transform();
176 }
177
178 void Transform()
179 {
180 JITDUMP("*** %s: transforming [%06u]\n", Name(), compiler->dspTreeID(stmt));
181 FixupRetExpr();
182 ClearFlag();
183 CreateRemainder();
184 CreateCheck();
185 CreateThen();
186 CreateElse();
187 RemoveOldStatement();
188 SetWeights();
189 ChainFlow();
190 }
191
192 protected:
193 virtual const char* Name() = 0;
194 virtual void ClearFlag() = 0;
195 virtual GenTreeCall* GetCall(GenTreeStmt* callStmt) = 0;
196 virtual void FixupRetExpr() = 0;
197
198 //------------------------------------------------------------------------
199 // CreateRemainder: split current block at the call stmt and
200 // insert statements after the call into remainderBlock.
201 //
202 void CreateRemainder()
203 {
204 remainderBlock = compiler->fgSplitBlockAfterStatement(currBlock, stmt);
205 unsigned propagateFlags = currBlock->bbFlags & BBF_GC_SAFE_POINT;
206 remainderBlock->bbFlags |= BBF_JMP_TARGET | BBF_HAS_LABEL | propagateFlags;
207 }
208
209 virtual void CreateCheck() = 0;
210
211 //------------------------------------------------------------------------
212 // CreateAndInsertBasicBlock: ask compiler to create new basic block.
213 // and insert in into the basic block list.
214 //
215 // Arguments:
216 // jumpKind - jump kind for the new basic block
217 // insertAfter - basic block, after which compiler has to insert the new one.
218 //
219 // Return Value:
220 // new basic block.
221 BasicBlock* CreateAndInsertBasicBlock(BBjumpKinds jumpKind, BasicBlock* insertAfter)
222 {
223 BasicBlock* block = compiler->fgNewBBafter(jumpKind, insertAfter, true);
224 if ((insertAfter->bbFlags & BBF_INTERNAL) == 0)
225 {
226 block->bbFlags &= ~BBF_INTERNAL;
227 block->bbFlags |= BBF_IMPORTED;
228 }
229 return block;
230 }
231
232 virtual void CreateThen() = 0;
233 virtual void CreateElse() = 0;
234
235 //------------------------------------------------------------------------
236 // RemoveOldStatement: remove original stmt from current block.
237 //
238 void RemoveOldStatement()
239 {
240 compiler->fgRemoveStmt(currBlock, stmt);
241 }
242
243 //------------------------------------------------------------------------
244 // SetWeights: set weights for new blocks.
245 //
246 void SetWeights()
247 {
248 remainderBlock->inheritWeight(currBlock);
249 checkBlock->inheritWeight(currBlock);
250 thenBlock->inheritWeightPercentage(currBlock, HIGH_PROBABILITY);
251 elseBlock->inheritWeightPercentage(currBlock, 100 - HIGH_PROBABILITY);
252 }
253
254 //------------------------------------------------------------------------
255 // ChainFlow: link new blocks into correct cfg.
256 //
257 void ChainFlow()
258 {
259 assert(!compiler->fgComputePredsDone);
260 checkBlock->bbJumpDest = elseBlock;
261 thenBlock->bbJumpDest = remainderBlock;
262 }
263
264 Compiler* compiler;
265 BasicBlock* currBlock;
266 BasicBlock* remainderBlock;
267 BasicBlock* checkBlock;
268 BasicBlock* thenBlock;
269 BasicBlock* elseBlock;
270 GenTreeStmt* stmt;
271 GenTreeCall* origCall;
272
273 const int HIGH_PROBABILITY = 80;
274 };
275
276 class FatPointerCallTransformer final : public Transformer
277 {
278 public:
279 FatPointerCallTransformer(Compiler* compiler, BasicBlock* block, GenTreeStmt* stmt)
280 : Transformer(compiler, block, stmt)
281 {
282 doesReturnValue = stmt->gtStmtExpr->OperIs(GT_ASG);
283 origCall = GetCall(stmt);
284 fptrAddress = origCall->gtCallAddr;
285 pointerType = fptrAddress->TypeGet();
286 }
287
288 protected:
289 virtual const char* Name()
290 {
291 return "FatPointerCall";
292 }
293
294 //------------------------------------------------------------------------
295 // GetCall: find a call in a statement.
296 //
297 // Arguments:
298 // callStmt - the statement with the call inside.
299 //
300 // Return Value:
301 // call tree node pointer.
302 virtual GenTreeCall* GetCall(GenTreeStmt* callStmt)
303 {
304 GenTree* tree = callStmt->gtStmtExpr;
305 GenTreeCall* call = nullptr;
306 if (doesReturnValue)
307 {
308 assert(tree->OperIs(GT_ASG));
309 call = tree->gtGetOp2()->AsCall();
310 }
311 else
312 {
313 call = tree->AsCall(); // call with void return type.
314 }
315 return call;
316 }
317
318 //------------------------------------------------------------------------
319 // ClearFlag: clear fat pointer candidate flag from the original call.
320 //
321 virtual void ClearFlag()
322 {
323 origCall->ClearFatPointerCandidate();
324 }
325
326 // FixupRetExpr: no action needed as we handle this in the importer.
327 virtual void FixupRetExpr()
328 {
329 }
330
331 //------------------------------------------------------------------------
332 // CreateCheck: create check block, that checks fat pointer bit set.
333 //
334 virtual void CreateCheck()
335 {
336 checkBlock = CreateAndInsertBasicBlock(BBJ_COND, currBlock);
337 GenTree* fatPointerMask = new (compiler, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, FAT_POINTER_MASK);
338 GenTree* fptrAddressCopy = compiler->gtCloneExpr(fptrAddress);
339 GenTree* fatPointerAnd = compiler->gtNewOperNode(GT_AND, TYP_I_IMPL, fptrAddressCopy, fatPointerMask);
340 GenTree* zero = new (compiler, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, 0);
341 GenTree* fatPointerCmp = compiler->gtNewOperNode(GT_NE, TYP_INT, fatPointerAnd, zero);
342 GenTree* jmpTree = compiler->gtNewOperNode(GT_JTRUE, TYP_VOID, fatPointerCmp);
343 GenTree* jmpStmt = compiler->fgNewStmtFromTree(jmpTree, stmt->gtStmt.gtStmtILoffsx);
344 compiler->fgInsertStmtAtEnd(checkBlock, jmpStmt);
345 }
346
347 //------------------------------------------------------------------------
348 // CreateThen: create then block, that is executed if the check succeeds.
349 // This simply executes the original call.
350 //
351 virtual void CreateThen()
352 {
353 thenBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, checkBlock);
354 GenTree* copyOfOriginalStmt = compiler->gtCloneExpr(stmt)->AsStmt();
355 compiler->fgInsertStmtAtEnd(thenBlock, copyOfOriginalStmt);
356 }
357
358 //------------------------------------------------------------------------
359 // CreateElse: create else block, that is executed if call address is fat pointer.
360 //
361 virtual void CreateElse()
362 {
363 elseBlock = CreateAndInsertBasicBlock(BBJ_NONE, thenBlock);
364
365 GenTree* fixedFptrAddress = GetFixedFptrAddress();
366 GenTree* actualCallAddress = compiler->gtNewOperNode(GT_IND, pointerType, fixedFptrAddress);
367 GenTree* hiddenArgument = GetHiddenArgument(fixedFptrAddress);
368
369 GenTreeStmt* fatStmt = CreateFatCallStmt(actualCallAddress, hiddenArgument);
370 compiler->fgInsertStmtAtEnd(elseBlock, fatStmt);
371 }
372
373 //------------------------------------------------------------------------
374 // GetFixedFptrAddress: clear fat pointer bit from fat pointer address.
375 //
376 // Return Value:
377 // address without fat pointer bit set.
378 GenTree* GetFixedFptrAddress()
379 {
380 GenTree* fptrAddressCopy = compiler->gtCloneExpr(fptrAddress);
381 GenTree* fatPointerMask = new (compiler, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, FAT_POINTER_MASK);
382 return compiler->gtNewOperNode(GT_SUB, pointerType, fptrAddressCopy, fatPointerMask);
383 }
384
385 //------------------------------------------------------------------------
386 // GetHiddenArgument: load hidden argument.
387 //
388 // Arguments:
389 // fixedFptrAddress - pointer to the tuple <methodPointer, instantiationArgumentPointer>
390 //
391 // Return Value:
392 // generic context hidden argument.
393 GenTree* GetHiddenArgument(GenTree* fixedFptrAddress)
394 {
395 GenTree* fixedFptrAddressCopy = compiler->gtCloneExpr(fixedFptrAddress);
396 GenTree* wordSize = new (compiler, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, genTypeSize(TYP_I_IMPL));
397 GenTree* hiddenArgumentPtrPtr =
398 compiler->gtNewOperNode(GT_ADD, pointerType, fixedFptrAddressCopy, wordSize);
399 GenTree* hiddenArgumentPtr = compiler->gtNewOperNode(GT_IND, pointerType, hiddenArgumentPtrPtr);
400 return compiler->gtNewOperNode(GT_IND, fixedFptrAddressCopy->TypeGet(), hiddenArgumentPtr);
401 }
402
403 //------------------------------------------------------------------------
404 // CreateFatCallStmt: create call with fixed call address and hidden argument in the args list.
405 //
406 // Arguments:
407 // actualCallAddress - fixed call address
408 // hiddenArgument - generic context hidden argument
409 //
410 // Return Value:
411 // created call node.
412 GenTreeStmt* CreateFatCallStmt(GenTree* actualCallAddress, GenTree* hiddenArgument)
413 {
414 GenTreeStmt* fatStmt = compiler->gtCloneExpr(stmt)->AsStmt();
415 GenTree* fatTree = fatStmt->gtStmtExpr;
416 GenTreeCall* fatCall = GetCall(fatStmt);
417 fatCall->gtCallAddr = actualCallAddress;
418 AddHiddenArgument(fatCall, hiddenArgument);
419 return fatStmt;
420 }
421
422 //------------------------------------------------------------------------
423 // AddHiddenArgument: add hidden argument to the call argument list.
424 //
425 // Arguments:
426 // fatCall - fat call node
427 // hiddenArgument - generic context hidden argument
428 //
429 void AddHiddenArgument(GenTreeCall* fatCall, GenTree* hiddenArgument)
430 {
431 GenTreeArgList* oldArgs = fatCall->gtCallArgs;
432 GenTreeArgList* newArgs;
433#if USER_ARGS_COME_LAST
434 if (fatCall->HasRetBufArg())
435 {
436 GenTree* retBuffer = oldArgs->Current();
437 GenTreeArgList* rest = oldArgs->Rest();
438 newArgs = compiler->gtNewListNode(hiddenArgument, rest);
439 newArgs = compiler->gtNewListNode(retBuffer, newArgs);
440 }
441 else
442 {
443 newArgs = compiler->gtNewListNode(hiddenArgument, oldArgs);
444 }
445#else
446 newArgs = oldArgs;
447 AddArgumentToTail(newArgs, hiddenArgument);
448#endif
449 fatCall->gtCallArgs = newArgs;
450 }
451
452 //------------------------------------------------------------------------
453 // AddArgumentToTail: add hidden argument to the tail of the call argument list.
454 //
455 // Arguments:
456 // argList - fat call node
457 // hiddenArgument - generic context hidden argument
458 //
459 void AddArgumentToTail(GenTreeArgList* argList, GenTree* hiddenArgument)
460 {
461 GenTreeArgList* iterator = argList;
462 while (iterator->Rest() != nullptr)
463 {
464 iterator = iterator->Rest();
465 }
466 iterator->Rest() = compiler->gtNewArgList(hiddenArgument);
467 }
468
469 private:
470 const int FAT_POINTER_MASK = 0x2;
471
472 GenTree* fptrAddress;
473 var_types pointerType;
474 bool doesReturnValue;
475 };
476
477 class GuardedDevirtualizationTransformer final : public Transformer
478 {
479 public:
480 GuardedDevirtualizationTransformer(Compiler* compiler, BasicBlock* block, GenTreeStmt* stmt)
481 : Transformer(compiler, block, stmt), returnTemp(BAD_VAR_NUM)
482 {
483 }
484
485 //------------------------------------------------------------------------
486 // Run: transform the statement as described above.
487 //
488 virtual void Run()
489 {
490 origCall = GetCall(stmt);
491
492 JITDUMP("*** %s contemplating [%06u]\n", Name(), compiler->dspTreeID(origCall));
493
494 // We currently need inline candidate info to guarded devirt.
495 if (!origCall->IsInlineCandidate())
496 {
497 JITDUMP("*** %s Bailing on [%06u] -- not an inline candidate\n", Name(), compiler->dspTreeID(origCall));
498 ClearFlag();
499 return;
500 }
501
502 // For now, bail on transforming calls that still appear
503 // to return structs by value as there is deferred work
504 // needed to fix up the return type.
505 //
506 // See for instance fgUpdateInlineReturnExpressionPlaceHolder.
507 if (origCall->TypeGet() == TYP_STRUCT)
508 {
509 JITDUMP("*** %s Bailing on [%06u] -- can't handle by-value struct returns yet\n", Name(),
510 compiler->dspTreeID(origCall));
511 ClearFlag();
512
513 // For stub calls restore the stub address
514 if (origCall->IsVirtualStub())
515 {
516 origCall->gtStubCallStubAddr = origCall->gtInlineCandidateInfo->stubAddr;
517 }
518 return;
519 }
520
521 Transform();
522 }
523
524 protected:
525 virtual const char* Name()
526 {
527 return "GuardedDevirtualization";
528 }
529
530 //------------------------------------------------------------------------
531 // GetCall: find a call in a statement.
532 //
533 // Arguments:
534 // callStmt - the statement with the call inside.
535 //
536 // Return Value:
537 // call tree node pointer.
538 virtual GenTreeCall* GetCall(GenTreeStmt* callStmt)
539 {
540 GenTree* tree = callStmt->gtStmtExpr;
541 assert(tree->IsCall());
542 GenTreeCall* call = tree->AsCall();
543 return call;
544 }
545
546 //------------------------------------------------------------------------
547 // ClearFlag: clear guarded devirtualization candidate flag from the original call.
548 //
549 virtual void ClearFlag()
550 {
551 origCall->ClearGuardedDevirtualizationCandidate();
552 }
553
554 //------------------------------------------------------------------------
555 // CreateCheck: create check block and check method table
556 //
557 virtual void CreateCheck()
558 {
559 checkBlock = CreateAndInsertBasicBlock(BBJ_COND, currBlock);
560
561 // Fetch method table from object arg to call.
562 GenTree* thisTree = compiler->gtCloneExpr(origCall->gtCallObjp);
563
564 // Create temp for this if the tree is costly.
565 if (!thisTree->IsLocal())
566 {
567 const unsigned thisTempNum = compiler->lvaGrabTemp(true DEBUGARG("guarded devirt this temp"));
568 // lvaSetClass(thisTempNum, ...);
569 GenTree* asgTree = compiler->gtNewTempAssign(thisTempNum, thisTree);
570 GenTree* asgStmt = compiler->fgNewStmtFromTree(asgTree, stmt->gtStmt.gtStmtILoffsx);
571 compiler->fgInsertStmtAtEnd(checkBlock, asgStmt);
572
573 thisTree = compiler->gtNewLclvNode(thisTempNum, TYP_REF);
574
575 // Propagate the new this to the call. Must be a new expr as the call
576 // will live on in the else block and thisTree is used below.
577 origCall->gtCallObjp = compiler->gtNewLclvNode(thisTempNum, TYP_REF);
578 }
579
580 GenTree* methodTable = compiler->gtNewIndir(TYP_I_IMPL, thisTree);
581 methodTable->gtFlags |= GTF_IND_INVARIANT;
582
583 // Find target method table
584 GuardedDevirtualizationCandidateInfo* guardedInfo = origCall->gtGuardedDevirtualizationCandidateInfo;
585 CORINFO_CLASS_HANDLE clsHnd = guardedInfo->guardedClassHandle;
586 GenTree* targetMethodTable = compiler->gtNewIconEmbClsHndNode(clsHnd);
587
588 // Compare and jump to else (which does the indirect call) if NOT equal
589 GenTree* methodTableCompare = compiler->gtNewOperNode(GT_NE, TYP_INT, targetMethodTable, methodTable);
590 GenTree* jmpTree = compiler->gtNewOperNode(GT_JTRUE, TYP_VOID, methodTableCompare);
591 GenTree* jmpStmt = compiler->fgNewStmtFromTree(jmpTree, stmt->gtStmt.gtStmtILoffsx);
592 compiler->fgInsertStmtAtEnd(checkBlock, jmpStmt);
593 }
594
595 //------------------------------------------------------------------------
596 // FixupRetExpr: set up to repair return value placeholder from call
597 //
598 virtual void FixupRetExpr()
599 {
600 // If call returns a value, we need to copy it to a temp, and
601 // update the associated GT_RET_EXPR to refer to the temp instead
602 // of the call.
603 //
604 // Note implicit by-ref returns should have already been converted
605 // so any struct copy we induce here should be cheap.
606 //
607 // Todo: make sure we understand how this interacts with return type
608 // munging for small structs.
609 InlineCandidateInfo* inlineInfo = origCall->gtInlineCandidateInfo;
610 GenTree* retExpr = inlineInfo->retExpr;
611
612 // Sanity check the ret expr if non-null: it should refer to the original call.
613 if (retExpr != nullptr)
614 {
615 assert(retExpr->gtRetExpr.gtInlineCandidate == origCall);
616 }
617
618 if (origCall->TypeGet() != TYP_VOID)
619 {
620 returnTemp = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt return temp"));
621 JITDUMP("Reworking call(s) to return value via a new temp V%02u\n", returnTemp);
622
623 if (varTypeIsStruct(origCall))
624 {
625 compiler->lvaSetStruct(returnTemp, origCall->gtRetClsHnd, false);
626 }
627
628 GenTree* tempTree = compiler->gtNewLclvNode(returnTemp, origCall->TypeGet());
629
630 JITDUMP("Updating GT_RET_EXPR [%06u] to refer to temp V%02u\n", compiler->dspTreeID(retExpr),
631 returnTemp);
632 retExpr->gtRetExpr.gtInlineCandidate = tempTree;
633 }
634 else if (retExpr != nullptr)
635 {
636 // We still oddly produce GT_RET_EXPRs for some void
637 // returning calls. Just patch the ret expr to a NOP.
638 //
639 // Todo: consider bagging creation of these RET_EXPRs. The only possible
640 // benefit they provide is stitching back larger trees for failed inlines
641 // of void-returning methods. But then the calls likely sit in commas and
642 // the benefit of a larger tree is unclear.
643 JITDUMP("Updating GT_RET_EXPR [%06u] for VOID return to refer to a NOP\n",
644 compiler->dspTreeID(retExpr));
645 GenTree* nopTree = compiler->gtNewNothingNode();
646 retExpr->gtRetExpr.gtInlineCandidate = nopTree;
647 }
648 else
649 {
650 // We do not produce GT_RET_EXPRs for CTOR calls, so there is nothing to patch.
651 }
652 }
653
654 //------------------------------------------------------------------------
655 // CreateThen: create else block with direct call to method
656 //
657 virtual void CreateThen()
658 {
659 thenBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, checkBlock);
660
661 InlineCandidateInfo* inlineInfo = origCall->gtInlineCandidateInfo;
662 CORINFO_CLASS_HANDLE clsHnd = inlineInfo->clsHandle;
663
664 // copy 'this' to temp with exact type.
665 const unsigned thisTemp = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt this exact temp"));
666 GenTree* clonedObj = compiler->gtCloneExpr(origCall->gtCallObjp);
667 GenTree* assign = compiler->gtNewTempAssign(thisTemp, clonedObj);
668 compiler->lvaSetClass(thisTemp, clsHnd, true);
669 GenTreeStmt* assignStmt = compiler->gtNewStmt(assign);
670 compiler->fgInsertStmtAtEnd(thenBlock, assignStmt);
671
672 // Clone call. Note we must use the special candidate helper.
673 GenTreeCall* call = compiler->gtCloneCandidateCall(origCall);
674 call->gtCallObjp = compiler->gtNewLclvNode(thisTemp, TYP_REF);
675 call->SetIsGuarded();
676
677 JITDUMP("Direct call [%06u] in block BB%02u\n", compiler->dspTreeID(call), thenBlock->bbNum);
678
679 // Then invoke impDevirtualizeCall do actually
680 // transform the call for us. It should succeed.... as we have
681 // now provided an exact typed this.
682 CORINFO_METHOD_HANDLE methodHnd = inlineInfo->methInfo.ftn;
683 unsigned methodFlags = inlineInfo->methAttr;
684 CORINFO_CONTEXT_HANDLE context = inlineInfo->exactContextHnd;
685 const bool isLateDevirtualization = true;
686 compiler->impDevirtualizeCall(call, &methodHnd, &methodFlags, &context, nullptr, isLateDevirtualization);
687
688 // Presumably devirt might fail? If so we should try and avoid
689 // making this a guarded devirt candidate instead of ending
690 // up here.
691 assert(!call->IsVirtual());
692
693 // Re-establish this call as an inline candidate.
694 GenTree* oldRetExpr = inlineInfo->retExpr;
695 inlineInfo->clsHandle = clsHnd;
696 inlineInfo->exactContextHnd = context;
697 call->gtInlineCandidateInfo = inlineInfo;
698
699 // Add the call
700 GenTreeStmt* callStmt = compiler->gtNewStmt(call);
701 compiler->fgInsertStmtAtEnd(thenBlock, callStmt);
702
703 // If there was a ret expr for this call, we need to create a new one
704 // and append it just after the call.
705 //
706 // Note the original GT_RET_EXPR is sitting at the join point of the
707 // guarded expansion and for non-void calls, and now refers to a temp local;
708 // we set all this up in FixupRetExpr().
709 if (oldRetExpr != nullptr)
710 {
711 GenTree* retExpr = compiler->gtNewInlineCandidateReturnExpr(call, call->TypeGet());
712 inlineInfo->retExpr = retExpr;
713
714 if (returnTemp != BAD_VAR_NUM)
715 {
716 retExpr = compiler->gtNewTempAssign(returnTemp, retExpr);
717 }
718 else
719 {
720 // We should always have a return temp if we return results by value
721 assert(origCall->TypeGet() == TYP_VOID);
722 }
723
724 GenTreeStmt* resultStmt = compiler->gtNewStmt(retExpr);
725 compiler->fgInsertStmtAtEnd(thenBlock, resultStmt);
726 }
727 }
728
729 //------------------------------------------------------------------------
730 // CreateElse: create else block. This executes the unaltered indirect call.
731 //
732 virtual void CreateElse()
733 {
734 elseBlock = CreateAndInsertBasicBlock(BBJ_NONE, thenBlock);
735 GenTreeCall* call = origCall;
736 GenTreeStmt* newStmt = compiler->gtNewStmt(call);
737
738 call->gtFlags &= ~GTF_CALL_INLINE_CANDIDATE;
739 call->SetIsGuarded();
740
741 JITDUMP("Residual call [%06u] moved to block BB%02u\n", compiler->dspTreeID(call), elseBlock->bbNum);
742
743 if (returnTemp != BAD_VAR_NUM)
744 {
745 GenTree* assign = compiler->gtNewTempAssign(returnTemp, call);
746 newStmt->gtStmtExpr = assign;
747 }
748
749 // For stub calls, restore the stub address. For everything else,
750 // null out the candidate info field.
751 if (call->IsVirtualStub())
752 {
753 JITDUMP("Restoring stub addr %p from candidate info\n", call->gtInlineCandidateInfo->stubAddr);
754 call->gtStubCallStubAddr = call->gtInlineCandidateInfo->stubAddr;
755 }
756 else
757 {
758 call->gtInlineCandidateInfo = nullptr;
759 }
760
761 compiler->fgInsertStmtAtEnd(elseBlock, newStmt);
762
763 // Set the original statement to a nop.
764 stmt->gtStmtExpr = compiler->gtNewNothingNode();
765 }
766
767 private:
768 unsigned returnTemp;
769 };
770
771 Compiler* compiler;
772};
773
774#ifdef DEBUG
775
776//------------------------------------------------------------------------
777// fgDebugCheckForTransformableIndirectCalls: callback to make sure there
778// are no more GTF_CALL_M_FAT_POINTER_CHECK or GTF_CALL_M_GUARDED_DEVIRT
779// calls remaining
780//
781Compiler::fgWalkResult Compiler::fgDebugCheckForTransformableIndirectCalls(GenTree** pTree, fgWalkData* data)
782{
783 GenTree* tree = *pTree;
784 if (tree->IsCall())
785 {
786 assert(!tree->AsCall()->IsFatPointerCandidate());
787 assert(!tree->AsCall()->IsGuardedDevirtualizationCandidate());
788 }
789 return WALK_CONTINUE;
790}
791
792//------------------------------------------------------------------------
793// CheckNoTransformableIndirectCallsRemain: walk through blocks and check
794// that there are no indirect call candidates left to transform.
795//
796void Compiler::CheckNoTransformableIndirectCallsRemain()
797{
798 assert(!doesMethodHaveFatPointer());
799 assert(!doesMethodHaveGuardedDevirtualization());
800
801 for (BasicBlock* block = fgFirstBB; block != nullptr; block = block->bbNext)
802 {
803 for (GenTreeStmt* stmt = fgFirstBB->firstStmt(); stmt != nullptr; stmt = stmt->gtNextStmt)
804 {
805 fgWalkTreePre(&stmt->gtStmtExpr, fgDebugCheckForTransformableIndirectCalls);
806 }
807 }
808}
809#endif
810
811//------------------------------------------------------------------------
812// fgTransformIndirectCalls: find and transform various indirect calls
813//
814// These transformations happen post-import because they may introduce
815// control flow.
816//
817void Compiler::fgTransformIndirectCalls()
818{
819 JITDUMP("\n*************** in fgTransformIndirectCalls(%s)\n", compIsForInlining() ? "inlinee" : "root");
820
821 if (doesMethodHaveFatPointer() || doesMethodHaveGuardedDevirtualization())
822 {
823 IndirectCallTransformer indirectCallTransformer(this);
824 int count = indirectCallTransformer.Run();
825
826 if (count > 0)
827 {
828 JITDUMP("\n*************** After fgTransformIndirectCalls() [%d calls transformed]\n", count);
829 INDEBUG(if (verbose) { fgDispBasicBlocks(true); });
830 }
831 else
832 {
833 JITDUMP(" -- no transforms done (?)\n");
834 }
835
836 clearMethodHasFatPointer();
837 clearMethodHasGuardedDevirtualization();
838 }
839 else
840 {
841 JITDUMP(" -- no candidates to transform\n");
842 }
843
844 INDEBUG(CheckNoTransformableIndirectCallsRemain(););
845}
846