1// Licensed to the .NET Foundation under one or more agreements.
2// The .NET Foundation licenses this file to you under the MIT license.
3// See the LICENSE file in the project root for more information.
4
5#include "jitpch.h"
6#ifdef _MSC_VER
7#pragma hdrstop
8#endif
9
10#include "inlinepolicy.h"
11#include "sm.h"
12
13//------------------------------------------------------------------------
14// getPolicy: Factory method for getting an InlinePolicy
15//
16// Arguments:
17// compiler - the compiler instance that will evaluate inlines
18// isPrejitRoot - true if this policy is evaluating a prejit root
19//
20// Return Value:
21// InlinePolicy to use in evaluating an inline.
22//
23// Notes:
24// Determines which of the various policies should apply,
25// and creates (or reuses) a policy instance to use.
26
27InlinePolicy* InlinePolicy::GetPolicy(Compiler* compiler, bool isPrejitRoot)
28{
29
30#if defined(DEBUG) || defined(INLINE_DATA)
31
32#if defined(DEBUG)
33 const bool useRandomPolicyForStress = compiler->compRandomInlineStress();
34#else
35 const bool useRandomPolicyForStress = false;
36#endif // defined(DEBUG)
37
38 const bool useRandomPolicy = (JitConfig.JitInlinePolicyRandom() != 0);
39
40 // Optionally install the RandomPolicy.
41 if (useRandomPolicyForStress || useRandomPolicy)
42 {
43 return new (compiler, CMK_Inlining) RandomPolicy(compiler, isPrejitRoot);
44 }
45
46 // Optionally install the ReplayPolicy.
47 bool useReplayPolicy = JitConfig.JitInlinePolicyReplay() != 0;
48
49 if (useReplayPolicy)
50 {
51 return new (compiler, CMK_Inlining) ReplayPolicy(compiler, isPrejitRoot);
52 }
53
54 // Optionally install the SizePolicy.
55 bool useSizePolicy = JitConfig.JitInlinePolicySize() != 0;
56
57 if (useSizePolicy)
58 {
59 return new (compiler, CMK_Inlining) SizePolicy(compiler, isPrejitRoot);
60 }
61
62 // Optionally install the FullPolicy.
63 bool useFullPolicy = JitConfig.JitInlinePolicyFull() != 0;
64
65 if (useFullPolicy)
66 {
67 return new (compiler, CMK_Inlining) FullPolicy(compiler, isPrejitRoot);
68 }
69
70 // Optionally install the DiscretionaryPolicy.
71 bool useDiscretionaryPolicy = JitConfig.JitInlinePolicyDiscretionary() != 0;
72
73 if (useDiscretionaryPolicy)
74 {
75 return new (compiler, CMK_Inlining) DiscretionaryPolicy(compiler, isPrejitRoot);
76 }
77
78#endif // defined(DEBUG) || defined(INLINE_DATA)
79
80 // Optionally install the ModelPolicy.
81 bool useModelPolicy = JitConfig.JitInlinePolicyModel() != 0;
82
83 if (useModelPolicy)
84 {
85 return new (compiler, CMK_Inlining) ModelPolicy(compiler, isPrejitRoot);
86 }
87
88 // Use the default policy by default
89 return new (compiler, CMK_Inlining) DefaultPolicy(compiler, isPrejitRoot);
90}
91
92//------------------------------------------------------------------------
93// NoteFatal: handle an observation with fatal impact
94//
95// Arguments:
96// obs - the current obsevation
97
98void LegalPolicy::NoteFatal(InlineObservation obs)
99{
100 // As a safeguard, all fatal impact must be
101 // reported via NoteFatal.
102 assert(InlGetImpact(obs) == InlineImpact::FATAL);
103 NoteInternal(obs);
104 assert(InlDecisionIsFailure(m_Decision));
105}
106
107#if defined(DEBUG) || defined(INLINE_DATA)
108
109//------------------------------------------------------------------------
110// NotePriorFailure: record reason for earlier inline failure
111//
112// Arguments:
113// obs - the current obsevation
114//
115// Notes:
116// Used to "resurrect" failure observations from the early inline
117// screen when building the inline context tree. Only used during
118// debug modes.
119
120void LegalPolicy::NotePriorFailure(InlineObservation obs)
121{
122 NoteInternal(obs);
123 assert(InlDecisionIsFailure(m_Decision));
124}
125
126#endif // defined(DEBUG) || defined(INLINE_DATA)
127
128//------------------------------------------------------------------------
129// NoteInternal: helper for handling an observation
130//
131// Arguments:
132// obs - the current obsevation
133
134void LegalPolicy::NoteInternal(InlineObservation obs)
135{
136 // Note any INFORMATION that reaches here will now cause failure.
137 // Non-fatal INFORMATION observations must be handled higher up.
138 InlineTarget target = InlGetTarget(obs);
139
140 if (target == InlineTarget::CALLEE)
141 {
142 this->SetNever(obs);
143 }
144 else
145 {
146 this->SetFailure(obs);
147 }
148}
149
150//------------------------------------------------------------------------
151// SetFailure: helper for setting a failing decision
152//
153// Arguments:
154// obs - the current obsevation
155
156void LegalPolicy::SetFailure(InlineObservation obs)
157{
158 // Expect a valid observation
159 assert(InlIsValidObservation(obs));
160
161 switch (m_Decision)
162 {
163 case InlineDecision::FAILURE:
164 // Repeated failure only ok if evaluating a prejit root
165 // (since we can't fail fast because we're not inlining)
166 // or if inlining and the observation is CALLSITE_TOO_MANY_LOCALS
167 // (since we can't fail fast from lvaGrabTemp).
168 assert(m_IsPrejitRoot || (obs == InlineObservation::CALLSITE_TOO_MANY_LOCALS));
169 break;
170 case InlineDecision::UNDECIDED:
171 case InlineDecision::CANDIDATE:
172 m_Decision = InlineDecision::FAILURE;
173 m_Observation = obs;
174 break;
175 default:
176 // SUCCESS, NEVER, or ??
177 assert(!"Unexpected m_Decision");
178 unreached();
179 }
180}
181
182//------------------------------------------------------------------------
183// SetNever: helper for setting a never decision
184//
185// Arguments:
186// obs - the current obsevation
187
188void LegalPolicy::SetNever(InlineObservation obs)
189{
190 // Expect a valid observation
191 assert(InlIsValidObservation(obs));
192
193 switch (m_Decision)
194 {
195 case InlineDecision::NEVER:
196 // Repeated never only ok if evaluating a prejit root
197 assert(m_IsPrejitRoot);
198 break;
199 case InlineDecision::UNDECIDED:
200 case InlineDecision::CANDIDATE:
201 m_Decision = InlineDecision::NEVER;
202 m_Observation = obs;
203 break;
204 default:
205 // SUCCESS, FAILURE or ??
206 assert(!"Unexpected m_Decision");
207 unreached();
208 }
209}
210
211//------------------------------------------------------------------------
212// SetCandidate: helper updating candidacy
213//
214// Arguments:
215// obs - the current obsevation
216//
217// Note:
218// Candidate observations are handled here. If the inline has already
219// failed, they're ignored. If there's already a candidate reason,
220// this new reason trumps it.
221
222void LegalPolicy::SetCandidate(InlineObservation obs)
223{
224 // Ignore if this inline is going to fail.
225 if (InlDecisionIsFailure(m_Decision))
226 {
227 return;
228 }
229
230 // We should not have declared success yet.
231 assert(!InlDecisionIsSuccess(m_Decision));
232
233 // Update, overriding any previous candidacy.
234 m_Decision = InlineDecision::CANDIDATE;
235 m_Observation = obs;
236}
237
238//------------------------------------------------------------------------
239// NoteSuccess: handle finishing all the inlining checks successfully
240
241void DefaultPolicy::NoteSuccess()
242{
243 assert(InlDecisionIsCandidate(m_Decision));
244 m_Decision = InlineDecision::SUCCESS;
245}
246
247//------------------------------------------------------------------------
248// NoteBool: handle a boolean observation with non-fatal impact
249//
250// Arguments:
251// obs - the current obsevation
252// value - the value of the observation
253void DefaultPolicy::NoteBool(InlineObservation obs, bool value)
254{
255 // Check the impact
256 InlineImpact impact = InlGetImpact(obs);
257
258 // As a safeguard, all fatal impact must be
259 // reported via NoteFatal.
260 assert(impact != InlineImpact::FATAL);
261
262 // Handle most information here
263 bool isInformation = (impact == InlineImpact::INFORMATION);
264 bool propagate = !isInformation;
265
266 if (isInformation)
267 {
268 switch (obs)
269 {
270 case InlineObservation::CALLEE_IS_FORCE_INLINE:
271 // We may make the force-inline observation more than
272 // once. All observations should agree.
273 assert(!m_IsForceInlineKnown || (m_IsForceInline == value));
274 m_IsForceInline = value;
275 m_IsForceInlineKnown = true;
276 break;
277
278 case InlineObservation::CALLEE_IS_INSTANCE_CTOR:
279 m_IsInstanceCtor = value;
280 break;
281
282 case InlineObservation::CALLEE_CLASS_PROMOTABLE:
283 m_IsFromPromotableValueClass = value;
284 break;
285
286 case InlineObservation::CALLEE_HAS_SIMD:
287 m_HasSimd = value;
288 break;
289
290 case InlineObservation::CALLEE_LOOKS_LIKE_WRAPPER:
291 m_LooksLikeWrapperMethod = value;
292 break;
293
294 case InlineObservation::CALLEE_ARG_FEEDS_TEST:
295 m_ArgFeedsTest++;
296 break;
297
298 case InlineObservation::CALLEE_ARG_FEEDS_CONSTANT_TEST:
299 m_ArgFeedsConstantTest++;
300 break;
301
302 case InlineObservation::CALLEE_ARG_FEEDS_RANGE_CHECK:
303 m_ArgFeedsRangeCheck++;
304 break;
305
306 case InlineObservation::CALLEE_HAS_SWITCH:
307 case InlineObservation::CALLEE_UNSUPPORTED_OPCODE:
308 propagate = true;
309 break;
310
311 case InlineObservation::CALLSITE_CONSTANT_ARG_FEEDS_TEST:
312 // We shouldn't see this for a prejit root since
313 // we don't know anything about callers.
314 assert(!m_IsPrejitRoot);
315 m_ConstantArgFeedsConstantTest++;
316 break;
317
318 case InlineObservation::CALLEE_BEGIN_OPCODE_SCAN:
319 {
320 // Set up the state machine, if this inline is
321 // discretionary and is still a candidate.
322 if (InlDecisionIsCandidate(m_Decision) &&
323 (m_Observation == InlineObservation::CALLEE_IS_DISCRETIONARY_INLINE))
324 {
325 // Better not have a state machine already.
326 assert(m_StateMachine == nullptr);
327 m_StateMachine = new (m_RootCompiler, CMK_Inlining) CodeSeqSM;
328 m_StateMachine->Start(m_RootCompiler);
329 }
330 break;
331 }
332
333 case InlineObservation::CALLEE_END_OPCODE_SCAN:
334 {
335 if (m_StateMachine != nullptr)
336 {
337 m_StateMachine->End();
338 }
339
340 // If this function is mostly loads and stores, we
341 // should try harder to inline it. You can't just use
342 // the percentage test because if the method has 8
343 // instructions and 6 are loads, it's only 75% loads.
344 // This allows for CALL, RET, and one more non-ld/st
345 // instruction.
346 if (((m_InstructionCount - m_LoadStoreCount) < 4) ||
347 (((double)m_LoadStoreCount / (double)m_InstructionCount) > .90))
348 {
349 m_MethodIsMostlyLoadStore = true;
350 }
351
352 // Budget check.
353 //
354 // Conceptually this should happen when we
355 // observe the candidate's IL size.
356 //
357 // However, we do this here to avoid potential
358 // inconsistency between the state of the budget
359 // during candidate scan and the state when the IL is
360 // being scanned.
361 //
362 // Consider the case where we're just below the budget
363 // during candidate scan, and we have three possible
364 // inlines, any two of which put us over budget. We
365 // allow them all to become candidates. We then move
366 // on to inlining and the first two get inlined and
367 // put us over budget. Now the third can't be inlined
368 // anymore, but we have a policy that when we replay
369 // the candidate IL size during the inlining pass it
370 // "reestablishes" candidacy rather than alters
371 // candidacy ... so instead we bail out here.
372
373 if (!m_IsPrejitRoot)
374 {
375 InlineStrategy* strategy = m_RootCompiler->m_inlineStrategy;
376 bool overBudget = strategy->BudgetCheck(m_CodeSize);
377 if (overBudget)
378 {
379 SetFailure(InlineObservation::CALLSITE_OVER_BUDGET);
380 }
381 }
382
383 break;
384 }
385
386 case InlineObservation::CALLSITE_IN_TRY_REGION:
387 m_CallsiteIsInTryRegion = true;
388 break;
389
390 case InlineObservation::CALLSITE_IN_LOOP:
391 m_CallsiteIsInLoop = true;
392 break;
393
394 case InlineObservation::CALLEE_DOES_NOT_RETURN:
395 m_IsNoReturn = value;
396 m_IsNoReturnKnown = true;
397 break;
398
399 case InlineObservation::CALLSITE_RARE_GC_STRUCT:
400 // If this is a discretionary or always inline candidate
401 // with a gc struct, we may change our mind about inlining
402 // if the call site is rare, to avoid costs associated with
403 // zeroing the GC struct up in the root prolog.
404 if (m_Observation == InlineObservation::CALLEE_BELOW_ALWAYS_INLINE_SIZE)
405 {
406 assert(m_CallsiteFrequency == InlineCallsiteFrequency::UNUSED);
407 SetFailure(obs);
408 return;
409 }
410 else if (m_Observation == InlineObservation::CALLEE_IS_DISCRETIONARY_INLINE)
411 {
412 assert(m_CallsiteFrequency == InlineCallsiteFrequency::RARE);
413 SetFailure(obs);
414 return;
415 }
416 break;
417
418 case InlineObservation::CALLEE_HAS_PINNED_LOCALS:
419 if (m_CallsiteIsInTryRegion)
420 {
421 // Inlining a method with pinned locals in a try
422 // region requires wrapping the inline body in a
423 // try/finally to ensure unpinning. Bail instead.
424 SetFailure(InlineObservation::CALLSITE_PIN_IN_TRY_REGION);
425 return;
426 }
427 break;
428
429 case InlineObservation::CALLEE_HAS_LOCALLOC:
430 // We see this during the IL prescan. Ignore for now, we will
431 // bail out, if necessary, during importation
432 break;
433
434 default:
435 // Ignore the remainder for now
436 break;
437 }
438 }
439
440 if (propagate)
441 {
442 NoteInternal(obs);
443 }
444}
445
446//------------------------------------------------------------------------
447// NoteInt: handle an observed integer value
448//
449// Arguments:
450// obs - the current obsevation
451// value - the value being observed
452
453void DefaultPolicy::NoteInt(InlineObservation obs, int value)
454{
455 switch (obs)
456 {
457 case InlineObservation::CALLEE_MAXSTACK:
458 {
459 assert(m_IsForceInlineKnown);
460
461 unsigned calleeMaxStack = static_cast<unsigned>(value);
462
463 if (!m_IsForceInline && (calleeMaxStack > SMALL_STACK_SIZE))
464 {
465 SetNever(InlineObservation::CALLEE_MAXSTACK_TOO_BIG);
466 }
467
468 break;
469 }
470
471 case InlineObservation::CALLEE_NUMBER_OF_BASIC_BLOCKS:
472 {
473 assert(m_IsForceInlineKnown);
474 assert(value != 0);
475 assert(m_IsNoReturnKnown);
476
477 //
478 // Let's be conservative for now and reject inlining of "no return" methods only
479 // if the callee contains a single basic block. This covers most of the use cases
480 // (typical throw helpers simply do "throw new X();" and so they have a single block)
481 // without affecting more exotic cases (loops that do actual work for example) where
482 // failure to inline could negatively impact code quality.
483 //
484
485 unsigned basicBlockCount = static_cast<unsigned>(value);
486
487 // CALLEE_IS_FORCE_INLINE overrides CALLEE_DOES_NOT_RETURN
488 if (!m_IsForceInline && m_IsNoReturn && (basicBlockCount == 1))
489 {
490 SetNever(InlineObservation::CALLEE_DOES_NOT_RETURN);
491 }
492 else if (!m_IsForceInline && (basicBlockCount > MAX_BASIC_BLOCKS))
493 {
494 SetNever(InlineObservation::CALLEE_TOO_MANY_BASIC_BLOCKS);
495 }
496
497 break;
498 }
499
500 case InlineObservation::CALLEE_IL_CODE_SIZE:
501 {
502 assert(m_IsForceInlineKnown);
503 assert(value != 0);
504 m_CodeSize = static_cast<unsigned>(value);
505
506 // Now that we know size and forceinline state,
507 // update candidacy.
508 if (m_IsForceInline)
509 {
510 // Candidate based on force inline
511 SetCandidate(InlineObservation::CALLEE_IS_FORCE_INLINE);
512 }
513 else if (m_CodeSize <= InlineStrategy::ALWAYS_INLINE_SIZE)
514 {
515 // Candidate based on small size
516 SetCandidate(InlineObservation::CALLEE_BELOW_ALWAYS_INLINE_SIZE);
517 }
518 else if (m_CodeSize <= m_RootCompiler->m_inlineStrategy->GetMaxInlineILSize())
519 {
520 // Candidate, pending profitability evaluation
521 SetCandidate(InlineObservation::CALLEE_IS_DISCRETIONARY_INLINE);
522 }
523 else
524 {
525 // Callee too big, not a candidate
526 SetNever(InlineObservation::CALLEE_TOO_MUCH_IL);
527 }
528
529 break;
530 }
531
532 case InlineObservation::CALLSITE_DEPTH:
533 {
534 unsigned depth = static_cast<unsigned>(value);
535
536 if (depth > m_RootCompiler->m_inlineStrategy->GetMaxInlineDepth())
537 {
538 SetFailure(InlineObservation::CALLSITE_IS_TOO_DEEP);
539 }
540
541 break;
542 }
543
544 case InlineObservation::CALLEE_OPCODE_NORMED:
545 case InlineObservation::CALLEE_OPCODE:
546 {
547 m_InstructionCount++;
548 OPCODE opcode = static_cast<OPCODE>(value);
549
550 if (m_StateMachine != nullptr)
551 {
552 SM_OPCODE smOpcode = CodeSeqSM::MapToSMOpcode(opcode);
553 noway_assert(smOpcode < SM_COUNT);
554 noway_assert(smOpcode != SM_PREFIX_N);
555 if (obs == InlineObservation::CALLEE_OPCODE_NORMED)
556 {
557 if (smOpcode == SM_LDARGA_S)
558 {
559 smOpcode = SM_LDARGA_S_NORMED;
560 }
561 else if (smOpcode == SM_LDLOCA_S)
562 {
563 smOpcode = SM_LDLOCA_S_NORMED;
564 }
565 }
566
567 m_StateMachine->Run(smOpcode DEBUGARG(0));
568 }
569
570 // Look for opcodes that imply loads and stores.
571 // Logic here is as it is to match legacy behavior.
572 if ((opcode >= CEE_LDARG_0 && opcode <= CEE_STLOC_S) || (opcode >= CEE_LDARG && opcode <= CEE_STLOC) ||
573 (opcode >= CEE_LDNULL && opcode <= CEE_LDC_R8) || (opcode >= CEE_LDIND_I1 && opcode <= CEE_STIND_R8) ||
574 (opcode >= CEE_LDFLD && opcode <= CEE_STOBJ) || (opcode >= CEE_LDELEMA && opcode <= CEE_STELEM) ||
575 (opcode == CEE_POP))
576 {
577 m_LoadStoreCount++;
578 }
579
580 break;
581 }
582
583 case InlineObservation::CALLSITE_FREQUENCY:
584 assert(m_CallsiteFrequency == InlineCallsiteFrequency::UNUSED);
585 m_CallsiteFrequency = static_cast<InlineCallsiteFrequency>(value);
586 assert(m_CallsiteFrequency != InlineCallsiteFrequency::UNUSED);
587 break;
588
589 default:
590 // Ignore all other information
591 break;
592 }
593}
594
595//------------------------------------------------------------------------
596// DetermineMultiplier: determine benefit multiplier for this inline
597//
598// Notes: uses the accumulated set of observations to compute a
599// profitability boost for the inline candidate.
600
601double DefaultPolicy::DetermineMultiplier()
602{
603 double multiplier = 0;
604
605 // Bump up the multiplier for instance constructors
606
607 if (m_IsInstanceCtor)
608 {
609 multiplier += 1.5;
610 JITDUMP("\nmultiplier in instance constructors increased to %g.", multiplier);
611 }
612
613 // Bump up the multiplier for methods in promotable struct
614
615 if (m_IsFromPromotableValueClass)
616 {
617 multiplier += 3;
618 JITDUMP("\nmultiplier in methods of promotable struct increased to %g.", multiplier);
619 }
620
621#ifdef FEATURE_SIMD
622
623 if (m_HasSimd)
624 {
625 multiplier += JitConfig.JitInlineSIMDMultiplier();
626 JITDUMP("\nInline candidate has SIMD type args, locals or return value. Multiplier increased to %g.",
627 multiplier);
628 }
629
630#endif // FEATURE_SIMD
631
632 if (m_LooksLikeWrapperMethod)
633 {
634 multiplier += 1.0;
635 JITDUMP("\nInline candidate looks like a wrapper method. Multiplier increased to %g.", multiplier);
636 }
637
638 if (m_ArgFeedsConstantTest > 0)
639 {
640 multiplier += 1.0;
641 JITDUMP("\nInline candidate has an arg that feeds a constant test. Multiplier increased to %g.", multiplier);
642 }
643
644 if (m_MethodIsMostlyLoadStore)
645 {
646 multiplier += 3.0;
647 JITDUMP("\nInline candidate is mostly loads and stores. Multiplier increased to %g.", multiplier);
648 }
649
650 if (m_ArgFeedsRangeCheck > 0)
651 {
652 multiplier += 0.5;
653 JITDUMP("\nInline candidate has arg that feeds range check. Multiplier increased to %g.", multiplier);
654 }
655
656 if (m_ConstantArgFeedsConstantTest > 0)
657 {
658 multiplier += 3.0;
659 JITDUMP("\nInline candidate has const arg that feeds a conditional. Multiplier increased to %g.", multiplier);
660 }
661 // For prejit roots we do not see the call sites. To be suitably optimistic
662 // assume that call sites may pass constants.
663 else if (m_IsPrejitRoot && ((m_ArgFeedsConstantTest > 0) || (m_ArgFeedsTest > 0)))
664 {
665 multiplier += 3.0;
666 JITDUMP("\nPrejit root candidate has arg that feeds a conditional. Multiplier increased to %g.", multiplier);
667 }
668
669 switch (m_CallsiteFrequency)
670 {
671 case InlineCallsiteFrequency::RARE:
672 // Note this one is not additive, it uses '=' instead of '+='
673 multiplier = 1.3;
674 JITDUMP("\nInline candidate callsite is rare. Multiplier limited to %g.", multiplier);
675 break;
676 case InlineCallsiteFrequency::BORING:
677 multiplier += 1.3;
678 JITDUMP("\nInline candidate callsite is boring. Multiplier increased to %g.", multiplier);
679 break;
680 case InlineCallsiteFrequency::WARM:
681 multiplier += 2.0;
682 JITDUMP("\nInline candidate callsite is warm. Multiplier increased to %g.", multiplier);
683 break;
684 case InlineCallsiteFrequency::LOOP:
685 multiplier += 3.0;
686 JITDUMP("\nInline candidate callsite is in a loop. Multiplier increased to %g.", multiplier);
687 break;
688 case InlineCallsiteFrequency::HOT:
689 multiplier += 3.0;
690 JITDUMP("\nInline candidate callsite is hot. Multiplier increased to %g.", multiplier);
691 break;
692 default:
693 assert(!"Unexpected callsite frequency");
694 break;
695 }
696
697#ifdef DEBUG
698
699 int additionalMultiplier = JitConfig.JitInlineAdditionalMultiplier();
700
701 if (additionalMultiplier != 0)
702 {
703 multiplier += additionalMultiplier;
704 JITDUMP("\nmultiplier increased via JitInlineAdditionalMultiplier=%d to %g.", additionalMultiplier, multiplier);
705 }
706
707 if (m_RootCompiler->compInlineStress())
708 {
709 multiplier += 10;
710 JITDUMP("\nmultiplier increased via inline stress to %g.", multiplier);
711 }
712
713#endif // DEBUG
714
715 return multiplier;
716}
717
718//------------------------------------------------------------------------
719// DetermineNativeSizeEstimate: return estimated native code size for
720// this inline candidate.
721//
722// Notes:
723// This is an estimate for the size of the inlined callee.
724// It does not include size impact on the caller side.
725//
726// Uses the results of a state machine model for discretionary
727// candidates. Should not be needed for forced or always
728// candidates.
729
730int DefaultPolicy::DetermineNativeSizeEstimate()
731{
732 // Should be a discretionary candidate.
733 assert(m_StateMachine != nullptr);
734
735 return m_StateMachine->NativeSize;
736}
737
738//------------------------------------------------------------------------
739// DetermineCallsiteNativeSizeEstimate: estimate native size for the
740// callsite.
741//
742// Arguments:
743// methInfo -- method info for the callee
744//
745// Notes:
746// Estimates the native size (in bytes, scaled up by 10x) for the
747// call site. While the quality of the estimate here is questionable
748// (especially for x64) it is being left as is for legacy compatibility.
749
750int DefaultPolicy::DetermineCallsiteNativeSizeEstimate(CORINFO_METHOD_INFO* methInfo)
751{
752 int callsiteSize = 55; // Direct call take 5 native bytes; indirect call takes 6 native bytes.
753
754 bool hasThis = methInfo->args.hasThis();
755
756 if (hasThis)
757 {
758 callsiteSize += 30; // "mov" or "lea"
759 }
760
761 CORINFO_ARG_LIST_HANDLE argLst = methInfo->args.args;
762 COMP_HANDLE comp = m_RootCompiler->info.compCompHnd;
763
764 for (unsigned i = (hasThis ? 1 : 0); i < methInfo->args.totalILArgs(); i++, argLst = comp->getArgNext(argLst))
765 {
766 var_types sigType = (var_types)m_RootCompiler->eeGetArgType(argLst, &methInfo->args);
767
768 if (sigType == TYP_STRUCT)
769 {
770 typeInfo verType = m_RootCompiler->verParseArgSigToTypeInfo(&methInfo->args, argLst);
771
772 /*
773
774 IN0028: 00009B lea EAX, bword ptr [EBP-14H]
775 IN0029: 00009E push dword ptr [EAX+4]
776 IN002a: 0000A1 push gword ptr [EAX]
777 IN002b: 0000A3 call [MyStruct.staticGetX2(struct):int]
778
779 */
780
781 callsiteSize += 10; // "lea EAX, bword ptr [EBP-14H]"
782
783 unsigned opsz = roundUp(comp->getClassSize(verType.GetClassHandle()), TARGET_POINTER_SIZE);
784 unsigned slots = opsz / TARGET_POINTER_SIZE;
785
786 callsiteSize += slots * 20; // "push gword ptr [EAX+offs] "
787 }
788 else
789 {
790 callsiteSize += 30; // push by average takes 3 bytes.
791 }
792 }
793
794 return callsiteSize;
795}
796
797//------------------------------------------------------------------------
798// DetermineProfitability: determine if this inline is profitable
799//
800// Arguments:
801// methodInfo -- method info for the callee
802//
803// Notes:
804// A profitable inline is one that is projected to have a beneficial
805// size/speed tradeoff.
806//
807// It is expected that this method is only invoked for discretionary
808// candidates, since it does not make sense to do this assessment for
809// failed, always, or forced inlines.
810
811void DefaultPolicy::DetermineProfitability(CORINFO_METHOD_INFO* methodInfo)
812{
813
814#if defined(DEBUG)
815
816 // Punt if we're inlining and we've reached the acceptance limit.
817 int limit = JitConfig.JitInlineLimit();
818 unsigned current = m_RootCompiler->m_inlineStrategy->GetInlineCount();
819
820 if (!m_IsPrejitRoot && (limit >= 0) && (current >= static_cast<unsigned>(limit)))
821 {
822 SetFailure(InlineObservation::CALLSITE_OVER_INLINE_LIMIT);
823 return;
824 }
825
826#endif // defined(DEBUG)
827
828 assert(InlDecisionIsCandidate(m_Decision));
829 assert(m_Observation == InlineObservation::CALLEE_IS_DISCRETIONARY_INLINE);
830
831 m_CalleeNativeSizeEstimate = DetermineNativeSizeEstimate();
832 m_CallsiteNativeSizeEstimate = DetermineCallsiteNativeSizeEstimate(methodInfo);
833 m_Multiplier = DetermineMultiplier();
834 const int threshold = (int)(m_CallsiteNativeSizeEstimate * m_Multiplier);
835
836 // Note the DefaultPolicy estimates are scaled up by SIZE_SCALE
837 JITDUMP("\ncalleeNativeSizeEstimate=%d\n", m_CalleeNativeSizeEstimate)
838 JITDUMP("callsiteNativeSizeEstimate=%d\n", m_CallsiteNativeSizeEstimate);
839 JITDUMP("benefit multiplier=%g\n", m_Multiplier);
840 JITDUMP("threshold=%d\n", threshold);
841
842 // Reject if callee size is over the threshold
843 if (m_CalleeNativeSizeEstimate > threshold)
844 {
845 // Inline appears to be unprofitable
846 JITLOG_THIS(m_RootCompiler,
847 (LL_INFO100000, "Native estimate for function size exceeds threshold"
848 " for inlining %g > %g (multiplier = %g)\n",
849 (double)m_CalleeNativeSizeEstimate / SIZE_SCALE, (double)threshold / SIZE_SCALE, m_Multiplier));
850
851 // Fail the inline
852 if (m_IsPrejitRoot)
853 {
854 SetNever(InlineObservation::CALLEE_NOT_PROFITABLE_INLINE);
855 }
856 else
857 {
858 SetFailure(InlineObservation::CALLSITE_NOT_PROFITABLE_INLINE);
859 }
860 }
861 else
862 {
863 // Inline appears to be profitable
864 JITLOG_THIS(m_RootCompiler,
865 (LL_INFO100000, "Native estimate for function size is within threshold"
866 " for inlining %g <= %g (multiplier = %g)\n",
867 (double)m_CalleeNativeSizeEstimate / SIZE_SCALE, (double)threshold / SIZE_SCALE, m_Multiplier));
868
869 // Update candidacy
870 if (m_IsPrejitRoot)
871 {
872 SetCandidate(InlineObservation::CALLEE_IS_PROFITABLE_INLINE);
873 }
874 else
875 {
876 SetCandidate(InlineObservation::CALLSITE_IS_PROFITABLE_INLINE);
877 }
878 }
879}
880
881//------------------------------------------------------------------------
882// CodeSizeEstimate: estimated code size impact of the inline
883//
884// Return Value:
885// Estimated code size impact, in bytes * 10
886//
887// Notes:
888// Only meaningful for discretionary inlines (whether successful or
889// not). For always or force inlines the legacy policy doesn't
890// estimate size impact.
891
892int DefaultPolicy::CodeSizeEstimate()
893{
894 if (m_StateMachine != nullptr)
895 {
896 // This is not something the DefaultPolicy explicitly computed,
897 // since it uses a blended evaluation model (mixing size and time
898 // together for overall profitability). But it's effecitvely an
899 // estimate of the size impact.
900 return (m_CalleeNativeSizeEstimate - m_CallsiteNativeSizeEstimate);
901 }
902 else
903 {
904 return 0;
905 }
906}
907
908//------------------------------------------------------------------------
909// PropagateNeverToRuntime: determine if a never result should cause the
910// method to be marked as un-inlinable.
911
912bool DefaultPolicy::PropagateNeverToRuntime() const
913{
914 //
915 // Do not propagate the "no return" observation. If we do this then future inlining
916 // attempts will fail immediately without marking the call node as "no return".
917 // This can have an adverse impact on caller's code quality as it may have to preserve
918 // registers across the call.
919 // TODO-Throughput: We should persist the "no return" information in the runtime
920 // so we don't need to re-analyze the inlinee all the time.
921 //
922
923 bool propagate = (m_Observation != InlineObservation::CALLEE_DOES_NOT_RETURN);
924
925 return propagate;
926}
927
928#if defined(DEBUG) || defined(INLINE_DATA)
929
930//------------------------------------------------------------------------
931// RandomPolicy: construct a new RandomPolicy
932//
933// Arguments:
934// compiler -- compiler instance doing the inlining (root compiler)
935// isPrejitRoot -- true if this compiler is prejitting the root method
936
937RandomPolicy::RandomPolicy(Compiler* compiler, bool isPrejitRoot) : DiscretionaryPolicy(compiler, isPrejitRoot)
938{
939 m_Random = compiler->m_inlineStrategy->GetRandom();
940}
941
942//------------------------------------------------------------------------
943// NoteInt: handle an observed integer value
944//
945// Arguments:
946// obs - the current obsevation
947// value - the value being observed
948
949void RandomPolicy::NoteInt(InlineObservation obs, int value)
950{
951 switch (obs)
952 {
953 case InlineObservation::CALLEE_IL_CODE_SIZE:
954 {
955 assert(m_IsForceInlineKnown);
956 assert(value != 0);
957 m_CodeSize = static_cast<unsigned>(value);
958
959 if (m_IsForceInline)
960 {
961 // Candidate based on force inline
962 SetCandidate(InlineObservation::CALLEE_IS_FORCE_INLINE);
963 }
964 else
965 {
966 // Candidate, pending profitability evaluation
967 SetCandidate(InlineObservation::CALLEE_IS_DISCRETIONARY_INLINE);
968 }
969
970 break;
971 }
972
973 default:
974 // Defer to superclass for all other information
975 DiscretionaryPolicy::NoteInt(obs, value);
976 break;
977 }
978}
979
980//------------------------------------------------------------------------
981// DetermineProfitability: determine if this inline is profitable
982//
983// Arguments:
984// methodInfo -- method info for the callee
985//
986// Notes:
987// The random policy makes random decisions about profitablity.
988// Generally we aspire to inline differently, not necessarily to
989// inline more.
990
991void RandomPolicy::DetermineProfitability(CORINFO_METHOD_INFO* methodInfo)
992{
993 assert(InlDecisionIsCandidate(m_Decision));
994 assert(m_Observation == InlineObservation::CALLEE_IS_DISCRETIONARY_INLINE);
995
996 // Budget check.
997 if (!m_IsPrejitRoot)
998 {
999 InlineStrategy* strategy = m_RootCompiler->m_inlineStrategy;
1000 bool overBudget = strategy->BudgetCheck(m_CodeSize);
1001 if (overBudget)
1002 {
1003 SetFailure(InlineObservation::CALLSITE_OVER_BUDGET);
1004 return;
1005 }
1006 }
1007
1008 // If we're also dumping inline data, make additional observations
1009 // based on the method info, and estimate code size and perf
1010 // impact, so that the reports have the necessary data.
1011 if (JitConfig.JitInlineDumpData() != 0)
1012 {
1013 MethodInfoObservations(methodInfo);
1014 EstimateCodeSize();
1015 EstimatePerformanceImpact();
1016 }
1017
1018 // Use a probability curve that roughly matches the observed
1019 // behavior of the DefaultPolicy. That way we're inlining
1020 // differently but not creating enormous methods.
1021 //
1022 // We vary a bit at the extremes. The RandomPolicy won't always
1023 // inline the small methods (<= 16 IL bytes) and won't always
1024 // reject the large methods (> 100 IL bytes).
1025
1026 unsigned threshold = 0;
1027
1028 if (m_CodeSize <= 16)
1029 {
1030 threshold = 75;
1031 }
1032 else if (m_CodeSize <= 30)
1033 {
1034 threshold = 50;
1035 }
1036 else if (m_CodeSize <= 40)
1037 {
1038 threshold = 40;
1039 }
1040 else if (m_CodeSize <= 50)
1041 {
1042 threshold = 30;
1043 }
1044 else if (m_CodeSize <= 75)
1045 {
1046 threshold = 20;
1047 }
1048 else if (m_CodeSize <= 100)
1049 {
1050 threshold = 10;
1051 }
1052 else if (m_CodeSize <= 200)
1053 {
1054 threshold = 5;
1055 }
1056 else
1057 {
1058 threshold = 1;
1059 }
1060
1061 unsigned randomValue = m_Random->Next(1, 100);
1062
1063 // Reject if callee size is over the threshold
1064 if (randomValue > threshold)
1065 {
1066 // Inline appears to be unprofitable
1067 JITLOG_THIS(m_RootCompiler, (LL_INFO100000, "Random rejection (r=%d > t=%d)\n", randomValue, threshold));
1068
1069 // Fail the inline
1070 if (m_IsPrejitRoot)
1071 {
1072 SetNever(InlineObservation::CALLEE_RANDOM_REJECT);
1073 }
1074 else
1075 {
1076 SetFailure(InlineObservation::CALLSITE_RANDOM_REJECT);
1077 }
1078 }
1079 else
1080 {
1081 // Inline appears to be profitable
1082 JITLOG_THIS(m_RootCompiler, (LL_INFO100000, "Random acceptance (r=%d <= t=%d)\n", randomValue, threshold));
1083
1084 // Update candidacy
1085 if (m_IsPrejitRoot)
1086 {
1087 SetCandidate(InlineObservation::CALLEE_RANDOM_ACCEPT);
1088 }
1089 else
1090 {
1091 SetCandidate(InlineObservation::CALLSITE_RANDOM_ACCEPT);
1092 }
1093 }
1094}
1095
1096#endif // defined(DEBUG) || defined(INLINE_DATA)
1097
1098#ifdef _MSC_VER
1099// Disable warning about new array member initialization behavior
1100#pragma warning(disable : 4351)
1101#endif
1102
1103//------------------------------------------------------------------------
1104// DiscretionaryPolicy: construct a new DiscretionaryPolicy
1105//
1106// Arguments:
1107// compiler -- compiler instance doing the inlining (root compiler)
1108// isPrejitRoot -- true if this compiler is prejitting the root method
1109
1110// clang-format off
1111DiscretionaryPolicy::DiscretionaryPolicy(Compiler* compiler, bool isPrejitRoot)
1112 : DefaultPolicy(compiler, isPrejitRoot)
1113 , m_Depth(0)
1114 , m_BlockCount(0)
1115 , m_Maxstack(0)
1116 , m_ArgCount(0)
1117 , m_ArgType()
1118 , m_ArgSize()
1119 , m_LocalCount(0)
1120 , m_ReturnType(CORINFO_TYPE_UNDEF)
1121 , m_ReturnSize(0)
1122 , m_ArgAccessCount(0)
1123 , m_LocalAccessCount(0)
1124 , m_IntConstantCount(0)
1125 , m_FloatConstantCount(0)
1126 , m_IntLoadCount(0)
1127 , m_FloatLoadCount(0)
1128 , m_IntStoreCount(0)
1129 , m_FloatStoreCount(0)
1130 , m_SimpleMathCount(0)
1131 , m_ComplexMathCount(0)
1132 , m_OverflowMathCount(0)
1133 , m_IntArrayLoadCount(0)
1134 , m_FloatArrayLoadCount(0)
1135 , m_RefArrayLoadCount(0)
1136 , m_StructArrayLoadCount(0)
1137 , m_IntArrayStoreCount(0)
1138 , m_FloatArrayStoreCount(0)
1139 , m_RefArrayStoreCount(0)
1140 , m_StructArrayStoreCount(0)
1141 , m_StructOperationCount(0)
1142 , m_ObjectModelCount(0)
1143 , m_FieldLoadCount(0)
1144 , m_FieldStoreCount(0)
1145 , m_StaticFieldLoadCount(0)
1146 , m_StaticFieldStoreCount(0)
1147 , m_LoadAddressCount(0)
1148 , m_ThrowCount(0)
1149 , m_ReturnCount(0)
1150 , m_CallCount(0)
1151 , m_CallSiteWeight(0)
1152 , m_ModelCodeSizeEstimate(0)
1153 , m_PerCallInstructionEstimate(0)
1154 , m_IsClassCtor(false)
1155 , m_IsSameThis(false)
1156 , m_CallerHasNewArray(false)
1157 , m_CallerHasNewObj(false)
1158 , m_CalleeHasGCStruct(false)
1159{
1160 // Empty
1161}
1162// clang-format on
1163
1164//------------------------------------------------------------------------
1165// NoteBool: handle an observed boolean value
1166//
1167// Arguments:
1168// obs - the current obsevation
1169// value - the value being observed
1170
1171void DiscretionaryPolicy::NoteBool(InlineObservation obs, bool value)
1172{
1173 switch (obs)
1174 {
1175 case InlineObservation::CALLEE_IS_CLASS_CTOR:
1176 m_IsClassCtor = value;
1177 break;
1178
1179 case InlineObservation::CALLSITE_IS_SAME_THIS:
1180 m_IsSameThis = value;
1181 break;
1182
1183 case InlineObservation::CALLER_HAS_NEWARRAY:
1184 m_CallerHasNewArray = value;
1185 break;
1186
1187 case InlineObservation::CALLER_HAS_NEWOBJ:
1188 m_CallerHasNewObj = value;
1189 break;
1190
1191 case InlineObservation::CALLEE_HAS_GC_STRUCT:
1192 m_CalleeHasGCStruct = value;
1193 break;
1194
1195 case InlineObservation::CALLSITE_RARE_GC_STRUCT:
1196 // This is redundant since this policy tracks call site
1197 // hotness for all candidates. So ignore.
1198 break;
1199
1200 default:
1201 DefaultPolicy::NoteBool(obs, value);
1202 break;
1203 }
1204}
1205
1206//------------------------------------------------------------------------
1207// NoteInt: handle an observed integer value
1208//
1209// Arguments:
1210// obs - the current obsevation
1211// value - the value being observed
1212
1213void DiscretionaryPolicy::NoteInt(InlineObservation obs, int value)
1214{
1215 switch (obs)
1216 {
1217 case InlineObservation::CALLEE_IL_CODE_SIZE:
1218 // Override how code size is handled
1219 {
1220 assert(m_IsForceInlineKnown);
1221 assert(value != 0);
1222 m_CodeSize = static_cast<unsigned>(value);
1223
1224 if (m_IsForceInline)
1225 {
1226 // Candidate based on force inline
1227 SetCandidate(InlineObservation::CALLEE_IS_FORCE_INLINE);
1228 }
1229 else
1230 {
1231 // Candidate, pending profitability evaluation
1232 SetCandidate(InlineObservation::CALLEE_IS_DISCRETIONARY_INLINE);
1233 }
1234
1235 break;
1236 }
1237
1238 case InlineObservation::CALLEE_OPCODE:
1239 {
1240 // This tries to do a rough binning of opcodes based
1241 // on similarity of impact on codegen.
1242 OPCODE opcode = static_cast<OPCODE>(value);
1243 ComputeOpcodeBin(opcode);
1244 DefaultPolicy::NoteInt(obs, value);
1245 break;
1246 }
1247
1248 case InlineObservation::CALLEE_MAXSTACK:
1249 m_Maxstack = value;
1250 break;
1251
1252 case InlineObservation::CALLEE_NUMBER_OF_BASIC_BLOCKS:
1253 m_BlockCount = value;
1254 break;
1255
1256 case InlineObservation::CALLSITE_DEPTH:
1257 m_Depth = value;
1258 break;
1259
1260 case InlineObservation::CALLSITE_WEIGHT:
1261 m_CallSiteWeight = static_cast<unsigned>(value);
1262 break;
1263
1264 default:
1265 // Delegate remainder to the super class.
1266 DefaultPolicy::NoteInt(obs, value);
1267 break;
1268 }
1269}
1270
1271//------------------------------------------------------------------------
1272// ComputeOpcodeBin: simple histogramming of opcodes based on presumably
1273// similar codegen impact.
1274//
1275// Arguments:
1276// opcode - an MSIL opcode from the callee
1277
1278void DiscretionaryPolicy::ComputeOpcodeBin(OPCODE opcode)
1279{
1280 switch (opcode)
1281 {
1282 case CEE_LDARG_0:
1283 case CEE_LDARG_1:
1284 case CEE_LDARG_2:
1285 case CEE_LDARG_3:
1286 case CEE_LDARG_S:
1287 case CEE_LDARG:
1288 case CEE_STARG_S:
1289 case CEE_STARG:
1290 m_ArgAccessCount++;
1291 break;
1292
1293 case CEE_LDLOC_0:
1294 case CEE_LDLOC_1:
1295 case CEE_LDLOC_2:
1296 case CEE_LDLOC_3:
1297 case CEE_LDLOC_S:
1298 case CEE_STLOC_0:
1299 case CEE_STLOC_1:
1300 case CEE_STLOC_2:
1301 case CEE_STLOC_3:
1302 case CEE_STLOC_S:
1303 case CEE_LDLOC:
1304 case CEE_STLOC:
1305 m_LocalAccessCount++;
1306 break;
1307
1308 case CEE_LDNULL:
1309 case CEE_LDC_I4_M1:
1310 case CEE_LDC_I4_0:
1311 case CEE_LDC_I4_1:
1312 case CEE_LDC_I4_2:
1313 case CEE_LDC_I4_3:
1314 case CEE_LDC_I4_4:
1315 case CEE_LDC_I4_5:
1316 case CEE_LDC_I4_6:
1317 case CEE_LDC_I4_7:
1318 case CEE_LDC_I4_8:
1319 case CEE_LDC_I4_S:
1320 m_IntConstantCount++;
1321 break;
1322
1323 case CEE_LDC_R4:
1324 case CEE_LDC_R8:
1325 m_FloatConstantCount++;
1326 break;
1327
1328 case CEE_LDIND_I1:
1329 case CEE_LDIND_U1:
1330 case CEE_LDIND_I2:
1331 case CEE_LDIND_U2:
1332 case CEE_LDIND_I4:
1333 case CEE_LDIND_U4:
1334 case CEE_LDIND_I8:
1335 case CEE_LDIND_I:
1336 m_IntLoadCount++;
1337 break;
1338
1339 case CEE_LDIND_R4:
1340 case CEE_LDIND_R8:
1341 m_FloatLoadCount++;
1342 break;
1343
1344 case CEE_STIND_I1:
1345 case CEE_STIND_I2:
1346 case CEE_STIND_I4:
1347 case CEE_STIND_I8:
1348 case CEE_STIND_I:
1349 m_IntStoreCount++;
1350 break;
1351
1352 case CEE_STIND_R4:
1353 case CEE_STIND_R8:
1354 m_FloatStoreCount++;
1355 break;
1356
1357 case CEE_SUB:
1358 case CEE_AND:
1359 case CEE_OR:
1360 case CEE_XOR:
1361 case CEE_SHL:
1362 case CEE_SHR:
1363 case CEE_SHR_UN:
1364 case CEE_NEG:
1365 case CEE_NOT:
1366 case CEE_CONV_I1:
1367 case CEE_CONV_I2:
1368 case CEE_CONV_I4:
1369 case CEE_CONV_I8:
1370 case CEE_CONV_U4:
1371 case CEE_CONV_U8:
1372 case CEE_CONV_U2:
1373 case CEE_CONV_U1:
1374 case CEE_CONV_I:
1375 case CEE_CONV_U:
1376 m_SimpleMathCount++;
1377 break;
1378
1379 case CEE_MUL:
1380 case CEE_DIV:
1381 case CEE_DIV_UN:
1382 case CEE_REM:
1383 case CEE_REM_UN:
1384 case CEE_CONV_R4:
1385 case CEE_CONV_R8:
1386 case CEE_CONV_R_UN:
1387 m_ComplexMathCount++;
1388 break;
1389
1390 case CEE_CONV_OVF_I1_UN:
1391 case CEE_CONV_OVF_I2_UN:
1392 case CEE_CONV_OVF_I4_UN:
1393 case CEE_CONV_OVF_I8_UN:
1394 case CEE_CONV_OVF_U1_UN:
1395 case CEE_CONV_OVF_U2_UN:
1396 case CEE_CONV_OVF_U4_UN:
1397 case CEE_CONV_OVF_U8_UN:
1398 case CEE_CONV_OVF_I_UN:
1399 case CEE_CONV_OVF_U_UN:
1400 case CEE_CONV_OVF_I1:
1401 case CEE_CONV_OVF_U1:
1402 case CEE_CONV_OVF_I2:
1403 case CEE_CONV_OVF_U2:
1404 case CEE_CONV_OVF_I4:
1405 case CEE_CONV_OVF_U4:
1406 case CEE_CONV_OVF_I8:
1407 case CEE_CONV_OVF_U8:
1408 case CEE_ADD_OVF:
1409 case CEE_ADD_OVF_UN:
1410 case CEE_MUL_OVF:
1411 case CEE_MUL_OVF_UN:
1412 case CEE_SUB_OVF:
1413 case CEE_SUB_OVF_UN:
1414 case CEE_CKFINITE:
1415 m_OverflowMathCount++;
1416 break;
1417
1418 case CEE_LDELEM_I1:
1419 case CEE_LDELEM_U1:
1420 case CEE_LDELEM_I2:
1421 case CEE_LDELEM_U2:
1422 case CEE_LDELEM_I4:
1423 case CEE_LDELEM_U4:
1424 case CEE_LDELEM_I8:
1425 case CEE_LDELEM_I:
1426 m_IntArrayLoadCount++;
1427 break;
1428
1429 case CEE_LDELEM_R4:
1430 case CEE_LDELEM_R8:
1431 m_FloatArrayLoadCount++;
1432 break;
1433
1434 case CEE_LDELEM_REF:
1435 m_RefArrayLoadCount++;
1436 break;
1437
1438 case CEE_LDELEM:
1439 m_StructArrayLoadCount++;
1440 break;
1441
1442 case CEE_STELEM_I:
1443 case CEE_STELEM_I1:
1444 case CEE_STELEM_I2:
1445 case CEE_STELEM_I4:
1446 case CEE_STELEM_I8:
1447 m_IntArrayStoreCount++;
1448 break;
1449
1450 case CEE_STELEM_R4:
1451 case CEE_STELEM_R8:
1452 m_FloatArrayStoreCount++;
1453 break;
1454
1455 case CEE_STELEM_REF:
1456 m_RefArrayStoreCount++;
1457 break;
1458
1459 case CEE_STELEM:
1460 m_StructArrayStoreCount++;
1461 break;
1462
1463 case CEE_CPOBJ:
1464 case CEE_LDOBJ:
1465 case CEE_CPBLK:
1466 case CEE_INITBLK:
1467 case CEE_STOBJ:
1468 m_StructOperationCount++;
1469 break;
1470
1471 case CEE_CASTCLASS:
1472 case CEE_ISINST:
1473 case CEE_UNBOX:
1474 case CEE_BOX:
1475 case CEE_UNBOX_ANY:
1476 case CEE_LDFTN:
1477 case CEE_LDVIRTFTN:
1478 case CEE_SIZEOF:
1479 m_ObjectModelCount++;
1480 break;
1481
1482 case CEE_LDFLD:
1483 case CEE_LDLEN:
1484 case CEE_REFANYTYPE:
1485 case CEE_REFANYVAL:
1486 m_FieldLoadCount++;
1487 break;
1488
1489 case CEE_STFLD:
1490 m_FieldStoreCount++;
1491 break;
1492
1493 case CEE_LDSFLD:
1494 m_StaticFieldLoadCount++;
1495 break;
1496
1497 case CEE_STSFLD:
1498 m_StaticFieldStoreCount++;
1499 break;
1500
1501 case CEE_LDELEMA:
1502 case CEE_LDSFLDA:
1503 case CEE_LDFLDA:
1504 case CEE_LDSTR:
1505 case CEE_LDARGA:
1506 case CEE_LDLOCA:
1507 m_LoadAddressCount++;
1508 break;
1509
1510 case CEE_CALL:
1511 case CEE_CALLI:
1512 case CEE_CALLVIRT:
1513 case CEE_NEWOBJ:
1514 case CEE_NEWARR:
1515 case CEE_JMP:
1516 m_CallCount++;
1517 break;
1518
1519 case CEE_THROW:
1520 case CEE_RETHROW:
1521 m_ThrowCount++;
1522 break;
1523
1524 case CEE_RET:
1525 m_ReturnCount++;
1526
1527 default:
1528 break;
1529 }
1530}
1531
1532//------------------------------------------------------------------------
1533// PropagateNeverToRuntime: determine if a never result should cause the
1534// method to be marked as un-inlinable.
1535
1536bool DiscretionaryPolicy::PropagateNeverToRuntime() const
1537{
1538 // Propagate most failures, but don't propagate when the inline
1539 // was viable but unprofitable.
1540 bool propagate = (m_Observation != InlineObservation::CALLEE_NOT_PROFITABLE_INLINE);
1541
1542 return propagate;
1543}
1544
1545//------------------------------------------------------------------------
1546// DetermineProfitability: determine if this inline is profitable
1547//
1548// Arguments:
1549// methodInfo -- method info for the callee
1550
1551void DiscretionaryPolicy::DetermineProfitability(CORINFO_METHOD_INFO* methodInfo)
1552{
1553
1554#if defined(DEBUG)
1555
1556 // Punt if we're inlining and we've reached the acceptance limit.
1557 int limit = JitConfig.JitInlineLimit();
1558 unsigned current = m_RootCompiler->m_inlineStrategy->GetInlineCount();
1559
1560 if (!m_IsPrejitRoot && (limit >= 0) && (current >= static_cast<unsigned>(limit)))
1561 {
1562 SetFailure(InlineObservation::CALLSITE_OVER_INLINE_LIMIT);
1563 return;
1564 }
1565
1566#endif // defined(DEBUG)
1567
1568 // Make additional observations based on the method info
1569 MethodInfoObservations(methodInfo);
1570
1571 // Estimate the code size impact. This is just for model
1572 // evaluation purposes -- we'll still use the legacy policy's
1573 // model for actual inlining.
1574 EstimateCodeSize();
1575
1576 // Estimate peformance impact. This is just for model
1577 // evaluation purposes -- we'll still use the legacy policy's
1578 // model for actual inlining.
1579 EstimatePerformanceImpact();
1580
1581 // Delegate to super class for the rest
1582 DefaultPolicy::DetermineProfitability(methodInfo);
1583}
1584
1585//------------------------------------------------------------------------
1586// MethodInfoObservations: make observations based on information from
1587// the method info for the callee.
1588//
1589// Arguments:
1590// methodInfo -- method info for the callee
1591
1592void DiscretionaryPolicy::MethodInfoObservations(CORINFO_METHOD_INFO* methodInfo)
1593{
1594 CORINFO_SIG_INFO& locals = methodInfo->locals;
1595 m_LocalCount = locals.numArgs;
1596
1597 CORINFO_SIG_INFO& args = methodInfo->args;
1598 const unsigned argCount = args.numArgs;
1599 m_ArgCount = argCount;
1600
1601 const unsigned pointerSize = TARGET_POINTER_SIZE;
1602 unsigned i = 0;
1603
1604 // Implicit arguments
1605
1606 const bool hasThis = args.hasThis();
1607
1608 if (hasThis)
1609 {
1610 m_ArgType[i] = CORINFO_TYPE_CLASS;
1611 m_ArgSize[i] = pointerSize;
1612 i++;
1613 m_ArgCount++;
1614 }
1615
1616 const bool hasTypeArg = args.hasTypeArg();
1617
1618 if (hasTypeArg)
1619 {
1620 m_ArgType[i] = CORINFO_TYPE_NATIVEINT;
1621 m_ArgSize[i] = pointerSize;
1622 i++;
1623 m_ArgCount++;
1624 }
1625
1626 // Explicit arguments
1627
1628 unsigned j = 0;
1629 CORINFO_ARG_LIST_HANDLE argListHandle = args.args;
1630 COMP_HANDLE comp = m_RootCompiler->info.compCompHnd;
1631
1632 while ((i < MAX_ARGS) && (j < argCount))
1633 {
1634 CORINFO_CLASS_HANDLE classHandle;
1635 CorInfoType type = strip(comp->getArgType(&args, argListHandle, &classHandle));
1636
1637 m_ArgType[i] = type;
1638
1639 if (type == CORINFO_TYPE_VALUECLASS)
1640 {
1641 assert(classHandle != nullptr);
1642 m_ArgSize[i] = roundUp(comp->getClassSize(classHandle), pointerSize);
1643 }
1644 else
1645 {
1646 m_ArgSize[i] = pointerSize;
1647 }
1648
1649 argListHandle = comp->getArgNext(argListHandle);
1650 i++;
1651 j++;
1652 }
1653
1654 while (i < MAX_ARGS)
1655 {
1656 m_ArgType[i] = CORINFO_TYPE_UNDEF;
1657 m_ArgSize[i] = 0;
1658 i++;
1659 }
1660
1661 // Return Type
1662
1663 m_ReturnType = args.retType;
1664
1665 if (m_ReturnType == CORINFO_TYPE_VALUECLASS)
1666 {
1667 assert(args.retTypeClass != nullptr);
1668 m_ReturnSize = roundUp(comp->getClassSize(args.retTypeClass), pointerSize);
1669 }
1670 else if (m_ReturnType == CORINFO_TYPE_VOID)
1671 {
1672 m_ReturnSize = 0;
1673 }
1674 else
1675 {
1676 m_ReturnSize = pointerSize;
1677 }
1678}
1679
1680//------------------------------------------------------------------------
1681// EstimateCodeSize: produce (various) code size estimates based on
1682// observations.
1683//
1684// The "Baseline" code size model used by the legacy policy is
1685// effectively
1686//
1687// 0.100 * m_CalleeNativeSizeEstimate +
1688// -0.100 * m_CallsiteNativeSizeEstimate
1689//
1690// On the inlines in CoreCLR's mscorlib, release windows x64, this
1691// yields scores of R=0.42, MSE=228, and MAE=7.25.
1692//
1693// This estimate can be improved slighly by refitting, resulting in
1694//
1695// -1.451 +
1696// 0.095 * m_CalleeNativeSizeEstimate +
1697// -0.104 * m_CallsiteNativeSizeEstimate
1698//
1699// With R=0.44, MSE=220, and MAE=6.93.
1700
1701void DiscretionaryPolicy::EstimateCodeSize()
1702{
1703 // Ensure we have this available.
1704 m_CalleeNativeSizeEstimate = DetermineNativeSizeEstimate();
1705
1706 // Size estimate based on GLMNET model.
1707 // R=0.55, MSE=177, MAE=6.59
1708 //
1709 // Suspect it doesn't handle factors properly...
1710 // clang-format off
1711 double sizeEstimate =
1712 -13.532 +
1713 0.359 * (int) m_CallsiteFrequency +
1714 -0.015 * m_ArgCount +
1715 -1.553 * m_ArgSize[5] +
1716 2.326 * m_LocalCount +
1717 0.287 * m_ReturnSize +
1718 0.561 * m_IntConstantCount +
1719 1.932 * m_FloatConstantCount +
1720 -0.822 * m_SimpleMathCount +
1721 -7.591 * m_IntArrayLoadCount +
1722 4.784 * m_RefArrayLoadCount +
1723 12.778 * m_StructArrayLoadCount +
1724 1.452 * m_FieldLoadCount +
1725 8.811 * m_StaticFieldLoadCount +
1726 2.752 * m_StaticFieldStoreCount +
1727 -6.566 * m_ThrowCount +
1728 6.021 * m_CallCount +
1729 -0.238 * m_IsInstanceCtor +
1730 -5.357 * m_IsFromPromotableValueClass +
1731 -7.901 * (m_ConstantArgFeedsConstantTest > 0 ? 1 : 0) +
1732 0.065 * m_CalleeNativeSizeEstimate;
1733 // clang-format on
1734
1735 // Scaled up and reported as an integer value.
1736 m_ModelCodeSizeEstimate = (int)(SIZE_SCALE * sizeEstimate);
1737}
1738
1739//------------------------------------------------------------------------
1740// EstimatePeformanceImpact: produce performance estimates based on
1741// observations.
1742//
1743// Notes:
1744// Attempts to predict the per-call savings in instructions executed.
1745//
1746// A negative value indicates the doing the inline will save instructions
1747// and likely time.
1748
1749void DiscretionaryPolicy::EstimatePerformanceImpact()
1750{
1751 // Performance estimate based on GLMNET model.
1752 // R=0.24, RMSE=16.1, MAE=8.9.
1753 // clang-format off
1754 double perCallSavingsEstimate =
1755 -7.35
1756 + (m_CallsiteFrequency == InlineCallsiteFrequency::BORING ? 0.76 : 0)
1757 + (m_CallsiteFrequency == InlineCallsiteFrequency::LOOP ? -2.02 : 0)
1758 + (m_ArgType[0] == CORINFO_TYPE_CLASS ? 3.51 : 0)
1759 + (m_ArgType[3] == CORINFO_TYPE_BOOL ? 20.7 : 0)
1760 + (m_ArgType[4] == CORINFO_TYPE_CLASS ? 0.38 : 0)
1761 + (m_ReturnType == CORINFO_TYPE_CLASS ? 2.32 : 0);
1762 // clang-format on
1763
1764 // Scaled up and reported as an integer value.
1765 m_PerCallInstructionEstimate = (int)(SIZE_SCALE * perCallSavingsEstimate);
1766}
1767
1768//------------------------------------------------------------------------
1769// CodeSizeEstimate: estimated code size impact of the inline
1770//
1771// Return Value:
1772// Estimated code size impact, in bytes * 10
1773
1774int DiscretionaryPolicy::CodeSizeEstimate()
1775{
1776 return m_ModelCodeSizeEstimate;
1777}
1778
1779#if defined(DEBUG) || defined(INLINE_DATA)
1780
1781//------------------------------------------------------------------------
1782// DumpSchema: dump names for all the supporting data for the
1783// inline decision in CSV format.
1784//
1785// Arguments:
1786// file -- file to write to
1787
1788void DiscretionaryPolicy::DumpSchema(FILE* file) const
1789{
1790 fprintf(file, "ILSize");
1791 fprintf(file, ",CallsiteFrequency");
1792 fprintf(file, ",InstructionCount");
1793 fprintf(file, ",LoadStoreCount");
1794 fprintf(file, ",Depth");
1795 fprintf(file, ",BlockCount");
1796 fprintf(file, ",Maxstack");
1797 fprintf(file, ",ArgCount");
1798
1799 for (unsigned i = 0; i < MAX_ARGS; i++)
1800 {
1801 fprintf(file, ",Arg%uType", i);
1802 }
1803
1804 for (unsigned i = 0; i < MAX_ARGS; i++)
1805 {
1806 fprintf(file, ",Arg%uSize", i);
1807 }
1808
1809 fprintf(file, ",LocalCount");
1810 fprintf(file, ",ReturnType");
1811 fprintf(file, ",ReturnSize");
1812 fprintf(file, ",ArgAccessCount");
1813 fprintf(file, ",LocalAccessCount");
1814 fprintf(file, ",IntConstantCount");
1815 fprintf(file, ",FloatConstantCount");
1816 fprintf(file, ",IntLoadCount");
1817 fprintf(file, ",FloatLoadCount");
1818 fprintf(file, ",IntStoreCount");
1819 fprintf(file, ",FloatStoreCount");
1820 fprintf(file, ",SimpleMathCount");
1821 fprintf(file, ",ComplexMathCount");
1822 fprintf(file, ",OverflowMathCount");
1823 fprintf(file, ",IntArrayLoadCount");
1824 fprintf(file, ",FloatArrayLoadCount");
1825 fprintf(file, ",RefArrayLoadCount");
1826 fprintf(file, ",StructArrayLoadCount");
1827 fprintf(file, ",IntArrayStoreCount");
1828 fprintf(file, ",FloatArrayStoreCount");
1829 fprintf(file, ",RefArrayStoreCount");
1830 fprintf(file, ",StructArrayStoreCount");
1831 fprintf(file, ",StructOperationCount");
1832 fprintf(file, ",ObjectModelCount");
1833 fprintf(file, ",FieldLoadCount");
1834 fprintf(file, ",FieldStoreCount");
1835 fprintf(file, ",StaticFieldLoadCount");
1836 fprintf(file, ",StaticFieldStoreCount");
1837 fprintf(file, ",LoadAddressCount");
1838 fprintf(file, ",ThrowCount");
1839 fprintf(file, ",ReturnCount");
1840 fprintf(file, ",CallCount");
1841 fprintf(file, ",CallSiteWeight");
1842 fprintf(file, ",IsForceInline");
1843 fprintf(file, ",IsInstanceCtor");
1844 fprintf(file, ",IsFromPromotableValueClass");
1845 fprintf(file, ",HasSimd");
1846 fprintf(file, ",LooksLikeWrapperMethod");
1847 fprintf(file, ",ArgFeedsConstantTest");
1848 fprintf(file, ",IsMostlyLoadStore");
1849 fprintf(file, ",ArgFeedsRangeCheck");
1850 fprintf(file, ",ConstantArgFeedsConstantTest");
1851 fprintf(file, ",CalleeNativeSizeEstimate");
1852 fprintf(file, ",CallsiteNativeSizeEstimate");
1853 fprintf(file, ",ModelCodeSizeEstimate");
1854 fprintf(file, ",ModelPerCallInstructionEstimate");
1855 fprintf(file, ",IsClassCtor");
1856 fprintf(file, ",IsSameThis");
1857 fprintf(file, ",CallerHasNewArray");
1858 fprintf(file, ",CallerHasNewObj");
1859 fprintf(file, ",CalleeDoesNotReturn");
1860 fprintf(file, ",CalleeHasGCStruct");
1861}
1862
1863//------------------------------------------------------------------------
1864// DumpData: dump all the supporting data for the inline decision
1865// in CSV format.
1866//
1867// Arguments:
1868// file -- file to write to
1869
1870void DiscretionaryPolicy::DumpData(FILE* file) const
1871{
1872 fprintf(file, "%u", m_CodeSize);
1873 fprintf(file, ",%u", m_CallsiteFrequency);
1874 fprintf(file, ",%u", m_InstructionCount);
1875 fprintf(file, ",%u", m_LoadStoreCount);
1876 fprintf(file, ",%u", m_Depth);
1877 fprintf(file, ",%u", m_BlockCount);
1878 fprintf(file, ",%u", m_Maxstack);
1879 fprintf(file, ",%u", m_ArgCount);
1880
1881 for (unsigned i = 0; i < MAX_ARGS; i++)
1882 {
1883 fprintf(file, ",%u", m_ArgType[i]);
1884 }
1885
1886 for (unsigned i = 0; i < MAX_ARGS; i++)
1887 {
1888 fprintf(file, ",%u", (unsigned)m_ArgSize[i]);
1889 }
1890
1891 fprintf(file, ",%u", m_LocalCount);
1892 fprintf(file, ",%u", m_ReturnType);
1893 fprintf(file, ",%u", (unsigned)m_ReturnSize);
1894 fprintf(file, ",%u", m_ArgAccessCount);
1895 fprintf(file, ",%u", m_LocalAccessCount);
1896 fprintf(file, ",%u", m_IntConstantCount);
1897 fprintf(file, ",%u", m_FloatConstantCount);
1898 fprintf(file, ",%u", m_IntLoadCount);
1899 fprintf(file, ",%u", m_FloatLoadCount);
1900 fprintf(file, ",%u", m_IntStoreCount);
1901 fprintf(file, ",%u", m_FloatStoreCount);
1902 fprintf(file, ",%u", m_SimpleMathCount);
1903 fprintf(file, ",%u", m_ComplexMathCount);
1904 fprintf(file, ",%u", m_OverflowMathCount);
1905 fprintf(file, ",%u", m_IntArrayLoadCount);
1906 fprintf(file, ",%u", m_FloatArrayLoadCount);
1907 fprintf(file, ",%u", m_RefArrayLoadCount);
1908 fprintf(file, ",%u", m_StructArrayLoadCount);
1909 fprintf(file, ",%u", m_IntArrayStoreCount);
1910 fprintf(file, ",%u", m_FloatArrayStoreCount);
1911 fprintf(file, ",%u", m_RefArrayStoreCount);
1912 fprintf(file, ",%u", m_StructArrayStoreCount);
1913 fprintf(file, ",%u", m_StructOperationCount);
1914 fprintf(file, ",%u", m_ObjectModelCount);
1915 fprintf(file, ",%u", m_FieldLoadCount);
1916 fprintf(file, ",%u", m_FieldStoreCount);
1917 fprintf(file, ",%u", m_StaticFieldLoadCount);
1918 fprintf(file, ",%u", m_StaticFieldStoreCount);
1919 fprintf(file, ",%u", m_LoadAddressCount);
1920 fprintf(file, ",%u", m_ReturnCount);
1921 fprintf(file, ",%u", m_ThrowCount);
1922 fprintf(file, ",%u", m_CallCount);
1923 fprintf(file, ",%u", m_CallSiteWeight);
1924 fprintf(file, ",%u", m_IsForceInline ? 1 : 0);
1925 fprintf(file, ",%u", m_IsInstanceCtor ? 1 : 0);
1926 fprintf(file, ",%u", m_IsFromPromotableValueClass ? 1 : 0);
1927 fprintf(file, ",%u", m_HasSimd ? 1 : 0);
1928 fprintf(file, ",%u", m_LooksLikeWrapperMethod ? 1 : 0);
1929 fprintf(file, ",%u", m_ArgFeedsConstantTest);
1930 fprintf(file, ",%u", m_MethodIsMostlyLoadStore ? 1 : 0);
1931 fprintf(file, ",%u", m_ArgFeedsRangeCheck);
1932 fprintf(file, ",%u", m_ConstantArgFeedsConstantTest);
1933 fprintf(file, ",%d", m_CalleeNativeSizeEstimate);
1934 fprintf(file, ",%d", m_CallsiteNativeSizeEstimate);
1935 fprintf(file, ",%d", m_ModelCodeSizeEstimate);
1936 fprintf(file, ",%d", m_PerCallInstructionEstimate);
1937 fprintf(file, ",%u", m_IsClassCtor ? 1 : 0);
1938 fprintf(file, ",%u", m_IsSameThis ? 1 : 0);
1939 fprintf(file, ",%u", m_CallerHasNewArray ? 1 : 0);
1940 fprintf(file, ",%u", m_CallerHasNewObj ? 1 : 0);
1941 fprintf(file, ",%u", m_IsNoReturn ? 1 : 0);
1942 fprintf(file, ",%u", m_CalleeHasGCStruct ? 1 : 0);
1943}
1944
1945#endif // defined(DEBUG) || defined(INLINE_DATA)
1946
1947//------------------------------------------------------------------------/
1948// ModelPolicy: construct a new ModelPolicy
1949//
1950// Arguments:
1951// compiler -- compiler instance doing the inlining (root compiler)
1952// isPrejitRoot -- true if this compiler is prejitting the root method
1953
1954ModelPolicy::ModelPolicy(Compiler* compiler, bool isPrejitRoot) : DiscretionaryPolicy(compiler, isPrejitRoot)
1955{
1956 // Empty
1957}
1958
1959//------------------------------------------------------------------------
1960// NoteInt: handle an observed integer value
1961//
1962// Arguments:
1963// obs - the current obsevation
1964// value - the value being observed
1965//
1966// Notes:
1967// The ILSize threshold used here should be large enough that
1968// it does not generally influence inlining decisions -- it only
1969// helps to make them faster.
1970//
1971// The value is determined as follows. We figure out the maximum
1972// possible code size estimate that will lead to an inline. This is
1973// found by determining the maximum possible inline benefit and
1974// working backwards.
1975//
1976// In the current ModelPolicy, the maximum benefit is -28.1, which
1977// comes from a CallSiteWeight of 3 and a per call benefit of
1978// -9.37. This implies that any candidate with code size larger
1979// than (28.1/0.2) will not pass the threshold. So maximum code
1980// size estimate (in bytes) for any inlinee is 140.55, and hence
1981// maximum estimate is 1405.
1982//
1983// Since we are trying to short circuit early in the evaluation
1984// process we don't have the code size estimate in hand. We need to
1985// estimate the possible code size estimate based on something we
1986// know cheaply and early -- the ILSize. So we use quantile
1987// regression to project how ILSize predicts the model code size
1988// estimate. Note that ILSize does not currently directly enter
1989// into the model.
1990//
1991// The median value for the model code size estimate based on
1992// ILSize is given by -107 + 12.6 * ILSize for the V9 data. This
1993// means an ILSize of 120 is likely to lead to a size estimate of
1994// at least 1405 at least 50% of the time. So we choose this as the
1995// early rejection threshold.
1996
1997void ModelPolicy::NoteInt(InlineObservation obs, int value)
1998{
1999 // Let underlying policy do its thing.
2000 DiscretionaryPolicy::NoteInt(obs, value);
2001
2002 // Fail fast for inlinees that are too large to ever inline.
2003 // The value of 120 is model-dependent; see notes above.
2004 if (!m_IsForceInline && (obs == InlineObservation::CALLEE_IL_CODE_SIZE) && (value >= 120))
2005 {
2006 // Callee too big, not a candidate
2007 SetNever(InlineObservation::CALLEE_TOO_MUCH_IL);
2008 return;
2009 }
2010
2011 // Safeguard against overly deep inlines
2012 if (obs == InlineObservation::CALLSITE_DEPTH)
2013 {
2014 unsigned depthLimit = m_RootCompiler->m_inlineStrategy->GetMaxInlineDepth();
2015
2016 if (m_Depth > depthLimit)
2017 {
2018 SetFailure(InlineObservation::CALLSITE_IS_TOO_DEEP);
2019 return;
2020 }
2021 }
2022}
2023
2024//------------------------------------------------------------------------
2025// DetermineProfitability: determine if this inline is profitable
2026//
2027// Arguments:
2028// methodInfo -- method info for the callee
2029//
2030// Notes:
2031// There are currently two parameters that are ad-hoc: the
2032// per-call-site weight and the size/speed threshold. Ideally this
2033// policy would have just one tunable parameter, the threshold,
2034// which describes how willing we are to trade size for speed.
2035
2036void ModelPolicy::DetermineProfitability(CORINFO_METHOD_INFO* methodInfo)
2037{
2038 // Do some homework
2039 MethodInfoObservations(methodInfo);
2040 EstimateCodeSize();
2041 EstimatePerformanceImpact();
2042
2043 // Preliminary inline model.
2044 //
2045 // If code size is estimated to increase, look at
2046 // the profitability model for guidance.
2047 //
2048 // If code size will decrease, just inline.
2049
2050 if (m_ModelCodeSizeEstimate <= 0)
2051 {
2052 // Inline will likely decrease code size
2053 JITLOG_THIS(m_RootCompiler, (LL_INFO100000, "Inline profitable, will decrease code size by %g bytes\n",
2054 (double)-m_ModelCodeSizeEstimate / SIZE_SCALE));
2055
2056 if (m_IsPrejitRoot)
2057 {
2058 SetCandidate(InlineObservation::CALLEE_IS_SIZE_DECREASING_INLINE);
2059 }
2060 else
2061 {
2062 SetCandidate(InlineObservation::CALLSITE_IS_SIZE_DECREASING_INLINE);
2063 }
2064 }
2065 else
2066 {
2067 // We estimate that this inline will increase code size. Only
2068 // inline if the performance win is sufficiently large to
2069 // justify bigger code.
2070
2071 // First compute the number of instruction executions saved
2072 // via inlining per call to the callee per byte of code size
2073 // impact.
2074 //
2075 // The per call instruction estimate is negative if the inline
2076 // will reduce instruction count. Flip the sign here to make
2077 // positive be better and negative worse.
2078 double perCallBenefit = -((double)m_PerCallInstructionEstimate / (double)m_ModelCodeSizeEstimate);
2079
2080 // Now estimate the local call frequency.
2081 //
2082 // Todo: use IBC data, or a better local profile estimate, or
2083 // try and incorporate this into the model. For instance if we
2084 // tried to predict the benefit per call to the root method
2085 // then the model would have to incorporate the local call
2086 // frequency, somehow.
2087 double callSiteWeight = 1.0;
2088
2089 switch (m_CallsiteFrequency)
2090 {
2091 case InlineCallsiteFrequency::RARE:
2092 callSiteWeight = 0.1;
2093 break;
2094 case InlineCallsiteFrequency::BORING:
2095 callSiteWeight = 1.0;
2096 break;
2097 case InlineCallsiteFrequency::WARM:
2098 callSiteWeight = 1.5;
2099 break;
2100 case InlineCallsiteFrequency::LOOP:
2101 case InlineCallsiteFrequency::HOT:
2102 callSiteWeight = 3.0;
2103 break;
2104 default:
2105 assert(false);
2106 break;
2107 }
2108
2109 // Determine the estimated number of instructions saved per
2110 // call to the root method per byte of code size impact. This
2111 // is our benefit figure of merit.
2112 double benefit = callSiteWeight * perCallBenefit;
2113
2114 // Compare this to the threshold, and inline if greater.
2115 //
2116 // The threshold is interpretable as a size/speed tradeoff:
2117 // the value of 0.2 below indicates we'll allow inlines that
2118 // grow code by as many as 5 bytes to save 1 instruction
2119 // execution (per call to the root method).
2120 double threshold = 0.20;
2121 bool shouldInline = (benefit > threshold);
2122
2123 JITLOG_THIS(m_RootCompiler,
2124 (LL_INFO100000, "Inline %s profitable: benefit=%g (weight=%g, percall=%g, size=%g)\n",
2125 shouldInline ? "is" : "is not", benefit, callSiteWeight,
2126 (double)m_PerCallInstructionEstimate / SIZE_SCALE, (double)m_ModelCodeSizeEstimate / SIZE_SCALE));
2127
2128 if (!shouldInline)
2129 {
2130 // Fail the inline
2131 if (m_IsPrejitRoot)
2132 {
2133 SetNever(InlineObservation::CALLEE_NOT_PROFITABLE_INLINE);
2134 }
2135 else
2136 {
2137 SetFailure(InlineObservation::CALLSITE_NOT_PROFITABLE_INLINE);
2138 }
2139 }
2140 else
2141 {
2142 // Update candidacy
2143 if (m_IsPrejitRoot)
2144 {
2145 SetCandidate(InlineObservation::CALLEE_IS_PROFITABLE_INLINE);
2146 }
2147 else
2148 {
2149 SetCandidate(InlineObservation::CALLSITE_IS_PROFITABLE_INLINE);
2150 }
2151 }
2152 }
2153}
2154
2155#if defined(DEBUG) || defined(INLINE_DATA)
2156
2157//------------------------------------------------------------------------/
2158// FullPolicy: construct a new FullPolicy
2159//
2160// Arguments:
2161// compiler -- compiler instance doing the inlining (root compiler)
2162// isPrejitRoot -- true if this compiler is prejitting the root method
2163
2164FullPolicy::FullPolicy(Compiler* compiler, bool isPrejitRoot) : DiscretionaryPolicy(compiler, isPrejitRoot)
2165{
2166 // Empty
2167}
2168
2169//------------------------------------------------------------------------
2170// DetermineProfitability: determine if this inline is profitable
2171//
2172// Arguments:
2173// methodInfo -- method info for the callee
2174
2175void FullPolicy::DetermineProfitability(CORINFO_METHOD_INFO* methodInfo)
2176{
2177 // Check depth
2178
2179 unsigned depthLimit = m_RootCompiler->m_inlineStrategy->GetMaxInlineDepth();
2180
2181 if (m_Depth > depthLimit)
2182 {
2183 SetFailure(InlineObservation::CALLSITE_IS_TOO_DEEP);
2184 return;
2185 }
2186
2187 // Check size
2188
2189 unsigned sizeLimit = m_RootCompiler->m_inlineStrategy->GetMaxInlineILSize();
2190
2191 if (m_CodeSize > sizeLimit)
2192 {
2193 SetFailure(InlineObservation::CALLEE_TOO_MUCH_IL);
2194 return;
2195 }
2196
2197 // Otherwise, we're good to go
2198
2199 if (m_IsPrejitRoot)
2200 {
2201 SetCandidate(InlineObservation::CALLEE_IS_PROFITABLE_INLINE);
2202 }
2203 else
2204 {
2205 SetCandidate(InlineObservation::CALLSITE_IS_PROFITABLE_INLINE);
2206 }
2207
2208 return;
2209}
2210
2211//------------------------------------------------------------------------/
2212// SizePolicy: construct a new SizePolicy
2213//
2214// Arguments:
2215// compiler -- compiler instance doing the inlining (root compiler)
2216// isPrejitRoot -- true if this compiler is prejitting the root method
2217
2218SizePolicy::SizePolicy(Compiler* compiler, bool isPrejitRoot) : DiscretionaryPolicy(compiler, isPrejitRoot)
2219{
2220 // Empty
2221}
2222
2223//------------------------------------------------------------------------
2224// DetermineProfitability: determine if this inline is profitable
2225//
2226// Arguments:
2227// methodInfo -- method info for the callee
2228
2229void SizePolicy::DetermineProfitability(CORINFO_METHOD_INFO* methodInfo)
2230{
2231 // Do some homework
2232 MethodInfoObservations(methodInfo);
2233 EstimateCodeSize();
2234
2235 // Does this inline increase the estimated size beyond
2236 // the original size estimate?
2237 const InlineStrategy* strategy = m_RootCompiler->m_inlineStrategy;
2238 const int initialSize = strategy->GetInitialSizeEstimate();
2239 const int currentSize = strategy->GetCurrentSizeEstimate();
2240 const int newSize = currentSize + m_ModelCodeSizeEstimate;
2241
2242 if (newSize <= initialSize)
2243 {
2244 // Estimated size impact is acceptable, so inline here.
2245 JITLOG_THIS(m_RootCompiler,
2246 (LL_INFO100000, "Inline profitable, root size estimate %d is less than initial size %d\n",
2247 newSize / SIZE_SCALE, initialSize / SIZE_SCALE));
2248
2249 if (m_IsPrejitRoot)
2250 {
2251 SetCandidate(InlineObservation::CALLEE_IS_SIZE_DECREASING_INLINE);
2252 }
2253 else
2254 {
2255 SetCandidate(InlineObservation::CALLSITE_IS_SIZE_DECREASING_INLINE);
2256 }
2257 }
2258 else
2259 {
2260 // Estimated size increase is too large, so no inline here.
2261 //
2262 // Note that we ought to reconsider this inline if we make
2263 // room in the budget by inlining a bunch of size decreasing
2264 // inlines after this one. But for now, we won't do this.
2265 if (m_IsPrejitRoot)
2266 {
2267 SetNever(InlineObservation::CALLEE_NOT_PROFITABLE_INLINE);
2268 }
2269 else
2270 {
2271 SetFailure(InlineObservation::CALLSITE_NOT_PROFITABLE_INLINE);
2272 }
2273 }
2274
2275 return;
2276}
2277
2278// Statics to track emission of the replay banner
2279// and provide file access to the inline xml
2280
2281bool ReplayPolicy::s_WroteReplayBanner = false;
2282FILE* ReplayPolicy::s_ReplayFile = nullptr;
2283CritSecObject ReplayPolicy::s_XmlReaderLock;
2284
2285//------------------------------------------------------------------------/
2286// ReplayPolicy: construct a new ReplayPolicy
2287//
2288// Arguments:
2289// compiler -- compiler instance doing the inlining (root compiler)
2290// isPrejitRoot -- true if this compiler is prejitting the root method
2291
2292ReplayPolicy::ReplayPolicy(Compiler* compiler, bool isPrejitRoot)
2293 : DiscretionaryPolicy(compiler, isPrejitRoot)
2294 , m_InlineContext(nullptr)
2295 , m_Offset(BAD_IL_OFFSET)
2296 , m_WasForceInline(false)
2297{
2298 // Is there a log file open already? If so, we can use it.
2299 if (s_ReplayFile == nullptr)
2300 {
2301 // Did we already try and open and fail?
2302 if (!s_WroteReplayBanner)
2303 {
2304 // Nope, open it up.
2305 const wchar_t* replayFileName = JitConfig.JitInlineReplayFile();
2306 s_ReplayFile = _wfopen(replayFileName, W("r"));
2307
2308 // Display banner to stderr, unless we're dumping inline Xml,
2309 // in which case the policy name is captured in the Xml.
2310 if (JitConfig.JitInlineDumpXml() == 0)
2311 {
2312 fprintf(stderr, "*** %s inlines from %ws\n", s_ReplayFile == nullptr ? "Unable to replay" : "Replaying",
2313 replayFileName);
2314 }
2315
2316 s_WroteReplayBanner = true;
2317 }
2318 }
2319}
2320
2321//------------------------------------------------------------------------
2322// ReplayPolicy: Finalize reading of inline Xml
2323//
2324// Notes:
2325// Called during jitShutdown()
2326
2327void ReplayPolicy::FinalizeXml()
2328{
2329 if (s_ReplayFile != nullptr)
2330 {
2331 fclose(s_ReplayFile);
2332 s_ReplayFile = nullptr;
2333 }
2334}
2335
2336//------------------------------------------------------------------------
2337// FindMethod: find the root method in the inline Xml
2338//
2339// ReturnValue:
2340// true if found. File position left pointing just after the
2341// <Token> entry for the method.
2342
2343bool ReplayPolicy::FindMethod()
2344{
2345 if (s_ReplayFile == nullptr)
2346 {
2347 return false;
2348 }
2349
2350 // See if we've already found this method.
2351 InlineStrategy* inlineStrategy = m_RootCompiler->m_inlineStrategy;
2352 long filePosition = inlineStrategy->GetMethodXmlFilePosition();
2353
2354 if (filePosition == -1)
2355 {
2356 // Past lookup failed
2357 return false;
2358 }
2359 else if (filePosition > 0)
2360 {
2361 // Past lookup succeeded, jump there
2362 fseek(s_ReplayFile, filePosition, SEEK_SET);
2363 return true;
2364 }
2365
2366 // Else, scan the file. Might be nice to build an index
2367 // or something, someday.
2368 const mdMethodDef methodToken =
2369 m_RootCompiler->info.compCompHnd->getMethodDefFromMethod(m_RootCompiler->info.compMethodHnd);
2370 const unsigned methodHash = m_RootCompiler->info.compMethodHash();
2371
2372 bool foundMethod = false;
2373 char buffer[256];
2374 fseek(s_ReplayFile, 0, SEEK_SET);
2375
2376 while (!foundMethod)
2377 {
2378 // Get next line
2379 if (fgets(buffer, sizeof(buffer), s_ReplayFile) == nullptr)
2380 {
2381 break;
2382 }
2383
2384 // Look for next method entry
2385 if (strstr(buffer, "<Method>") == nullptr)
2386 {
2387 continue;
2388 }
2389
2390 // Get next line
2391 if (fgets(buffer, sizeof(buffer), s_ReplayFile) == nullptr)
2392 {
2393 break;
2394 }
2395
2396 // See if token matches
2397 unsigned token = 0;
2398 int count = sscanf_s(buffer, " <Token>%u</Token> ", &token);
2399 if ((count != 1) || (token != methodToken))
2400 {
2401 continue;
2402 }
2403
2404 // Get next line
2405 if (fgets(buffer, sizeof(buffer), s_ReplayFile) == nullptr)
2406 {
2407 break;
2408 }
2409
2410 // See if hash matches
2411 unsigned hash = 0;
2412 count = sscanf_s(buffer, " <Hash>%u</Hash> ", &hash);
2413 if ((count != 1) || (hash != methodHash))
2414 {
2415 continue;
2416 }
2417
2418 // Found a match...
2419 foundMethod = true;
2420 break;
2421 }
2422
2423 // Update file position cache for this method
2424 long foundPosition = -1;
2425
2426 if (foundMethod)
2427 {
2428 foundPosition = ftell(s_ReplayFile);
2429 }
2430
2431 inlineStrategy->SetMethodXmlFilePosition(foundPosition);
2432
2433 return foundMethod;
2434}
2435
2436//------------------------------------------------------------------------
2437// FindContext: find an inline context in the inline Xml
2438//
2439// Notes:
2440// Assumes file position within the relevant method has just been
2441// set by a successful call to FindMethod().
2442//
2443// Arguments:
2444// context -- context of interest
2445//
2446// ReturnValue:
2447// true if found. File position left pointing just after the
2448// <Token> entry for the context.
2449
2450bool ReplayPolicy::FindContext(InlineContext* context)
2451{
2452 // Make sure we've found the parent context.
2453 if (context->IsRoot())
2454 {
2455 // We've already found the method context so we're good.
2456 return true;
2457 }
2458
2459 bool foundParent = FindContext(context->GetParent());
2460
2461 if (!foundParent)
2462 {
2463 return false;
2464 }
2465
2466 // File pointer should be pointing at the parent context level.
2467 // See if we see an inline entry for this context.
2468 //
2469 // Token and Hash we're looking for.
2470 mdMethodDef contextToken = m_RootCompiler->info.compCompHnd->getMethodDefFromMethod(context->GetCallee());
2471 unsigned contextHash = m_RootCompiler->info.compCompHnd->getMethodHash(context->GetCallee());
2472 unsigned contextOffset = (unsigned)context->GetOffset();
2473
2474 return FindInline(contextToken, contextHash, contextOffset);
2475}
2476
2477//------------------------------------------------------------------------
2478// FindInline: find entry for the current inline in inline Xml.
2479//
2480// Arguments:
2481// token -- token describing the inline
2482// hash -- hash describing the inline
2483// offset -- IL offset of the call site in the parent method
2484//
2485// ReturnValue:
2486// true if the inline entry was found
2487//
2488// Notes:
2489// Assumes file position has just been set by a successful call to
2490// FindMethod or FindContext.
2491//
2492// Token and Hash will not be sufficiently unique to identify a
2493// particular inline, if there are multiple calls to the same
2494// method.
2495
2496bool ReplayPolicy::FindInline(unsigned token, unsigned hash, unsigned offset)
2497{
2498 char buffer[256];
2499 bool foundInline = false;
2500 int depth = 0;
2501
2502 while (!foundInline)
2503 {
2504 // Get next line
2505 if (fgets(buffer, sizeof(buffer), s_ReplayFile) == nullptr)
2506 {
2507 break;
2508 }
2509
2510 // If we hit </Method> we've gone too far,
2511 // and the XML is messed up.
2512 if (strstr(buffer, "</Method>") != nullptr)
2513 {
2514 break;
2515 }
2516
2517 // Look for <Inlines />....
2518 if (strstr(buffer, "<Inlines />") != nullptr)
2519 {
2520 if (depth == 0)
2521 {
2522 // Exited depth 1, failed to find the context
2523 break;
2524 }
2525 else
2526 {
2527 // Exited nested, keep looking
2528 continue;
2529 }
2530 }
2531
2532 // Look for <Inlines>....
2533 if (strstr(buffer, "<Inlines>") != nullptr)
2534 {
2535 depth++;
2536 continue;
2537 }
2538
2539 // If we hit </Inlines> we've exited a nested entry
2540 // or the current entry.
2541 if (strstr(buffer, "</Inlines>") != nullptr)
2542 {
2543 depth--;
2544
2545 if (depth == 0)
2546 {
2547 // Exited depth 1, failed to find the context
2548 break;
2549 }
2550 else
2551 {
2552 // Exited nested, keep looking
2553 continue;
2554 }
2555 }
2556
2557 // Look for start of inline section at the right depth
2558 if ((depth != 1) || (strstr(buffer, "<Inline>") == nullptr))
2559 {
2560 continue;
2561 }
2562
2563 // Get next line
2564 if (fgets(buffer, sizeof(buffer), s_ReplayFile) == nullptr)
2565 {
2566 break;
2567 }
2568
2569 // Match token
2570 unsigned inlineToken = 0;
2571 int count = sscanf_s(buffer, " <Token>%u</Token> ", &inlineToken);
2572
2573 if ((count != 1) || (inlineToken != token))
2574 {
2575 continue;
2576 }
2577
2578 // Get next line
2579 if (fgets(buffer, sizeof(buffer), s_ReplayFile) == nullptr)
2580 {
2581 break;
2582 }
2583
2584 // Match hash
2585 unsigned inlineHash = 0;
2586 count = sscanf_s(buffer, " <Hash>%u</Hash> ", &inlineHash);
2587
2588 if ((count != 1) || (inlineHash != hash))
2589 {
2590 continue;
2591 }
2592
2593 // Get next line
2594 if (fgets(buffer, sizeof(buffer), s_ReplayFile) == nullptr)
2595 {
2596 break;
2597 }
2598
2599 // Match offset
2600 unsigned inlineOffset = 0;
2601 count = sscanf_s(buffer, " <Offset>%u</Offset> ", &inlineOffset);
2602 if ((count != 1) || (inlineOffset != offset))
2603 {
2604 continue;
2605 }
2606
2607 // Token,Hash,Offset may still not be unique enough, but it's
2608 // all we have right now.
2609
2610 // We're good!
2611 foundInline = true;
2612
2613 // Check for a data collection marker. This does not affect
2614 // matching...
2615
2616 // Get next line
2617 if (fgets(buffer, sizeof(buffer), s_ReplayFile) != nullptr)
2618 {
2619 unsigned collectData = 0;
2620 count = sscanf_s(buffer, " <CollectData>%u</CollectData> ", &collectData);
2621
2622 if (count == 1)
2623 {
2624 m_IsDataCollectionTarget = (collectData == 1);
2625 }
2626 }
2627
2628 break;
2629 }
2630
2631 return foundInline;
2632}
2633
2634//------------------------------------------------------------------------
2635// FindInline: find entry for a particular callee in inline Xml.
2636//
2637// Arguments:
2638// callee -- handle for the callee method
2639//
2640// ReturnValue:
2641// true if the inline should be performed.
2642//
2643// Notes:
2644// Assumes file position has just been set by a successful call to
2645// FindContext(...);
2646//
2647// callee handle will not be sufficiently unique to identify a
2648// particular inline, if there are multiple calls to the same
2649// method.
2650
2651bool ReplayPolicy::FindInline(CORINFO_METHOD_HANDLE callee)
2652{
2653 // Token and Hash we're looking for
2654 mdMethodDef calleeToken = m_RootCompiler->info.compCompHnd->getMethodDefFromMethod(callee);
2655 unsigned calleeHash = m_RootCompiler->info.compCompHnd->getMethodHash(callee);
2656
2657 // Abstract this or just pass through raw bits
2658 // See matching code in xml writer
2659 int offset = -1;
2660 if (m_Offset != BAD_IL_OFFSET)
2661 {
2662 offset = (int)jitGetILoffs(m_Offset);
2663 }
2664
2665 unsigned calleeOffset = (unsigned)offset;
2666
2667 bool foundInline = FindInline(calleeToken, calleeHash, calleeOffset);
2668
2669 return foundInline;
2670}
2671
2672//------------------------------------------------------------------------
2673// NoteBool: handle an observed boolean value
2674//
2675// Arguments:
2676// obs - the current obsevation
2677// value - the value being observed
2678//
2679// Notes:
2680// Overrides parent so Replay can control force inlines.
2681
2682void ReplayPolicy::NoteBool(InlineObservation obs, bool value)
2683{
2684 // When inlining, let log override force inline.
2685 // Make a note of the actual value for later reporting during observations.
2686 if (!m_IsPrejitRoot && (obs == InlineObservation::CALLEE_IS_FORCE_INLINE))
2687 {
2688 m_WasForceInline = value;
2689 value = false;
2690 }
2691
2692 DiscretionaryPolicy::NoteBool(obs, value);
2693}
2694
2695//------------------------------------------------------------------------
2696// DetermineProfitability: determine if this inline is profitable
2697//
2698// Arguments:
2699// methodInfo -- method info for the callee
2700
2701void ReplayPolicy::DetermineProfitability(CORINFO_METHOD_INFO* methodInfo)
2702{
2703 // TODO: handle prejit root case....need to record this in the
2704 // root method XML.
2705 if (m_IsPrejitRoot)
2706 {
2707 // Fall back to discretionary policy for now.
2708 return DiscretionaryPolicy::DetermineProfitability(methodInfo);
2709 }
2710
2711 // If we're also dumping inline data, make additional observations
2712 // based on the method info, and estimate code size and perf
2713 // impact, so that the reports have the necessary data.
2714 if (JitConfig.JitInlineDumpData() != 0)
2715 {
2716 MethodInfoObservations(methodInfo);
2717 EstimateCodeSize();
2718 EstimatePerformanceImpact();
2719 m_IsForceInline = m_WasForceInline;
2720 }
2721
2722 // Try and find this candiate in the Xml.
2723 // If we fail to find it, then don't inline.
2724 bool accept = false;
2725
2726 // Grab the reader lock, since we'll be manipulating
2727 // the file pointer as we look for the relevant inline xml.
2728 {
2729 CritSecHolder readerLock(s_XmlReaderLock);
2730
2731 // First, locate the entries for the root method.
2732 bool foundMethod = FindMethod();
2733
2734 if (foundMethod && (m_InlineContext != nullptr))
2735 {
2736 // Next, navigate the context tree to find the entries
2737 // for the context that contains this candidate.
2738 bool foundContext = FindContext(m_InlineContext);
2739
2740 if (foundContext)
2741 {
2742 // Finally, find this candidate within its context
2743 CORINFO_METHOD_HANDLE calleeHandle = methodInfo->ftn;
2744 accept = FindInline(calleeHandle);
2745 }
2746 }
2747 }
2748
2749 if (accept)
2750 {
2751 JITLOG_THIS(m_RootCompiler, (LL_INFO100000, "Inline accepted via log replay"));
2752
2753 if (m_IsPrejitRoot)
2754 {
2755 SetCandidate(InlineObservation::CALLEE_LOG_REPLAY_ACCEPT);
2756 }
2757 else
2758 {
2759 SetCandidate(InlineObservation::CALLSITE_LOG_REPLAY_ACCEPT);
2760 }
2761 }
2762 else
2763 {
2764 JITLOG_THIS(m_RootCompiler, (LL_INFO100000, "Inline rejected via log replay"));
2765
2766 if (m_IsPrejitRoot)
2767 {
2768 SetNever(InlineObservation::CALLEE_LOG_REPLAY_REJECT);
2769 }
2770 else
2771 {
2772 SetFailure(InlineObservation::CALLSITE_LOG_REPLAY_REJECT);
2773 }
2774 }
2775
2776 return;
2777}
2778
2779#endif // defined(DEBUG) || defined(INLINE_DATA)
2780