1// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file
2// for details. All rights reserved. Use of this source code is governed by a
3// BSD-style license that can be found in the LICENSE file.
4
5#include "vm/compiler/backend/inliner.h"
6
7#include "vm/compiler/aot/aot_call_specializer.h"
8#include "vm/compiler/aot/precompiler.h"
9#include "vm/compiler/backend/block_scheduler.h"
10#include "vm/compiler/backend/branch_optimizer.h"
11#include "vm/compiler/backend/flow_graph_checker.h"
12#include "vm/compiler/backend/flow_graph_compiler.h"
13#include "vm/compiler/backend/il_printer.h"
14#include "vm/compiler/backend/type_propagator.h"
15#include "vm/compiler/compiler_pass.h"
16#include "vm/compiler/frontend/flow_graph_builder.h"
17#include "vm/compiler/frontend/kernel_to_il.h"
18#include "vm/compiler/jit/compiler.h"
19#include "vm/compiler/jit/jit_call_specializer.h"
20#include "vm/flags.h"
21#include "vm/kernel.h"
22#include "vm/longjump.h"
23#include "vm/object.h"
24#include "vm/object_store.h"
25
26namespace dart {
27
28DEFINE_FLAG(int,
29 deoptimization_counter_inlining_threshold,
30 12,
31 "How many times we allow deoptimization before we stop inlining.");
32DEFINE_FLAG(bool, trace_inlining, false, "Trace inlining");
33DEFINE_FLAG(charp, inlining_filter, NULL, "Inline only in named function");
34
35// Flags for inlining heuristics.
36DEFINE_FLAG(int,
37 inline_getters_setters_smaller_than,
38 10,
39 "Always inline getters and setters that have fewer instructions");
40DEFINE_FLAG(int,
41 inlining_depth_threshold,
42 6,
43 "Inline function calls up to threshold nesting depth");
44DEFINE_FLAG(
45 int,
46 inlining_size_threshold,
47 25,
48 "Always inline functions that have threshold or fewer instructions");
49DEFINE_FLAG(int,
50 inlining_callee_call_sites_threshold,
51 1,
52 "Always inline functions containing threshold or fewer calls.");
53DEFINE_FLAG(int,
54 inlining_callee_size_threshold,
55 160,
56 "Do not inline callees larger than threshold");
57DEFINE_FLAG(int,
58 inlining_small_leaf_size_threshold,
59 50,
60 "Do not inline leaf callees larger than threshold");
61DEFINE_FLAG(int,
62 inlining_caller_size_threshold,
63 50000,
64 "Stop inlining once caller reaches the threshold.");
65DEFINE_FLAG(int,
66 inlining_hotness,
67 10,
68 "Inline only hotter calls, in percents (0 .. 100); "
69 "default 10%: calls above-equal 10% of max-count are inlined.");
70DEFINE_FLAG(int,
71 inlining_recursion_depth_threshold,
72 1,
73 "Inline recursive function calls up to threshold recursion depth.");
74DEFINE_FLAG(int,
75 max_inlined_per_depth,
76 500,
77 "Max. number of inlined calls per depth");
78DEFINE_FLAG(bool, print_inlining_tree, false, "Print inlining tree");
79
80DECLARE_FLAG(int, max_deoptimization_counter_threshold);
81DECLARE_FLAG(bool, print_flow_graph);
82DECLARE_FLAG(bool, print_flow_graph_optimized);
83
84// Quick access to the current zone.
85#define Z (zone())
86#define I (isolate())
87
88#define TRACE_INLINING(statement) \
89 do { \
90 if (trace_inlining()) statement; \
91 } while (false)
92
93#define PRINT_INLINING_TREE(comment, caller, target, instance_call) \
94 do { \
95 if (FLAG_print_inlining_tree) { \
96 inlined_info_.Add(InlinedInfo(caller, target, inlining_depth_, \
97 instance_call, comment)); \
98 } \
99 } while (false)
100
101// Test and obtain Smi value.
102static bool IsSmiValue(Value* val, intptr_t* int_val) {
103 if (val->BindsToConstant() && val->BoundConstant().IsSmi()) {
104 *int_val = Smi::Cast(val->BoundConstant()).Value();
105 return true;
106 }
107 return false;
108}
109
110// Test if a call is recursive by looking in the deoptimization environment.
111static bool IsCallRecursive(const Function& function, Definition* call) {
112 Environment* env = call->env();
113 while (env != NULL) {
114 if (function.raw() == env->function().raw()) {
115 return true;
116 }
117 env = env->outer();
118 }
119 return false;
120}
121
122// Helper to get the default value of a formal parameter.
123static ConstantInstr* GetDefaultValue(intptr_t i,
124 const ParsedFunction& parsed_function) {
125 return new ConstantInstr(parsed_function.DefaultParameterValueAt(i));
126}
127
128// Helper to get result type from call (or nullptr otherwise).
129static CompileType* ResultType(Definition* call) {
130 if (auto static_call = call->AsStaticCall()) {
131 return static_call->result_type();
132 } else if (auto instance_call = call->AsInstanceCall()) {
133 return instance_call->result_type();
134 }
135 return nullptr;
136}
137
138// Pair of an argument name and its value.
139struct NamedArgument {
140 String* name;
141 Value* value;
142 NamedArgument(String* name, Value* value) : name(name), value(value) {}
143};
144
145// Ensures we only inline callee graphs which are safe. There are certain
146// instructions which cannot be inlined and we ensure here that we don't do
147// that.
148class CalleeGraphValidator : public AllStatic {
149 public:
150 static void Validate(FlowGraph* callee_graph) {
151#ifdef DEBUG
152 for (BlockIterator block_it = callee_graph->reverse_postorder_iterator();
153 !block_it.Done(); block_it.Advance()) {
154 BlockEntryInstr* entry = block_it.Current();
155
156 for (ForwardInstructionIterator it(entry); !it.Done(); it.Advance()) {
157 Instruction* current = it.Current();
158 if (current->IsBranch()) {
159 current = current->AsBranch()->comparison();
160 }
161 // The following instructions are not safe to inline, since they make
162 // assumptions about the frame layout.
163 ASSERT(!current->IsTailCall());
164 ASSERT(!current->IsLoadIndexedUnsafe());
165 ASSERT(!current->IsStoreIndexedUnsafe());
166 }
167 }
168#endif // DEBUG
169 }
170};
171
172// Helper to collect information about a callee graph when considering it for
173// inlining.
174class GraphInfoCollector : public ValueObject {
175 public:
176 GraphInfoCollector() : call_site_count_(0), instruction_count_(0) {}
177
178 void Collect(const FlowGraph& graph) {
179 call_site_count_ = 0;
180 instruction_count_ = 0;
181 for (BlockIterator block_it = graph.postorder_iterator(); !block_it.Done();
182 block_it.Advance()) {
183 // Skip any blocks from the prologue to make them not count towards the
184 // inlining instruction budget.
185 const intptr_t block_id = block_it.Current()->block_id();
186 if (graph.prologue_info().Contains(block_id)) {
187 continue;
188 }
189
190 for (ForwardInstructionIterator it(block_it.Current()); !it.Done();
191 it.Advance()) {
192 Instruction* current = it.Current();
193 // Don't count instructions that won't generate any code.
194 if (current->IsRedefinition()) {
195 continue;
196 }
197 // UnboxedConstant is often folded into the indexing
198 // instructions (similar to Constant instructions which
199 // belong to initial definitions and not counted here).
200 if (current->IsUnboxedConstant()) {
201 continue;
202 }
203 ++instruction_count_;
204 // Count inputs of certain instructions as if separate PushArgument
205 // instructions are used for inputs. This is done in order to
206 // preserve inlining behavior and avoid code size growth after
207 // PushArgument instructions are eliminated.
208 if (current->IsAllocateObject()) {
209 instruction_count_ += current->InputCount();
210 } else if (current->ArgumentCount() > 0) {
211 ASSERT(!current->HasPushArguments());
212 instruction_count_ += current->ArgumentCount();
213 }
214 if (current->IsInstanceCall() || current->IsStaticCall() ||
215 current->IsClosureCall()) {
216 ++call_site_count_;
217 continue;
218 }
219 if (current->IsPolymorphicInstanceCall()) {
220 PolymorphicInstanceCallInstr* call =
221 current->AsPolymorphicInstanceCall();
222 // These checks make sure that the number of call-sites counted does
223 // not change relative to the time when the current set of inlining
224 // parameters was fixed.
225 // TODO(fschneider): Determine new heuristic parameters that avoid
226 // these checks entirely.
227 if (!call->IsSureToCallSingleRecognizedTarget() &&
228 (call->token_kind() != Token::kEQ)) {
229 ++call_site_count_;
230 }
231 }
232 }
233 }
234 }
235
236 intptr_t call_site_count() const { return call_site_count_; }
237 intptr_t instruction_count() const { return instruction_count_; }
238
239 private:
240 intptr_t call_site_count_;
241 intptr_t instruction_count_;
242};
243
244// Structure for collecting inline data needed to print inlining tree.
245struct InlinedInfo {
246 const Function* caller;
247 const Function* inlined;
248 intptr_t inlined_depth;
249 const Definition* call_instr;
250 const char* bailout_reason;
251 InlinedInfo(const Function* caller_function,
252 const Function* inlined_function,
253 const intptr_t depth,
254 const Definition* call,
255 const char* reason)
256 : caller(caller_function),
257 inlined(inlined_function),
258 inlined_depth(depth),
259 call_instr(call),
260 bailout_reason(reason) {}
261};
262
263// A collection of call sites to consider for inlining.
264class CallSites : public ValueObject {
265 public:
266 explicit CallSites(intptr_t threshold)
267 : inlining_depth_threshold_(threshold),
268 static_calls_(),
269 closure_calls_(),
270 instance_calls_() {}
271
272 struct InstanceCallInfo {
273 PolymorphicInstanceCallInstr* call;
274 double ratio;
275 const FlowGraph* caller_graph;
276 intptr_t nesting_depth;
277 InstanceCallInfo(PolymorphicInstanceCallInstr* call_arg,
278 FlowGraph* flow_graph,
279 intptr_t depth)
280 : call(call_arg),
281 ratio(0.0),
282 caller_graph(flow_graph),
283 nesting_depth(depth) {}
284 const Function& caller() const { return caller_graph->function(); }
285 };
286
287 struct StaticCallInfo {
288 StaticCallInstr* call;
289 double ratio;
290 FlowGraph* caller_graph;
291 intptr_t nesting_depth;
292 StaticCallInfo(StaticCallInstr* value,
293 FlowGraph* flow_graph,
294 intptr_t depth)
295 : call(value),
296 ratio(0.0),
297 caller_graph(flow_graph),
298 nesting_depth(depth) {}
299 const Function& caller() const { return caller_graph->function(); }
300 };
301
302 struct ClosureCallInfo {
303 ClosureCallInstr* call;
304 FlowGraph* caller_graph;
305 ClosureCallInfo(ClosureCallInstr* value, FlowGraph* flow_graph)
306 : call(value), caller_graph(flow_graph) {}
307 const Function& caller() const { return caller_graph->function(); }
308 };
309
310 const GrowableArray<InstanceCallInfo>& instance_calls() const {
311 return instance_calls_;
312 }
313
314 const GrowableArray<StaticCallInfo>& static_calls() const {
315 return static_calls_;
316 }
317
318 const GrowableArray<ClosureCallInfo>& closure_calls() const {
319 return closure_calls_;
320 }
321
322 bool HasCalls() const {
323 return !(static_calls_.is_empty() && closure_calls_.is_empty() &&
324 instance_calls_.is_empty());
325 }
326
327 intptr_t NumCalls() const {
328 return instance_calls_.length() + static_calls_.length() +
329 closure_calls_.length();
330 }
331
332 void Clear() {
333 static_calls_.Clear();
334 closure_calls_.Clear();
335 instance_calls_.Clear();
336 }
337
338 // Heuristic that maps the loop nesting depth to a static estimate of number
339 // of times code at that depth is executed (code at each higher nesting
340 // depth is assumed to execute 10x more often up to depth 3).
341 static intptr_t AotCallCountApproximation(intptr_t nesting_depth) {
342 switch (nesting_depth) {
343 case 0:
344 // The value 1 makes most sense, but it may give a high ratio to call
345 // sites outside loops. Therefore, such call sites are subject to
346 // subsequent stricter heuristic to limit code size increase.
347 return 1;
348 case 1:
349 return 10;
350 case 2:
351 return 10 * 10;
352 default:
353 return 10 * 10 * 10;
354 }
355 }
356
357 // Computes the ratio for each call site in a method, defined as the
358 // number of times a call site is executed over the maximum number of
359 // times any call site is executed in the method. JIT uses actual call
360 // counts whereas AOT uses a static estimate based on nesting depth.
361 void ComputeCallSiteRatio(intptr_t static_call_start_ix,
362 intptr_t instance_call_start_ix) {
363 const intptr_t num_static_calls =
364 static_calls_.length() - static_call_start_ix;
365 const intptr_t num_instance_calls =
366 instance_calls_.length() - instance_call_start_ix;
367
368 intptr_t max_count = 0;
369 GrowableArray<intptr_t> instance_call_counts(num_instance_calls);
370 for (intptr_t i = 0; i < num_instance_calls; ++i) {
371 const InstanceCallInfo& info =
372 instance_calls_[i + instance_call_start_ix];
373 intptr_t aggregate_count =
374 CompilerState::Current().is_aot()
375 ? AotCallCountApproximation(info.nesting_depth)
376 : info.call->CallCount();
377 instance_call_counts.Add(aggregate_count);
378 if (aggregate_count > max_count) max_count = aggregate_count;
379 }
380
381 GrowableArray<intptr_t> static_call_counts(num_static_calls);
382 for (intptr_t i = 0; i < num_static_calls; ++i) {
383 const StaticCallInfo& info = static_calls_[i + static_call_start_ix];
384 intptr_t aggregate_count =
385 CompilerState::Current().is_aot()
386 ? AotCallCountApproximation(info.nesting_depth)
387 : info.call->CallCount();
388 static_call_counts.Add(aggregate_count);
389 if (aggregate_count > max_count) max_count = aggregate_count;
390 }
391
392 // Note that max_count can be 0 if none of the calls was executed.
393 for (intptr_t i = 0; i < num_instance_calls; ++i) {
394 const double ratio =
395 (max_count == 0)
396 ? 0.0
397 : static_cast<double>(instance_call_counts[i]) / max_count;
398 instance_calls_[i + instance_call_start_ix].ratio = ratio;
399 }
400 for (intptr_t i = 0; i < num_static_calls; ++i) {
401 const double ratio =
402 (max_count == 0)
403 ? 0.0
404 : static_cast<double>(static_call_counts[i]) / max_count;
405 static_calls_[i + static_call_start_ix].ratio = ratio;
406 }
407 }
408
409 static void RecordAllNotInlinedFunction(
410 FlowGraph* graph,
411 intptr_t depth,
412 GrowableArray<InlinedInfo>* inlined_info) {
413 const Function* caller = &graph->function();
414 Function& target = Function::ZoneHandle();
415 for (BlockIterator block_it = graph->postorder_iterator(); !block_it.Done();
416 block_it.Advance()) {
417 for (ForwardInstructionIterator it(block_it.Current()); !it.Done();
418 it.Advance()) {
419 Instruction* current = it.Current();
420 Definition* call = NULL;
421 if (current->IsPolymorphicInstanceCall()) {
422 PolymorphicInstanceCallInstr* instance_call =
423 current->AsPolymorphicInstanceCall();
424 target = instance_call->targets().FirstTarget().raw();
425 call = instance_call;
426 } else if (current->IsStaticCall()) {
427 StaticCallInstr* static_call = current->AsStaticCall();
428 target = static_call->function().raw();
429 call = static_call;
430 } else if (current->IsClosureCall()) {
431 // TODO(srdjan): Add data for closure calls.
432 }
433 if (call != NULL) {
434 inlined_info->Add(
435 InlinedInfo(caller, &target, depth + 1, call, "Too deep"));
436 }
437 }
438 }
439 }
440
441 void FindCallSites(FlowGraph* graph,
442 intptr_t depth,
443 GrowableArray<InlinedInfo>* inlined_info) {
444 ASSERT(graph != NULL);
445 if (depth > inlining_depth_threshold_) {
446 if (FLAG_print_inlining_tree) {
447 RecordAllNotInlinedFunction(graph, depth, inlined_info);
448 }
449 return;
450 }
451
452 // At the maximum inlining depth, only profitable methods
453 // are further considered for inlining.
454 const bool inline_only_profitable_methods =
455 (depth >= inlining_depth_threshold_);
456
457 // In AOT, compute loop hierarchy.
458 if (CompilerState::Current().is_aot()) {
459 graph->GetLoopHierarchy();
460 }
461
462 const intptr_t instance_call_start_ix = instance_calls_.length();
463 const intptr_t static_call_start_ix = static_calls_.length();
464 for (BlockIterator block_it = graph->postorder_iterator(); !block_it.Done();
465 block_it.Advance()) {
466 BlockEntryInstr* entry = block_it.Current();
467 const intptr_t depth = entry->NestingDepth();
468 for (ForwardInstructionIterator it(entry); !it.Done(); it.Advance()) {
469 Instruction* current = it.Current();
470 if (current->IsPolymorphicInstanceCall()) {
471 PolymorphicInstanceCallInstr* instance_call =
472 current->AsPolymorphicInstanceCall();
473 if (!inline_only_profitable_methods ||
474 instance_call->IsSureToCallSingleRecognizedTarget() ||
475 instance_call->HasOnlyDispatcherOrImplicitAccessorTargets()) {
476 // Consider instance call for further inlining. Note that it will
477 // still be subject to all the inlining heuristics.
478 instance_calls_.Add(InstanceCallInfo(instance_call, graph, depth));
479 } else {
480 // No longer consider the instance call because inlining is too
481 // deep and the method is not deemed profitable by other criteria.
482 if (FLAG_print_inlining_tree) {
483 const Function* caller = &graph->function();
484 const Function* target = &instance_call->targets().FirstTarget();
485 inlined_info->Add(InlinedInfo(caller, target, depth + 1,
486 instance_call, "Too deep"));
487 }
488 }
489 } else if (current->IsStaticCall()) {
490 StaticCallInstr* static_call = current->AsStaticCall();
491 const Function& function = static_call->function();
492 if (!inline_only_profitable_methods || function.IsRecognized() ||
493 function.IsDispatcherOrImplicitAccessor() ||
494 (function.is_const() && function.IsGenerativeConstructor())) {
495 // Consider static call for further inlining. Note that it will
496 // still be subject to all the inlining heuristics.
497 static_calls_.Add(StaticCallInfo(static_call, graph, depth));
498 } else {
499 // No longer consider the static call because inlining is too
500 // deep and the method is not deemed profitable by other criteria.
501 if (FLAG_print_inlining_tree) {
502 const Function* caller = &graph->function();
503 const Function* target = &static_call->function();
504 inlined_info->Add(InlinedInfo(caller, target, depth + 1,
505 static_call, "Too deep"));
506 }
507 }
508 } else if (current->IsClosureCall()) {
509 if (!inline_only_profitable_methods) {
510 // Consider closure for further inlining. Note that it will
511 // still be subject to all the inlining heuristics.
512 ClosureCallInstr* closure_call = current->AsClosureCall();
513 closure_calls_.Add(ClosureCallInfo(closure_call, graph));
514 } else {
515 // No longer consider the closure because inlining is too deep.
516 }
517 }
518 }
519 }
520 ComputeCallSiteRatio(static_call_start_ix, instance_call_start_ix);
521 }
522
523 private:
524 intptr_t inlining_depth_threshold_;
525 GrowableArray<StaticCallInfo> static_calls_;
526 GrowableArray<ClosureCallInfo> closure_calls_;
527 GrowableArray<InstanceCallInfo> instance_calls_;
528
529 DISALLOW_COPY_AND_ASSIGN(CallSites);
530};
531
532// Determines if inlining this graph yields a small leaf node.
533static bool IsSmallLeaf(FlowGraph* graph) {
534 intptr_t instruction_count = 0;
535 for (BlockIterator block_it = graph->postorder_iterator(); !block_it.Done();
536 block_it.Advance()) {
537 BlockEntryInstr* entry = block_it.Current();
538 for (ForwardInstructionIterator it(entry); !it.Done(); it.Advance()) {
539 Instruction* current = it.Current();
540 ++instruction_count;
541 if (current->IsInstanceCall() || current->IsPolymorphicInstanceCall() ||
542 current->IsClosureCall()) {
543 return false;
544 } else if (current->IsStaticCall()) {
545 const Function& function = current->AsStaticCall()->function();
546 const intptr_t inl_size = function.optimized_instruction_count();
547 const bool always_inline =
548 FlowGraphInliner::FunctionHasPreferInlinePragma(function);
549 // Accept a static call is always inlined in some way and add the
550 // cached size to the total instruction count. A reasonable guess
551 // is made if the count has not been collected yet (listed methods
552 // are never very large).
553 if (!always_inline && !function.IsRecognized()) {
554 return false;
555 }
556 if (!always_inline) {
557 static constexpr intptr_t kAvgListedMethodSize = 20;
558 instruction_count +=
559 (inl_size == 0 ? kAvgListedMethodSize : inl_size);
560 }
561 }
562 }
563 }
564 return instruction_count <= FLAG_inlining_small_leaf_size_threshold;
565}
566
567struct InlinedCallData {
568 InlinedCallData(Definition* call,
569 const Array& arguments_descriptor,
570 intptr_t first_arg_index, // 1 if type args are passed.
571 GrowableArray<Value*>* arguments,
572 const Function& caller,
573 intptr_t caller_inlining_id)
574 : call(call),
575 arguments_descriptor(arguments_descriptor),
576 first_arg_index(first_arg_index),
577 arguments(arguments),
578 callee_graph(NULL),
579 parameter_stubs(NULL),
580 exit_collector(NULL),
581 caller(caller),
582 caller_inlining_id(caller_inlining_id) {}
583
584 Definition* call;
585 const Array& arguments_descriptor;
586 const intptr_t first_arg_index;
587 GrowableArray<Value*>* arguments;
588 FlowGraph* callee_graph;
589 ZoneGrowableArray<Definition*>* parameter_stubs;
590 InlineExitCollector* exit_collector;
591 const Function& caller;
592 const intptr_t caller_inlining_id;
593};
594
595class CallSiteInliner;
596
597class PolymorphicInliner : public ValueObject {
598 public:
599 PolymorphicInliner(CallSiteInliner* owner,
600 PolymorphicInstanceCallInstr* call,
601 const Function& caller_function,
602 intptr_t caller_inlining_id);
603
604 bool Inline();
605
606 private:
607 bool CheckInlinedDuplicate(const Function& target);
608 bool CheckNonInlinedDuplicate(const Function& target);
609
610 bool TryInliningPoly(const TargetInfo& target);
611 bool TryInlineRecognizedMethod(intptr_t receiver_cid, const Function& target);
612
613 TargetEntryInstr* BuildDecisionGraph();
614
615 Isolate* isolate() const;
616 Zone* zone() const;
617 intptr_t AllocateBlockId() const;
618 inline bool trace_inlining() const;
619
620 CallSiteInliner* const owner_;
621 PolymorphicInstanceCallInstr* const call_;
622 const intptr_t num_variants_;
623 const CallTargets& variants_;
624
625 CallTargets inlined_variants_;
626 // The non_inlined_variants_ can be used in a long-lived instruction object,
627 // so they are not embedded into the shorter-lived PolymorphicInliner object.
628 CallTargets* non_inlined_variants_;
629 GrowableArray<BlockEntryInstr*> inlined_entries_;
630 InlineExitCollector* exit_collector_;
631
632 const Function& caller_function_;
633 const intptr_t caller_inlining_id_;
634};
635
636static void ReplaceParameterStubs(Zone* zone,
637 FlowGraph* caller_graph,
638 InlinedCallData* call_data,
639 const TargetInfo* target_info) {
640 const bool is_polymorphic = call_data->call->IsPolymorphicInstanceCall();
641 ASSERT(is_polymorphic == (target_info != NULL));
642 FlowGraph* callee_graph = call_data->callee_graph;
643 auto callee_entry = callee_graph->graph_entry()->normal_entry();
644
645 // Replace each stub with the actual argument or the caller's constant.
646 // Nulls denote optional parameters for which no actual was given.
647 const intptr_t first_arg_index = call_data->first_arg_index;
648 // When first_arg_index > 0, the stub and actual argument processed in the
649 // first loop iteration represent a passed-in type argument vector.
650 GrowableArray<Value*>* arguments = call_data->arguments;
651 intptr_t first_arg_stub_index = 0;
652 if (arguments->length() != call_data->parameter_stubs->length()) {
653 ASSERT(arguments->length() == call_data->parameter_stubs->length() - 1);
654 ASSERT(first_arg_index == 0);
655 // The first parameter stub accepts an optional type argument vector, but
656 // none was provided in arguments.
657 first_arg_stub_index = 1;
658 }
659 for (intptr_t i = 0; i < arguments->length(); ++i) {
660 Value* actual = (*arguments)[i];
661 Definition* defn = NULL;
662 if (is_polymorphic && (i == first_arg_index)) {
663 // Replace the receiver argument with a redefinition to prevent code from
664 // the inlined body from being hoisted above the inlined entry.
665 RedefinitionInstr* redefinition =
666 new (zone) RedefinitionInstr(actual->Copy(zone));
667 redefinition->set_ssa_temp_index(caller_graph->alloc_ssa_temp_index());
668 if (FlowGraph::NeedsPairLocation(redefinition->representation())) {
669 caller_graph->alloc_ssa_temp_index();
670 }
671 if (target_info->IsSingleCid()) {
672 redefinition->UpdateType(CompileType::FromCid(target_info->cid_start));
673 }
674 redefinition->InsertAfter(callee_entry);
675 defn = redefinition;
676 // Since the redefinition does not dominate the callee entry, replace
677 // uses of the receiver argument in this entry with the redefined value.
678 callee_entry->ReplaceInEnvironment(
679 call_data->parameter_stubs->At(first_arg_stub_index + i),
680 actual->definition());
681 } else if (actual != NULL) {
682 defn = actual->definition();
683 }
684 if (defn != NULL) {
685 call_data->parameter_stubs->At(first_arg_stub_index + i)
686 ->ReplaceUsesWith(defn);
687 }
688 }
689
690 // Replace remaining constants with uses by constants in the caller's
691 // initial definitions.
692 auto defns = callee_graph->graph_entry()->initial_definitions();
693 for (intptr_t i = 0; i < defns->length(); ++i) {
694 ConstantInstr* constant = (*defns)[i]->AsConstant();
695 if (constant != NULL && constant->HasUses()) {
696 constant->ReplaceUsesWith(caller_graph->GetConstant(constant->value()));
697 }
698 }
699
700 defns = callee_graph->graph_entry()->normal_entry()->initial_definitions();
701 for (intptr_t i = 0; i < defns->length(); ++i) {
702 ConstantInstr* constant = (*defns)[i]->AsConstant();
703 if (constant != NULL && constant->HasUses()) {
704 constant->ReplaceUsesWith(caller_graph->GetConstant(constant->value()));
705 }
706
707 SpecialParameterInstr* param = (*defns)[i]->AsSpecialParameter();
708 if (param != NULL && param->HasUses()) {
709 switch (param->kind()) {
710 case SpecialParameterInstr::kContext: {
711 ASSERT(!is_polymorphic);
712 // We do not support polymorphic inlining of closure calls.
713 ASSERT(call_data->call->IsClosureCall());
714 LoadFieldInstr* context_load = new (zone) LoadFieldInstr(
715 new Value((*arguments)[first_arg_index]->definition()),
716 Slot::Closure_context(), call_data->call->token_pos());
717 context_load->set_ssa_temp_index(
718 caller_graph->alloc_ssa_temp_index());
719 if (FlowGraph::NeedsPairLocation(context_load->representation())) {
720 caller_graph->alloc_ssa_temp_index();
721 }
722 context_load->InsertBefore(callee_entry->next());
723 param->ReplaceUsesWith(context_load);
724 break;
725 }
726 case SpecialParameterInstr::kTypeArgs: {
727 Definition* type_args;
728 if (first_arg_index > 0) {
729 type_args = (*arguments)[0]->definition();
730 } else {
731 type_args = caller_graph->constant_null();
732 }
733 param->ReplaceUsesWith(type_args);
734 break;
735 }
736 case SpecialParameterInstr::kArgDescriptor: {
737 param->ReplaceUsesWith(
738 caller_graph->GetConstant(call_data->arguments_descriptor));
739 break;
740 }
741 default: {
742 UNREACHABLE();
743 break;
744 }
745 }
746 }
747 }
748}
749
750class CallSiteInliner : public ValueObject {
751 public:
752 explicit CallSiteInliner(FlowGraphInliner* inliner, intptr_t threshold)
753 : inliner_(inliner),
754 caller_graph_(inliner->flow_graph()),
755 inlined_(false),
756 initial_size_(inliner->flow_graph()->InstructionCount()),
757 inlined_size_(0),
758 inlined_recursive_call_(false),
759 inlining_depth_(1),
760 inlining_recursion_depth_(0),
761 inlining_depth_threshold_(threshold),
762 collected_call_sites_(NULL),
763 inlining_call_sites_(NULL),
764 function_cache_(),
765 inlined_info_() {}
766
767 FlowGraph* caller_graph() const { return caller_graph_; }
768
769 Thread* thread() const { return caller_graph_->thread(); }
770 Isolate* isolate() const { return caller_graph_->isolate(); }
771 Zone* zone() const { return caller_graph_->zone(); }
772
773 bool trace_inlining() const { return inliner_->trace_inlining(); }
774
775 int inlining_depth() { return inlining_depth_; }
776
777 struct InliningDecision {
778 InliningDecision(bool b, const char* r) : value(b), reason(r) {}
779 bool value;
780 const char* reason;
781 static InliningDecision Yes(const char* reason) {
782 return InliningDecision(true, reason);
783 }
784 static InliningDecision No(const char* reason) {
785 return InliningDecision(false, reason);
786 }
787 };
788
789 // Inlining heuristics based on Cooper et al. 2008.
790 InliningDecision ShouldWeInline(const Function& callee,
791 intptr_t instr_count,
792 intptr_t call_site_count) {
793 // Pragma or size heuristics.
794 if (inliner_->AlwaysInline(callee)) {
795 return InliningDecision::Yes("AlwaysInline");
796 } else if (inlined_size_ > FLAG_inlining_caller_size_threshold) {
797 // Prevent caller methods becoming humongous and thus slow to compile.
798 return InliningDecision::No("--inlining-caller-size-threshold");
799 } else if (instr_count > FLAG_inlining_callee_size_threshold) {
800 // Prevent inlining of callee methods that exceed certain size.
801 return InliningDecision::No("--inlining-callee-size-threshold");
802 }
803 // Inlining depth.
804 const int callee_inlining_depth = callee.inlining_depth();
805 if (callee_inlining_depth > 0 &&
806 ((callee_inlining_depth + inlining_depth_) >
807 FLAG_inlining_depth_threshold)) {
808 return InliningDecision::No("--inlining-depth-threshold");
809 }
810 // Situation instr_count == 0 denotes no counts have been computed yet.
811 // In that case, we say ok to the early heuristic and come back with the
812 // late heuristic.
813 if (instr_count == 0) {
814 return InliningDecision::Yes("need to count first");
815 } else if (instr_count <= FLAG_inlining_size_threshold) {
816 return InliningDecision::Yes("--inlining-size-threshold");
817 } else if (call_site_count <= FLAG_inlining_callee_call_sites_threshold) {
818 return InliningDecision::Yes("--inlining-callee-call-sites-threshold");
819 }
820 return InliningDecision::No("default");
821 }
822
823 void InlineCalls() {
824 // If inlining depth is less than one abort.
825 if (inlining_depth_threshold_ < 1) return;
826 if (caller_graph_->function().deoptimization_counter() >=
827 FLAG_deoptimization_counter_inlining_threshold) {
828 return;
829 }
830 // Create two call site collections to swap between.
831 CallSites sites1(inlining_depth_threshold_);
832 CallSites sites2(inlining_depth_threshold_);
833 CallSites* call_sites_temp = NULL;
834 collected_call_sites_ = &sites1;
835 inlining_call_sites_ = &sites2;
836 // Collect initial call sites.
837 collected_call_sites_->FindCallSites(caller_graph_, inlining_depth_,
838 &inlined_info_);
839 while (collected_call_sites_->HasCalls()) {
840 TRACE_INLINING(
841 THR_Print(" Depth %" Pd " ----------\n", inlining_depth_));
842 if (FLAG_print_inlining_tree) {
843 THR_Print("**Depth % " Pd " calls to inline %" Pd " (threshold % " Pd
844 ")\n",
845 inlining_depth_, collected_call_sites_->NumCalls(),
846 static_cast<intptr_t>(FLAG_max_inlined_per_depth));
847 }
848 if (collected_call_sites_->NumCalls() > FLAG_max_inlined_per_depth) {
849 break;
850 }
851 // Swap collected and inlining arrays and clear the new collecting array.
852 call_sites_temp = collected_call_sites_;
853 collected_call_sites_ = inlining_call_sites_;
854 inlining_call_sites_ = call_sites_temp;
855 collected_call_sites_->Clear();
856 // Inline call sites at the current depth.
857 bool inlined_instance = InlineInstanceCalls();
858 bool inlined_statics = InlineStaticCalls();
859 bool inlined_closures = InlineClosureCalls();
860 if (inlined_instance || inlined_statics || inlined_closures) {
861 // Increment the inlining depths. Checked before subsequent inlining.
862 ++inlining_depth_;
863 if (inlined_recursive_call_) {
864 ++inlining_recursion_depth_;
865 inlined_recursive_call_ = false;
866 }
867 thread()->CheckForSafepoint();
868 }
869 }
870
871 collected_call_sites_ = NULL;
872 inlining_call_sites_ = NULL;
873 }
874
875 bool inlined() const { return inlined_; }
876
877 double GrowthFactor() const {
878 return static_cast<double>(inlined_size_) /
879 static_cast<double>(initial_size_);
880 }
881
882 // Helper to create a parameter stub from an actual argument.
883 Definition* CreateParameterStub(intptr_t i,
884 Value* argument,
885 FlowGraph* graph) {
886 ConstantInstr* constant = argument->definition()->AsConstant();
887 if (constant != NULL) {
888 return new (Z) ConstantInstr(constant->value());
889 } else {
890 ParameterInstr* param = new (Z)
891 ParameterInstr(i, -1, graph->graph_entry(), kNoRepresentation);
892 param->UpdateType(*argument->Type());
893 return param;
894 }
895 }
896
897 bool TryInlining(const Function& function,
898 const Array& argument_names,
899 InlinedCallData* call_data,
900 bool stricter_heuristic) {
901 if (trace_inlining()) {
902 String& name = String::Handle(function.QualifiedUserVisibleName());
903 THR_Print(" => %s (deopt count %d)\n", name.ToCString(),
904 function.deoptimization_counter());
905 }
906
907 // Abort if the inlinable bit on the function is low.
908 if (!function.CanBeInlined()) {
909 TRACE_INLINING(THR_Print(" Bailout: not inlinable\n"));
910 PRINT_INLINING_TREE("Not inlinable", &call_data->caller, &function,
911 call_data->call);
912 return false;
913 }
914
915 if (FlowGraphInliner::FunctionHasNeverInlinePragma(function)) {
916 TRACE_INLINING(THR_Print(" Bailout: vm:never-inline pragma\n"));
917 return false;
918 }
919
920 // Don't inline any intrinsified functions in precompiled mode
921 // to reduce code size and make sure we use the intrinsic code.
922 if (CompilerState::Current().is_aot() && function.is_intrinsic() &&
923 !inliner_->AlwaysInline(function)) {
924 TRACE_INLINING(THR_Print(" Bailout: intrinisic\n"));
925 PRINT_INLINING_TREE("intrinsic", &call_data->caller, &function,
926 call_data->call);
927 return false;
928 }
929
930 // Do not rely on function type feedback or presence of code to determine
931 // if a function was compiled.
932 if (!CompilerState::Current().is_aot() && !function.WasCompiled()) {
933 TRACE_INLINING(THR_Print(" Bailout: not compiled yet\n"));
934 PRINT_INLINING_TREE("Not compiled", &call_data->caller, &function,
935 call_data->call);
936 return false;
937 }
938
939 // Type feedback may have been cleared for this function (ClearICDataArray),
940 // but we need it for inlining.
941 if (!CompilerState::Current().is_aot() &&
942 (function.ic_data_array() == Array::null())) {
943 TRACE_INLINING(THR_Print(" Bailout: type feedback cleared\n"));
944 PRINT_INLINING_TREE("Not compiled", &call_data->caller, &function,
945 call_data->call);
946 return false;
947 }
948
949 // Abort if this function has deoptimized too much.
950 if (function.deoptimization_counter() >=
951 FLAG_max_deoptimization_counter_threshold) {
952 function.set_is_inlinable(false);
953 TRACE_INLINING(THR_Print(" Bailout: deoptimization threshold\n"));
954 PRINT_INLINING_TREE("Deoptimization threshold exceeded",
955 &call_data->caller, &function, call_data->call);
956 return false;
957 }
958
959 // Apply early heuristics. For a specialized case
960 // (constants_arg_counts > 0), don't use a previously
961 // estimate of the call site and instruction counts.
962 // Note that at this point, optional constant parameters
963 // are not counted yet, which makes this decision approximate.
964 GrowableArray<Value*>* arguments = call_data->arguments;
965 const intptr_t constant_arg_count = CountConstants(*arguments);
966 const intptr_t instruction_count =
967 constant_arg_count == 0 ? function.optimized_instruction_count() : 0;
968 const intptr_t call_site_count =
969 constant_arg_count == 0 ? function.optimized_call_site_count() : 0;
970 InliningDecision decision =
971 ShouldWeInline(function, instruction_count, call_site_count);
972 if (!decision.value) {
973 TRACE_INLINING(
974 THR_Print(" Bailout: early heuristics (%s) with "
975 "code size: %" Pd ", "
976 "call sites: %" Pd ", "
977 "inlining depth of callee: %d, "
978 "const args: %" Pd "\n",
979 decision.reason, instruction_count, call_site_count,
980 function.inlining_depth(), constant_arg_count));
981 PRINT_INLINING_TREE("Early heuristic", &call_data->caller, &function,
982 call_data->call);
983 return false;
984 }
985
986 // Abort if this is a recursive occurrence.
987 Definition* call = call_data->call;
988 // Added 'volatile' works around a possible GCC 4.9 compiler bug.
989 volatile bool is_recursive_call = IsCallRecursive(function, call);
990 if (is_recursive_call &&
991 inlining_recursion_depth_ >= FLAG_inlining_recursion_depth_threshold) {
992 TRACE_INLINING(THR_Print(" Bailout: recursive function\n"));
993 PRINT_INLINING_TREE("Recursive function", &call_data->caller, &function,
994 call_data->call);
995 return false;
996 }
997
998 Error& error = Error::Handle();
999 {
1000 // Save and clear deopt id.
1001 DeoptIdScope deopt_id_scope(thread(), 0);
1002
1003 // Install bailout jump.
1004 LongJumpScope jump;
1005 if (setjmp(*jump.Set()) == 0) {
1006 // Load IC data for the callee.
1007 ZoneGrowableArray<const ICData*>* ic_data_array =
1008 new (Z) ZoneGrowableArray<const ICData*>();
1009 const bool clone_ic_data = Compiler::IsBackgroundCompilation();
1010 function.RestoreICDataMap(ic_data_array, clone_ic_data);
1011 if (Compiler::IsBackgroundCompilation() &&
1012 (function.ic_data_array() == Array::null())) {
1013 Compiler::AbortBackgroundCompilation(DeoptId::kNone,
1014 "ICData cleared while inlining");
1015 }
1016
1017 // Parse the callee function.
1018 bool in_cache;
1019 ParsedFunction* parsed_function;
1020 {
1021 parsed_function = GetParsedFunction(function, &in_cache);
1022 if (!function.CanBeInlined()) {
1023 // As a side effect of parsing the function, it may be marked
1024 // as not inlinable. This happens for async and async* functions
1025 // when causal stack traces are being tracked.
1026 return false;
1027 }
1028 }
1029
1030 // Build the callee graph.
1031 InlineExitCollector* exit_collector =
1032 new (Z) InlineExitCollector(caller_graph_, call);
1033 FlowGraph* callee_graph;
1034 Code::EntryKind entry_kind = Code::EntryKind::kNormal;
1035 if (StaticCallInstr* instr = call_data->call->AsStaticCall()) {
1036 entry_kind = instr->entry_kind();
1037 } else if (InstanceCallInstr* instr =
1038 call_data->call->AsInstanceCall()) {
1039 entry_kind = instr->entry_kind();
1040 } else if (PolymorphicInstanceCallInstr* instr =
1041 call_data->call->AsPolymorphicInstanceCall()) {
1042 entry_kind = instr->entry_kind();
1043 } else if (ClosureCallInstr* instr = call_data->call->AsClosureCall()) {
1044 entry_kind = instr->entry_kind();
1045 }
1046 kernel::FlowGraphBuilder builder(
1047 parsed_function, ic_data_array, /* not building var desc */ NULL,
1048 exit_collector,
1049 /* optimized = */ true, Compiler::kNoOSRDeoptId,
1050 caller_graph_->max_block_id() + 1,
1051 entry_kind == Code::EntryKind::kUnchecked);
1052 {
1053 callee_graph = builder.BuildGraph();
1054#if defined(DEBUG)
1055 FlowGraphChecker(callee_graph).Check("Builder (callee)");
1056#endif
1057 CalleeGraphValidator::Validate(callee_graph);
1058 }
1059#if defined(DART_PRECOMPILER) && !defined(TARGET_ARCH_IA32)
1060 if (CompilerState::Current().is_aot()) {
1061 callee_graph->PopulateWithICData(parsed_function->function());
1062 }
1063#endif
1064
1065 // If we inline a function which is intrinsified without a fall-through
1066 // to IR code, we will not have any ICData attached, so we do it
1067 // manually here.
1068 if (!CompilerState::Current().is_aot() && function.is_intrinsic()) {
1069 callee_graph->PopulateWithICData(parsed_function->function());
1070 }
1071
1072 // The parameter stubs are a copy of the actual arguments providing
1073 // concrete information about the values, for example constant values,
1074 // without linking between the caller and callee graphs.
1075 // TODO(zerny): Put more information in the stubs, eg, type information.
1076 const intptr_t first_actual_param_index = call_data->first_arg_index;
1077 const intptr_t inlined_type_args_param = function.IsGeneric() ? 1 : 0;
1078 const intptr_t num_inlined_params =
1079 inlined_type_args_param + function.NumParameters();
1080 ZoneGrowableArray<Definition*>* param_stubs =
1081 new (Z) ZoneGrowableArray<Definition*>(num_inlined_params);
1082
1083 // Create a ConstantInstr as Definition for the type arguments, if any.
1084 if (first_actual_param_index > 0) {
1085 // A type argument vector is explicitly passed.
1086 param_stubs->Add(
1087 CreateParameterStub(-1, (*arguments)[0], callee_graph));
1088 } else if (inlined_type_args_param > 0) {
1089 // No type argument vector is passed to the generic function,
1090 // pass a null vector, which is the same as a vector of dynamic types.
1091 param_stubs->Add(callee_graph->GetConstant(Object::ZoneHandle()));
1092 }
1093 // Create a parameter stub for each fixed positional parameter.
1094 for (intptr_t i = 0; i < function.num_fixed_parameters(); ++i) {
1095 param_stubs->Add(CreateParameterStub(
1096 i, (*arguments)[first_actual_param_index + i], callee_graph));
1097 }
1098
1099 // If the callee has optional parameters, rebuild the argument and stub
1100 // arrays so that actual arguments are in one-to-one with the formal
1101 // parameters.
1102 if (function.HasOptionalParameters()) {
1103 TRACE_INLINING(THR_Print(" adjusting for optional parameters\n"));
1104 if (!AdjustForOptionalParameters(
1105 *parsed_function, first_actual_param_index, argument_names,
1106 arguments, param_stubs, callee_graph)) {
1107 function.set_is_inlinable(false);
1108 TRACE_INLINING(THR_Print(" Bailout: optional arg mismatch\n"));
1109 PRINT_INLINING_TREE("Optional arg mismatch", &call_data->caller,
1110 &function, call_data->call);
1111 return false;
1112 }
1113 }
1114
1115 // After treating optional parameters the actual/formal count must
1116 // match.
1117 ASSERT(arguments->length() ==
1118 first_actual_param_index + function.NumParameters());
1119
1120 // Update try-index of the callee graph.
1121 BlockEntryInstr* call_block = call_data->call->GetBlock();
1122 if (call_block->InsideTryBlock()) {
1123 intptr_t try_index = call_block->try_index();
1124 for (BlockIterator it = callee_graph->reverse_postorder_iterator();
1125 !it.Done(); it.Advance()) {
1126 BlockEntryInstr* block = it.Current();
1127 block->set_try_index(try_index);
1128 }
1129 }
1130
1131 BlockScheduler::AssignEdgeWeights(callee_graph);
1132
1133 {
1134 // Compute SSA on the callee graph, catching bailouts.
1135 callee_graph->ComputeSSA(caller_graph_->max_virtual_register_number(),
1136 param_stubs);
1137#if defined(DEBUG)
1138 FlowGraphChecker(callee_graph).Check("SSA (callee)");
1139#endif
1140 }
1141
1142 if (FLAG_support_il_printer && trace_inlining() &&
1143 (FLAG_print_flow_graph || FLAG_print_flow_graph_optimized)) {
1144 THR_Print("Callee graph for inlining %s (unoptimized)\n",
1145 function.ToFullyQualifiedCString());
1146 FlowGraphPrinter printer(*callee_graph);
1147 printer.PrintBlocks();
1148 }
1149
1150 {
1151 // TODO(fschneider): Improve suppression of speculative inlining.
1152 // Deopt-ids overlap between caller and callee.
1153 if (CompilerState::Current().is_aot()) {
1154#if defined(DART_PRECOMPILER) && !defined(TARGET_ARCH_IA32)
1155 AotCallSpecializer call_specializer(inliner_->precompiler_,
1156 callee_graph,
1157 inliner_->speculative_policy_);
1158
1159 CompilerPassState state(Thread::Current(), callee_graph,
1160 inliner_->speculative_policy_);
1161 state.call_specializer = &call_specializer;
1162 CompilerPass::RunInliningPipeline(CompilerPass::kAOT, &state);
1163#else
1164 UNREACHABLE();
1165#endif // defined(DART_PRECOMPILER) && !defined(TARGET_ARCH_IA32)
1166 } else {
1167 JitCallSpecializer call_specializer(callee_graph,
1168 inliner_->speculative_policy_);
1169
1170 CompilerPassState state(Thread::Current(), callee_graph,
1171 inliner_->speculative_policy_);
1172 state.call_specializer = &call_specializer;
1173 CompilerPass::RunInliningPipeline(CompilerPass::kJIT, &state);
1174 }
1175 }
1176
1177 if (FLAG_support_il_printer && trace_inlining() &&
1178 (FLAG_print_flow_graph || FLAG_print_flow_graph_optimized)) {
1179 THR_Print("Callee graph for inlining %s (optimized)\n",
1180 function.ToFullyQualifiedCString());
1181 FlowGraphPrinter printer(*callee_graph);
1182 printer.PrintBlocks();
1183 }
1184
1185 // Collect information about the call site and caller graph. At this
1186 // point, optional constant parameters are counted too, making the
1187 // specialized vs. non-specialized decision accurate.
1188 intptr_t constants_count = 0;
1189 for (intptr_t i = 0, n = param_stubs->length(); i < n; ++i) {
1190 if ((*param_stubs)[i]->IsConstant()) ++constants_count;
1191 }
1192 intptr_t instruction_count = 0;
1193 intptr_t call_site_count = 0;
1194 FlowGraphInliner::CollectGraphInfo(callee_graph, constants_count,
1195 /*force*/ false, &instruction_count,
1196 &call_site_count);
1197
1198 // Use heuristics do decide if this call should be inlined.
1199 InliningDecision decision =
1200 ShouldWeInline(function, instruction_count, call_site_count);
1201 if (!decision.value) {
1202 // If size is larger than all thresholds, don't consider it again.
1203 if ((instruction_count > FLAG_inlining_size_threshold) &&
1204 (call_site_count > FLAG_inlining_callee_call_sites_threshold)) {
1205 function.set_is_inlinable(false);
1206 }
1207 TRACE_INLINING(
1208 THR_Print(" Bailout: heuristics (%s) with "
1209 "code size: %" Pd ", "
1210 "call sites: %" Pd ", "
1211 "inlining depth of callee: %d, "
1212 "const args: %" Pd "\n",
1213 decision.reason, instruction_count, call_site_count,
1214 function.inlining_depth(), constants_count));
1215 PRINT_INLINING_TREE("Heuristic fail", &call_data->caller, &function,
1216 call_data->call);
1217 return false;
1218 }
1219
1220 // If requested, a stricter heuristic is applied to this inlining. This
1221 // heuristic always scans the method (rather than possibly reusing
1222 // cached results) to make sure all specializations are accounted for.
1223 // TODO(ajcbik): with the now better bookkeeping, explore removing this
1224 if (stricter_heuristic) {
1225 if (!IsSmallLeaf(callee_graph)) {
1226 TRACE_INLINING(
1227 THR_Print(" Bailout: heuristics (no small leaf)\n"));
1228 PRINT_INLINING_TREE("Heuristic fail (no small leaf)",
1229 &call_data->caller, &function, call_data->call);
1230 return false;
1231 }
1232 }
1233
1234 // Inline dispatcher methods regardless of the current depth.
1235 const intptr_t depth =
1236 function.IsDispatcherOrImplicitAccessor() ? 0 : inlining_depth_;
1237 collected_call_sites_->FindCallSites(callee_graph, depth,
1238 &inlined_info_);
1239
1240 // Add the function to the cache.
1241 if (!in_cache) {
1242 function_cache_.Add(parsed_function);
1243 }
1244
1245 // Build succeeded so we restore the bailout jump.
1246 inlined_ = true;
1247 inlined_size_ += instruction_count;
1248 if (is_recursive_call) {
1249 inlined_recursive_call_ = true;
1250 }
1251
1252 call_data->callee_graph = callee_graph;
1253 call_data->parameter_stubs = param_stubs;
1254 call_data->exit_collector = exit_collector;
1255
1256 // When inlined, we add the guarded fields of the callee to the caller's
1257 // list of guarded fields.
1258 const ZoneGrowableArray<const Field*>& callee_guarded_fields =
1259 *callee_graph->parsed_function().guarded_fields();
1260 for (intptr_t i = 0; i < callee_guarded_fields.length(); ++i) {
1261 caller_graph()->parsed_function().AddToGuardedFields(
1262 callee_guarded_fields[i]);
1263 }
1264
1265 FlowGraphInliner::SetInliningId(
1266 callee_graph,
1267 inliner_->NextInlineId(callee_graph->function(),
1268 call_data->call->token_pos(),
1269 call_data->caller_inlining_id));
1270 TRACE_INLINING(THR_Print(" Success\n"));
1271 TRACE_INLINING(THR_Print(
1272 " with reason %s, code size %" Pd ", call sites: %" Pd "\n",
1273 decision.reason, instruction_count, call_site_count));
1274 PRINT_INLINING_TREE(NULL, &call_data->caller, &function, call);
1275 return true;
1276 } else {
1277 error = thread()->StealStickyError();
1278
1279 if (error.IsLanguageError() &&
1280 (LanguageError::Cast(error).kind() == Report::kBailout)) {
1281 if (error.raw() == Object::background_compilation_error().raw()) {
1282 // Fall through to exit the compilation, and retry it later.
1283 } else {
1284 TRACE_INLINING(
1285 THR_Print(" Bailout: %s\n", error.ToErrorCString()));
1286 PRINT_INLINING_TREE("Bailout", &call_data->caller, &function, call);
1287 return false;
1288 }
1289 } else {
1290 // Fall through to exit long jump scope.
1291 }
1292 }
1293 }
1294
1295 // Propagate a compile-time error. In precompilation we attempt to
1296 // inline functions that have never been compiled before; when JITing we
1297 // should only see language errors in unoptimized compilation.
1298 // Otherwise, there can be an out-of-memory error (unhandled exception).
1299 // In background compilation we may abort compilation as the state
1300 // changes while compiling. Propagate that 'error' and retry compilation
1301 // later.
1302 ASSERT(CompilerState::Current().is_aot() ||
1303 Compiler::IsBackgroundCompilation() || error.IsUnhandledException());
1304 Thread::Current()->long_jump_base()->Jump(1, error);
1305 UNREACHABLE();
1306 return false;
1307 }
1308
1309 void PrintInlinedInfo(const Function& top) {
1310 if (inlined_info_.length() > 0) {
1311 THR_Print("Inlining into: '%s'\n growth: %f (%" Pd " -> %" Pd ")\n",
1312 top.ToFullyQualifiedCString(), GrowthFactor(), initial_size_,
1313 inlined_size_);
1314 PrintInlinedInfoFor(top, 1);
1315 }
1316 }
1317
1318 private:
1319 friend class PolymorphicInliner;
1320
1321 static bool Contains(const GrowableArray<intptr_t>& a, intptr_t deopt_id) {
1322 for (intptr_t i = 0; i < a.length(); i++) {
1323 if (a[i] == deopt_id) return true;
1324 }
1325 return false;
1326 }
1327
1328 void PrintInlinedInfoFor(const Function& caller, intptr_t depth) {
1329 // Prevent duplicate printing as inlined_info aggregates all inlinining.
1330 GrowableArray<intptr_t> call_instructions_printed;
1331 // Print those that were inlined.
1332 for (intptr_t i = 0; i < inlined_info_.length(); i++) {
1333 const InlinedInfo& info = inlined_info_[i];
1334 if (info.bailout_reason != NULL) {
1335 continue;
1336 }
1337 if ((info.inlined_depth == depth) &&
1338 (info.caller->raw() == caller.raw()) &&
1339 !Contains(call_instructions_printed, info.call_instr->GetDeoptId())) {
1340 for (int t = 0; t < depth; t++) {
1341 THR_Print(" ");
1342 }
1343 THR_Print("%" Pd " %s\n", info.call_instr->GetDeoptId(),
1344 info.inlined->ToQualifiedCString());
1345 PrintInlinedInfoFor(*info.inlined, depth + 1);
1346 call_instructions_printed.Add(info.call_instr->GetDeoptId());
1347 }
1348 }
1349 call_instructions_printed.Clear();
1350 // Print those that were not inlined.
1351 for (intptr_t i = 0; i < inlined_info_.length(); i++) {
1352 const InlinedInfo& info = inlined_info_[i];
1353 if (info.bailout_reason == NULL) {
1354 continue;
1355 }
1356 if ((info.inlined_depth == depth) &&
1357 (info.caller->raw() == caller.raw()) &&
1358 !Contains(call_instructions_printed, info.call_instr->GetDeoptId())) {
1359 for (int t = 0; t < depth; t++) {
1360 THR_Print(" ");
1361 }
1362 THR_Print("NO %" Pd " %s - %s\n", info.call_instr->GetDeoptId(),
1363 info.inlined->ToQualifiedCString(), info.bailout_reason);
1364 call_instructions_printed.Add(info.call_instr->GetDeoptId());
1365 }
1366 }
1367 }
1368
1369 void InlineCall(InlinedCallData* call_data) {
1370 FlowGraph* callee_graph = call_data->callee_graph;
1371 auto callee_function_entry = callee_graph->graph_entry()->normal_entry();
1372
1373 // Plug result in the caller graph.
1374 InlineExitCollector* exit_collector = call_data->exit_collector;
1375 exit_collector->PrepareGraphs(callee_graph);
1376 exit_collector->ReplaceCall(callee_function_entry);
1377
1378 ReplaceParameterStubs(zone(), caller_graph_, call_data, NULL);
1379
1380 ASSERT(!call_data->call->HasPushArguments());
1381 }
1382
1383 static intptr_t CountConstants(const GrowableArray<Value*>& arguments) {
1384 intptr_t count = 0;
1385 for (intptr_t i = 0; i < arguments.length(); i++) {
1386 if (arguments[i]->BindsToConstant()) count++;
1387 }
1388 return count;
1389 }
1390
1391 // Parse a function reusing the cache if possible.
1392 ParsedFunction* GetParsedFunction(const Function& function, bool* in_cache) {
1393 // TODO(zerny): Use a hash map for the cache.
1394 for (intptr_t i = 0; i < function_cache_.length(); ++i) {
1395 ParsedFunction* parsed_function = function_cache_[i];
1396 if (parsed_function->function().raw() == function.raw()) {
1397 *in_cache = true;
1398 return parsed_function;
1399 }
1400 }
1401 *in_cache = false;
1402 ParsedFunction* parsed_function =
1403 new (Z) ParsedFunction(thread(), function);
1404 return parsed_function;
1405 }
1406
1407 bool InlineStaticCalls() {
1408 bool inlined = false;
1409 const GrowableArray<CallSites::StaticCallInfo>& call_info =
1410 inlining_call_sites_->static_calls();
1411 TRACE_INLINING(THR_Print(" Static Calls (%" Pd ")\n", call_info.length()));
1412 for (intptr_t call_idx = 0; call_idx < call_info.length(); ++call_idx) {
1413 StaticCallInstr* call = call_info[call_idx].call;
1414
1415 if (FlowGraphInliner::TryReplaceStaticCallWithInline(
1416 inliner_->flow_graph(), NULL, call,
1417 inliner_->speculative_policy_)) {
1418 inlined = true;
1419 continue;
1420 }
1421
1422 const Function& target = call->function();
1423 if (!inliner_->AlwaysInline(target) &&
1424 (call_info[call_idx].ratio * 100) < FLAG_inlining_hotness) {
1425 if (trace_inlining()) {
1426 String& name = String::Handle(target.QualifiedUserVisibleName());
1427 THR_Print(" => %s (deopt count %d)\n Bailout: cold %f\n",
1428 name.ToCString(), target.deoptimization_counter(),
1429 call_info[call_idx].ratio);
1430 }
1431 PRINT_INLINING_TREE("Too cold", &call_info[call_idx].caller(),
1432 &call->function(), call);
1433 continue;
1434 }
1435
1436 GrowableArray<Value*> arguments(call->ArgumentCount());
1437 for (int i = 0; i < call->ArgumentCount(); ++i) {
1438 arguments.Add(call->ArgumentValueAt(i));
1439 }
1440 InlinedCallData call_data(
1441 call, Array::ZoneHandle(Z, call->GetArgumentsDescriptor()),
1442 call->FirstArgIndex(), &arguments, call_info[call_idx].caller(),
1443 call_info[call_idx].caller_graph->inlining_id());
1444
1445 // Under AOT, calls outside loops may pass our regular heuristics due
1446 // to a relatively high ratio. So, unless we are optimizing solely for
1447 // speed, such call sites are subject to subsequent stricter heuristic
1448 // to limit code size increase.
1449 bool stricter_heuristic = CompilerState::Current().is_aot() &&
1450 FLAG_optimization_level <= 2 &&
1451 !inliner_->AlwaysInline(target) &&
1452 call_info[call_idx].nesting_depth == 0;
1453 if (TryInlining(call->function(), call->argument_names(), &call_data,
1454 stricter_heuristic)) {
1455 InlineCall(&call_data);
1456 inlined = true;
1457 }
1458 }
1459 return inlined;
1460 }
1461
1462 bool InlineClosureCalls() {
1463 // Under this flag, tear off testing closure calls appear before the
1464 // StackOverflowInstr, which breaks assertions in our compiler when inlined.
1465 // TODO(sjindel): move testing closure calls after first check
1466 if (FLAG_enable_testing_pragmas) return false; // keep all closures
1467 bool inlined = false;
1468 const GrowableArray<CallSites::ClosureCallInfo>& call_info =
1469 inlining_call_sites_->closure_calls();
1470 TRACE_INLINING(
1471 THR_Print(" Closure Calls (%" Pd ")\n", call_info.length()));
1472 for (intptr_t call_idx = 0; call_idx < call_info.length(); ++call_idx) {
1473 ClosureCallInstr* call = call_info[call_idx].call;
1474 // Find the closure of the callee.
1475 ASSERT(call->ArgumentCount() > 0);
1476 Function& target = Function::ZoneHandle();
1477 Definition* receiver =
1478 call->Receiver()->definition()->OriginalDefinition();
1479 if (AllocateObjectInstr* alloc = receiver->AsAllocateObject()) {
1480 if (!alloc->closure_function().IsNull()) {
1481 target = alloc->closure_function().raw();
1482 ASSERT(alloc->cls().IsClosureClass());
1483 }
1484 } else if (ConstantInstr* constant = receiver->AsConstant()) {
1485 if (constant->value().IsClosure()) {
1486 target = Closure::Cast(constant->value()).function();
1487 }
1488 }
1489
1490 if (target.IsNull()) {
1491 TRACE_INLINING(THR_Print(" Bailout: non-closure operator\n"));
1492 continue;
1493 }
1494
1495 if (call->ArgumentCount() > target.NumParameters() ||
1496 call->ArgumentCount() < target.num_fixed_parameters()) {
1497 TRACE_INLINING(THR_Print(" Bailout: wrong parameter count\n"));
1498 continue;
1499 }
1500
1501 GrowableArray<Value*> arguments(call->ArgumentCount());
1502 for (int i = 0; i < call->ArgumentCount(); ++i) {
1503 arguments.Add(call->ArgumentValueAt(i));
1504 }
1505 const Array& arguments_descriptor =
1506 Array::ZoneHandle(Z, call->GetArgumentsDescriptor());
1507 InlinedCallData call_data(
1508 call, arguments_descriptor, call->FirstArgIndex(), &arguments,
1509 call_info[call_idx].caller(),
1510 call_info[call_idx].caller_graph->inlining_id());
1511 if (TryInlining(target, call->argument_names(), &call_data, false)) {
1512 InlineCall(&call_data);
1513 inlined = true;
1514 }
1515 }
1516 return inlined;
1517 }
1518
1519 bool InlineInstanceCalls() {
1520 bool inlined = false;
1521 const GrowableArray<CallSites::InstanceCallInfo>& call_info =
1522 inlining_call_sites_->instance_calls();
1523 TRACE_INLINING(THR_Print(" Polymorphic Instance Calls (%" Pd ")\n",
1524 call_info.length()));
1525 for (intptr_t call_idx = 0; call_idx < call_info.length(); ++call_idx) {
1526 PolymorphicInstanceCallInstr* call = call_info[call_idx].call;
1527 // PolymorphicInliner introduces deoptimization paths.
1528 if (!call->complete() && !FLAG_polymorphic_with_deopt) {
1529 TRACE_INLINING(THR_Print(" => %s\n Bailout: call with checks\n",
1530 call->function_name().ToCString()));
1531 continue;
1532 }
1533 const Function& cl = call_info[call_idx].caller();
1534 intptr_t caller_inlining_id =
1535 call_info[call_idx].caller_graph->inlining_id();
1536 PolymorphicInliner inliner(this, call, cl, caller_inlining_id);
1537 if (inliner.Inline()) inlined = true;
1538 }
1539 return inlined;
1540 }
1541
1542 bool AdjustForOptionalParameters(const ParsedFunction& parsed_function,
1543 intptr_t first_arg_index,
1544 const Array& argument_names,
1545 GrowableArray<Value*>* arguments,
1546 ZoneGrowableArray<Definition*>* param_stubs,
1547 FlowGraph* callee_graph) {
1548 const Function& function = parsed_function.function();
1549 // The language and this code does not support both optional positional
1550 // and optional named parameters for the same function.
1551 ASSERT(!function.HasOptionalPositionalParameters() ||
1552 !function.HasOptionalNamedParameters());
1553
1554 intptr_t arg_count = arguments->length();
1555 intptr_t param_count = function.NumParameters();
1556 intptr_t fixed_param_count = function.num_fixed_parameters();
1557 ASSERT(fixed_param_count <= arg_count - first_arg_index);
1558 ASSERT(arg_count - first_arg_index <= param_count);
1559
1560 if (function.HasOptionalPositionalParameters()) {
1561 // Create a stub for each optional positional parameters with an actual.
1562 for (intptr_t i = first_arg_index + fixed_param_count; i < arg_count;
1563 ++i) {
1564 param_stubs->Add(CreateParameterStub(i, (*arguments)[i], callee_graph));
1565 }
1566 ASSERT(function.NumOptionalPositionalParameters() ==
1567 (param_count - fixed_param_count));
1568 // For each optional positional parameter without an actual, add its
1569 // default value.
1570 for (intptr_t i = arg_count - first_arg_index; i < param_count; ++i) {
1571 const Instance& object =
1572 parsed_function.DefaultParameterValueAt(i - fixed_param_count);
1573 ConstantInstr* constant = new (Z) ConstantInstr(object);
1574 arguments->Add(NULL);
1575 param_stubs->Add(constant);
1576 }
1577 return true;
1578 }
1579
1580 ASSERT(function.HasOptionalNamedParameters());
1581
1582 // Passed arguments (not counting optional type args) must match fixed
1583 // parameters plus named arguments.
1584 intptr_t argument_names_count =
1585 (argument_names.IsNull()) ? 0 : argument_names.Length();
1586 ASSERT((arg_count - first_arg_index) ==
1587 (fixed_param_count + argument_names_count));
1588
1589 // Fast path when no optional named parameters are given.
1590 if (argument_names_count == 0) {
1591 for (intptr_t i = 0; i < param_count - fixed_param_count; ++i) {
1592 arguments->Add(NULL);
1593 param_stubs->Add(GetDefaultValue(i, parsed_function));
1594 }
1595 return true;
1596 }
1597
1598 // Otherwise, build a collection of name/argument pairs.
1599 GrowableArray<NamedArgument> named_args(argument_names_count);
1600 for (intptr_t i = 0; i < argument_names.Length(); ++i) {
1601 String& arg_name = String::Handle(caller_graph_->zone());
1602 arg_name ^= argument_names.At(i);
1603 named_args.Add(NamedArgument(
1604 &arg_name, (*arguments)[first_arg_index + fixed_param_count + i]));
1605 }
1606
1607 // Truncate the arguments array to just type args and fixed parameters.
1608 arguments->TruncateTo(first_arg_index + fixed_param_count);
1609
1610 // For each optional named parameter, add the actual argument or its
1611 // default if no argument is passed.
1612 intptr_t match_count = 0;
1613 for (intptr_t i = fixed_param_count; i < param_count; ++i) {
1614 String& param_name = String::Handle(function.ParameterNameAt(i));
1615 // Search for and add the named argument.
1616 Value* arg = NULL;
1617 for (intptr_t j = 0; j < named_args.length(); ++j) {
1618 if (param_name.Equals(*named_args[j].name)) {
1619 arg = named_args[j].value;
1620 match_count++;
1621 break;
1622 }
1623 }
1624 arguments->Add(arg);
1625 // Create a stub for the argument or use the parameter's default value.
1626 if (arg != NULL) {
1627 param_stubs->Add(
1628 CreateParameterStub(first_arg_index + i, arg, callee_graph));
1629 } else {
1630 param_stubs->Add(
1631 GetDefaultValue(i - fixed_param_count, parsed_function));
1632 }
1633 }
1634 return argument_names_count == match_count;
1635 }
1636
1637 FlowGraphInliner* inliner_;
1638 FlowGraph* caller_graph_;
1639 bool inlined_;
1640 const intptr_t initial_size_;
1641 intptr_t inlined_size_;
1642 bool inlined_recursive_call_;
1643 intptr_t inlining_depth_;
1644 intptr_t inlining_recursion_depth_;
1645 intptr_t inlining_depth_threshold_;
1646 CallSites* collected_call_sites_;
1647 CallSites* inlining_call_sites_;
1648 GrowableArray<ParsedFunction*> function_cache_;
1649 GrowableArray<InlinedInfo> inlined_info_;
1650
1651 DISALLOW_COPY_AND_ASSIGN(CallSiteInliner);
1652};
1653
1654PolymorphicInliner::PolymorphicInliner(CallSiteInliner* owner,
1655 PolymorphicInstanceCallInstr* call,
1656 const Function& caller_function,
1657 intptr_t caller_inlining_id)
1658 : owner_(owner),
1659 call_(call),
1660 num_variants_(call->NumberOfChecks()),
1661 variants_(call->targets_),
1662 inlined_variants_(zone()),
1663 non_inlined_variants_(new (zone()) CallTargets(zone())),
1664 inlined_entries_(num_variants_),
1665 exit_collector_(new (Z) InlineExitCollector(owner->caller_graph(), call)),
1666 caller_function_(caller_function),
1667 caller_inlining_id_(caller_inlining_id) {}
1668
1669Isolate* PolymorphicInliner::isolate() const {
1670 return owner_->caller_graph()->isolate();
1671}
1672
1673Zone* PolymorphicInliner::zone() const {
1674 return owner_->caller_graph()->zone();
1675}
1676
1677intptr_t PolymorphicInliner::AllocateBlockId() const {
1678 return owner_->caller_graph()->allocate_block_id();
1679}
1680
1681// Inlined bodies are shared if two different class ids have the same
1682// inlined target. This sharing is represented by using three different
1683// types of entries in the inlined_entries_ array:
1684//
1685// * GraphEntry: the inlined body is not shared.
1686//
1687// * TargetEntry: the inlined body is shared and this is the first variant.
1688//
1689// * JoinEntry: the inlined body is shared and this is a subsequent variant.
1690bool PolymorphicInliner::CheckInlinedDuplicate(const Function& target) {
1691 for (intptr_t i = 0; i < inlined_variants_.length(); ++i) {
1692 if ((target.raw() == inlined_variants_.TargetAt(i)->target->raw()) &&
1693 !target.is_polymorphic_target()) {
1694 // The call target is shared with a previous inlined variant. Share
1695 // the graph. This requires a join block at the entry, and edge-split
1696 // form requires a target for each branch.
1697 //
1698 // Represent the sharing by recording a fresh target for the first
1699 // variant and the shared join for all later variants.
1700 if (inlined_entries_[i]->IsGraphEntry()) {
1701 // Convert the old target entry to a new join entry.
1702 auto old_entry = inlined_entries_[i]->AsGraphEntry()->normal_entry();
1703 BlockEntryInstr* old_target = old_entry;
1704
1705 // Unuse all inputs in the old graph entry since it is not part of
1706 // the graph anymore. A new target be created instead.
1707 inlined_entries_[i]->AsGraphEntry()->UnuseAllInputs();
1708
1709 JoinEntryInstr* new_join =
1710 BranchSimplifier::ToJoinEntry(zone(), old_target);
1711 old_target->ReplaceAsPredecessorWith(new_join);
1712 for (intptr_t j = 0; j < old_target->dominated_blocks().length(); ++j) {
1713 BlockEntryInstr* block = old_target->dominated_blocks()[j];
1714 new_join->AddDominatedBlock(block);
1715 }
1716 // Since we are reusing the same inlined body across multiple cids,
1717 // reset the type information on the redefinition of the receiver
1718 // in case it was originally given a concrete type.
1719 ASSERT(new_join->next()->IsRedefinition());
1720 new_join->next()->AsRedefinition()->UpdateType(CompileType::Dynamic());
1721 // Create a new target with the join as unconditional successor.
1722 TargetEntryInstr* new_target = new TargetEntryInstr(
1723 AllocateBlockId(), old_target->try_index(), DeoptId::kNone);
1724 new_target->InheritDeoptTarget(zone(), new_join);
1725 GotoInstr* new_goto = new (Z) GotoInstr(new_join, DeoptId::kNone);
1726 new_goto->InheritDeoptTarget(zone(), new_join);
1727 new_target->LinkTo(new_goto);
1728 new_target->set_last_instruction(new_goto);
1729 new_join->predecessors_.Add(new_target);
1730
1731 // Record the new target for the first variant.
1732 inlined_entries_[i] = new_target;
1733 }
1734 ASSERT(inlined_entries_[i]->IsTargetEntry());
1735 // Record the shared join for this variant.
1736 BlockEntryInstr* join =
1737 inlined_entries_[i]->last_instruction()->SuccessorAt(0);
1738 ASSERT(join->IsJoinEntry());
1739 inlined_entries_.Add(join);
1740 return true;
1741 }
1742 }
1743
1744 return false;
1745}
1746
1747bool PolymorphicInliner::CheckNonInlinedDuplicate(const Function& target) {
1748 for (intptr_t i = 0; i < non_inlined_variants_->length(); ++i) {
1749 if (target.raw() == non_inlined_variants_->TargetAt(i)->target->raw()) {
1750 return true;
1751 }
1752 }
1753
1754 return false;
1755}
1756
1757bool PolymorphicInliner::TryInliningPoly(const TargetInfo& target_info) {
1758 if ((!CompilerState::Current().is_aot() ||
1759 owner_->inliner_->speculative_policy()->AllowsSpeculativeInlining()) &&
1760 target_info.IsSingleCid() &&
1761 TryInlineRecognizedMethod(target_info.cid_start, *target_info.target)) {
1762 owner_->inlined_ = true;
1763 return true;
1764 }
1765
1766 GrowableArray<Value*> arguments(call_->ArgumentCount());
1767 for (int i = 0; i < call_->ArgumentCount(); ++i) {
1768 arguments.Add(call_->ArgumentValueAt(i));
1769 }
1770 const Array& arguments_descriptor =
1771 Array::ZoneHandle(Z, call_->GetArgumentsDescriptor());
1772 InlinedCallData call_data(call_, arguments_descriptor, call_->FirstArgIndex(),
1773 &arguments, caller_function_, caller_inlining_id_);
1774 Function& target = Function::ZoneHandle(zone(), target_info.target->raw());
1775 if (!owner_->TryInlining(target, call_->argument_names(), &call_data,
1776 false)) {
1777 return false;
1778 }
1779
1780 FlowGraph* callee_graph = call_data.callee_graph;
1781 call_data.exit_collector->PrepareGraphs(callee_graph);
1782 inlined_entries_.Add(callee_graph->graph_entry());
1783 exit_collector_->Union(call_data.exit_collector);
1784
1785 ReplaceParameterStubs(zone(), owner_->caller_graph(), &call_data,
1786 &target_info);
1787 return true;
1788}
1789
1790static Instruction* AppendInstruction(Instruction* first, Instruction* second) {
1791 for (intptr_t i = second->InputCount() - 1; i >= 0; --i) {
1792 Value* input = second->InputAt(i);
1793 input->definition()->AddInputUse(input);
1794 }
1795 first->LinkTo(second);
1796 return second;
1797}
1798
1799bool PolymorphicInliner::TryInlineRecognizedMethod(intptr_t receiver_cid,
1800 const Function& target) {
1801 auto temp_parsed_function = new (Z) ParsedFunction(Thread::Current(), target);
1802 auto graph_entry =
1803 new (Z) GraphEntryInstr(*temp_parsed_function, Compiler::kNoOSRDeoptId);
1804
1805 FunctionEntryInstr* entry = nullptr;
1806 Instruction* last = nullptr;
1807 Definition* result = nullptr;
1808 // Replace the receiver argument with a redefinition to prevent code from
1809 // the inlined body from being hoisted above the inlined entry.
1810 GrowableArray<Definition*> arguments(call_->ArgumentCount());
1811 Definition* receiver = call_->Receiver()->definition();
1812 RedefinitionInstr* redefinition =
1813 new (Z) RedefinitionInstr(new (Z) Value(receiver));
1814 redefinition->set_ssa_temp_index(
1815 owner_->caller_graph()->alloc_ssa_temp_index());
1816 if (FlowGraph::NeedsPairLocation(redefinition->representation())) {
1817 owner_->caller_graph()->alloc_ssa_temp_index();
1818 }
1819 if (FlowGraphInliner::TryInlineRecognizedMethod(
1820 owner_->caller_graph(), receiver_cid, target, call_, redefinition,
1821 call_->token_pos(), call_->ic_data(), graph_entry, &entry, &last,
1822 &result, owner_->inliner_->speculative_policy())) {
1823 // The empty Object constructor is the only case where the inlined body is
1824 // empty and there is no result.
1825 ASSERT((last != nullptr && result != nullptr) ||
1826 (target.recognized_kind() == MethodRecognizer::kObjectConstructor));
1827 graph_entry->set_normal_entry(entry);
1828 // Create a graph fragment.
1829 redefinition->InsertAfter(entry);
1830 InlineExitCollector* exit_collector =
1831 new (Z) InlineExitCollector(owner_->caller_graph(), call_);
1832 ReturnInstr* return_result = new (Z)
1833 ReturnInstr(call_->token_pos(), new (Z) Value(result), DeoptId::kNone);
1834 owner_->caller_graph()->AppendTo(
1835 last, return_result,
1836 call_->env(), // Return can become deoptimization target.
1837 FlowGraph::kEffect);
1838 entry->set_last_instruction(return_result);
1839 exit_collector->AddExit(return_result);
1840
1841 // Update polymorphic inliner state.
1842 inlined_entries_.Add(graph_entry);
1843 exit_collector_->Union(exit_collector);
1844 return true;
1845 }
1846 return false;
1847}
1848
1849// Build a DAG to dispatch to the inlined function bodies. Load the class
1850// id of the receiver and make explicit comparisons for each inlined body,
1851// in frequency order. If all variants are inlined, the entry to the last
1852// inlined body is guarded by a CheckClassId instruction which can deopt.
1853// If not all variants are inlined, we add a PolymorphicInstanceCall
1854// instruction to handle the non-inlined variants.
1855TargetEntryInstr* PolymorphicInliner::BuildDecisionGraph() {
1856 const intptr_t try_idx = call_->GetBlock()->try_index();
1857
1858 // Start with a fresh target entry.
1859 TargetEntryInstr* entry = new (Z) TargetEntryInstr(
1860 AllocateBlockId(), try_idx, CompilerState::Current().GetNextDeoptId());
1861 entry->InheritDeoptTarget(zone(), call_);
1862
1863 // This function uses a cursor (a pointer to the 'current' instruction) to
1864 // build the graph. The next instruction will be inserted after the
1865 // cursor.
1866 BlockEntryInstr* current_block = entry;
1867 Instruction* cursor = entry;
1868
1869 Definition* receiver = call_->Receiver()->definition();
1870 // There are at least two variants including non-inlined ones, so we have
1871 // at least one branch on the class id.
1872 LoadClassIdInstr* load_cid =
1873 new (Z) LoadClassIdInstr(new (Z) Value(receiver));
1874 load_cid->set_ssa_temp_index(owner_->caller_graph()->alloc_ssa_temp_index());
1875 cursor = AppendInstruction(cursor, load_cid);
1876 for (intptr_t i = 0; i < inlined_variants_.length(); ++i) {
1877 const CidRange& variant = inlined_variants_[i];
1878 bool test_is_range = !variant.IsSingleCid();
1879 bool is_last_test = (i == inlined_variants_.length() - 1);
1880 // 1. Guard the body with a class id check. We don't need any check if
1881 // it's the last test and global analysis has told us that the call is
1882 // complete.
1883 if (is_last_test && non_inlined_variants_->is_empty()) {
1884 // If it is the last variant use a check class id instruction which can
1885 // deoptimize, followed unconditionally by the body. Omit the check if
1886 // we know that we have covered all possible classes.
1887 if (!call_->complete()) {
1888 RedefinitionInstr* cid_redefinition =
1889 new RedefinitionInstr(new (Z) Value(load_cid));
1890 cid_redefinition->set_ssa_temp_index(
1891 owner_->caller_graph()->alloc_ssa_temp_index());
1892 cursor = AppendInstruction(cursor, cid_redefinition);
1893 CheckClassIdInstr* check_class_id = new (Z) CheckClassIdInstr(
1894 new (Z) Value(cid_redefinition), variant, call_->deopt_id());
1895 check_class_id->InheritDeoptTarget(zone(), call_);
1896 cursor = AppendInstruction(cursor, check_class_id);
1897 }
1898
1899 // The next instruction is the first instruction of the inlined body.
1900 // Handle the two possible cases (unshared and shared subsequent
1901 // predecessors) separately.
1902 BlockEntryInstr* callee_entry = inlined_entries_[i];
1903 if (callee_entry->IsGraphEntry()) {
1904 // Unshared. Graft the normal entry on after the check class
1905 // instruction.
1906 auto target = callee_entry->AsGraphEntry()->normal_entry();
1907 ASSERT(cursor != nullptr);
1908 cursor->LinkTo(target->next());
1909 target->ReplaceAsPredecessorWith(current_block);
1910 // Unuse all inputs of the graph entry and the normal entry. They are
1911 // not in the graph anymore.
1912 callee_entry->UnuseAllInputs();
1913 target->UnuseAllInputs();
1914 // All blocks that were dominated by the normal entry are now
1915 // dominated by the current block.
1916 for (intptr_t j = 0; j < target->dominated_blocks().length(); ++j) {
1917 BlockEntryInstr* block = target->dominated_blocks()[j];
1918 current_block->AddDominatedBlock(block);
1919 }
1920 } else if (callee_entry->IsJoinEntry()) {
1921 // Shared inlined body and this is a subsequent entry. We have
1922 // already constructed a join and set its dominator. Add a jump to
1923 // the join.
1924 JoinEntryInstr* join = callee_entry->AsJoinEntry();
1925 ASSERT(join->dominator() != NULL);
1926 GotoInstr* goto_join = new GotoInstr(join, DeoptId::kNone);
1927 goto_join->InheritDeoptTarget(zone(), join);
1928 cursor->LinkTo(goto_join);
1929 current_block->set_last_instruction(goto_join);
1930 } else {
1931 // There is no possibility of a TargetEntry (the first entry to a
1932 // shared inlined body) because this is the last inlined entry.
1933 UNREACHABLE();
1934 }
1935 cursor = NULL;
1936 } else {
1937 // For all variants except the last, use a branch on the loaded class
1938 // id.
1939 //
1940 // TODO(ajcbik): see if this can use the NewDiamond() utility.
1941 //
1942 const Smi& cid = Smi::ZoneHandle(Smi::New(variant.cid_start));
1943 ConstantInstr* cid_constant = owner_->caller_graph()->GetConstant(cid);
1944 BranchInstr* branch;
1945 BranchInstr* upper_limit_branch = NULL;
1946 BlockEntryInstr* cid_test_entry_block = current_block;
1947 if (test_is_range) {
1948 // Double branch for testing a range of Cids.
1949 // TODO(ajcbik): Make a special instruction that uses subtraction
1950 // and unsigned comparison to do this with a single branch.
1951 const Smi& cid_end = Smi::ZoneHandle(Smi::New(variant.cid_end));
1952 ConstantInstr* cid_constant_end =
1953 owner_->caller_graph()->GetConstant(cid_end);
1954 RelationalOpInstr* compare_top = new RelationalOpInstr(
1955 call_->token_pos(), Token::kLTE, new Value(load_cid),
1956 new Value(cid_constant_end), kSmiCid, call_->deopt_id());
1957 BranchInstr* branch_top = upper_limit_branch =
1958 new BranchInstr(compare_top, DeoptId::kNone);
1959 branch_top->InheritDeoptTarget(zone(), call_);
1960 cursor = AppendInstruction(cursor, branch_top);
1961 current_block->set_last_instruction(branch_top);
1962
1963 TargetEntryInstr* below_target =
1964 new TargetEntryInstr(AllocateBlockId(), try_idx, DeoptId::kNone);
1965 below_target->InheritDeoptTarget(zone(), call_);
1966 current_block->AddDominatedBlock(below_target);
1967 ASSERT(cursor != nullptr); // read before overwrite
1968 cursor = current_block = below_target;
1969 *branch_top->true_successor_address() = below_target;
1970
1971 RelationalOpInstr* compare_bottom = new RelationalOpInstr(
1972 call_->token_pos(), Token::kGTE, new Value(load_cid),
1973 new Value(cid_constant), kSmiCid, call_->deopt_id());
1974 branch = new BranchInstr(compare_bottom, DeoptId::kNone);
1975 } else {
1976 StrictCompareInstr* compare =
1977 new StrictCompareInstr(call_->token_pos(), Token::kEQ_STRICT,
1978 new Value(load_cid), new Value(cid_constant),
1979 /* number_check = */ false, DeoptId::kNone);
1980 branch = new BranchInstr(compare, DeoptId::kNone);
1981 }
1982
1983 branch->InheritDeoptTarget(zone(), call_);
1984 AppendInstruction(cursor, branch);
1985 cursor = nullptr;
1986 current_block->set_last_instruction(branch);
1987
1988 // 2. Handle a match by linking to the inlined body. There are three
1989 // cases (unshared, shared first predecessor, and shared subsequent
1990 // predecessors).
1991 BlockEntryInstr* callee_entry = inlined_entries_[i];
1992 TargetEntryInstr* true_target = NULL;
1993 if (callee_entry->IsGraphEntry()) {
1994 // Unshared.
1995 auto graph_entry = callee_entry->AsGraphEntry();
1996 auto function_entry = graph_entry->normal_entry();
1997
1998 true_target = BranchSimplifier::ToTargetEntry(zone(), function_entry);
1999 function_entry->ReplaceAsPredecessorWith(true_target);
2000 for (intptr_t j = 0; j < function_entry->dominated_blocks().length();
2001 ++j) {
2002 BlockEntryInstr* block = function_entry->dominated_blocks()[j];
2003 true_target->AddDominatedBlock(block);
2004 }
2005
2006 // Unuse all inputs of the graph entry. It is not in the graph anymore.
2007 graph_entry->UnuseAllInputs();
2008 } else if (callee_entry->IsTargetEntry()) {
2009 ASSERT(!callee_entry->IsFunctionEntry());
2010 // Shared inlined body and this is the first entry. We have already
2011 // constructed a join and this target jumps to it.
2012 true_target = callee_entry->AsTargetEntry();
2013 BlockEntryInstr* join = true_target->last_instruction()->SuccessorAt(0);
2014 current_block->AddDominatedBlock(join);
2015 } else {
2016 // Shared inlined body and this is a subsequent entry. We have
2017 // already constructed a join. We need a fresh target that jumps to
2018 // the join.
2019 JoinEntryInstr* join = callee_entry->AsJoinEntry();
2020 ASSERT(join != NULL);
2021 ASSERT(join->dominator() != NULL);
2022 true_target =
2023 new TargetEntryInstr(AllocateBlockId(), try_idx, DeoptId::kNone);
2024 true_target->InheritDeoptTarget(zone(), join);
2025 GotoInstr* goto_join = new GotoInstr(join, DeoptId::kNone);
2026 goto_join->InheritDeoptTarget(zone(), join);
2027 true_target->LinkTo(goto_join);
2028 true_target->set_last_instruction(goto_join);
2029 }
2030 *branch->true_successor_address() = true_target;
2031 current_block->AddDominatedBlock(true_target);
2032
2033 // 3. Prepare to handle a match failure on the next iteration or the
2034 // fall-through code below for non-inlined variants.
2035
2036 TargetEntryInstr* false_target =
2037 new TargetEntryInstr(AllocateBlockId(), try_idx, DeoptId::kNone);
2038 false_target->InheritDeoptTarget(zone(), call_);
2039 *branch->false_successor_address() = false_target;
2040 cid_test_entry_block->AddDominatedBlock(false_target);
2041
2042 cursor = current_block = false_target;
2043
2044 if (test_is_range) {
2045 // If we tested against a range of Cids there are two different tests
2046 // that can go to the no-cid-match target.
2047 JoinEntryInstr* join =
2048 new JoinEntryInstr(AllocateBlockId(), try_idx, DeoptId::kNone);
2049 TargetEntryInstr* false_target2 =
2050 new TargetEntryInstr(AllocateBlockId(), try_idx, DeoptId::kNone);
2051 *upper_limit_branch->false_successor_address() = false_target2;
2052 cid_test_entry_block->AddDominatedBlock(false_target2);
2053 cid_test_entry_block->AddDominatedBlock(join);
2054 GotoInstr* goto_1 = new GotoInstr(join, DeoptId::kNone);
2055 GotoInstr* goto_2 = new GotoInstr(join, DeoptId::kNone);
2056 false_target->LinkTo(goto_1);
2057 false_target2->LinkTo(goto_2);
2058 false_target->set_last_instruction(goto_1);
2059 false_target2->set_last_instruction(goto_2);
2060
2061 join->InheritDeoptTarget(zone(), call_);
2062 false_target2->InheritDeoptTarget(zone(), call_);
2063 goto_1->InheritDeoptTarget(zone(), call_);
2064 goto_2->InheritDeoptTarget(zone(), call_);
2065
2066 cursor = current_block = join;
2067 }
2068 }
2069 }
2070
2071 ASSERT(!call_->HasPushArguments());
2072
2073 // Handle any non-inlined variants.
2074 if (!non_inlined_variants_->is_empty()) {
2075 PolymorphicInstanceCallInstr* fallback_call =
2076 PolymorphicInstanceCallInstr::FromCall(Z, call_, *non_inlined_variants_,
2077 call_->complete());
2078 fallback_call->set_ssa_temp_index(
2079 owner_->caller_graph()->alloc_ssa_temp_index());
2080 if (FlowGraph::NeedsPairLocation(fallback_call->representation())) {
2081 owner_->caller_graph()->alloc_ssa_temp_index();
2082 }
2083 fallback_call->InheritDeoptTarget(zone(), call_);
2084 fallback_call->set_total_call_count(call_->CallCount());
2085 ReturnInstr* fallback_return = new ReturnInstr(
2086 call_->token_pos(), new Value(fallback_call), DeoptId::kNone);
2087 fallback_return->InheritDeoptTargetAfter(owner_->caller_graph(), call_,
2088 fallback_call);
2089 AppendInstruction(AppendInstruction(cursor, fallback_call),
2090 fallback_return);
2091 exit_collector_->AddExit(fallback_return);
2092 cursor = nullptr;
2093 }
2094 return entry;
2095}
2096
2097static void TracePolyInlining(const CallTargets& targets,
2098 intptr_t idx,
2099 intptr_t total,
2100 const char* message) {
2101 String& name =
2102 String::Handle(targets.TargetAt(idx)->target->QualifiedUserVisibleName());
2103 int percent = total == 0 ? 0 : (100 * targets.TargetAt(idx)->count) / total;
2104 THR_Print("%s cid %" Pd "-%" Pd ": %" Pd "/%" Pd " %d%% %s\n",
2105 name.ToCString(), targets[idx].cid_start, targets[idx].cid_end,
2106 targets.TargetAt(idx)->count, total, percent, message);
2107}
2108
2109bool PolymorphicInliner::trace_inlining() const {
2110 return owner_->trace_inlining();
2111}
2112
2113bool PolymorphicInliner::Inline() {
2114 ASSERT(&variants_ == &call_->targets_);
2115
2116 intptr_t total = call_->total_call_count();
2117 for (intptr_t var_idx = 0; var_idx < variants_.length(); ++var_idx) {
2118 TargetInfo* info = variants_.TargetAt(var_idx);
2119 if (variants_.length() > FLAG_max_polymorphic_checks) {
2120 non_inlined_variants_->Add(info);
2121 continue;
2122 }
2123
2124 const Function& target = *variants_.TargetAt(var_idx)->target;
2125 const intptr_t count = variants_.TargetAt(var_idx)->count;
2126
2127 // We we almost inlined all the cases then try a little harder to inline
2128 // the last two, because it's a big win if we inline all of them (compiler
2129 // can see all side effects).
2130 const bool try_harder = (var_idx >= variants_.length() - 2) &&
2131 non_inlined_variants_->length() == 0;
2132
2133 intptr_t size = target.optimized_instruction_count();
2134 bool small = (size != 0 && size < FLAG_inlining_size_threshold);
2135
2136 // If it's less than 3% of the dispatches, we won't even consider
2137 // checking for the class ID and branching to another already-inlined
2138 // version.
2139 if (!try_harder && count < (total >> 5)) {
2140 TRACE_INLINING(
2141 TracePolyInlining(variants_, var_idx, total, "way too infrequent"));
2142 non_inlined_variants_->Add(info);
2143 continue;
2144 }
2145
2146 // First check if this is the same target as an earlier inlined variant.
2147 if (CheckInlinedDuplicate(target)) {
2148 TRACE_INLINING(TracePolyInlining(variants_, var_idx, total,
2149 "duplicate already inlined"));
2150 inlined_variants_.Add(info);
2151 continue;
2152 }
2153
2154 // If it's less than 12% of the dispatches and it's not already inlined, we
2155 // don't consider inlining. For very small functions we are willing to
2156 // consider inlining for 6% of the cases.
2157 if (!try_harder && count < (total >> (small ? 4 : 3))) {
2158 TRACE_INLINING(
2159 TracePolyInlining(variants_, var_idx, total, "too infrequent"));
2160 non_inlined_variants_->Add(&variants_[var_idx]);
2161 continue;
2162 }
2163
2164 // Also check if this is the same target as an earlier non-inlined
2165 // variant. If so and since inlining decisions are costly, do not try
2166 // to inline this variant.
2167 if (CheckNonInlinedDuplicate(target)) {
2168 TRACE_INLINING(
2169 TracePolyInlining(variants_, var_idx, total, "already not inlined"));
2170 non_inlined_variants_->Add(&variants_[var_idx]);
2171 continue;
2172 }
2173
2174 // Make an inlining decision.
2175 if (TryInliningPoly(*info)) {
2176 TRACE_INLINING(TracePolyInlining(variants_, var_idx, total, "inlined"));
2177 inlined_variants_.Add(&variants_[var_idx]);
2178 } else {
2179 TRACE_INLINING(
2180 TracePolyInlining(variants_, var_idx, total, "not inlined"));
2181 non_inlined_variants_->Add(&variants_[var_idx]);
2182 }
2183 }
2184
2185 // If there are no inlined variants, leave the call in place.
2186 if (inlined_variants_.is_empty()) return false;
2187
2188 // Now build a decision tree (a DAG because of shared inline variants) and
2189 // inline it at the call site.
2190 TargetEntryInstr* entry = BuildDecisionGraph();
2191 exit_collector_->ReplaceCall(entry);
2192 return true;
2193}
2194
2195FlowGraphInliner::FlowGraphInliner(
2196 FlowGraph* flow_graph,
2197 GrowableArray<const Function*>* inline_id_to_function,
2198 GrowableArray<TokenPosition>* inline_id_to_token_pos,
2199 GrowableArray<intptr_t>* caller_inline_id,
2200 SpeculativeInliningPolicy* speculative_policy,
2201 Precompiler* precompiler)
2202 : flow_graph_(flow_graph),
2203 inline_id_to_function_(inline_id_to_function),
2204 inline_id_to_token_pos_(inline_id_to_token_pos),
2205 caller_inline_id_(caller_inline_id),
2206 trace_inlining_(FLAG_trace_inlining && flow_graph->should_print()),
2207 speculative_policy_(speculative_policy),
2208 precompiler_(precompiler) {}
2209
2210void FlowGraphInliner::CollectGraphInfo(FlowGraph* flow_graph,
2211 intptr_t constants_count,
2212 bool force,
2213 intptr_t* instruction_count,
2214 intptr_t* call_site_count) {
2215 const Function& function = flow_graph->function();
2216 // For OSR, don't even bother.
2217 if (flow_graph->IsCompiledForOsr()) {
2218 *instruction_count = 0;
2219 *call_site_count = 0;
2220 return;
2221 }
2222 // Specialized case: always recompute, never cache.
2223 if (constants_count > 0) {
2224 ASSERT(!force);
2225 GraphInfoCollector info;
2226 info.Collect(*flow_graph);
2227 *instruction_count = info.instruction_count();
2228 *call_site_count = info.call_site_count();
2229 return;
2230 }
2231 // Non-specialized case: unless forced, only recompute on a cache miss.
2232 ASSERT(constants_count == 0);
2233 if (force || (function.optimized_instruction_count() == 0)) {
2234 GraphInfoCollector info;
2235 info.Collect(*flow_graph);
2236 function.SetOptimizedInstructionCountClamped(info.instruction_count());
2237 function.SetOptimizedCallSiteCountClamped(info.call_site_count());
2238 }
2239 *instruction_count = function.optimized_instruction_count();
2240 *call_site_count = function.optimized_call_site_count();
2241}
2242
2243// TODO(srdjan): This is only needed when disassembling and/or profiling.
2244// Sets inlining id for all instructions of this flow-graph, as well for the
2245// FlowGraph itself.
2246void FlowGraphInliner::SetInliningId(FlowGraph* flow_graph,
2247 intptr_t inlining_id) {
2248 ASSERT(flow_graph->inlining_id() < 0);
2249 flow_graph->set_inlining_id(inlining_id);
2250 for (BlockIterator block_it = flow_graph->postorder_iterator();
2251 !block_it.Done(); block_it.Advance()) {
2252 for (ForwardInstructionIterator it(block_it.Current()); !it.Done();
2253 it.Advance()) {
2254 Instruction* current = it.Current();
2255 // Do not overwrite owner function.
2256 ASSERT(!current->has_inlining_id());
2257 current->set_inlining_id(inlining_id);
2258 }
2259 }
2260}
2261
2262// Use function name to determine if inlineable operator.
2263// Add names as necessary.
2264static bool IsInlineableOperator(const Function& function) {
2265 return (function.name() == Symbols::IndexToken().raw()) ||
2266 (function.name() == Symbols::AssignIndexToken().raw()) ||
2267 (function.name() == Symbols::Plus().raw()) ||
2268 (function.name() == Symbols::Minus().raw());
2269}
2270
2271bool FlowGraphInliner::FunctionHasPreferInlinePragma(const Function& function) {
2272 Object& options = Object::Handle();
2273 return Library::FindPragma(dart::Thread::Current(), /*only_core=*/false,
2274 function, Symbols::vm_prefer_inline(), &options);
2275}
2276
2277bool FlowGraphInliner::FunctionHasNeverInlinePragma(const Function& function) {
2278 Object& options = Object::Handle();
2279 return Library::FindPragma(dart::Thread::Current(), /*only_core=*/false,
2280 function, Symbols::vm_never_inline(), &options);
2281}
2282
2283bool FlowGraphInliner::AlwaysInline(const Function& function) {
2284 if (FunctionHasPreferInlinePragma(function)) {
2285 TRACE_INLINING(
2286 THR_Print("vm:prefer-inline pragma for %s\n", function.ToCString()));
2287 return true;
2288 }
2289
2290 // We don't want to inline DIFs for recognized methods because we would rather
2291 // replace them with inline FG before inlining introduces any superfluous
2292 // AssertAssignable instructions.
2293 if (function.IsDispatcherOrImplicitAccessor() &&
2294 !(function.kind() == FunctionLayout::kDynamicInvocationForwarder &&
2295 function.IsRecognized())) {
2296 // Smaller or same size as the call.
2297 return true;
2298 }
2299
2300 if (function.is_const()) {
2301 // Inlined const fields are smaller than a call.
2302 return true;
2303 }
2304
2305 if (function.IsGetterFunction() || function.IsSetterFunction() ||
2306 IsInlineableOperator(function) ||
2307 (function.kind() == FunctionLayout::kConstructor)) {
2308 const intptr_t count = function.optimized_instruction_count();
2309 if ((count != 0) && (count < FLAG_inline_getters_setters_smaller_than)) {
2310 return true;
2311 }
2312 }
2313 return false;
2314}
2315
2316int FlowGraphInliner::Inline() {
2317 // Collect some early graph information assuming it is non-specialized
2318 // so that the cached approximation may be used later for an early
2319 // bailout from inlining.
2320 intptr_t instruction_count = 0;
2321 intptr_t call_site_count = 0;
2322 FlowGraphInliner::CollectGraphInfo(flow_graph_,
2323 /*constants_count*/ 0,
2324 /*force*/ false, &instruction_count,
2325 &call_site_count);
2326
2327 const Function& top = flow_graph_->function();
2328 if ((FLAG_inlining_filter != NULL) &&
2329 (strstr(top.ToFullyQualifiedCString(), FLAG_inlining_filter) == NULL)) {
2330 return 0;
2331 }
2332
2333 if (trace_inlining()) {
2334 String& name = String::Handle(top.QualifiedUserVisibleName());
2335 THR_Print("Inlining calls in %s\n", name.ToCString());
2336 }
2337
2338 if (FLAG_support_il_printer && trace_inlining() &&
2339 (FLAG_print_flow_graph || FLAG_print_flow_graph_optimized)) {
2340 THR_Print("Before Inlining of %s\n",
2341 flow_graph_->function().ToFullyQualifiedCString());
2342 FlowGraphPrinter printer(*flow_graph_);
2343 printer.PrintBlocks();
2344 }
2345
2346 intptr_t inlining_depth_threshold = FLAG_inlining_depth_threshold;
2347
2348 CallSiteInliner inliner(this, inlining_depth_threshold);
2349 inliner.InlineCalls();
2350 if (FLAG_print_inlining_tree) {
2351 inliner.PrintInlinedInfo(top);
2352 }
2353
2354 if (inliner.inlined()) {
2355 flow_graph_->DiscoverBlocks();
2356 if (trace_inlining()) {
2357 THR_Print("Inlining growth factor: %f\n", inliner.GrowthFactor());
2358 if (FLAG_support_il_printer &&
2359 (FLAG_print_flow_graph || FLAG_print_flow_graph_optimized)) {
2360 THR_Print("After Inlining of %s\n",
2361 flow_graph_->function().ToFullyQualifiedCString());
2362 FlowGraphPrinter printer(*flow_graph_);
2363 printer.PrintBlocks();
2364 }
2365 }
2366 }
2367 return inliner.inlining_depth();
2368}
2369
2370intptr_t FlowGraphInliner::NextInlineId(const Function& function,
2371 TokenPosition tp,
2372 intptr_t parent_id) {
2373 const intptr_t id = inline_id_to_function_->length();
2374 // TODO(johnmccutchan): Do not allow IsNoSource once all nodes have proper
2375 // source positions.
2376 ASSERT(tp.IsReal() || tp.IsSynthetic() || tp.IsNoSource());
2377 RELEASE_ASSERT(!function.IsNull());
2378 inline_id_to_function_->Add(&function);
2379 inline_id_to_token_pos_->Add(tp);
2380 caller_inline_id_->Add(parent_id);
2381 // We always have one less token position than functions.
2382 ASSERT(inline_id_to_token_pos_->length() ==
2383 (inline_id_to_function_->length() - 1));
2384 return id;
2385}
2386
2387static bool ShouldInlineSimd() {
2388 return FlowGraphCompiler::SupportsUnboxedSimd128();
2389}
2390
2391static bool CanUnboxDouble() {
2392 return FlowGraphCompiler::SupportsUnboxedDoubles();
2393}
2394
2395static bool ShouldInlineInt64ArrayOps() {
2396 return FlowGraphCompiler::SupportsUnboxedInt64();
2397}
2398
2399static bool CanUnboxInt32() {
2400 // Int32/Uint32 can be unboxed if it fits into a smi or the platform
2401 // supports unboxed mints.
2402 return (compiler::target::kSmiBits >= 32) ||
2403 FlowGraphCompiler::SupportsUnboxedInt64();
2404}
2405
2406// Quick access to the current one.
2407#undef Z
2408#define Z (flow_graph->zone())
2409
2410static intptr_t PrepareInlineIndexedOp(FlowGraph* flow_graph,
2411 Instruction* call,
2412 intptr_t array_cid,
2413 Definition** array,
2414 Definition** index,
2415 Instruction** cursor) {
2416 // Insert array length load and bounds check.
2417 LoadFieldInstr* length = new (Z) LoadFieldInstr(
2418 new (Z) Value(*array), Slot::GetLengthFieldForArrayCid(array_cid),
2419 call->token_pos());
2420 *cursor = flow_graph->AppendTo(*cursor, length, NULL, FlowGraph::kValue);
2421 *index = flow_graph->CreateCheckBound(length, *index, call->deopt_id());
2422 *cursor =
2423 flow_graph->AppendTo(*cursor, *index, call->env(), FlowGraph::kValue);
2424
2425 if (array_cid == kGrowableObjectArrayCid) {
2426 // Insert data elements load.
2427 LoadFieldInstr* elements = new (Z)
2428 LoadFieldInstr(new (Z) Value(*array), Slot::GrowableObjectArray_data(),
2429 call->token_pos());
2430 *cursor = flow_graph->AppendTo(*cursor, elements, NULL, FlowGraph::kValue);
2431 // Load from the data from backing store which is a fixed-length array.
2432 *array = elements;
2433 array_cid = kArrayCid;
2434 } else if (IsExternalTypedDataClassId(array_cid)) {
2435 LoadUntaggedInstr* elements = new (Z)
2436 LoadUntaggedInstr(new (Z) Value(*array),
2437 compiler::target::TypedDataBase::data_field_offset());
2438 *cursor = flow_graph->AppendTo(*cursor, elements, NULL, FlowGraph::kValue);
2439 *array = elements;
2440 }
2441 return array_cid;
2442}
2443
2444static bool InlineGetIndexed(FlowGraph* flow_graph,
2445 bool can_speculate,
2446 bool is_dynamic_call,
2447 MethodRecognizer::Kind kind,
2448 Definition* call,
2449 Definition* receiver,
2450 GraphEntryInstr* graph_entry,
2451 FunctionEntryInstr** entry,
2452 Instruction** last,
2453 Definition** result) {
2454 intptr_t array_cid = MethodRecognizer::MethodKindToReceiverCid(kind);
2455
2456 Definition* array = receiver;
2457 Definition* index = call->ArgumentAt(1);
2458
2459 if (!can_speculate && is_dynamic_call && !index->Type()->IsInt()) {
2460 return false;
2461 }
2462
2463 *entry =
2464 new (Z) FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
2465 call->GetBlock()->try_index(), DeoptId::kNone);
2466 (*entry)->InheritDeoptTarget(Z, call);
2467 Instruction* cursor = *entry;
2468
2469 array_cid = PrepareInlineIndexedOp(flow_graph, call, array_cid, &array,
2470 &index, &cursor);
2471
2472 intptr_t deopt_id = DeoptId::kNone;
2473 if ((array_cid == kTypedDataInt32ArrayCid) ||
2474 (array_cid == kTypedDataUint32ArrayCid)) {
2475 // Deoptimization may be needed if result does not always fit in a Smi.
2476 deopt_id =
2477 (compiler::target::kSmiBits >= 32) ? DeoptId::kNone : call->deopt_id();
2478 }
2479
2480 // Array load and return.
2481 intptr_t index_scale = compiler::target::Instance::ElementSizeFor(array_cid);
2482 LoadIndexedInstr* load = new (Z) LoadIndexedInstr(
2483 new (Z) Value(array), new (Z) Value(index), /*index_unboxed=*/false,
2484 index_scale, array_cid, kAlignedAccess, deopt_id, call->token_pos(),
2485 ResultType(call));
2486
2487 *last = load;
2488 cursor = flow_graph->AppendTo(cursor, load,
2489 deopt_id != DeoptId::kNone ? call->env() : NULL,
2490 FlowGraph::kValue);
2491
2492 const bool value_needs_boxing =
2493 array_cid == kTypedDataInt8ArrayCid ||
2494 array_cid == kTypedDataInt16ArrayCid ||
2495 array_cid == kTypedDataUint8ArrayCid ||
2496 array_cid == kTypedDataUint8ClampedArrayCid ||
2497 array_cid == kTypedDataUint16ArrayCid ||
2498 array_cid == kExternalTypedDataUint8ArrayCid ||
2499 array_cid == kExternalTypedDataUint8ClampedArrayCid;
2500
2501 if (array_cid == kTypedDataFloat32ArrayCid) {
2502 *last = new (Z) FloatToDoubleInstr(new (Z) Value(load), deopt_id);
2503 flow_graph->AppendTo(cursor, *last,
2504 deopt_id != DeoptId::kNone ? call->env() : NULL,
2505 FlowGraph::kValue);
2506 } else if (value_needs_boxing) {
2507 *last = BoxInstr::Create(kUnboxedIntPtr, new Value(load));
2508 flow_graph->AppendTo(cursor, *last, nullptr, FlowGraph::kValue);
2509 }
2510 *result = (*last)->AsDefinition();
2511 return true;
2512}
2513
2514static bool InlineSetIndexed(FlowGraph* flow_graph,
2515 MethodRecognizer::Kind kind,
2516 const Function& target,
2517 Instruction* call,
2518 Definition* receiver,
2519 TokenPosition token_pos,
2520 const Cids* value_check,
2521 FlowGraphInliner::ExactnessInfo* exactness,
2522 GraphEntryInstr* graph_entry,
2523 FunctionEntryInstr** entry,
2524 Instruction** last,
2525 Definition** result) {
2526 intptr_t array_cid = MethodRecognizer::MethodKindToReceiverCid(kind);
2527
2528 Definition* array = receiver;
2529 Definition* index = call->ArgumentAt(1);
2530 Definition* stored_value = call->ArgumentAt(2);
2531
2532 *entry =
2533 new (Z) FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
2534 call->GetBlock()->try_index(), DeoptId::kNone);
2535 (*entry)->InheritDeoptTarget(Z, call);
2536
2537 bool is_unchecked_call = false;
2538 if (StaticCallInstr* static_call = call->AsStaticCall()) {
2539 is_unchecked_call =
2540 static_call->entry_kind() == Code::EntryKind::kUnchecked;
2541 } else if (InstanceCallInstr* instance_call = call->AsInstanceCall()) {
2542 is_unchecked_call =
2543 instance_call->entry_kind() == Code::EntryKind::kUnchecked;
2544 } else if (PolymorphicInstanceCallInstr* instance_call =
2545 call->AsPolymorphicInstanceCall()) {
2546 is_unchecked_call =
2547 instance_call->entry_kind() == Code::EntryKind::kUnchecked;
2548 }
2549
2550 Instruction* cursor = *entry;
2551 if (!is_unchecked_call &&
2552 (kind != MethodRecognizer::kObjectArraySetIndexedUnchecked &&
2553 kind != MethodRecognizer::kGrowableArraySetIndexedUnchecked)) {
2554 // Only type check for the value. A type check for the index is not
2555 // needed here because we insert a deoptimizing smi-check for the case
2556 // the index is not a smi.
2557 const AbstractType& value_type =
2558 AbstractType::ZoneHandle(Z, target.ParameterTypeAt(2));
2559 Definition* type_args = NULL;
2560 switch (array_cid) {
2561 case kArrayCid:
2562 case kGrowableObjectArrayCid: {
2563 const Class& instantiator_class = Class::Handle(Z, target.Owner());
2564 LoadFieldInstr* load_type_args = new (Z)
2565 LoadFieldInstr(new (Z) Value(array),
2566 Slot::GetTypeArgumentsSlotFor(flow_graph->thread(),
2567 instantiator_class),
2568 call->token_pos());
2569 cursor = flow_graph->AppendTo(cursor, load_type_args, NULL,
2570 FlowGraph::kValue);
2571 type_args = load_type_args;
2572 break;
2573 }
2574 case kTypedDataInt8ArrayCid:
2575 FALL_THROUGH;
2576 case kTypedDataUint8ArrayCid:
2577 FALL_THROUGH;
2578 case kTypedDataUint8ClampedArrayCid:
2579 FALL_THROUGH;
2580 case kExternalTypedDataUint8ArrayCid:
2581 FALL_THROUGH;
2582 case kExternalTypedDataUint8ClampedArrayCid:
2583 FALL_THROUGH;
2584 case kTypedDataInt16ArrayCid:
2585 FALL_THROUGH;
2586 case kTypedDataUint16ArrayCid:
2587 FALL_THROUGH;
2588 case kTypedDataInt32ArrayCid:
2589 FALL_THROUGH;
2590 case kTypedDataUint32ArrayCid:
2591 FALL_THROUGH;
2592 case kTypedDataInt64ArrayCid:
2593 FALL_THROUGH;
2594 case kTypedDataUint64ArrayCid:
2595 ASSERT(value_type.IsIntType());
2596 FALL_THROUGH;
2597 case kTypedDataFloat32ArrayCid:
2598 FALL_THROUGH;
2599 case kTypedDataFloat64ArrayCid: {
2600 type_args = flow_graph->constant_null();
2601 ASSERT((array_cid != kTypedDataFloat32ArrayCid &&
2602 array_cid != kTypedDataFloat64ArrayCid) ||
2603 value_type.IsDoubleType());
2604 ASSERT(value_type.IsInstantiated());
2605 break;
2606 }
2607 case kTypedDataFloat32x4ArrayCid: {
2608 type_args = flow_graph->constant_null();
2609 ASSERT(value_type.IsFloat32x4Type());
2610 ASSERT(value_type.IsInstantiated());
2611 break;
2612 }
2613 case kTypedDataFloat64x2ArrayCid: {
2614 type_args = flow_graph->constant_null();
2615 ASSERT(value_type.IsFloat64x2Type());
2616 ASSERT(value_type.IsInstantiated());
2617 break;
2618 }
2619 default:
2620 // TODO(fschneider): Add support for other array types.
2621 UNREACHABLE();
2622 }
2623
2624 if (exactness != nullptr && exactness->is_exact) {
2625 exactness->emit_exactness_guard = true;
2626 } else {
2627 auto const function_type_args = flow_graph->constant_null();
2628 auto const dst_type = flow_graph->GetConstant(value_type);
2629 AssertAssignableInstr* assert_value = new (Z) AssertAssignableInstr(
2630 token_pos, new (Z) Value(stored_value), new (Z) Value(dst_type),
2631 new (Z) Value(type_args), new (Z) Value(function_type_args),
2632 Symbols::Value(), call->deopt_id());
2633 cursor = flow_graph->AppendTo(cursor, assert_value, call->env(),
2634 FlowGraph::kValue);
2635 }
2636 }
2637
2638 array_cid = PrepareInlineIndexedOp(flow_graph, call, array_cid, &array,
2639 &index, &cursor);
2640
2641 // Check if store barrier is needed. Byte arrays don't need a store barrier.
2642 StoreBarrierType needs_store_barrier =
2643 (IsTypedDataClassId(array_cid) || IsTypedDataViewClassId(array_cid) ||
2644 IsExternalTypedDataClassId(array_cid))
2645 ? kNoStoreBarrier
2646 : kEmitStoreBarrier;
2647
2648 const bool value_needs_unboxing =
2649 array_cid == kTypedDataInt8ArrayCid ||
2650 array_cid == kTypedDataInt16ArrayCid ||
2651 array_cid == kTypedDataInt32ArrayCid ||
2652 array_cid == kTypedDataUint8ArrayCid ||
2653 array_cid == kTypedDataUint8ClampedArrayCid ||
2654 array_cid == kTypedDataUint16ArrayCid ||
2655 array_cid == kTypedDataUint32ArrayCid ||
2656 array_cid == kExternalTypedDataUint8ArrayCid ||
2657 array_cid == kExternalTypedDataUint8ClampedArrayCid;
2658
2659 if (value_check != NULL) {
2660 // No store barrier needed because checked value is a smi, an unboxed mint,
2661 // an unboxed double, an unboxed Float32x4, or unboxed Int32x4.
2662 needs_store_barrier = kNoStoreBarrier;
2663 Instruction* check = flow_graph->CreateCheckClass(
2664 stored_value, *value_check, call->deopt_id(), call->token_pos());
2665 cursor =
2666 flow_graph->AppendTo(cursor, check, call->env(), FlowGraph::kEffect);
2667 }
2668
2669 if (array_cid == kTypedDataFloat32ArrayCid) {
2670 stored_value = new (Z)
2671 DoubleToFloatInstr(new (Z) Value(stored_value), call->deopt_id());
2672 cursor =
2673 flow_graph->AppendTo(cursor, stored_value, NULL, FlowGraph::kValue);
2674 } else if (value_needs_unboxing) {
2675 Representation representation = kNoRepresentation;
2676 switch (array_cid) {
2677 case kTypedDataInt32ArrayCid:
2678 representation = kUnboxedInt32;
2679 break;
2680 case kTypedDataUint32ArrayCid:
2681 representation = kUnboxedUint32;
2682 break;
2683 case kTypedDataInt8ArrayCid:
2684 case kTypedDataInt16ArrayCid:
2685 case kTypedDataUint8ArrayCid:
2686 case kTypedDataUint8ClampedArrayCid:
2687 case kTypedDataUint16ArrayCid:
2688 case kExternalTypedDataUint8ArrayCid:
2689 case kExternalTypedDataUint8ClampedArrayCid:
2690 representation = kUnboxedIntPtr;
2691 break;
2692 default:
2693 UNREACHABLE();
2694 }
2695 stored_value = UnboxInstr::Create(
2696 representation, new (Z) Value(stored_value), call->deopt_id());
2697 stored_value->AsUnboxInteger()->mark_truncating();
2698 cursor = flow_graph->AppendTo(cursor, stored_value, call->env(),
2699 FlowGraph::kValue);
2700 }
2701
2702 const intptr_t index_scale =
2703 compiler::target::Instance::ElementSizeFor(array_cid);
2704 *last = new (Z) StoreIndexedInstr(
2705 new (Z) Value(array), new (Z) Value(index), new (Z) Value(stored_value),
2706 needs_store_barrier, /*index_unboxed=*/false, index_scale, array_cid,
2707 kAlignedAccess, call->deopt_id(), call->token_pos());
2708 flow_graph->AppendTo(cursor, *last, call->env(), FlowGraph::kEffect);
2709 // We need a return value to replace uses of the original definition. However,
2710 // the final instruction is a use of 'void operator[]=()', so we use null.
2711 *result = flow_graph->constant_null();
2712 return true;
2713}
2714
2715static bool InlineDoubleOp(FlowGraph* flow_graph,
2716 Token::Kind op_kind,
2717 Instruction* call,
2718 Definition* receiver,
2719 GraphEntryInstr* graph_entry,
2720 FunctionEntryInstr** entry,
2721 Instruction** last,
2722 Definition** result) {
2723 if (!CanUnboxDouble()) {
2724 return false;
2725 }
2726 Definition* left = receiver;
2727 Definition* right = call->ArgumentAt(1);
2728
2729 *entry =
2730 new (Z) FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
2731 call->GetBlock()->try_index(), DeoptId::kNone);
2732 (*entry)->InheritDeoptTarget(Z, call);
2733 // Arguments are checked. No need for class check.
2734 BinaryDoubleOpInstr* double_bin_op = new (Z)
2735 BinaryDoubleOpInstr(op_kind, new (Z) Value(left), new (Z) Value(right),
2736 call->deopt_id(), call->token_pos());
2737 flow_graph->AppendTo(*entry, double_bin_op, call->env(), FlowGraph::kValue);
2738 *last = double_bin_op;
2739 *result = double_bin_op->AsDefinition();
2740
2741 return true;
2742}
2743
2744static bool InlineDoubleTestOp(FlowGraph* flow_graph,
2745 Instruction* call,
2746 Definition* receiver,
2747 MethodRecognizer::Kind kind,
2748 GraphEntryInstr* graph_entry,
2749 FunctionEntryInstr** entry,
2750 Instruction** last,
2751 Definition** result) {
2752 if (!CanUnboxDouble()) {
2753 return false;
2754 }
2755
2756 *entry =
2757 new (Z) FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
2758 call->GetBlock()->try_index(), DeoptId::kNone);
2759 (*entry)->InheritDeoptTarget(Z, call);
2760 // Arguments are checked. No need for class check.
2761
2762 DoubleTestOpInstr* double_test_op = new (Z) DoubleTestOpInstr(
2763 kind, new (Z) Value(receiver), call->deopt_id(), call->token_pos());
2764 flow_graph->AppendTo(*entry, double_test_op, call->env(), FlowGraph::kValue);
2765 *last = double_test_op;
2766 *result = double_test_op->AsDefinition();
2767
2768 return true;
2769}
2770
2771static bool InlineSmiBitAndFromSmi(FlowGraph* flow_graph,
2772 Instruction* call,
2773 Definition* receiver,
2774 GraphEntryInstr* graph_entry,
2775 FunctionEntryInstr** entry,
2776 Instruction** last,
2777 Definition** result) {
2778 Definition* left = receiver;
2779 Definition* right = call->ArgumentAt(1);
2780
2781 *entry =
2782 new (Z) FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
2783 call->GetBlock()->try_index(), DeoptId::kNone);
2784 (*entry)->InheritDeoptTarget(Z, call);
2785 // Right arguments is known to be smi: other._bitAndFromSmi(this);
2786 BinarySmiOpInstr* smi_op =
2787 new (Z) BinarySmiOpInstr(Token::kBIT_AND, new (Z) Value(left),
2788 new (Z) Value(right), call->deopt_id());
2789 flow_graph->AppendTo(*entry, smi_op, call->env(), FlowGraph::kValue);
2790 *last = smi_op;
2791 *result = smi_op->AsDefinition();
2792
2793 return true;
2794}
2795
2796static bool InlineGrowableArraySetter(FlowGraph* flow_graph,
2797 const Slot& field,
2798 StoreBarrierType store_barrier_type,
2799 Instruction* call,
2800 Definition* receiver,
2801 GraphEntryInstr* graph_entry,
2802 FunctionEntryInstr** entry,
2803 Instruction** last,
2804 Definition** result) {
2805 Definition* array = receiver;
2806 Definition* value = call->ArgumentAt(1);
2807
2808 *entry =
2809 new (Z) FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
2810 call->GetBlock()->try_index(), DeoptId::kNone);
2811 (*entry)->InheritDeoptTarget(Z, call);
2812
2813 // This is an internal method, no need to check argument types.
2814 StoreInstanceFieldInstr* store = new (Z)
2815 StoreInstanceFieldInstr(field, new (Z) Value(array), new (Z) Value(value),
2816 store_barrier_type, call->token_pos());
2817 flow_graph->AppendTo(*entry, store, call->env(), FlowGraph::kEffect);
2818 *last = store;
2819 // We need a return value to replace uses of the original definition. However,
2820 // the last instruction is a field setter, which returns void, so we use null.
2821 *result = flow_graph->constant_null();
2822
2823 return true;
2824}
2825
2826static bool InlineLoadClassId(FlowGraph* flow_graph,
2827 Instruction* call,
2828 GraphEntryInstr* graph_entry,
2829 FunctionEntryInstr** entry,
2830 Instruction** last,
2831 Definition** result) {
2832 *entry =
2833 new (Z) FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
2834 call->GetBlock()->try_index(), DeoptId::kNone);
2835 (*entry)->InheritDeoptTarget(Z, call);
2836 auto load_cid =
2837 new (Z) LoadClassIdInstr(call->ArgumentValueAt(0)->CopyWithType(Z));
2838 flow_graph->InsertBefore(call, load_cid, nullptr, FlowGraph::kValue);
2839 *last = load_cid;
2840 *result = load_cid->AsDefinition();
2841 return true;
2842}
2843
2844// Emits preparatory code for a typed getter/setter.
2845// Handles three cases:
2846// (1) dynamic: generates load untagged (internal or external)
2847// (2) external: generates load untagged
2848// (3) internal: no code required.
2849static void PrepareInlineByteArrayBaseOp(FlowGraph* flow_graph,
2850 Instruction* call,
2851 Definition* receiver,
2852 intptr_t array_cid,
2853 Definition** array,
2854 Instruction** cursor) {
2855 if (array_cid == kDynamicCid || IsExternalTypedDataClassId(array_cid)) {
2856 // Internal or External typed data: load untagged.
2857 auto elements = new (Z)
2858 LoadUntaggedInstr(new (Z) Value(*array),
2859 compiler::target::TypedDataBase::data_field_offset());
2860 *cursor = flow_graph->AppendTo(*cursor, elements, NULL, FlowGraph::kValue);
2861 *array = elements;
2862 } else {
2863 // Internal typed data: no action.
2864 ASSERT(IsTypedDataClassId(array_cid));
2865 }
2866}
2867
2868static bool InlineByteArrayBaseLoad(FlowGraph* flow_graph,
2869 Definition* call,
2870 Definition* receiver,
2871 intptr_t array_cid,
2872 intptr_t view_cid,
2873 GraphEntryInstr* graph_entry,
2874 FunctionEntryInstr** entry,
2875 Instruction** last,
2876 Definition** result) {
2877 ASSERT(array_cid != kIllegalCid);
2878
2879 // Dynamic calls are polymorphic due to:
2880 // (A) extra bounds check computations (length stored in receiver),
2881 // (B) external/internal typed data in receiver.
2882 // Both issues are resolved in the inlined code.
2883 // All getters that go through InlineByteArrayBaseLoad() have explicit
2884 // bounds checks in all their clients in the library, so we can omit yet
2885 // another inlined bounds check.
2886 if (array_cid == kDynamicCid) {
2887 ASSERT(call->IsStaticCall());
2888 }
2889
2890 Definition* array = receiver;
2891 Definition* index = call->ArgumentAt(1);
2892 *entry =
2893 new (Z) FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
2894 call->GetBlock()->try_index(), DeoptId::kNone);
2895 (*entry)->InheritDeoptTarget(Z, call);
2896 Instruction* cursor = *entry;
2897
2898
2899 // Generates a template for the load, either a dynamic conditional
2900 // that dispatches on external and internal storage, or a single
2901 // case that deals with either external or internal storage.
2902 PrepareInlineByteArrayBaseOp(flow_graph, call, receiver, array_cid, &array,
2903 &cursor);
2904
2905 // Fill out the generated template with loads.
2906 // Load from either external or internal.
2907 LoadIndexedInstr* load = new (Z) LoadIndexedInstr(
2908 new (Z) Value(array), new (Z) Value(index),
2909 /*index_unboxed=*/false, /*index_scale=*/1, view_cid, kUnalignedAccess,
2910 DeoptId::kNone, call->token_pos(), ResultType(call));
2911 flow_graph->AppendTo(
2912 cursor, load, call->deopt_id() != DeoptId::kNone ? call->env() : nullptr,
2913 FlowGraph::kValue);
2914 cursor = *last = load;
2915
2916 if (view_cid == kTypedDataFloat32ArrayCid) {
2917 *last = new (Z) FloatToDoubleInstr(new (Z) Value((*last)->AsDefinition()),
2918 DeoptId::kNone);
2919 flow_graph->AppendTo(cursor, *last, nullptr, FlowGraph::kValue);
2920 }
2921 *result = (*last)->AsDefinition();
2922 return true;
2923}
2924
2925static StoreIndexedInstr* NewStore(FlowGraph* flow_graph,
2926 Instruction* call,
2927 Definition* array,
2928 Definition* index,
2929 Definition* stored_value,
2930 intptr_t view_cid) {
2931 return new (Z) StoreIndexedInstr(
2932 new (Z) Value(array), new (Z) Value(index), new (Z) Value(stored_value),
2933 kNoStoreBarrier, /*index_unboxed=*/false,
2934 /*index_scale=*/1, view_cid, kUnalignedAccess, call->deopt_id(),
2935 call->token_pos());
2936}
2937
2938static bool InlineByteArrayBaseStore(FlowGraph* flow_graph,
2939 const Function& target,
2940 Instruction* call,
2941 Definition* receiver,
2942 intptr_t array_cid,
2943 intptr_t view_cid,
2944 GraphEntryInstr* graph_entry,
2945 FunctionEntryInstr** entry,
2946 Instruction** last,
2947 Definition** result) {
2948 ASSERT(array_cid != kIllegalCid);
2949
2950 // Dynamic calls are polymorphic due to:
2951 // (A) extra bounds check computations (length stored in receiver),
2952 // (B) external/internal typed data in receiver.
2953 // Both issues are resolved in the inlined code.
2954 // All setters that go through InlineByteArrayBaseLoad() have explicit
2955 // bounds checks in all their clients in the library, so we can omit yet
2956 // another inlined bounds check.
2957 if (array_cid == kDynamicCid) {
2958 ASSERT(call->IsStaticCall());
2959 }
2960
2961 Definition* array = receiver;
2962 Definition* index = call->ArgumentAt(1);
2963 *entry =
2964 new (Z) FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
2965 call->GetBlock()->try_index(), DeoptId::kNone);
2966 (*entry)->InheritDeoptTarget(Z, call);
2967 Instruction* cursor = *entry;
2968
2969 // Prepare additional checks. In AOT Dart2, we use an explicit null check and
2970 // non-speculative unboxing for most value types.
2971 Cids* value_check = nullptr;
2972 bool needs_null_check = false;
2973 switch (view_cid) {
2974 case kTypedDataInt8ArrayCid:
2975 case kTypedDataUint8ArrayCid:
2976 case kTypedDataUint8ClampedArrayCid:
2977 case kExternalTypedDataUint8ArrayCid:
2978 case kExternalTypedDataUint8ClampedArrayCid:
2979 case kTypedDataInt16ArrayCid:
2980 case kTypedDataUint16ArrayCid: {
2981 if (CompilerState::Current().is_aot()) {
2982 needs_null_check = true;
2983 } else {
2984 // Check that value is always smi.
2985 value_check = Cids::CreateMonomorphic(Z, kSmiCid);
2986 }
2987 break;
2988 }
2989 case kTypedDataInt32ArrayCid:
2990 case kTypedDataUint32ArrayCid:
2991 if (CompilerState::Current().is_aot()) {
2992 needs_null_check = true;
2993 } else {
2994 // On 64-bit platforms assume that stored value is always a smi.
2995 if (compiler::target::kSmiBits >= 32) {
2996 value_check = Cids::CreateMonomorphic(Z, kSmiCid);
2997 }
2998 }
2999 break;
3000 case kTypedDataFloat32ArrayCid:
3001 case kTypedDataFloat64ArrayCid: {
3002 // Check that value is always double.
3003 if (CompilerState::Current().is_aot()) {
3004 needs_null_check = true;
3005 } else {
3006 value_check = Cids::CreateMonomorphic(Z, kDoubleCid);
3007 }
3008 break;
3009 }
3010 case kTypedDataInt32x4ArrayCid: {
3011 // Check that value is always Int32x4.
3012 value_check = Cids::CreateMonomorphic(Z, kInt32x4Cid);
3013 break;
3014 }
3015 case kTypedDataFloat32x4ArrayCid: {
3016 // Check that value is always Float32x4.
3017 value_check = Cids::CreateMonomorphic(Z, kFloat32x4Cid);
3018 break;
3019 }
3020 case kTypedDataInt64ArrayCid:
3021 case kTypedDataUint64ArrayCid:
3022 // StoreIndexedInstr takes unboxed int64, so value is
3023 // checked when unboxing. In AOT, we use an
3024 // explicit null check and non-speculative unboxing.
3025 needs_null_check = CompilerState::Current().is_aot();
3026 break;
3027 default:
3028 // Array cids are already checked in the caller.
3029 UNREACHABLE();
3030 }
3031
3032 Definition* stored_value = call->ArgumentAt(2);
3033
3034 // Handle value check.
3035 if (value_check != nullptr) {
3036 Instruction* check = flow_graph->CreateCheckClass(
3037 stored_value, *value_check, call->deopt_id(), call->token_pos());
3038 cursor =
3039 flow_graph->AppendTo(cursor, check, call->env(), FlowGraph::kEffect);
3040 }
3041
3042 // Handle null check.
3043 if (needs_null_check) {
3044 String& name = String::ZoneHandle(Z, target.name());
3045 Instruction* check = new (Z) CheckNullInstr(
3046 new (Z) Value(stored_value), name, call->deopt_id(), call->token_pos());
3047 cursor =
3048 flow_graph->AppendTo(cursor, check, call->env(), FlowGraph::kEffect);
3049 // With an explicit null check, a non-speculative unbox suffices.
3050 switch (view_cid) {
3051 case kTypedDataFloat32ArrayCid:
3052 case kTypedDataFloat64ArrayCid:
3053 stored_value =
3054 UnboxInstr::Create(kUnboxedDouble, new (Z) Value(stored_value),
3055 call->deopt_id(), Instruction::kNotSpeculative);
3056 cursor = flow_graph->AppendTo(cursor, stored_value, call->env(),
3057 FlowGraph::kValue);
3058 break;
3059 case kTypedDataInt64ArrayCid:
3060 case kTypedDataUint64ArrayCid:
3061 stored_value = new (Z)
3062 UnboxInt64Instr(new (Z) Value(stored_value), call->deopt_id(),
3063 Instruction::kNotSpeculative);
3064 cursor = flow_graph->AppendTo(cursor, stored_value, call->env(),
3065 FlowGraph::kValue);
3066 break;
3067 }
3068 }
3069
3070 // Handle conversions and special unboxing (to ensure unboxing instructions
3071 // are marked as truncating, since [SelectRepresentations] does not take care
3072 // of that).
3073 switch (view_cid) {
3074 case kTypedDataInt8ArrayCid:
3075 case kTypedDataInt16ArrayCid:
3076 case kTypedDataUint8ArrayCid:
3077 case kTypedDataUint8ClampedArrayCid:
3078 case kTypedDataUint16ArrayCid:
3079 case kExternalTypedDataUint8ArrayCid:
3080 case kExternalTypedDataUint8ClampedArrayCid: {
3081 stored_value =
3082 UnboxInstr::Create(kUnboxedIntPtr, new (Z) Value(stored_value),
3083 call->deopt_id(), Instruction::kNotSpeculative);
3084 stored_value->AsUnboxInteger()->mark_truncating();
3085 cursor = flow_graph->AppendTo(cursor, stored_value, call->env(),
3086 FlowGraph::kValue);
3087 break;
3088 }
3089 case kTypedDataFloat32ArrayCid: {
3090 stored_value = new (Z)
3091 DoubleToFloatInstr(new (Z) Value(stored_value), call->deopt_id());
3092 cursor = flow_graph->AppendTo(cursor, stored_value, nullptr,
3093 FlowGraph::kValue);
3094 break;
3095 }
3096 case kTypedDataInt32ArrayCid: {
3097 stored_value = new (Z)
3098 UnboxInt32Instr(UnboxInt32Instr::kTruncate,
3099 new (Z) Value(stored_value), call->deopt_id());
3100 cursor = flow_graph->AppendTo(cursor, stored_value, call->env(),
3101 FlowGraph::kValue);
3102 break;
3103 }
3104 case kTypedDataUint32ArrayCid: {
3105 stored_value = new (Z)
3106 UnboxUint32Instr(new (Z) Value(stored_value), call->deopt_id());
3107 ASSERT(stored_value->AsUnboxInteger()->is_truncating());
3108 cursor = flow_graph->AppendTo(cursor, stored_value, call->env(),
3109 FlowGraph::kValue);
3110 break;
3111 }
3112 default:
3113 break;
3114 }
3115
3116 // Generates a template for the store, either a dynamic conditional
3117 // that dispatches on external and internal storage, or a single
3118 // case that deals with either external or internal storage.
3119 PrepareInlineByteArrayBaseOp(flow_graph, call, receiver, array_cid, &array,
3120 &cursor);
3121
3122 // Fill out the generated template with stores.
3123 {
3124 // Store on either external or internal.
3125 StoreIndexedInstr* store =
3126 NewStore(flow_graph, call, array, index, stored_value, view_cid);
3127 flow_graph->AppendTo(
3128 cursor, store,
3129 call->deopt_id() != DeoptId::kNone ? call->env() : nullptr,
3130 FlowGraph::kEffect);
3131 *last = store;
3132 }
3133 // We need a return value to replace uses of the original definition. However,
3134 // the final instruction is a use of 'void operator[]=()', so we use null.
3135 *result = flow_graph->constant_null();
3136 return true;
3137}
3138
3139// Returns the LoadIndexedInstr.
3140static Definition* PrepareInlineStringIndexOp(FlowGraph* flow_graph,
3141 Instruction* call,
3142 intptr_t cid,
3143 Definition* str,
3144 Definition* index,
3145 Instruction* cursor) {
3146 LoadFieldInstr* length = new (Z)
3147 LoadFieldInstr(new (Z) Value(str), Slot::GetLengthFieldForArrayCid(cid),
3148 str->token_pos());
3149 cursor = flow_graph->AppendTo(cursor, length, NULL, FlowGraph::kValue);
3150
3151 // Bounds check.
3152 index = flow_graph->CreateCheckBound(length, index, call->deopt_id());
3153 cursor = flow_graph->AppendTo(cursor, index, call->env(), FlowGraph::kValue);
3154
3155 // For external strings: Load backing store.
3156 if (cid == kExternalOneByteStringCid) {
3157 str = new LoadUntaggedInstr(
3158 new Value(str),
3159 compiler::target::ExternalOneByteString::external_data_offset());
3160 cursor = flow_graph->AppendTo(cursor, str, NULL, FlowGraph::kValue);
3161 } else if (cid == kExternalTwoByteStringCid) {
3162 str = new LoadUntaggedInstr(
3163 new Value(str),
3164 compiler::target::ExternalTwoByteString::external_data_offset());
3165 cursor = flow_graph->AppendTo(cursor, str, NULL, FlowGraph::kValue);
3166 }
3167
3168 LoadIndexedInstr* load_indexed = new (Z) LoadIndexedInstr(
3169 new (Z) Value(str), new (Z) Value(index), /*index_unboxed=*/false,
3170 compiler::target::Instance::ElementSizeFor(cid), cid, kAlignedAccess,
3171 DeoptId::kNone, call->token_pos());
3172 cursor = flow_graph->AppendTo(cursor, load_indexed, NULL, FlowGraph::kValue);
3173
3174 auto box = BoxInstr::Create(kUnboxedIntPtr, new Value(load_indexed));
3175 cursor = flow_graph->AppendTo(cursor, box, nullptr, FlowGraph::kValue);
3176
3177 ASSERT(box == cursor);
3178 return box;
3179}
3180
3181static bool InlineStringBaseCharAt(FlowGraph* flow_graph,
3182 Instruction* call,
3183 Definition* receiver,
3184 intptr_t cid,
3185 GraphEntryInstr* graph_entry,
3186 FunctionEntryInstr** entry,
3187 Instruction** last,
3188 Definition** result) {
3189 if ((cid != kOneByteStringCid) && (cid != kExternalOneByteStringCid)) {
3190 return false;
3191 }
3192 Definition* str = receiver;
3193 Definition* index = call->ArgumentAt(1);
3194
3195 *entry =
3196 new (Z) FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
3197 call->GetBlock()->try_index(), DeoptId::kNone);
3198 (*entry)->InheritDeoptTarget(Z, call);
3199
3200 *last = PrepareInlineStringIndexOp(flow_graph, call, cid, str, index, *entry);
3201
3202 OneByteStringFromCharCodeInstr* char_at = new (Z)
3203 OneByteStringFromCharCodeInstr(new (Z) Value((*last)->AsDefinition()));
3204
3205 flow_graph->AppendTo(*last, char_at, NULL, FlowGraph::kValue);
3206 *last = char_at;
3207 *result = char_at->AsDefinition();
3208
3209 return true;
3210}
3211
3212static bool InlineStringCodeUnitAt(FlowGraph* flow_graph,
3213 Instruction* call,
3214 Definition* receiver,
3215 intptr_t cid,
3216 GraphEntryInstr* graph_entry,
3217 FunctionEntryInstr** entry,
3218 Instruction** last,
3219 Definition** result) {
3220 if (cid == kDynamicCid) {
3221 ASSERT(call->IsStaticCall());
3222 return false;
3223 } else if ((cid != kOneByteStringCid) && (cid != kTwoByteStringCid) &&
3224 (cid != kExternalOneByteStringCid) &&
3225 (cid != kExternalTwoByteStringCid)) {
3226 return false;
3227 }
3228 Definition* str = receiver;
3229 Definition* index = call->ArgumentAt(1);
3230
3231 *entry =
3232 new (Z) FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
3233 call->GetBlock()->try_index(), DeoptId::kNone);
3234 (*entry)->InheritDeoptTarget(Z, call);
3235
3236 *last = PrepareInlineStringIndexOp(flow_graph, call, cid, str, index, *entry);
3237 *result = (*last)->AsDefinition();
3238
3239 return true;
3240}
3241
3242// Only used for monomorphic calls.
3243bool FlowGraphInliner::TryReplaceInstanceCallWithInline(
3244 FlowGraph* flow_graph,
3245 ForwardInstructionIterator* iterator,
3246 InstanceCallInstr* call,
3247 SpeculativeInliningPolicy* policy) {
3248 const CallTargets& targets = call->Targets();
3249 ASSERT(targets.IsMonomorphic());
3250 const intptr_t receiver_cid = targets.MonomorphicReceiverCid();
3251 const Function& target = targets.FirstTarget();
3252 const auto exactness = targets.MonomorphicExactness();
3253 ExactnessInfo exactness_info{exactness.IsExact(), false};
3254
3255 FunctionEntryInstr* entry = nullptr;
3256 Instruction* last = nullptr;
3257 Definition* result = nullptr;
3258 if (FlowGraphInliner::TryInlineRecognizedMethod(
3259 flow_graph, receiver_cid, target, call,
3260 call->Receiver()->definition(), call->token_pos(), call->ic_data(),
3261 /*graph_entry=*/nullptr, &entry, &last, &result, policy,
3262 &exactness_info)) {
3263 // The empty Object constructor is the only case where the inlined body is
3264 // empty and there is no result.
3265 ASSERT((last != nullptr && result != nullptr) ||
3266 (target.recognized_kind() == MethodRecognizer::kObjectConstructor));
3267 // Determine if inlining instance methods needs a check.
3268 FlowGraph::ToCheck check = FlowGraph::ToCheck::kNoCheck;
3269 if (target.is_polymorphic_target()) {
3270 check = FlowGraph::ToCheck::kCheckCid;
3271 } else {
3272 check = flow_graph->CheckForInstanceCall(call, target.kind());
3273 }
3274
3275 // Insert receiver class or null check if needed.
3276 switch (check) {
3277 case FlowGraph::ToCheck::kCheckCid: {
3278 Instruction* check_class = flow_graph->CreateCheckClass(
3279 call->Receiver()->definition(), targets, call->deopt_id(),
3280 call->token_pos());
3281 flow_graph->InsertBefore(call, check_class, call->env(),
3282 FlowGraph::kEffect);
3283 break;
3284 }
3285 case FlowGraph::ToCheck::kCheckNull: {
3286 Instruction* check_null = new (Z) CheckNullInstr(
3287 call->Receiver()->CopyWithType(Z), call->function_name(),
3288 call->deopt_id(), call->token_pos());
3289 flow_graph->InsertBefore(call, check_null, call->env(),
3290 FlowGraph::kEffect);
3291 break;
3292 }
3293 case FlowGraph::ToCheck::kNoCheck:
3294 break;
3295 }
3296
3297 if (exactness_info.emit_exactness_guard && exactness.IsTriviallyExact()) {
3298 flow_graph->AddExactnessGuard(call, receiver_cid);
3299 }
3300
3301 ASSERT(!call->HasPushArguments());
3302
3303 // Replace all uses of this definition with the result.
3304 if (call->HasUses()) {
3305 ASSERT(result != nullptr && result->HasSSATemp());
3306 call->ReplaceUsesWith(result);
3307 }
3308 // Finally insert the sequence other definition in place of this one in the
3309 // graph.
3310 if (entry->next() != NULL) {
3311 call->previous()->LinkTo(entry->next());
3312 }
3313 entry->UnuseAllInputs(); // Entry block is not in the graph.
3314 if (last != NULL) {
3315 ASSERT(call->GetBlock() == last->GetBlock());
3316 last->LinkTo(call);
3317 }
3318 // Remove through the iterator.
3319 ASSERT(iterator->Current() == call);
3320 iterator->RemoveCurrentFromGraph();
3321 call->set_previous(NULL);
3322 call->set_next(NULL);
3323 return true;
3324 }
3325 return false;
3326}
3327
3328bool FlowGraphInliner::TryReplaceStaticCallWithInline(
3329 FlowGraph* flow_graph,
3330 ForwardInstructionIterator* iterator,
3331 StaticCallInstr* call,
3332 SpeculativeInliningPolicy* policy) {
3333 FunctionEntryInstr* entry = nullptr;
3334 Instruction* last = nullptr;
3335 Definition* result = nullptr;
3336 Definition* receiver = nullptr;
3337 intptr_t receiver_cid = kIllegalCid;
3338 if (!call->function().is_static()) {
3339 receiver = call->Receiver()->definition();
3340 receiver_cid = call->Receiver()->Type()->ToCid();
3341 }
3342 if (FlowGraphInliner::TryInlineRecognizedMethod(
3343 flow_graph, receiver_cid, call->function(), call, receiver,
3344 call->token_pos(), call->ic_data(), /*graph_entry=*/nullptr, &entry,
3345 &last, &result, policy)) {
3346 // The empty Object constructor is the only case where the inlined body is
3347 // empty and there is no result.
3348 ASSERT((last != nullptr && result != nullptr) ||
3349 (call->function().recognized_kind() ==
3350 MethodRecognizer::kObjectConstructor));
3351 ASSERT(!call->HasPushArguments());
3352 // Replace all uses of this definition with the result.
3353 if (call->HasUses()) {
3354 ASSERT(result->HasSSATemp());
3355 call->ReplaceUsesWith(result);
3356 }
3357 // Finally insert the sequence other definition in place of this one in the
3358 // graph.
3359 if (entry != nullptr) {
3360 if (entry->next() != nullptr) {
3361 call->previous()->LinkTo(entry->next());
3362 }
3363 entry->UnuseAllInputs(); // Entry block is not in the graph.
3364 if (last != NULL) {
3365 BlockEntryInstr* link = call->GetBlock();
3366 BlockEntryInstr* exit = last->GetBlock();
3367 if (link != exit) {
3368 // Dominance relation and SSA are updated incrementally when
3369 // conditionals are inserted. But succ/pred and ordering needs
3370 // to be redone. TODO(ajcbik): do this incrementally too.
3371 for (intptr_t i = 0, n = link->dominated_blocks().length(); i < n;
3372 ++i) {
3373 exit->AddDominatedBlock(link->dominated_blocks()[i]);
3374 }
3375 link->ClearDominatedBlocks();
3376 for (intptr_t i = 0, n = entry->dominated_blocks().length(); i < n;
3377 ++i) {
3378 link->AddDominatedBlock(entry->dominated_blocks()[i]);
3379 }
3380 Instruction* scan = exit;
3381 while (scan->next() != nullptr) {
3382 scan = scan->next();
3383 }
3384 scan->LinkTo(call);
3385 flow_graph->DiscoverBlocks();
3386 } else {
3387 last->LinkTo(call);
3388 }
3389 }
3390 }
3391 // Remove through the iterator.
3392 if (iterator != NULL) {
3393 ASSERT(iterator->Current() == call);
3394 iterator->RemoveCurrentFromGraph();
3395 } else {
3396 call->RemoveFromGraph();
3397 }
3398 return true;
3399 }
3400 return false;
3401}
3402
3403static bool CheckMask(Definition* definition, intptr_t* mask_ptr) {
3404 if (!definition->IsConstant()) return false;
3405 ConstantInstr* constant_instruction = definition->AsConstant();
3406 const Object& constant_mask = constant_instruction->value();
3407 if (!constant_mask.IsSmi()) return false;
3408 const intptr_t mask = Smi::Cast(constant_mask).Value();
3409 if ((mask < 0) || (mask > 255)) {
3410 return false; // Not a valid mask.
3411 }
3412 *mask_ptr = mask;
3413 return true;
3414}
3415
3416static bool InlineSimdOp(FlowGraph* flow_graph,
3417 bool is_dynamic_call,
3418 Instruction* call,
3419 Definition* receiver,
3420 MethodRecognizer::Kind kind,
3421 GraphEntryInstr* graph_entry,
3422 FunctionEntryInstr** entry,
3423 Instruction** last,
3424 Definition** result) {
3425 if (!ShouldInlineSimd()) {
3426 return false;
3427 }
3428 if (is_dynamic_call && call->ArgumentCount() > 1) {
3429 // Issue(dartbug.com/37737): Dynamic invocation forwarders have the
3430 // same recognized kind as the method they are forwarding to.
3431 // That causes us to inline the recognized method and not the
3432 // dyn: forwarder itself.
3433 // This is only safe if all arguments are checked in the flow graph we
3434 // build.
3435 // For double/int arguments speculative unboxing instructions should ensure
3436 // to bailout in AOT (or deoptimize in JIT) if the incoming values are not
3437 // correct. Though for user-implementable types, like
3438 // operator+(Float32x4 other), this is not safe and we therefore bailout.
3439 return false;
3440 }
3441
3442 *entry =
3443 new (Z) FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
3444 call->GetBlock()->try_index(), DeoptId::kNone);
3445 Instruction* cursor = *entry;
3446 switch (kind) {
3447 case MethodRecognizer::kInt32x4Shuffle:
3448 case MethodRecognizer::kInt32x4ShuffleMix:
3449 case MethodRecognizer::kFloat32x4Shuffle:
3450 case MethodRecognizer::kFloat32x4ShuffleMix: {
3451 Definition* mask_definition = call->ArgumentAt(call->ArgumentCount() - 1);
3452 intptr_t mask = 0;
3453 if (!CheckMask(mask_definition, &mask)) {
3454 return false;
3455 }
3456 *last = SimdOpInstr::CreateFromCall(Z, kind, receiver, call, mask);
3457 break;
3458 }
3459
3460 case MethodRecognizer::kFloat32x4WithX:
3461 case MethodRecognizer::kFloat32x4WithY:
3462 case MethodRecognizer::kFloat32x4WithZ:
3463 case MethodRecognizer::kFloat32x4WithW:
3464 case MethodRecognizer::kFloat32x4Scale: {
3465 Definition* left = receiver;
3466 Definition* right = call->ArgumentAt(1);
3467 // Note: left and right values are swapped when handed to the instruction,
3468 // this is done so that the double value is loaded into the output
3469 // register and can be destroyed.
3470 // TODO(dartbug.com/31035) this swapping is only needed because register
3471 // allocator has SameAsFirstInput policy and not SameAsNthInput(n).
3472 *last = SimdOpInstr::Create(kind, new (Z) Value(right),
3473 new (Z) Value(left), call->deopt_id());
3474 break;
3475 }
3476
3477 case MethodRecognizer::kFloat32x4Zero:
3478 case MethodRecognizer::kFloat32x4ToFloat64x2:
3479 case MethodRecognizer::kFloat64x2ToFloat32x4:
3480 case MethodRecognizer::kFloat32x4ToInt32x4:
3481 case MethodRecognizer::kInt32x4ToFloat32x4:
3482 case MethodRecognizer::kFloat64x2Zero:
3483 *last = SimdOpInstr::CreateFromFactoryCall(Z, kind, call);
3484 break;
3485 case MethodRecognizer::kFloat32x4Mul:
3486 case MethodRecognizer::kFloat32x4Div:
3487 case MethodRecognizer::kFloat32x4Add:
3488 case MethodRecognizer::kFloat32x4Sub:
3489 case MethodRecognizer::kFloat64x2Mul:
3490 case MethodRecognizer::kFloat64x2Div:
3491 case MethodRecognizer::kFloat64x2Add:
3492 case MethodRecognizer::kFloat64x2Sub:
3493 *last = SimdOpInstr::CreateFromCall(Z, kind, receiver, call);
3494 if (CompilerState::Current().is_aot()) {
3495 // Add null-checks in case of the arguments are known to be compatible
3496 // but they are possibly nullable.
3497 // By inserting the null-check, we can allow the unbox instruction later
3498 // inserted to be non-speculative.
3499 CheckNullInstr* check1 =
3500 new (Z) CheckNullInstr(new (Z) Value(receiver), Symbols::FirstArg(),
3501 call->deopt_id(), call->token_pos());
3502
3503 CheckNullInstr* check2 = new (Z)
3504 CheckNullInstr(new (Z) Value(call->ArgumentAt(1)),
3505 Symbols::SecondArg(), call->deopt_id(),
3506 call->token_pos(), CheckNullInstr::kArgumentError);
3507
3508 (*last)->SetInputAt(0, new (Z) Value(check1));
3509 (*last)->SetInputAt(1, new (Z) Value(check2));
3510
3511 flow_graph->InsertBefore(call, check1, call->env(), FlowGraph::kValue);
3512 flow_graph->InsertBefore(call, check2, call->env(), FlowGraph::kValue);
3513 }
3514 break;
3515 default:
3516 *last = SimdOpInstr::CreateFromCall(Z, kind, receiver, call);
3517 break;
3518 }
3519 // InheritDeoptTarget also inherits environment (which may add 'entry' into
3520 // env_use_list()), so InheritDeoptTarget should be done only after decided
3521 // to inline.
3522 (*entry)->InheritDeoptTarget(Z, call);
3523 flow_graph->AppendTo(cursor, *last,
3524 call->deopt_id() != DeoptId::kNone ? call->env() : NULL,
3525 FlowGraph::kValue);
3526 *result = (*last)->AsDefinition();
3527 return true;
3528}
3529
3530static bool InlineMathCFunction(FlowGraph* flow_graph,
3531 Instruction* call,
3532 MethodRecognizer::Kind kind,
3533 GraphEntryInstr* graph_entry,
3534 FunctionEntryInstr** entry,
3535 Instruction** last,
3536 Definition** result) {
3537 if (!CanUnboxDouble()) {
3538 return false;
3539 }
3540
3541 for (intptr_t i = 0; i < call->ArgumentCount(); i++) {
3542 if (call->ArgumentAt(i)->Type()->ToCid() != kDoubleCid) {
3543 return false;
3544 }
3545 }
3546
3547 *entry =
3548 new (Z) FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
3549 call->GetBlock()->try_index(), DeoptId::kNone);
3550 (*entry)->InheritDeoptTarget(Z, call);
3551 Instruction* cursor = *entry;
3552
3553 switch (kind) {
3554 case MethodRecognizer::kMathSqrt: {
3555 *last = new (Z)
3556 MathUnaryInstr(MathUnaryInstr::kSqrt,
3557 new (Z) Value(call->ArgumentAt(0)), call->deopt_id());
3558 break;
3559 }
3560 default: {
3561 ZoneGrowableArray<Value*>* args =
3562 new (Z) ZoneGrowableArray<Value*>(call->ArgumentCount());
3563 for (intptr_t i = 0; i < call->ArgumentCount(); i++) {
3564 args->Add(new (Z) Value(call->ArgumentAt(i)));
3565 }
3566 *last = new (Z) InvokeMathCFunctionInstr(args, call->deopt_id(), kind,
3567 call->token_pos());
3568 break;
3569 }
3570 }
3571 flow_graph->AppendTo(cursor, *last,
3572 call->deopt_id() != DeoptId::kNone ? call->env() : NULL,
3573 FlowGraph::kValue);
3574 *result = (*last)->AsDefinition();
3575 return true;
3576}
3577
3578static Instruction* InlineMul(FlowGraph* flow_graph,
3579 Instruction* cursor,
3580 Definition* x,
3581 Definition* y) {
3582 BinaryInt64OpInstr* mul = new (Z)
3583 BinaryInt64OpInstr(Token::kMUL, new (Z) Value(x), new (Z) Value(y),
3584 DeoptId::kNone, Instruction::kNotSpeculative);
3585 return flow_graph->AppendTo(cursor, mul, nullptr, FlowGraph::kValue);
3586}
3587
3588static bool InlineMathIntPow(FlowGraph* flow_graph,
3589 Instruction* call,
3590 GraphEntryInstr* graph_entry,
3591 FunctionEntryInstr** entry,
3592 Instruction** last,
3593 Definition** result) {
3594 // Invoking the _intPow(x, y) implies that both:
3595 // (1) x, y are int
3596 // (2) y >= 0.
3597 // Thus, try to inline some very obvious cases.
3598 // TODO(ajcbik): useful to generalize?
3599 intptr_t val = 0;
3600 Value* x = call->ArgumentValueAt(0);
3601 Value* y = call->ArgumentValueAt(1);
3602 // Use x^0 == 1, x^1 == x, and x^c == x * .. * x for small c.
3603 const intptr_t small_exponent = 5;
3604 if (IsSmiValue(y, &val)) {
3605 if (val == 0) {
3606 *last = flow_graph->GetConstant(Smi::ZoneHandle(Smi::New(1)));
3607 *result = (*last)->AsDefinition();
3608 return true;
3609 } else if (val == 1) {
3610 *last = x->definition();
3611 *result = (*last)->AsDefinition();
3612 return true;
3613 } else if (1 < val && val <= small_exponent) {
3614 // Lazily construct entry only in this case.
3615 *entry = new (Z)
3616 FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
3617 call->GetBlock()->try_index(), DeoptId::kNone);
3618 (*entry)->InheritDeoptTarget(Z, call);
3619 Definition* x_def = x->definition();
3620 Definition* square =
3621 InlineMul(flow_graph, *entry, x_def, x_def)->AsDefinition();
3622 *last = square;
3623 *result = square;
3624 switch (val) {
3625 case 2:
3626 return true;
3627 case 3:
3628 *last = InlineMul(flow_graph, *last, x_def, square);
3629 *result = (*last)->AsDefinition();
3630 return true;
3631 case 4:
3632 *last = InlineMul(flow_graph, *last, square, square);
3633 *result = (*last)->AsDefinition();
3634 return true;
3635 case 5:
3636 *last = InlineMul(flow_graph, *last, square, square);
3637 *last = InlineMul(flow_graph, *last, x_def, (*last)->AsDefinition());
3638 *result = (*last)->AsDefinition();
3639 return true;
3640 }
3641 }
3642 }
3643 // Use 0^y == 0 (only for y != 0) and 1^y == 1.
3644 if (IsSmiValue(x, &val)) {
3645 if (val == 1) {
3646 *last = x->definition();
3647 *result = x->definition();
3648 return true;
3649 }
3650 }
3651 return false;
3652}
3653
3654bool FlowGraphInliner::TryInlineRecognizedMethod(
3655 FlowGraph* flow_graph,
3656 intptr_t receiver_cid,
3657 const Function& target,
3658 Definition* call,
3659 Definition* receiver,
3660 TokenPosition token_pos,
3661 const ICData* ic_data,
3662 GraphEntryInstr* graph_entry,
3663 FunctionEntryInstr** entry,
3664 Instruction** last,
3665 Definition** result,
3666 SpeculativeInliningPolicy* policy,
3667 FlowGraphInliner::ExactnessInfo* exactness) {
3668 if (receiver_cid == kNeverCid) {
3669 // Receiver was defined in dead code and was replaced by the sentinel.
3670 // Original receiver cid is lost, so don't try to inline recognized
3671 // methods.
3672 return false;
3673 }
3674
3675 const bool can_speculate = policy->IsAllowedForInlining(call->deopt_id());
3676 const bool is_dynamic_call = Function::IsDynamicInvocationForwarderName(
3677 String::Handle(flow_graph->zone(), target.name()));
3678
3679 const MethodRecognizer::Kind kind = target.recognized_kind();
3680 switch (kind) {
3681 // Recognized [] operators.
3682 case MethodRecognizer::kImmutableArrayGetIndexed:
3683 case MethodRecognizer::kObjectArrayGetIndexed:
3684 case MethodRecognizer::kGrowableArrayGetIndexed:
3685 case MethodRecognizer::kInt8ArrayGetIndexed:
3686 case MethodRecognizer::kUint8ArrayGetIndexed:
3687 case MethodRecognizer::kUint8ClampedArrayGetIndexed:
3688 case MethodRecognizer::kExternalUint8ArrayGetIndexed:
3689 case MethodRecognizer::kExternalUint8ClampedArrayGetIndexed:
3690 case MethodRecognizer::kInt16ArrayGetIndexed:
3691 case MethodRecognizer::kUint16ArrayGetIndexed:
3692 return InlineGetIndexed(flow_graph, can_speculate, is_dynamic_call, kind,
3693 call, receiver, graph_entry, entry, last, result);
3694 case MethodRecognizer::kFloat32ArrayGetIndexed:
3695 case MethodRecognizer::kFloat64ArrayGetIndexed:
3696 if (!CanUnboxDouble()) {
3697 return false;
3698 }
3699 return InlineGetIndexed(flow_graph, can_speculate, is_dynamic_call, kind,
3700 call, receiver, graph_entry, entry, last, result);
3701 case MethodRecognizer::kFloat32x4ArrayGetIndexed:
3702 case MethodRecognizer::kFloat64x2ArrayGetIndexed:
3703 if (!ShouldInlineSimd()) {
3704 return false;
3705 }
3706 return InlineGetIndexed(flow_graph, can_speculate, is_dynamic_call, kind,
3707 call, receiver, graph_entry, entry, last, result);
3708 case MethodRecognizer::kInt32ArrayGetIndexed:
3709 case MethodRecognizer::kUint32ArrayGetIndexed:
3710 if (!CanUnboxInt32()) {
3711 return false;
3712 }
3713 return InlineGetIndexed(flow_graph, can_speculate, is_dynamic_call, kind,
3714 call, receiver, graph_entry, entry, last, result);
3715 case MethodRecognizer::kInt64ArrayGetIndexed:
3716 case MethodRecognizer::kUint64ArrayGetIndexed:
3717 if (!ShouldInlineInt64ArrayOps()) {
3718 return false;
3719 }
3720 return InlineGetIndexed(flow_graph, can_speculate, is_dynamic_call, kind,
3721 call, receiver, graph_entry, entry, last, result);
3722 case MethodRecognizer::kClassIDgetID:
3723 return InlineLoadClassId(flow_graph, call, graph_entry, entry, last,
3724 result);
3725 default:
3726 break;
3727 }
3728
3729 // The following ones need to speculate.
3730 if (!can_speculate) {
3731 return false;
3732 }
3733
3734 switch (kind) {
3735 // Recognized []= operators.
3736 case MethodRecognizer::kObjectArraySetIndexed:
3737 case MethodRecognizer::kGrowableArraySetIndexed:
3738 case MethodRecognizer::kObjectArraySetIndexedUnchecked:
3739 case MethodRecognizer::kGrowableArraySetIndexedUnchecked:
3740 return InlineSetIndexed(flow_graph, kind, target, call, receiver,
3741 token_pos, /* value_check = */ NULL, exactness,
3742 graph_entry, entry, last, result);
3743 case MethodRecognizer::kInt8ArraySetIndexed:
3744 case MethodRecognizer::kUint8ArraySetIndexed:
3745 case MethodRecognizer::kUint8ClampedArraySetIndexed:
3746 case MethodRecognizer::kExternalUint8ArraySetIndexed:
3747 case MethodRecognizer::kExternalUint8ClampedArraySetIndexed:
3748 case MethodRecognizer::kInt16ArraySetIndexed:
3749 case MethodRecognizer::kUint16ArraySetIndexed: {
3750 // Optimistically assume Smi.
3751 if (ic_data != NULL && ic_data->HasDeoptReason(ICData::kDeoptCheckSmi)) {
3752 // Optimistic assumption failed at least once.
3753 return false;
3754 }
3755 Cids* value_check = Cids::CreateMonomorphic(Z, kSmiCid);
3756 return InlineSetIndexed(flow_graph, kind, target, call, receiver,
3757 token_pos, value_check, exactness, graph_entry,
3758 entry, last, result);
3759 }
3760 case MethodRecognizer::kInt32ArraySetIndexed:
3761 case MethodRecognizer::kUint32ArraySetIndexed: {
3762 // Value check not needed for Int32 and Uint32 arrays because they
3763 // implicitly contain unboxing instructions which check for right type.
3764 return InlineSetIndexed(flow_graph, kind, target, call, receiver,
3765 token_pos, /* value_check = */ NULL, exactness,
3766 graph_entry, entry, last, result);
3767 }
3768 case MethodRecognizer::kInt64ArraySetIndexed:
3769 case MethodRecognizer::kUint64ArraySetIndexed:
3770 if (!ShouldInlineInt64ArrayOps()) {
3771 return false;
3772 }
3773 return InlineSetIndexed(flow_graph, kind, target, call, receiver,
3774 token_pos, /* value_check = */ NULL, exactness,
3775 graph_entry, entry, last, result);
3776 case MethodRecognizer::kFloat32ArraySetIndexed:
3777 case MethodRecognizer::kFloat64ArraySetIndexed: {
3778 if (!CanUnboxDouble()) {
3779 return false;
3780 }
3781 Cids* value_check = Cids::CreateMonomorphic(Z, kDoubleCid);
3782 return InlineSetIndexed(flow_graph, kind, target, call, receiver,
3783 token_pos, value_check, exactness, graph_entry,
3784 entry, last, result);
3785 }
3786 case MethodRecognizer::kFloat32x4ArraySetIndexed: {
3787 if (!ShouldInlineSimd()) {
3788 return false;
3789 }
3790 Cids* value_check = Cids::CreateMonomorphic(Z, kFloat32x4Cid);
3791 return InlineSetIndexed(flow_graph, kind, target, call, receiver,
3792 token_pos, value_check, exactness, graph_entry,
3793 entry, last, result);
3794 }
3795 case MethodRecognizer::kFloat64x2ArraySetIndexed: {
3796 if (!ShouldInlineSimd()) {
3797 return false;
3798 }
3799 Cids* value_check = Cids::CreateMonomorphic(Z, kFloat64x2Cid);
3800 return InlineSetIndexed(flow_graph, kind, target, call, receiver,
3801 token_pos, value_check, exactness, graph_entry,
3802 entry, last, result);
3803 }
3804 case MethodRecognizer::kByteArrayBaseGetInt8:
3805 return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
3806 kTypedDataInt8ArrayCid, graph_entry, entry,
3807 last, result);
3808 case MethodRecognizer::kByteArrayBaseGetUint8:
3809 return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
3810 kTypedDataUint8ArrayCid, graph_entry,
3811 entry, last, result);
3812 case MethodRecognizer::kByteArrayBaseGetInt16:
3813 return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
3814 kTypedDataInt16ArrayCid, graph_entry,
3815 entry, last, result);
3816 case MethodRecognizer::kByteArrayBaseGetUint16:
3817 return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
3818 kTypedDataUint16ArrayCid, graph_entry,
3819 entry, last, result);
3820 case MethodRecognizer::kByteArrayBaseGetInt32:
3821 if (!CanUnboxInt32()) {
3822 return false;
3823 }
3824 return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
3825 kTypedDataInt32ArrayCid, graph_entry,
3826 entry, last, result);
3827 case MethodRecognizer::kByteArrayBaseGetUint32:
3828 if (!CanUnboxInt32()) {
3829 return false;
3830 }
3831 return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
3832 kTypedDataUint32ArrayCid, graph_entry,
3833 entry, last, result);
3834 case MethodRecognizer::kByteArrayBaseGetInt64:
3835 if (!ShouldInlineInt64ArrayOps()) {
3836 return false;
3837 }
3838 return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
3839 kTypedDataInt64ArrayCid, graph_entry,
3840 entry, last, result);
3841 case MethodRecognizer::kByteArrayBaseGetUint64:
3842 if (!ShouldInlineInt64ArrayOps()) {
3843 return false;
3844 }
3845 return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
3846 kTypedDataUint64ArrayCid, graph_entry,
3847 entry, last, result);
3848 case MethodRecognizer::kByteArrayBaseGetFloat32:
3849 if (!CanUnboxDouble()) {
3850 return false;
3851 }
3852 return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
3853 kTypedDataFloat32ArrayCid, graph_entry,
3854 entry, last, result);
3855 case MethodRecognizer::kByteArrayBaseGetFloat64:
3856 if (!CanUnboxDouble()) {
3857 return false;
3858 }
3859 return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
3860 kTypedDataFloat64ArrayCid, graph_entry,
3861 entry, last, result);
3862 case MethodRecognizer::kByteArrayBaseGetFloat32x4:
3863 if (!ShouldInlineSimd()) {
3864 return false;
3865 }
3866 return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
3867 kTypedDataFloat32x4ArrayCid, graph_entry,
3868 entry, last, result);
3869 case MethodRecognizer::kByteArrayBaseGetInt32x4:
3870 if (!ShouldInlineSimd()) {
3871 return false;
3872 }
3873 return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
3874 kTypedDataInt32x4ArrayCid, graph_entry,
3875 entry, last, result);
3876 case MethodRecognizer::kByteArrayBaseSetInt8:
3877 return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
3878 receiver_cid, kTypedDataInt8ArrayCid,
3879 graph_entry, entry, last, result);
3880 case MethodRecognizer::kByteArrayBaseSetUint8:
3881 return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
3882 receiver_cid, kTypedDataUint8ArrayCid,
3883 graph_entry, entry, last, result);
3884 case MethodRecognizer::kByteArrayBaseSetInt16:
3885 return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
3886 receiver_cid, kTypedDataInt16ArrayCid,
3887 graph_entry, entry, last, result);
3888 case MethodRecognizer::kByteArrayBaseSetUint16:
3889 return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
3890 receiver_cid, kTypedDataUint16ArrayCid,
3891 graph_entry, entry, last, result);
3892 case MethodRecognizer::kByteArrayBaseSetInt32:
3893 return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
3894 receiver_cid, kTypedDataInt32ArrayCid,
3895 graph_entry, entry, last, result);
3896 case MethodRecognizer::kByteArrayBaseSetUint32:
3897 return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
3898 receiver_cid, kTypedDataUint32ArrayCid,
3899 graph_entry, entry, last, result);
3900 case MethodRecognizer::kByteArrayBaseSetInt64:
3901 if (!ShouldInlineInt64ArrayOps()) {
3902 return false;
3903 }
3904 return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
3905 receiver_cid, kTypedDataInt64ArrayCid,
3906 graph_entry, entry, last, result);
3907 case MethodRecognizer::kByteArrayBaseSetUint64:
3908 if (!ShouldInlineInt64ArrayOps()) {
3909 return false;
3910 }
3911 return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
3912 receiver_cid, kTypedDataUint64ArrayCid,
3913 graph_entry, entry, last, result);
3914 case MethodRecognizer::kByteArrayBaseSetFloat32:
3915 if (!CanUnboxDouble()) {
3916 return false;
3917 }
3918 return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
3919 receiver_cid, kTypedDataFloat32ArrayCid,
3920 graph_entry, entry, last, result);
3921 case MethodRecognizer::kByteArrayBaseSetFloat64:
3922 if (!CanUnboxDouble()) {
3923 return false;
3924 }
3925 return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
3926 receiver_cid, kTypedDataFloat64ArrayCid,
3927 graph_entry, entry, last, result);
3928 case MethodRecognizer::kByteArrayBaseSetFloat32x4:
3929 if (!ShouldInlineSimd()) {
3930 return false;
3931 }
3932 return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
3933 receiver_cid, kTypedDataFloat32x4ArrayCid,
3934 graph_entry, entry, last, result);
3935 case MethodRecognizer::kByteArrayBaseSetInt32x4:
3936 if (!ShouldInlineSimd()) {
3937 return false;
3938 }
3939 return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
3940 receiver_cid, kTypedDataInt32x4ArrayCid,
3941 graph_entry, entry, last, result);
3942 case MethodRecognizer::kOneByteStringCodeUnitAt:
3943 case MethodRecognizer::kTwoByteStringCodeUnitAt:
3944 case MethodRecognizer::kExternalOneByteStringCodeUnitAt:
3945 case MethodRecognizer::kExternalTwoByteStringCodeUnitAt:
3946 return InlineStringCodeUnitAt(flow_graph, call, receiver, receiver_cid,
3947 graph_entry, entry, last, result);
3948 case MethodRecognizer::kStringBaseCharAt:
3949 return InlineStringBaseCharAt(flow_graph, call, receiver, receiver_cid,
3950 graph_entry, entry, last, result);
3951 case MethodRecognizer::kDoubleAdd:
3952 return InlineDoubleOp(flow_graph, Token::kADD, call, receiver,
3953 graph_entry, entry, last, result);
3954 case MethodRecognizer::kDoubleSub:
3955 return InlineDoubleOp(flow_graph, Token::kSUB, call, receiver,
3956 graph_entry, entry, last, result);
3957 case MethodRecognizer::kDoubleMul:
3958 return InlineDoubleOp(flow_graph, Token::kMUL, call, receiver,
3959 graph_entry, entry, last, result);
3960 case MethodRecognizer::kDoubleDiv:
3961 return InlineDoubleOp(flow_graph, Token::kDIV, call, receiver,
3962 graph_entry, entry, last, result);
3963 case MethodRecognizer::kDouble_getIsNaN:
3964 case MethodRecognizer::kDouble_getIsInfinite:
3965 return InlineDoubleTestOp(flow_graph, call, receiver, kind, graph_entry,
3966 entry, last, result);
3967 case MethodRecognizer::kGrowableArraySetData:
3968 ASSERT((receiver_cid == kGrowableObjectArrayCid) ||
3969 ((receiver_cid == kDynamicCid) && call->IsStaticCall()));
3970 return InlineGrowableArraySetter(
3971 flow_graph, Slot::GrowableObjectArray_data(), kEmitStoreBarrier, call,
3972 receiver, graph_entry, entry, last, result);
3973 case MethodRecognizer::kGrowableArraySetLength:
3974 ASSERT((receiver_cid == kGrowableObjectArrayCid) ||
3975 ((receiver_cid == kDynamicCid) && call->IsStaticCall()));
3976 return InlineGrowableArraySetter(
3977 flow_graph, Slot::GrowableObjectArray_length(), kNoStoreBarrier, call,
3978 receiver, graph_entry, entry, last, result);
3979 case MethodRecognizer::kSmi_bitAndFromSmi:
3980 return InlineSmiBitAndFromSmi(flow_graph, call, receiver, graph_entry,
3981 entry, last, result);
3982
3983 case MethodRecognizer::kFloat32x4Abs:
3984 case MethodRecognizer::kFloat32x4Clamp:
3985 case MethodRecognizer::kFloat32x4FromDoubles:
3986 case MethodRecognizer::kFloat32x4Equal:
3987 case MethodRecognizer::kFloat32x4GetSignMask:
3988 case MethodRecognizer::kFloat32x4GreaterThan:
3989 case MethodRecognizer::kFloat32x4GreaterThanOrEqual:
3990 case MethodRecognizer::kFloat32x4LessThan:
3991 case MethodRecognizer::kFloat32x4LessThanOrEqual:
3992 case MethodRecognizer::kFloat32x4Max:
3993 case MethodRecognizer::kFloat32x4Min:
3994 case MethodRecognizer::kFloat32x4Negate:
3995 case MethodRecognizer::kFloat32x4NotEqual:
3996 case MethodRecognizer::kFloat32x4Reciprocal:
3997 case MethodRecognizer::kFloat32x4ReciprocalSqrt:
3998 case MethodRecognizer::kFloat32x4Scale:
3999 case MethodRecognizer::kFloat32x4ShuffleW:
4000 case MethodRecognizer::kFloat32x4ShuffleX:
4001 case MethodRecognizer::kFloat32x4ShuffleY:
4002 case MethodRecognizer::kFloat32x4ShuffleZ:
4003 case MethodRecognizer::kFloat32x4Splat:
4004 case MethodRecognizer::kFloat32x4Sqrt:
4005 case MethodRecognizer::kFloat32x4ToFloat64x2:
4006 case MethodRecognizer::kFloat32x4ToInt32x4:
4007 case MethodRecognizer::kFloat32x4WithW:
4008 case MethodRecognizer::kFloat32x4WithX:
4009 case MethodRecognizer::kFloat32x4WithY:
4010 case MethodRecognizer::kFloat32x4WithZ:
4011 case MethodRecognizer::kFloat32x4Zero:
4012 case MethodRecognizer::kFloat64x2Abs:
4013 case MethodRecognizer::kFloat64x2FromDoubles:
4014 case MethodRecognizer::kFloat64x2GetSignMask:
4015 case MethodRecognizer::kFloat64x2GetX:
4016 case MethodRecognizer::kFloat64x2GetY:
4017 case MethodRecognizer::kFloat64x2Max:
4018 case MethodRecognizer::kFloat64x2Min:
4019 case MethodRecognizer::kFloat64x2Negate:
4020 case MethodRecognizer::kFloat64x2Scale:
4021 case MethodRecognizer::kFloat64x2Splat:
4022 case MethodRecognizer::kFloat64x2Sqrt:
4023 case MethodRecognizer::kFloat64x2ToFloat32x4:
4024 case MethodRecognizer::kFloat64x2WithX:
4025 case MethodRecognizer::kFloat64x2WithY:
4026 case MethodRecognizer::kFloat64x2Zero:
4027 case MethodRecognizer::kInt32x4FromBools:
4028 case MethodRecognizer::kInt32x4FromInts:
4029 case MethodRecognizer::kInt32x4GetFlagW:
4030 case MethodRecognizer::kInt32x4GetFlagX:
4031 case MethodRecognizer::kInt32x4GetFlagY:
4032 case MethodRecognizer::kInt32x4GetFlagZ:
4033 case MethodRecognizer::kInt32x4GetSignMask:
4034 case MethodRecognizer::kInt32x4Select:
4035 case MethodRecognizer::kInt32x4ToFloat32x4:
4036 case MethodRecognizer::kInt32x4WithFlagW:
4037 case MethodRecognizer::kInt32x4WithFlagX:
4038 case MethodRecognizer::kInt32x4WithFlagY:
4039 case MethodRecognizer::kInt32x4WithFlagZ:
4040 case MethodRecognizer::kFloat32x4ShuffleMix:
4041 case MethodRecognizer::kInt32x4ShuffleMix:
4042 case MethodRecognizer::kFloat32x4Shuffle:
4043 case MethodRecognizer::kInt32x4Shuffle:
4044 case MethodRecognizer::kFloat32x4Mul:
4045 case MethodRecognizer::kFloat32x4Div:
4046 case MethodRecognizer::kFloat32x4Add:
4047 case MethodRecognizer::kFloat32x4Sub:
4048 case MethodRecognizer::kFloat64x2Mul:
4049 case MethodRecognizer::kFloat64x2Div:
4050 case MethodRecognizer::kFloat64x2Add:
4051 case MethodRecognizer::kFloat64x2Sub:
4052 return InlineSimdOp(flow_graph, is_dynamic_call, call, receiver, kind,
4053 graph_entry, entry, last, result);
4054
4055 case MethodRecognizer::kMathSqrt:
4056 case MethodRecognizer::kMathDoublePow:
4057 case MethodRecognizer::kMathSin:
4058 case MethodRecognizer::kMathCos:
4059 case MethodRecognizer::kMathTan:
4060 case MethodRecognizer::kMathAsin:
4061 case MethodRecognizer::kMathAcos:
4062 case MethodRecognizer::kMathAtan:
4063 case MethodRecognizer::kMathAtan2:
4064 return InlineMathCFunction(flow_graph, call, kind, graph_entry, entry,
4065 last, result);
4066
4067 case MethodRecognizer::kMathIntPow:
4068 return InlineMathIntPow(flow_graph, call, graph_entry, entry, last,
4069 result);
4070
4071 case MethodRecognizer::kObjectConstructor: {
4072 *entry = new (Z)
4073 FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
4074 call->GetBlock()->try_index(), DeoptId::kNone);
4075 (*entry)->InheritDeoptTarget(Z, call);
4076 ASSERT(!call->HasUses());
4077 *last = NULL; // Empty body.
4078 *result = NULL; // Since no uses of original call, result will be unused.
4079 return true;
4080 }
4081
4082 case MethodRecognizer::kListFactory: {
4083 // We only want to inline new List(n) which decreases code size and
4084 // improves performance. We don't want to inline new List().
4085 if (call->ArgumentCount() != 2) {
4086 return false;
4087 }
4088
4089 const auto type = new (Z) Value(call->ArgumentAt(0));
4090 const auto num_elements = new (Z) Value(call->ArgumentAt(1));
4091 *entry = new (Z)
4092 FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
4093 call->GetBlock()->try_index(), DeoptId::kNone);
4094 (*entry)->InheritDeoptTarget(Z, call);
4095 *last = new (Z) CreateArrayInstr(call->token_pos(), type, num_elements,
4096 call->deopt_id());
4097 flow_graph->AppendTo(
4098 *entry, *last,
4099 call->deopt_id() != DeoptId::kNone ? call->env() : NULL,
4100 FlowGraph::kValue);
4101 *result = (*last)->AsDefinition();
4102 return true;
4103 }
4104
4105 case MethodRecognizer::kObjectArrayAllocate: {
4106 Value* num_elements = new (Z) Value(call->ArgumentAt(1));
4107 intptr_t length = 0;
4108 if (IsSmiValue(num_elements, &length)) {
4109 if (Array::IsValidLength(length)) {
4110 Value* type = new (Z) Value(call->ArgumentAt(0));
4111 *entry = new (Z)
4112 FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
4113 call->GetBlock()->try_index(), DeoptId::kNone);
4114 (*entry)->InheritDeoptTarget(Z, call);
4115 *last = new (Z) CreateArrayInstr(call->token_pos(), type,
4116 num_elements, call->deopt_id());
4117 flow_graph->AppendTo(
4118 *entry, *last,
4119 call->deopt_id() != DeoptId::kNone ? call->env() : NULL,
4120 FlowGraph::kValue);
4121 *result = (*last)->AsDefinition();
4122 return true;
4123 }
4124 }
4125 return false;
4126 }
4127
4128 case MethodRecognizer::kObjectRuntimeType: {
4129 Type& type = Type::ZoneHandle(Z);
4130 if (receiver_cid == kDynamicCid) {
4131 return false;
4132 } else if (IsStringClassId(receiver_cid)) {
4133 type = Type::StringType();
4134 } else if (receiver_cid == kDoubleCid) {
4135 type = Type::Double();
4136 } else if (IsIntegerClassId(receiver_cid)) {
4137 type = Type::IntType();
4138 } else if (receiver_cid != kClosureCid) {
4139 const Class& cls = Class::Handle(
4140 Z, flow_graph->isolate()->class_table()->At(receiver_cid));
4141 if (!cls.IsGeneric()) {
4142 type = cls.DeclarationType();
4143 }
4144 }
4145
4146 if (!type.IsNull()) {
4147 *entry = new (Z)
4148 FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
4149 call->GetBlock()->try_index(), DeoptId::kNone);
4150 (*entry)->InheritDeoptTarget(Z, call);
4151 ConstantInstr* ctype = flow_graph->GetConstant(type);
4152 // Create a synthetic (re)definition for return to flag insertion.
4153 // TODO(ajcbik): avoid this mechanism altogether
4154 RedefinitionInstr* redef =
4155 new (Z) RedefinitionInstr(new (Z) Value(ctype));
4156 flow_graph->AppendTo(
4157 *entry, redef,
4158 call->deopt_id() != DeoptId::kNone ? call->env() : NULL,
4159 FlowGraph::kValue);
4160 *last = *result = redef;
4161 return true;
4162 }
4163 return false;
4164 }
4165
4166 case MethodRecognizer::kWriteIntoOneByteString:
4167 case MethodRecognizer::kWriteIntoTwoByteString: {
4168 // This is an internal method, no need to check argument types nor
4169 // range.
4170 *entry = new (Z)
4171 FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
4172 call->GetBlock()->try_index(), DeoptId::kNone);
4173 (*entry)->InheritDeoptTarget(Z, call);
4174 Definition* str = call->ArgumentAt(0);
4175 Definition* index = call->ArgumentAt(1);
4176 Definition* value = call->ArgumentAt(2);
4177
4178 auto env = call->deopt_id() != DeoptId::kNone ? call->env() : nullptr;
4179
4180 // Insert explicit unboxing instructions with truncation to avoid relying
4181 // on [SelectRepresentations] which doesn't mark them as truncating.
4182 value =
4183 UnboxInstr::Create(kUnboxedIntPtr, new (Z) Value(value),
4184 call->deopt_id(), Instruction::kNotSpeculative);
4185 value->AsUnboxInteger()->mark_truncating();
4186 flow_graph->AppendTo(*entry, value, env, FlowGraph::kValue);
4187
4188 const bool is_onebyte = kind == MethodRecognizer::kWriteIntoOneByteString;
4189 const intptr_t index_scale = is_onebyte ? 1 : 2;
4190 const intptr_t cid = is_onebyte ? kOneByteStringCid : kTwoByteStringCid;
4191 *last = new (Z) StoreIndexedInstr(
4192 new (Z) Value(str), new (Z) Value(index), new (Z) Value(value),
4193 kNoStoreBarrier, /*index_unboxed=*/false, index_scale, cid,
4194 kAlignedAccess, call->deopt_id(), call->token_pos());
4195 flow_graph->AppendTo(value, *last, env, FlowGraph::kEffect);
4196
4197 // We need a return value to replace uses of the original definition.
4198 // The final instruction is a use of 'void operator[]=()', so we use null.
4199 *result = flow_graph->constant_null();
4200 return true;
4201 }
4202
4203 default:
4204 return false;
4205 }
4206}
4207
4208} // namespace dart
4209