1// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file
2// for details. All rights reserved. Use of this source code is governed by a
3// BSD-style license that can be found in the LICENSE file.
4
5#include "vm/globals.h" // Needed here to get TARGET_ARCH_ARM64.
6#if defined(TARGET_ARCH_ARM64)
7
8#include "vm/compiler/backend/flow_graph_compiler.h"
9
10#include "vm/compiler/api/type_check_mode.h"
11#include "vm/compiler/backend/il_printer.h"
12#include "vm/compiler/backend/locations.h"
13#include "vm/compiler/jit/compiler.h"
14#include "vm/cpu.h"
15#include "vm/dart_entry.h"
16#include "vm/deopt_instructions.h"
17#include "vm/dispatch_table.h"
18#include "vm/instructions.h"
19#include "vm/object_store.h"
20#include "vm/parser.h"
21#include "vm/stack_frame.h"
22#include "vm/stub_code.h"
23#include "vm/symbols.h"
24
25namespace dart {
26
27DEFINE_FLAG(bool, trap_on_deoptimization, false, "Trap on deoptimization.");
28DECLARE_FLAG(bool, enable_simd_inline);
29DEFINE_FLAG(bool, unbox_mints, true, "Optimize 64-bit integer arithmetic.");
30
31void FlowGraphCompiler::ArchSpecificInitialization() {
32 if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
33 auto object_store = isolate()->object_store();
34
35 const auto& stub =
36 Code::ZoneHandle(object_store->write_barrier_wrappers_stub());
37 if (CanPcRelativeCall(stub)) {
38 assembler_->generate_invoke_write_barrier_wrapper_ = [&](Register reg) {
39 const intptr_t offset_into_target =
40 Thread::WriteBarrierWrappersOffsetForRegister(reg);
41 assembler_->GenerateUnRelocatedPcRelativeCall(offset_into_target);
42 AddPcRelativeCallStubTarget(stub);
43 };
44 }
45
46 const auto& array_stub =
47 Code::ZoneHandle(object_store->array_write_barrier_stub());
48 if (CanPcRelativeCall(stub)) {
49 assembler_->generate_invoke_array_write_barrier_ = [&]() {
50 assembler_->GenerateUnRelocatedPcRelativeCall();
51 AddPcRelativeCallStubTarget(array_stub);
52 };
53 }
54 }
55}
56
57FlowGraphCompiler::~FlowGraphCompiler() {
58 // BlockInfos are zone-allocated, so their destructors are not called.
59 // Verify the labels explicitly here.
60 for (int i = 0; i < block_info_.length(); ++i) {
61 ASSERT(!block_info_[i]->jump_label()->IsLinked());
62 }
63}
64
65bool FlowGraphCompiler::SupportsUnboxedDoubles() {
66 return true;
67}
68
69bool FlowGraphCompiler::SupportsUnboxedInt64() {
70 return FLAG_unbox_mints;
71}
72
73bool FlowGraphCompiler::SupportsUnboxedSimd128() {
74 return FLAG_enable_simd_inline;
75}
76
77bool FlowGraphCompiler::CanConvertInt64ToDouble() {
78 return true;
79}
80
81bool FlowGraphCompiler::SupportsHardwareDivision() {
82 return true;
83}
84
85void FlowGraphCompiler::EnterIntrinsicMode() {
86 ASSERT(!intrinsic_mode());
87 intrinsic_mode_ = true;
88 ASSERT(!assembler()->constant_pool_allowed());
89}
90
91void FlowGraphCompiler::ExitIntrinsicMode() {
92 ASSERT(intrinsic_mode());
93 intrinsic_mode_ = false;
94}
95
96TypedDataPtr CompilerDeoptInfo::CreateDeoptInfo(FlowGraphCompiler* compiler,
97 DeoptInfoBuilder* builder,
98 const Array& deopt_table) {
99 if (deopt_env_ == NULL) {
100 ++builder->current_info_number_;
101 return TypedData::null();
102 }
103
104 intptr_t stack_height = compiler->StackSize();
105 AllocateIncomingParametersRecursive(deopt_env_, &stack_height);
106
107 intptr_t slot_ix = 0;
108 Environment* current = deopt_env_;
109
110 // Emit all kMaterializeObject instructions describing objects to be
111 // materialized on the deoptimization as a prefix to the deoptimization info.
112 EmitMaterializations(deopt_env_, builder);
113
114 // The real frame starts here.
115 builder->MarkFrameStart();
116
117 Zone* zone = compiler->zone();
118
119 builder->AddPp(current->function(), slot_ix++);
120 builder->AddPcMarker(Function::ZoneHandle(zone), slot_ix++);
121 builder->AddCallerFp(slot_ix++);
122 builder->AddReturnAddress(current->function(), deopt_id(), slot_ix++);
123
124 // Emit all values that are needed for materialization as a part of the
125 // expression stack for the bottom-most frame. This guarantees that GC
126 // will be able to find them during materialization.
127 slot_ix = builder->EmitMaterializationArguments(slot_ix);
128
129 // For the innermost environment, set outgoing arguments and the locals.
130 for (intptr_t i = current->Length() - 1;
131 i >= current->fixed_parameter_count(); i--) {
132 builder->AddCopy(current->ValueAt(i), current->LocationAt(i), slot_ix++);
133 }
134
135 Environment* previous = current;
136 current = current->outer();
137 while (current != NULL) {
138 builder->AddPp(current->function(), slot_ix++);
139 builder->AddPcMarker(previous->function(), slot_ix++);
140 builder->AddCallerFp(slot_ix++);
141
142 // For any outer environment the deopt id is that of the call instruction
143 // which is recorded in the outer environment.
144 builder->AddReturnAddress(current->function(),
145 DeoptId::ToDeoptAfter(current->deopt_id()),
146 slot_ix++);
147
148 // The values of outgoing arguments can be changed from the inlined call so
149 // we must read them from the previous environment.
150 for (intptr_t i = previous->fixed_parameter_count() - 1; i >= 0; i--) {
151 builder->AddCopy(previous->ValueAt(i), previous->LocationAt(i),
152 slot_ix++);
153 }
154
155 // Set the locals, note that outgoing arguments are not in the environment.
156 for (intptr_t i = current->Length() - 1;
157 i >= current->fixed_parameter_count(); i--) {
158 builder->AddCopy(current->ValueAt(i), current->LocationAt(i), slot_ix++);
159 }
160
161 // Iterate on the outer environment.
162 previous = current;
163 current = current->outer();
164 }
165 // The previous pointer is now the outermost environment.
166 ASSERT(previous != NULL);
167
168 // Add slots for the outermost environment.
169 builder->AddCallerPp(slot_ix++);
170 builder->AddPcMarker(previous->function(), slot_ix++);
171 builder->AddCallerFp(slot_ix++);
172 builder->AddCallerPc(slot_ix++);
173
174 // For the outermost environment, set the incoming arguments.
175 for (intptr_t i = previous->fixed_parameter_count() - 1; i >= 0; i--) {
176 builder->AddCopy(previous->ValueAt(i), previous->LocationAt(i), slot_ix++);
177 }
178
179 return builder->CreateDeoptInfo(deopt_table);
180}
181
182void CompilerDeoptInfoWithStub::GenerateCode(FlowGraphCompiler* compiler,
183 intptr_t stub_ix) {
184 // Calls do not need stubs, they share a deoptimization trampoline.
185 ASSERT(reason() != ICData::kDeoptAtCall);
186 compiler::Assembler* assembler = compiler->assembler();
187#define __ assembler->
188 __ Comment("%s", Name());
189 __ Bind(entry_label());
190 if (FLAG_trap_on_deoptimization) {
191 __ brk(0);
192 }
193
194 ASSERT(deopt_env() != NULL);
195 __ ldr(LR, compiler::Address(THR, Thread::deoptimize_entry_offset()));
196 __ blr(LR);
197 set_pc_offset(assembler->CodeSize());
198#undef __
199}
200
201#define __ assembler()->
202
203// Fall through if bool_register contains null.
204void FlowGraphCompiler::GenerateBoolToJump(Register bool_register,
205 compiler::Label* is_true,
206 compiler::Label* is_false) {
207 compiler::Label fall_through;
208 __ CompareObject(bool_register, Object::null_object());
209 __ b(&fall_through, EQ);
210 BranchLabels labels = {is_true, is_false, &fall_through};
211 Condition true_condition =
212 EmitBoolTest(bool_register, labels, /*invert=*/false);
213 ASSERT(true_condition == kInvalidCondition);
214 __ Bind(&fall_through);
215}
216
217// R0: instance (must be preserved).
218// R2: instantiator type arguments (if used).
219// R1: function type arguments (if used).
220SubtypeTestCachePtr FlowGraphCompiler::GenerateCallSubtypeTestStub(
221 TypeTestStubKind test_kind,
222 Register instance_reg,
223 Register instantiator_type_arguments_reg,
224 Register function_type_arguments_reg,
225 Register temp_reg,
226 compiler::Label* is_instance_lbl,
227 compiler::Label* is_not_instance_lbl) {
228 ASSERT(instance_reg == R0);
229 ASSERT(temp_reg == kNoRegister); // Unused on ARM64.
230 const SubtypeTestCache& type_test_cache =
231 SubtypeTestCache::ZoneHandle(zone(), SubtypeTestCache::New());
232 __ LoadUniqueObject(R3, type_test_cache);
233 if (test_kind == kTestTypeOneArg) {
234 ASSERT(instantiator_type_arguments_reg == kNoRegister);
235 ASSERT(function_type_arguments_reg == kNoRegister);
236 __ BranchLink(StubCode::Subtype1TestCache());
237 } else if (test_kind == kTestTypeTwoArgs) {
238 ASSERT(instantiator_type_arguments_reg == kNoRegister);
239 ASSERT(function_type_arguments_reg == kNoRegister);
240 __ BranchLink(StubCode::Subtype2TestCache());
241 } else if (test_kind == kTestTypeFourArgs) {
242 ASSERT(instantiator_type_arguments_reg ==
243 TypeTestABI::kInstantiatorTypeArgumentsReg);
244 ASSERT(function_type_arguments_reg ==
245 TypeTestABI::kFunctionTypeArgumentsReg);
246 __ BranchLink(StubCode::Subtype4TestCache());
247 } else if (test_kind == kTestTypeSixArgs) {
248 ASSERT(instantiator_type_arguments_reg ==
249 TypeTestABI::kInstantiatorTypeArgumentsReg);
250 ASSERT(function_type_arguments_reg ==
251 TypeTestABI::kFunctionTypeArgumentsReg);
252 __ BranchLink(StubCode::Subtype6TestCache());
253 } else {
254 UNREACHABLE();
255 }
256 // Result is in R1: null -> not found, otherwise Bool::True or Bool::False.
257 GenerateBoolToJump(R1, is_instance_lbl, is_not_instance_lbl);
258 return type_test_cache.raw();
259}
260
261// Jumps to labels 'is_instance' or 'is_not_instance' respectively, if
262// type test is conclusive, otherwise fallthrough if a type test could not
263// be completed.
264// R0: instance being type checked (preserved).
265// Clobbers R1, R2.
266SubtypeTestCachePtr
267FlowGraphCompiler::GenerateInstantiatedTypeWithArgumentsTest(
268 TokenPosition token_pos,
269 const AbstractType& type,
270 compiler::Label* is_instance_lbl,
271 compiler::Label* is_not_instance_lbl) {
272 __ Comment("InstantiatedTypeWithArgumentsTest");
273 ASSERT(type.IsInstantiated());
274 ASSERT(!type.IsFunctionType());
275 const Class& type_class = Class::ZoneHandle(zone(), type.type_class());
276 ASSERT(type_class.NumTypeArguments() > 0);
277 const Type& smi_type = Type::Handle(zone(), Type::SmiType());
278 const bool smi_is_ok = smi_type.IsSubtypeOf(type, Heap::kOld);
279 // Fast case for type = FutureOr<int/num/top-type>.
280 __ BranchIfSmi(TypeTestABI::kInstanceReg,
281 smi_is_ok ? is_instance_lbl : is_not_instance_lbl);
282
283 const intptr_t num_type_args = type_class.NumTypeArguments();
284 const intptr_t num_type_params = type_class.NumTypeParameters();
285 const intptr_t from_index = num_type_args - num_type_params;
286 const TypeArguments& type_arguments =
287 TypeArguments::ZoneHandle(zone(), type.arguments());
288 const bool is_raw_type = type_arguments.IsNull() ||
289 type_arguments.IsRaw(from_index, num_type_params);
290 if (is_raw_type) {
291 const Register kClassIdReg = R2;
292 // dynamic type argument, check only classes.
293 __ LoadClassId(kClassIdReg, TypeTestABI::kInstanceReg);
294 __ CompareImmediate(kClassIdReg, type_class.id());
295 __ b(is_instance_lbl, EQ);
296 // List is a very common case.
297 if (IsListClass(type_class)) {
298 GenerateListTypeCheck(kClassIdReg, is_instance_lbl);
299 }
300 return GenerateSubtype1TestCacheLookup(
301 token_pos, type_class, is_instance_lbl, is_not_instance_lbl);
302 }
303 // If one type argument only, check if type argument is a top type.
304 if (type_arguments.Length() == 1) {
305 const AbstractType& tp_argument =
306 AbstractType::ZoneHandle(zone(), type_arguments.TypeAt(0));
307 if (tp_argument.IsTopTypeForSubtyping()) {
308 // Instance class test only necessary.
309 return GenerateSubtype1TestCacheLookup(
310 token_pos, type_class, is_instance_lbl, is_not_instance_lbl);
311 }
312 }
313 // Regular subtype test cache involving instance's type arguments.
314 const Register kInstantiatorTypeArgumentsReg = kNoRegister;
315 const Register kFunctionTypeArgumentsReg = kNoRegister;
316 const Register kTempReg = kNoRegister;
317 // R0: instance (must be preserved).
318 return GenerateCallSubtypeTestStub(
319 kTestTypeTwoArgs, TypeTestABI::kInstanceReg,
320 kInstantiatorTypeArgumentsReg, kFunctionTypeArgumentsReg, kTempReg,
321 is_instance_lbl, is_not_instance_lbl);
322}
323
324void FlowGraphCompiler::CheckClassIds(Register class_id_reg,
325 const GrowableArray<intptr_t>& class_ids,
326 compiler::Label* is_equal_lbl,
327 compiler::Label* is_not_equal_lbl) {
328 for (intptr_t i = 0; i < class_ids.length(); i++) {
329 __ CompareImmediate(class_id_reg, class_ids[i]);
330 __ b(is_equal_lbl, EQ);
331 }
332 __ b(is_not_equal_lbl);
333}
334
335// Testing against an instantiated type with no arguments, without
336// SubtypeTestCache.
337// R0: instance being type checked (preserved).
338// Clobbers R2, R3.
339// Returns true if there is a fallthrough.
340bool FlowGraphCompiler::GenerateInstantiatedTypeNoArgumentsTest(
341 TokenPosition token_pos,
342 const AbstractType& type,
343 compiler::Label* is_instance_lbl,
344 compiler::Label* is_not_instance_lbl) {
345 __ Comment("InstantiatedTypeNoArgumentsTest");
346 ASSERT(type.IsInstantiated());
347 ASSERT(!type.IsFunctionType());
348 const Class& type_class = Class::Handle(zone(), type.type_class());
349 ASSERT(type_class.NumTypeArguments() == 0);
350
351 // If instance is Smi, check directly.
352 const Class& smi_class = Class::Handle(zone(), Smi::Class());
353 if (Class::IsSubtypeOf(smi_class, Object::null_type_arguments(),
354 Nullability::kNonNullable, type, Heap::kOld)) {
355 // Fast case for type = int/num/top-type.
356 __ BranchIfSmi(TypeTestABI::kInstanceReg, is_instance_lbl);
357 } else {
358 __ BranchIfSmi(TypeTestABI::kInstanceReg, is_not_instance_lbl);
359 }
360 const Register kClassIdReg = R2;
361 __ LoadClassId(kClassIdReg, TypeTestABI::kInstanceReg);
362 // Bool interface can be implemented only by core class Bool.
363 if (type.IsBoolType()) {
364 __ CompareImmediate(kClassIdReg, kBoolCid);
365 __ b(is_instance_lbl, EQ);
366 __ b(is_not_instance_lbl);
367 return false;
368 }
369 // Custom checking for numbers (Smi, Mint and Double).
370 // Note that instance is not Smi (checked above).
371 if (type.IsNumberType() || type.IsIntType() || type.IsDoubleType()) {
372 GenerateNumberTypeCheck(kClassIdReg, type, is_instance_lbl,
373 is_not_instance_lbl);
374 return false;
375 }
376 if (type.IsStringType()) {
377 GenerateStringTypeCheck(kClassIdReg, is_instance_lbl, is_not_instance_lbl);
378 return false;
379 }
380 if (type.IsDartFunctionType()) {
381 // Check if instance is a closure.
382 __ CompareImmediate(kClassIdReg, kClosureCid);
383 __ b(is_instance_lbl, EQ);
384 return true; // Fall through
385 }
386
387 // Fast case for cid-range based checks.
388 // Warning: This code destroys the contents of [kClassIdReg].
389 if (GenerateSubtypeRangeCheck(kClassIdReg, type_class, is_instance_lbl)) {
390 return false;
391 }
392
393 // Otherwise fallthrough, result non-conclusive.
394 return true;
395}
396
397// Uses SubtypeTestCache to store instance class and result.
398// R0: instance to test.
399// Clobbers R1-R5.
400// Immediate class test already done.
401// TODO(srdjan): Implement a quicker subtype check, as type test
402// arrays can grow too high, but they may be useful when optimizing
403// code (type-feedback).
404SubtypeTestCachePtr FlowGraphCompiler::GenerateSubtype1TestCacheLookup(
405 TokenPosition token_pos,
406 const Class& type_class,
407 compiler::Label* is_instance_lbl,
408 compiler::Label* is_not_instance_lbl) {
409 __ Comment("Subtype1TestCacheLookup");
410#if defined(DEBUG)
411 compiler::Label ok;
412 __ BranchIfNotSmi(TypeTestABI::kInstanceReg, &ok);
413 __ Breakpoint();
414 __ Bind(&ok);
415#endif
416 __ LoadClassId(TMP, TypeTestABI::kInstanceReg);
417 __ LoadClassById(R1, TMP);
418 // R1: instance class.
419 // Check immediate superclass equality. If type_class is Object, then testing
420 // supertype may yield a wrong result for Null in NNBD strong mode (because
421 // Null also extends Object).
422 if (!type_class.IsObjectClass() || !Isolate::Current()->null_safety()) {
423 __ LoadFieldFromOffset(R2, R1, Class::super_type_offset());
424 __ LoadFieldFromOffset(R2, R2, Type::type_class_id_offset());
425 __ CompareImmediate(R2, Smi::RawValue(type_class.id()));
426 __ b(is_instance_lbl, EQ);
427 }
428
429 const Register kInstantiatorTypeArgumentsReg = kNoRegister;
430 const Register kFunctionTypeArgumentsReg = kNoRegister;
431 const Register kTempReg = kNoRegister;
432 return GenerateCallSubtypeTestStub(kTestTypeOneArg, TypeTestABI::kInstanceReg,
433 kInstantiatorTypeArgumentsReg,
434 kFunctionTypeArgumentsReg, kTempReg,
435 is_instance_lbl, is_not_instance_lbl);
436}
437
438// Generates inlined check if 'type' is a type parameter or type itself
439// R0: instance (preserved).
440SubtypeTestCachePtr FlowGraphCompiler::GenerateUninstantiatedTypeTest(
441 TokenPosition token_pos,
442 const AbstractType& type,
443 compiler::Label* is_instance_lbl,
444 compiler::Label* is_not_instance_lbl) {
445 __ Comment("UninstantiatedTypeTest");
446 const Register kTempReg = kNoRegister;
447 ASSERT(!type.IsInstantiated());
448 ASSERT(!type.IsFunctionType());
449 // Skip check if destination is a dynamic type.
450 if (type.IsTypeParameter()) {
451 const TypeParameter& type_param = TypeParameter::Cast(type);
452
453 // Get instantiator type args (high) and function type args (low).
454 __ ldp(TypeTestABI::kFunctionTypeArgumentsReg,
455 TypeTestABI::kInstantiatorTypeArgumentsReg,
456 compiler::Address(SP, 0 * kWordSize, compiler::Address::PairOffset));
457 const Register kTypeArgumentsReg =
458 type_param.IsClassTypeParameter()
459 ? TypeTestABI::kInstantiatorTypeArgumentsReg
460 : TypeTestABI::kFunctionTypeArgumentsReg;
461 // Check if type arguments are null, i.e. equivalent to vector of dynamic.
462 __ CompareObject(kTypeArgumentsReg, Object::null_object());
463 __ b(is_instance_lbl, EQ);
464 __ LoadFieldFromOffset(R3, kTypeArgumentsReg,
465 TypeArguments::type_at_offset(type_param.index()));
466 // R3: concrete type of type.
467 // Check if type argument is dynamic, Object?, or void.
468 __ CompareObject(R3, Object::dynamic_type());
469 __ b(is_instance_lbl, EQ);
470 __ CompareObject(
471 R3, Type::ZoneHandle(
472 zone(), isolate()->object_store()->nullable_object_type()));
473 __ b(is_instance_lbl, EQ);
474 __ CompareObject(R3, Object::void_type());
475 __ b(is_instance_lbl, EQ);
476
477 // For Smi check quickly against int and num interfaces.
478 compiler::Label not_smi;
479 __ BranchIfNotSmi(R0, &not_smi);
480 __ CompareObject(R3, Type::ZoneHandle(zone(), Type::IntType()));
481 __ b(is_instance_lbl, EQ);
482 __ CompareObject(R3, Type::ZoneHandle(zone(), Type::Number()));
483 __ b(is_instance_lbl, EQ);
484 // Smi can be handled by type test cache.
485 __ Bind(&not_smi);
486
487 const auto test_kind = GetTypeTestStubKindForTypeParameter(type_param);
488 const SubtypeTestCache& type_test_cache = SubtypeTestCache::ZoneHandle(
489 zone(), GenerateCallSubtypeTestStub(
490 test_kind, TypeTestABI::kInstanceReg,
491 TypeTestABI::kInstantiatorTypeArgumentsReg,
492 TypeTestABI::kFunctionTypeArgumentsReg, kTempReg,
493 is_instance_lbl, is_not_instance_lbl));
494 return type_test_cache.raw();
495 }
496 if (type.IsType()) {
497 // Smi is FutureOr<T>, when T is a top type or int or num.
498 if (!type.IsFutureOrType()) {
499 __ BranchIfSmi(TypeTestABI::kInstanceReg, is_not_instance_lbl);
500 }
501 __ ldp(TypeTestABI::kFunctionTypeArgumentsReg,
502 TypeTestABI::kInstantiatorTypeArgumentsReg,
503 compiler::Address(SP, 0 * kWordSize, compiler::Address::PairOffset));
504 // Uninstantiated type class is known at compile time, but the type
505 // arguments are determined at runtime by the instantiator.
506 return GenerateCallSubtypeTestStub(
507 kTestTypeFourArgs, TypeTestABI::kInstanceReg,
508 TypeTestABI::kInstantiatorTypeArgumentsReg,
509 TypeTestABI::kFunctionTypeArgumentsReg, kTempReg, is_instance_lbl,
510 is_not_instance_lbl);
511 }
512 return SubtypeTestCache::null();
513}
514
515// Generates function type check.
516//
517// See [GenerateUninstantiatedTypeTest] for calling convention.
518SubtypeTestCachePtr FlowGraphCompiler::GenerateFunctionTypeTest(
519 TokenPosition token_pos,
520 const AbstractType& type,
521 compiler::Label* is_instance_lbl,
522 compiler::Label* is_not_instance_lbl) {
523 __ BranchIfSmi(TypeTestABI::kInstanceReg, is_not_instance_lbl);
524 __ ldp(TypeTestABI::kFunctionTypeArgumentsReg,
525 TypeTestABI::kInstantiatorTypeArgumentsReg,
526 compiler::Address(SP, 0 * kWordSize, compiler::Address::PairOffset));
527 // Uninstantiated type class is known at compile time, but the type
528 // arguments are determined at runtime by the instantiator(s).
529 const Register kTempReg = kNoRegister;
530 return GenerateCallSubtypeTestStub(
531 kTestTypeSixArgs, TypeTestABI::kInstanceReg,
532 TypeTestABI::kInstantiatorTypeArgumentsReg,
533 TypeTestABI::kFunctionTypeArgumentsReg, kTempReg, is_instance_lbl,
534 is_not_instance_lbl);
535}
536
537// Inputs:
538// - R0: instance being type checked (preserved).
539// - R2: optional instantiator type arguments (preserved).
540// - R1: optional function type arguments (preserved).
541// Clobbers R3, R4, R8, R9.
542// Returns:
543// - preserved instance in R0, optional instantiator type arguments in R2, and
544// optional function type arguments in R1.
545// Note that this inlined code must be followed by the runtime_call code, as it
546// may fall through to it. Otherwise, this inline code will jump to the label
547// is_instance or to the label is_not_instance.
548SubtypeTestCachePtr FlowGraphCompiler::GenerateInlineInstanceof(
549 TokenPosition token_pos,
550 const AbstractType& type,
551 compiler::Label* is_instance_lbl,
552 compiler::Label* is_not_instance_lbl) {
553 __ Comment("InlineInstanceof");
554
555 if (type.IsFunctionType()) {
556 return GenerateFunctionTypeTest(token_pos, type, is_instance_lbl,
557 is_not_instance_lbl);
558 }
559
560 if (type.IsInstantiated()) {
561 const Class& type_class = Class::ZoneHandle(zone(), type.type_class());
562 // A class equality check is only applicable with a dst type (not a
563 // function type) of a non-parameterized class or with a raw dst type of
564 // a parameterized class.
565 if (type_class.NumTypeArguments() > 0) {
566 return GenerateInstantiatedTypeWithArgumentsTest(
567 token_pos, type, is_instance_lbl, is_not_instance_lbl);
568 // Fall through to runtime call.
569 }
570 const bool has_fall_through = GenerateInstantiatedTypeNoArgumentsTest(
571 token_pos, type, is_instance_lbl, is_not_instance_lbl);
572 if (has_fall_through) {
573 // If test non-conclusive so far, try the inlined type-test cache.
574 // 'type' is known at compile time.
575 return GenerateSubtype1TestCacheLookup(
576 token_pos, type_class, is_instance_lbl, is_not_instance_lbl);
577 } else {
578 return SubtypeTestCache::null();
579 }
580 }
581 return GenerateUninstantiatedTypeTest(token_pos, type, is_instance_lbl,
582 is_not_instance_lbl);
583}
584
585// If instanceof type test cannot be performed successfully at compile time and
586// therefore eliminated, optimize it by adding inlined tests for:
587// - Null -> see comment below.
588// - Smi -> compile time subtype check (only if dst class is not parameterized).
589// - Class equality (only if class is not parameterized).
590// Inputs:
591// - R0: object.
592// - R2: instantiator type arguments or raw_null.
593// - R1: function type arguments or raw_null.
594// Returns:
595// - true or false in R0.
596void FlowGraphCompiler::GenerateInstanceOf(TokenPosition token_pos,
597 intptr_t deopt_id,
598 const AbstractType& type,
599 LocationSummary* locs) {
600 ASSERT(type.IsFinalized());
601 ASSERT(!type.IsTopTypeForInstanceOf()); // Already checked.
602 __ PushPair(TypeTestABI::kFunctionTypeArgumentsReg,
603 TypeTestABI::kInstantiatorTypeArgumentsReg);
604
605 compiler::Label is_instance, is_not_instance;
606 // 'null' is an instance of Null, Object*, Never*, void, and dynamic.
607 // In addition, 'null' is an instance of any nullable type.
608 // It is also an instance of FutureOr<T> if it is an instance of T.
609 const AbstractType& unwrapped_type =
610 AbstractType::Handle(type.UnwrapFutureOr());
611 if (!unwrapped_type.IsTypeParameter() || unwrapped_type.IsNullable()) {
612 // Only nullable type parameter remains nullable after instantiation.
613 // See NullIsInstanceOf().
614 __ CompareObject(TypeTestABI::kInstanceReg, Object::null_object());
615 __ b((unwrapped_type.IsNullable() ||
616 (unwrapped_type.IsLegacy() && unwrapped_type.IsNeverType()))
617 ? &is_instance
618 : &is_not_instance,
619 EQ);
620 }
621
622 // Generate inline instanceof test.
623 SubtypeTestCache& test_cache = SubtypeTestCache::ZoneHandle(zone());
624 test_cache =
625 GenerateInlineInstanceof(token_pos, type, &is_instance, &is_not_instance);
626
627 // test_cache is null if there is no fall-through.
628 compiler::Label done;
629 if (!test_cache.IsNull()) {
630 // Generate runtime call.
631 __ ldp(TypeTestABI::kFunctionTypeArgumentsReg,
632 TypeTestABI::kInstantiatorTypeArgumentsReg,
633 compiler::Address(SP, 0 * kWordSize, compiler::Address::PairOffset));
634 __ LoadUniqueObject(TypeTestABI::kDstTypeReg, type);
635 __ LoadUniqueObject(TypeTestABI::kSubtypeTestCacheReg, test_cache);
636 GenerateStubCall(token_pos, StubCode::InstanceOf(),
637 /*kind=*/PcDescriptorsLayout::kOther, locs, deopt_id);
638 __ b(&done);
639 }
640 __ Bind(&is_not_instance);
641 __ LoadObject(R0, Bool::Get(false));
642 __ b(&done);
643
644 __ Bind(&is_instance);
645 __ LoadObject(R0, Bool::Get(true));
646 __ Bind(&done);
647 // Remove instantiator type arguments and function type arguments.
648 __ Drop(2);
649}
650
651// Optimize assignable type check by adding inlined tests for:
652// - NULL -> return NULL.
653// - Smi -> compile time subtype check (only if dst class is not parameterized).
654// - Class equality (only if class is not parameterized).
655// Inputs:
656// - R0: instance being type checked.
657// - R8: destination type (if non-constant).
658// - R2: instantiator type arguments or raw_null.
659// - R1: function type arguments or raw_null.
660// Returns:
661// - object in R0 for successful assignable check (or throws TypeError).
662// Performance notes: positive checks must be quick, negative checks can be slow
663// as they throw an exception.
664void FlowGraphCompiler::GenerateAssertAssignable(CompileType* receiver_type,
665 TokenPosition token_pos,
666 intptr_t deopt_id,
667 const String& dst_name,
668 LocationSummary* locs) {
669 ASSERT(!TokenPosition(token_pos).IsClassifying());
670 ASSERT(CheckAssertAssignableTypeTestingABILocations(*locs));
671
672 compiler::Label is_assignable_fast, is_assignable, runtime_call;
673 // Generate inline type check, linking to runtime call if not assignable.
674 SubtypeTestCache& test_cache = SubtypeTestCache::ZoneHandle(zone());
675
676 if (locs->in(1).IsConstant()) {
677 const auto& dst_type = AbstractType::Cast(locs->in(1).constant());
678 ASSERT(dst_type.IsFinalized());
679
680 if (dst_type.IsTopTypeForSubtyping()) return; // No code needed.
681
682 if (ShouldUseTypeTestingStubFor(is_optimizing(), dst_type)) {
683 GenerateAssertAssignableViaTypeTestingStub(receiver_type, token_pos,
684 deopt_id, dst_name, locs);
685 return;
686 }
687
688 if (Instance::NullIsAssignableTo(dst_type)) {
689 __ CompareObject(TypeTestABI::kInstanceReg, Object::null_object());
690 __ b(&is_assignable_fast, EQ);
691 }
692
693 __ PushPair(TypeTestABI::kFunctionTypeArgumentsReg,
694 TypeTestABI::kInstantiatorTypeArgumentsReg);
695
696 test_cache = GenerateInlineInstanceof(token_pos, dst_type, &is_assignable,
697 &runtime_call);
698 } else {
699 // TODO(dartbug.com/40813): Handle setting up the non-constant case.
700 UNREACHABLE();
701 }
702
703 __ Bind(&runtime_call);
704 __ ldp(TypeTestABI::kFunctionTypeArgumentsReg,
705 TypeTestABI::kInstantiatorTypeArgumentsReg,
706 compiler::Address(SP, 0 * kWordSize, compiler::Address::PairOffset));
707 // Make room for the result and push the source object.
708 __ PushPair(TypeTestABI::kInstanceReg, NULL_REG);
709 // Push the destination type and the instantiator type arguments.
710 if (locs->in(1).IsConstant()) {
711 __ LoadObject(TMP, locs->in(1).constant());
712 __ PushPair(TypeTestABI::kInstantiatorTypeArgumentsReg, TMP);
713 } else {
714 // TODO(dartbug.com/40813): Handle setting up the non-constant case.
715 UNREACHABLE();
716 }
717 // Push the function type arguments and the name of the destination.
718 __ LoadObject(TMP, dst_name);
719 __ PushPair(TMP, TypeTestABI::kFunctionTypeArgumentsReg);
720
721 __ LoadUniqueObject(R0, test_cache);
722 __ LoadObject(TMP, Smi::ZoneHandle(zone(), Smi::New(kTypeCheckFromInline)));
723 __ PushPair(TMP, R0);
724 GenerateRuntimeCall(token_pos, deopt_id, kTypeCheckRuntimeEntry, 7, locs);
725 // Pop the parameters supplied to the runtime entry. The result of the
726 // type check runtime call is the checked value.
727 __ Drop(7);
728 __ Pop(TypeTestABI::kInstanceReg);
729 __ Bind(&is_assignable);
730 __ PopPair(TypeTestABI::kFunctionTypeArgumentsReg,
731 TypeTestABI::kInstantiatorTypeArgumentsReg);
732 __ Bind(&is_assignable_fast);
733}
734
735void FlowGraphCompiler::GenerateAssertAssignableViaTypeTestingStub(
736 CompileType* receiver_type,
737 TokenPosition token_pos,
738 intptr_t deopt_id,
739 const String& dst_name,
740 LocationSummary* locs) {
741 ASSERT(CheckAssertAssignableTypeTestingABILocations(*locs));
742 // We must have a constant dst_type for generating a call to the stub.
743 ASSERT(locs->in(1).IsConstant());
744 const auto& dst_type = AbstractType::Cast(locs->in(1).constant());
745
746 // If the dst_type is instantiated we know the target TTS stub at
747 // compile-time and can therefore use a pc-relative call.
748 const bool use_pc_relative_call =
749 dst_type.IsInstantiated() && CanPcRelativeCall(dst_type);
750
751 const Register kRegToCall =
752 use_pc_relative_call
753 ? kNoRegister
754 : (dst_type.IsTypeParameter() ? R9 : TypeTestABI::kDstTypeReg);
755 const Register kScratchReg = R4;
756
757 compiler::Label done;
758
759 GenerateAssertAssignableViaTypeTestingStub(receiver_type, dst_type, dst_name,
760 kRegToCall, kScratchReg, &done);
761
762 // We use 2 consecutive entries in the pool for the subtype cache and the
763 // destination name. The second entry, namely [dst_name] seems to be unused,
764 // but it will be used by the code throwing a TypeError if the type test fails
765 // (see runtime/vm/runtime_entry.cc:TypeCheck). It will use pattern matching
766 // on the call site to find out at which pool index the destination name is
767 // located.
768 const intptr_t sub_type_cache_index = __ object_pool_builder().AddObject(
769 Object::null_object(), ObjectPool::Patchability::kPatchable);
770 const intptr_t sub_type_cache_offset =
771 ObjectPool::element_offset(sub_type_cache_index);
772 const intptr_t dst_name_index = __ object_pool_builder().AddObject(
773 dst_name, ObjectPool::Patchability::kPatchable);
774 ASSERT((sub_type_cache_index + 1) == dst_name_index);
775 ASSERT(__ constant_pool_allowed());
776
777 if (use_pc_relative_call) {
778 __ LoadWordFromPoolOffset(TypeTestABI::kSubtypeTestCacheReg,
779 sub_type_cache_offset);
780 __ GenerateUnRelocatedPcRelativeCall();
781 AddPcRelativeTTSCallTypeTarget(dst_type);
782 } else {
783 __ LoadField(
784 R9, compiler::FieldAddress(
785 kRegToCall, AbstractType::type_test_stub_entry_point_offset()));
786 __ LoadWordFromPoolOffset(TypeTestABI::kSubtypeTestCacheReg,
787 sub_type_cache_offset);
788 __ blr(R9);
789 }
790 EmitCallsiteMetadata(token_pos, deopt_id, PcDescriptorsLayout::kOther, locs);
791 __ Bind(&done);
792}
793
794void FlowGraphCompiler::EmitInstructionEpilogue(Instruction* instr) {
795 if (is_optimizing()) {
796 return;
797 }
798 Definition* defn = instr->AsDefinition();
799 if ((defn != NULL) && defn->HasTemp()) {
800 __ Push(defn->locs()->out(0).reg());
801 }
802}
803
804void FlowGraphCompiler::GenerateMethodExtractorIntrinsic(
805 const Function& extracted_method,
806 intptr_t type_arguments_field_offset) {
807 // No frame has been setup here.
808 ASSERT(!__ constant_pool_allowed());
809 ASSERT(extracted_method.IsZoneHandle());
810
811 const Code& build_method_extractor = Code::ZoneHandle(
812 isolate()->object_store()->build_method_extractor_code());
813
814 const intptr_t stub_index = __ object_pool_builder().AddObject(
815 build_method_extractor, ObjectPool::Patchability::kNotPatchable);
816 const intptr_t function_index = __ object_pool_builder().AddObject(
817 extracted_method, ObjectPool::Patchability::kNotPatchable);
818
819 // We use a custom pool register to preserve caller PP.
820 Register kPoolReg = R0;
821
822 // R1 = extracted function
823 // R4 = offset of type argument vector (or 0 if class is not generic)
824 intptr_t pp_offset = 0;
825 if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
826 // PP is not tagged on arm64.
827 kPoolReg = PP;
828 pp_offset = kHeapObjectTag;
829 } else {
830 __ LoadFieldFromOffset(kPoolReg, CODE_REG, Code::object_pool_offset());
831 }
832 __ LoadImmediate(R4, type_arguments_field_offset);
833 __ LoadFieldFromOffset(
834 R1, kPoolReg, ObjectPool::element_offset(function_index) + pp_offset);
835 __ LoadFieldFromOffset(CODE_REG, kPoolReg,
836 ObjectPool::element_offset(stub_index) + pp_offset);
837 __ LoadFieldFromOffset(R0, CODE_REG,
838 Code::entry_point_offset(Code::EntryKind::kUnchecked));
839 __ br(R0);
840}
841
842void FlowGraphCompiler::EmitFrameEntry() {
843 const Function& function = parsed_function().function();
844 if (CanOptimizeFunction() && function.IsOptimizable() &&
845 (!is_optimizing() || may_reoptimize())) {
846 __ Comment("Invocation Count Check");
847 const Register function_reg = R6;
848 __ ldr(function_reg,
849 compiler::FieldAddress(CODE_REG, Code::owner_offset()));
850
851 __ LoadFieldFromOffset(R7, function_reg, Function::usage_counter_offset(),
852 kWord);
853 // Reoptimization of an optimized function is triggered by counting in
854 // IC stubs, but not at the entry of the function.
855 if (!is_optimizing()) {
856 __ add(R7, R7, compiler::Operand(1));
857 __ StoreFieldToOffset(R7, function_reg, Function::usage_counter_offset(),
858 kWord);
859 }
860 __ CompareImmediate(R7, GetOptimizationThreshold());
861 ASSERT(function_reg == R6);
862 compiler::Label dont_optimize;
863 __ b(&dont_optimize, LT);
864 __ ldr(TMP, compiler::Address(THR, Thread::optimize_entry_offset()));
865 __ br(TMP);
866 __ Bind(&dont_optimize);
867 }
868 __ Comment("Enter frame");
869 if (flow_graph().IsCompiledForOsr()) {
870 const intptr_t extra_slots = ExtraStackSlotsOnOsrEntry();
871 ASSERT(extra_slots >= 0);
872 __ EnterOsrFrame(extra_slots * kWordSize);
873 } else {
874 ASSERT(StackSize() >= 0);
875 __ EnterDartFrame(StackSize() * kWordSize);
876 }
877}
878
879void FlowGraphCompiler::EmitPrologue() {
880 EmitFrameEntry();
881 ASSERT(assembler()->constant_pool_allowed());
882
883 // In unoptimized code, initialize (non-argument) stack allocated slots.
884 if (!is_optimizing()) {
885 const int num_locals = parsed_function().num_stack_locals();
886
887 intptr_t args_desc_slot = -1;
888 if (parsed_function().has_arg_desc_var()) {
889 args_desc_slot = compiler::target::frame_layout.FrameSlotForVariable(
890 parsed_function().arg_desc_var());
891 }
892
893 __ Comment("Initialize spill slots");
894 if (num_locals > 1 || (num_locals == 1 && args_desc_slot == -1)) {
895 __ LoadObject(R0, Object::null_object());
896 }
897 for (intptr_t i = 0; i < num_locals; ++i) {
898 const intptr_t slot_index =
899 compiler::target::frame_layout.FrameSlotForVariableIndex(-i);
900 Register value_reg = slot_index == args_desc_slot ? ARGS_DESC_REG : R0;
901 __ StoreToOffset(value_reg, FP, slot_index * kWordSize);
902 }
903 }
904
905 EndCodeSourceRange(TokenPosition::kDartCodePrologue);
906}
907
908// Input parameters:
909// LR: return address.
910// SP: address of last argument.
911// FP: caller's frame pointer.
912// PP: caller's pool pointer.
913// R4: arguments descriptor array.
914void FlowGraphCompiler::CompileGraph() {
915 InitCompiler();
916
917 // For JIT we have multiple entrypoints functionality which moved the frame
918 // setup into the [TargetEntryInstr] (which will set the constant pool
919 // allowed bit to true). Despite this we still have to set the
920 // constant pool allowed bit to true here as well, because we can generate
921 // code for [CatchEntryInstr]s, which need the pool.
922 __ set_constant_pool_allowed(true);
923
924 VisitBlocks();
925
926#if defined(DEBUG)
927 __ brk(0);
928#endif
929
930 if (!skip_body_compilation()) {
931 ASSERT(assembler()->constant_pool_allowed());
932 GenerateDeferredCode();
933 }
934
935 for (intptr_t i = 0; i < indirect_gotos_.length(); ++i) {
936 indirect_gotos_[i]->ComputeOffsetTable(this);
937 }
938}
939
940void FlowGraphCompiler::EmitCallToStub(const Code& stub) {
941 ASSERT(!stub.IsNull());
942 if (CanPcRelativeCall(stub)) {
943 __ GenerateUnRelocatedPcRelativeCall();
944 AddPcRelativeCallStubTarget(stub);
945 } else {
946 __ BranchLink(stub);
947 AddStubCallTarget(stub);
948 }
949}
950
951void FlowGraphCompiler::EmitTailCallToStub(const Code& stub) {
952 ASSERT(!stub.IsNull());
953 if (CanPcRelativeCall(stub)) {
954 __ LeaveDartFrame();
955 __ GenerateUnRelocatedPcRelativeTailCall();
956 AddPcRelativeTailCallStubTarget(stub);
957#if defined(DEBUG)
958 __ Breakpoint();
959#endif
960 } else {
961 __ LoadObject(CODE_REG, stub);
962 __ LeaveDartFrame();
963 __ ldr(TMP, compiler::FieldAddress(
964 CODE_REG, compiler::target::Code::entry_point_offset()));
965 __ br(TMP);
966 AddStubCallTarget(stub);
967 }
968}
969
970void FlowGraphCompiler::GeneratePatchableCall(TokenPosition token_pos,
971 const Code& stub,
972 PcDescriptorsLayout::Kind kind,
973 LocationSummary* locs) {
974 __ BranchLinkPatchable(stub);
975 EmitCallsiteMetadata(token_pos, DeoptId::kNone, kind, locs);
976}
977
978void FlowGraphCompiler::GenerateDartCall(intptr_t deopt_id,
979 TokenPosition token_pos,
980 const Code& stub,
981 PcDescriptorsLayout::Kind kind,
982 LocationSummary* locs,
983 Code::EntryKind entry_kind) {
984 ASSERT(CanCallDart());
985 __ BranchLinkPatchable(stub, entry_kind);
986 EmitCallsiteMetadata(token_pos, deopt_id, kind, locs);
987}
988
989void FlowGraphCompiler::GenerateStaticDartCall(intptr_t deopt_id,
990 TokenPosition token_pos,
991 PcDescriptorsLayout::Kind kind,
992 LocationSummary* locs,
993 const Function& target,
994 Code::EntryKind entry_kind) {
995 ASSERT(CanCallDart());
996 if (CanPcRelativeCall(target)) {
997 __ GenerateUnRelocatedPcRelativeCall();
998 AddPcRelativeCallTarget(target, entry_kind);
999 EmitCallsiteMetadata(token_pos, deopt_id, kind, locs);
1000 } else {
1001 // Call sites to the same target can share object pool entries. These
1002 // call sites are never patched for breakpoints: the function is deoptimized
1003 // and the unoptimized code with IC calls for static calls is patched
1004 // instead.
1005 ASSERT(is_optimizing());
1006 const auto& stub = StubCode::CallStaticFunction();
1007 __ BranchLinkWithEquivalence(stub, target, entry_kind);
1008 EmitCallsiteMetadata(token_pos, deopt_id, kind, locs);
1009 AddStaticCallTarget(target, entry_kind);
1010 }
1011}
1012
1013void FlowGraphCompiler::GenerateRuntimeCall(TokenPosition token_pos,
1014 intptr_t deopt_id,
1015 const RuntimeEntry& entry,
1016 intptr_t argument_count,
1017 LocationSummary* locs) {
1018 __ CallRuntime(entry, argument_count);
1019 EmitCallsiteMetadata(token_pos, deopt_id, PcDescriptorsLayout::kOther, locs);
1020}
1021
1022void FlowGraphCompiler::EmitEdgeCounter(intptr_t edge_id) {
1023 // We do not check for overflow when incrementing the edge counter. The
1024 // function should normally be optimized long before the counter can
1025 // overflow; and though we do not reset the counters when we optimize or
1026 // deoptimize, there is a bound on the number of
1027 // optimization/deoptimization cycles we will attempt.
1028 ASSERT(!edge_counters_array_.IsNull());
1029 ASSERT(assembler_->constant_pool_allowed());
1030 __ Comment("Edge counter");
1031 __ LoadObject(R0, edge_counters_array_);
1032 __ LoadFieldFromOffset(TMP, R0, Array::element_offset(edge_id));
1033 __ add(TMP, TMP, compiler::Operand(Smi::RawValue(1)));
1034 __ StoreFieldToOffset(TMP, R0, Array::element_offset(edge_id));
1035}
1036
1037void FlowGraphCompiler::EmitOptimizedInstanceCall(const Code& stub,
1038 const ICData& ic_data,
1039 intptr_t deopt_id,
1040 TokenPosition token_pos,
1041 LocationSummary* locs,
1042 Code::EntryKind entry_kind) {
1043 ASSERT(CanCallDart());
1044 ASSERT(Array::Handle(zone(), ic_data.arguments_descriptor()).Length() > 0);
1045 // Each ICData propagated from unoptimized to optimized code contains the
1046 // function that corresponds to the Dart function of that IC call. Due
1047 // to inlining in optimized code, that function may not correspond to the
1048 // top-level function (parsed_function().function()) which could be
1049 // reoptimized and which counter needs to be incremented.
1050 // Pass the function explicitly, it is used in IC stub.
1051
1052 __ LoadObject(R6, parsed_function().function());
1053 __ LoadFromOffset(R0, SP, (ic_data.SizeWithoutTypeArgs() - 1) * kWordSize);
1054 __ LoadUniqueObject(R5, ic_data);
1055 GenerateDartCall(deopt_id, token_pos, stub, PcDescriptorsLayout::kIcCall,
1056 locs, entry_kind);
1057 __ Drop(ic_data.SizeWithTypeArgs());
1058}
1059
1060void FlowGraphCompiler::EmitInstanceCallJIT(const Code& stub,
1061 const ICData& ic_data,
1062 intptr_t deopt_id,
1063 TokenPosition token_pos,
1064 LocationSummary* locs,
1065 Code::EntryKind entry_kind) {
1066 ASSERT(CanCallDart());
1067 ASSERT(entry_kind == Code::EntryKind::kNormal ||
1068 entry_kind == Code::EntryKind::kUnchecked);
1069 ASSERT(Array::Handle(zone(), ic_data.arguments_descriptor()).Length() > 0);
1070 __ LoadFromOffset(R0, SP, (ic_data.SizeWithoutTypeArgs() - 1) * kWordSize);
1071
1072 compiler::ObjectPoolBuilder& op = __ object_pool_builder();
1073 const intptr_t ic_data_index =
1074 op.AddObject(ic_data, ObjectPool::Patchability::kPatchable);
1075 const intptr_t stub_index =
1076 op.AddObject(stub, ObjectPool::Patchability::kPatchable);
1077 ASSERT((ic_data_index + 1) == stub_index);
1078 __ LoadDoubleWordFromPoolOffset(R5, CODE_REG,
1079 ObjectPool::element_offset(ic_data_index));
1080 const intptr_t entry_point_offset =
1081 entry_kind == Code::EntryKind::kNormal
1082 ? Code::entry_point_offset(Code::EntryKind::kMonomorphic)
1083 : Code::entry_point_offset(Code::EntryKind::kMonomorphicUnchecked);
1084 __ ldr(LR, compiler::FieldAddress(CODE_REG, entry_point_offset));
1085 __ blr(LR);
1086 EmitCallsiteMetadata(token_pos, deopt_id, PcDescriptorsLayout::kIcCall, locs);
1087 __ Drop(ic_data.SizeWithTypeArgs());
1088}
1089
1090void FlowGraphCompiler::EmitMegamorphicInstanceCall(
1091 const String& name,
1092 const Array& arguments_descriptor,
1093 intptr_t deopt_id,
1094 TokenPosition token_pos,
1095 LocationSummary* locs,
1096 intptr_t try_index,
1097 intptr_t slow_path_argument_count) {
1098 ASSERT(CanCallDart());
1099 ASSERT(!arguments_descriptor.IsNull() && (arguments_descriptor.Length() > 0));
1100 const ArgumentsDescriptor args_desc(arguments_descriptor);
1101 const MegamorphicCache& cache = MegamorphicCache::ZoneHandle(
1102 zone(),
1103 MegamorphicCacheTable::Lookup(thread(), name, arguments_descriptor));
1104
1105 __ Comment("MegamorphicCall");
1106 // Load receiver into R0.
1107 __ LoadFromOffset(R0, SP, (args_desc.Count() - 1) * kWordSize);
1108
1109 // Use same code pattern as instance call so it can be parsed by code patcher.
1110 compiler::ObjectPoolBuilder& op = __ object_pool_builder();
1111 const intptr_t data_index =
1112 op.AddObject(cache, ObjectPool::Patchability::kPatchable);
1113 const intptr_t stub_index = op.AddObject(
1114 StubCode::MegamorphicCall(), ObjectPool::Patchability::kPatchable);
1115 ASSERT((data_index + 1) == stub_index);
1116 if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
1117 // The AOT runtime will replace the slot in the object pool with the
1118 // entrypoint address - see clustered_snapshot.cc.
1119 __ LoadDoubleWordFromPoolOffset(R5, LR,
1120 ObjectPool::element_offset(data_index));
1121 } else {
1122 __ LoadDoubleWordFromPoolOffset(R5, CODE_REG,
1123 ObjectPool::element_offset(data_index));
1124 __ ldr(LR, compiler::FieldAddress(
1125 CODE_REG,
1126 Code::entry_point_offset(Code::EntryKind::kMonomorphic)));
1127 }
1128 __ blr(LR);
1129
1130 RecordSafepoint(locs, slow_path_argument_count);
1131 const intptr_t deopt_id_after = DeoptId::ToDeoptAfter(deopt_id);
1132 if (FLAG_precompiled_mode) {
1133 // Megamorphic calls may occur in slow path stubs.
1134 // If valid use try_index argument.
1135 if (try_index == kInvalidTryIndex) {
1136 try_index = CurrentTryIndex();
1137 }
1138 AddDescriptor(PcDescriptorsLayout::kOther, assembler()->CodeSize(),
1139 DeoptId::kNone, token_pos, try_index);
1140 } else if (is_optimizing()) {
1141 AddCurrentDescriptor(PcDescriptorsLayout::kOther, DeoptId::kNone,
1142 token_pos);
1143 AddDeoptIndexAtCall(deopt_id_after);
1144 } else {
1145 AddCurrentDescriptor(PcDescriptorsLayout::kOther, DeoptId::kNone,
1146 token_pos);
1147 // Add deoptimization continuation point after the call and before the
1148 // arguments are removed.
1149 AddCurrentDescriptor(PcDescriptorsLayout::kDeopt, deopt_id_after,
1150 token_pos);
1151 }
1152 RecordCatchEntryMoves(pending_deoptimization_env_, try_index);
1153 __ Drop(args_desc.SizeWithTypeArgs());
1154}
1155
1156void FlowGraphCompiler::EmitInstanceCallAOT(const ICData& ic_data,
1157 intptr_t deopt_id,
1158 TokenPosition token_pos,
1159 LocationSummary* locs,
1160 Code::EntryKind entry_kind,
1161 bool receiver_can_be_smi) {
1162 ASSERT(CanCallDart());
1163 ASSERT(ic_data.NumArgsTested() == 1);
1164 const Code& initial_stub = StubCode::SwitchableCallMiss();
1165 const char* switchable_call_mode = "smiable";
1166 if (!receiver_can_be_smi) {
1167 switchable_call_mode = "non-smi";
1168 ic_data.set_receiver_cannot_be_smi(true);
1169 }
1170 const UnlinkedCall& data =
1171 UnlinkedCall::ZoneHandle(zone(), ic_data.AsUnlinkedCall());
1172
1173 compiler::ObjectPoolBuilder& op = __ object_pool_builder();
1174
1175 __ Comment("InstanceCallAOT (%s)", switchable_call_mode);
1176 // Clear argument descriptor to keep gc happy when it gets pushed on to
1177 // the stack.
1178 __ LoadImmediate(R4, 0);
1179 __ LoadFromOffset(R0, SP, (ic_data.SizeWithoutTypeArgs() - 1) * kWordSize);
1180
1181 const intptr_t data_index =
1182 op.AddObject(data, ObjectPool::Patchability::kPatchable);
1183 const intptr_t initial_stub_index =
1184 op.AddObject(initial_stub, ObjectPool::Patchability::kPatchable);
1185 ASSERT((data_index + 1) == initial_stub_index);
1186
1187 if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
1188 // The AOT runtime will replace the slot in the object pool with the
1189 // entrypoint address - see clustered_snapshot.cc.
1190 __ LoadDoubleWordFromPoolOffset(R5, LR,
1191 ObjectPool::element_offset(data_index));
1192 } else {
1193 __ LoadDoubleWordFromPoolOffset(R5, CODE_REG,
1194 ObjectPool::element_offset(data_index));
1195 const intptr_t entry_point_offset =
1196 entry_kind == Code::EntryKind::kNormal
1197 ? compiler::target::Code::entry_point_offset(
1198 Code::EntryKind::kMonomorphic)
1199 : compiler::target::Code::entry_point_offset(
1200 Code::EntryKind::kMonomorphicUnchecked);
1201 __ ldr(LR, compiler::FieldAddress(CODE_REG, entry_point_offset));
1202 }
1203 __ blr(LR);
1204
1205 EmitCallsiteMetadata(token_pos, DeoptId::kNone, PcDescriptorsLayout::kOther,
1206 locs);
1207 __ Drop(ic_data.SizeWithTypeArgs());
1208}
1209
1210void FlowGraphCompiler::EmitUnoptimizedStaticCall(intptr_t size_with_type_args,
1211 intptr_t deopt_id,
1212 TokenPosition token_pos,
1213 LocationSummary* locs,
1214 const ICData& ic_data,
1215 Code::EntryKind entry_kind) {
1216 ASSERT(CanCallDart());
1217 const Code& stub =
1218 StubCode::UnoptimizedStaticCallEntry(ic_data.NumArgsTested());
1219 __ LoadObject(R5, ic_data);
1220 GenerateDartCall(deopt_id, token_pos, stub,
1221 PcDescriptorsLayout::kUnoptStaticCall, locs, entry_kind);
1222 __ Drop(size_with_type_args);
1223}
1224
1225void FlowGraphCompiler::EmitOptimizedStaticCall(
1226 const Function& function,
1227 const Array& arguments_descriptor,
1228 intptr_t size_with_type_args,
1229 intptr_t deopt_id,
1230 TokenPosition token_pos,
1231 LocationSummary* locs,
1232 Code::EntryKind entry_kind) {
1233 ASSERT(CanCallDart());
1234 ASSERT(!function.IsClosureFunction());
1235 if (function.HasOptionalParameters() || function.IsGeneric()) {
1236 __ LoadObject(R4, arguments_descriptor);
1237 } else {
1238 if (!(FLAG_precompiled_mode && FLAG_use_bare_instructions)) {
1239 __ LoadImmediate(R4, 0); // GC safe smi zero because of stub.
1240 }
1241 }
1242 // Do not use the code from the function, but let the code be patched so that
1243 // we can record the outgoing edges to other code.
1244 GenerateStaticDartCall(deopt_id, token_pos, PcDescriptorsLayout::kOther, locs,
1245 function, entry_kind);
1246 __ Drop(size_with_type_args);
1247}
1248
1249void FlowGraphCompiler::EmitDispatchTableCall(
1250 Register cid_reg,
1251 int32_t selector_offset,
1252 const Array& arguments_descriptor) {
1253 ASSERT(CanCallDart());
1254 ASSERT(cid_reg != ARGS_DESC_REG);
1255 if (!arguments_descriptor.IsNull()) {
1256 __ LoadObject(ARGS_DESC_REG, arguments_descriptor);
1257 }
1258 const intptr_t offset = selector_offset - DispatchTable::OriginElement();
1259 __ AddImmediate(cid_reg, cid_reg, offset);
1260 __ ldr(LR, compiler::Address(DISPATCH_TABLE_REG, cid_reg, UXTX,
1261 compiler::Address::Scaled));
1262 __ blr(LR);
1263}
1264
1265Condition FlowGraphCompiler::EmitEqualityRegConstCompare(
1266 Register reg,
1267 const Object& obj,
1268 bool needs_number_check,
1269 TokenPosition token_pos,
1270 intptr_t deopt_id) {
1271 if (needs_number_check) {
1272 ASSERT(!obj.IsMint() && !obj.IsDouble());
1273 __ LoadObject(TMP, obj);
1274 __ PushPair(TMP, reg);
1275 if (is_optimizing()) {
1276 __ BranchLinkPatchable(StubCode::OptimizedIdenticalWithNumberCheck());
1277 } else {
1278 __ BranchLinkPatchable(StubCode::UnoptimizedIdenticalWithNumberCheck());
1279 }
1280 AddCurrentDescriptor(PcDescriptorsLayout::kRuntimeCall, deopt_id,
1281 token_pos);
1282 // Stub returns result in flags (result of a cmp, we need Z computed).
1283 // Discard constant.
1284 // Restore 'reg'.
1285 __ PopPair(ZR, reg);
1286 } else {
1287 __ CompareObject(reg, obj);
1288 }
1289 return EQ;
1290}
1291
1292Condition FlowGraphCompiler::EmitEqualityRegRegCompare(Register left,
1293 Register right,
1294 bool needs_number_check,
1295 TokenPosition token_pos,
1296 intptr_t deopt_id) {
1297 if (needs_number_check) {
1298 __ PushPair(right, left);
1299 if (is_optimizing()) {
1300 __ BranchLinkPatchable(StubCode::OptimizedIdenticalWithNumberCheck());
1301 } else {
1302 __ BranchLinkPatchable(StubCode::UnoptimizedIdenticalWithNumberCheck());
1303 }
1304 AddCurrentDescriptor(PcDescriptorsLayout::kRuntimeCall, deopt_id,
1305 token_pos);
1306 // Stub returns result in flags (result of a cmp, we need Z computed).
1307 __ PopPair(right, left);
1308 } else {
1309 __ CompareRegisters(left, right);
1310 }
1311 return EQ;
1312}
1313
1314Condition FlowGraphCompiler::EmitBoolTest(Register value,
1315 BranchLabels labels,
1316 bool invert) {
1317 __ Comment("BoolTest");
1318 if (labels.true_label == nullptr || labels.false_label == nullptr) {
1319 __ tsti(value, compiler::Immediate(
1320 compiler::target::ObjectAlignment::kBoolValueMask));
1321 return invert ? NE : EQ;
1322 }
1323 const intptr_t bool_bit =
1324 compiler::target::ObjectAlignment::kBoolValueBitPosition;
1325 if (labels.fall_through == labels.false_label) {
1326 if (invert) {
1327 __ tbnz(labels.true_label, value, bool_bit);
1328 } else {
1329 __ tbz(labels.true_label, value, bool_bit);
1330 }
1331 } else {
1332 if (invert) {
1333 __ tbz(labels.false_label, value, bool_bit);
1334 } else {
1335 __ tbnz(labels.false_label, value, bool_bit);
1336 }
1337 if (labels.fall_through != labels.true_label) {
1338 __ b(labels.true_label);
1339 }
1340 }
1341 return kInvalidCondition;
1342}
1343
1344// This function must be in sync with FlowGraphCompiler::RecordSafepoint and
1345// FlowGraphCompiler::SlowPathEnvironmentFor.
1346void FlowGraphCompiler::SaveLiveRegisters(LocationSummary* locs) {
1347#if defined(DEBUG)
1348 locs->CheckWritableInputs();
1349 ClobberDeadTempRegisters(locs);
1350#endif
1351 // TODO(vegorov): consider saving only caller save (volatile) registers.
1352 __ PushRegisters(*locs->live_registers());
1353}
1354
1355void FlowGraphCompiler::RestoreLiveRegisters(LocationSummary* locs) {
1356 __ PopRegisters(*locs->live_registers());
1357}
1358
1359#if defined(DEBUG)
1360void FlowGraphCompiler::ClobberDeadTempRegisters(LocationSummary* locs) {
1361 // Clobber temporaries that have not been manually preserved.
1362 for (intptr_t i = 0; i < locs->temp_count(); ++i) {
1363 Location tmp = locs->temp(i);
1364 // TODO(zerny): clobber non-live temporary FPU registers.
1365 if (tmp.IsRegister() &&
1366 !locs->live_registers()->ContainsRegister(tmp.reg())) {
1367 __ movz(tmp.reg(), compiler::Immediate(0xf7), 0);
1368 }
1369 }
1370}
1371#endif
1372
1373Register FlowGraphCompiler::EmitTestCidRegister() {
1374 return R2;
1375}
1376
1377void FlowGraphCompiler::EmitTestAndCallLoadReceiver(
1378 intptr_t count_without_type_args,
1379 const Array& arguments_descriptor) {
1380 __ Comment("EmitTestAndCall");
1381 // Load receiver into R0.
1382 __ LoadFromOffset(R0, SP, (count_without_type_args - 1) * kWordSize);
1383 __ LoadObject(R4, arguments_descriptor);
1384}
1385
1386void FlowGraphCompiler::EmitTestAndCallSmiBranch(compiler::Label* label,
1387 bool if_smi) {
1388 if (if_smi) {
1389 __ BranchIfSmi(R0, label);
1390 } else {
1391 __ BranchIfNotSmi(R0, label);
1392 }
1393}
1394
1395void FlowGraphCompiler::EmitTestAndCallLoadCid(Register class_id_reg) {
1396 ASSERT(class_id_reg != R0);
1397 __ LoadClassId(class_id_reg, R0);
1398}
1399
1400#undef __
1401#define __ assembler->
1402
1403int FlowGraphCompiler::EmitTestAndCallCheckCid(compiler::Assembler* assembler,
1404 compiler::Label* label,
1405 Register class_id_reg,
1406 const CidRangeValue& range,
1407 int bias,
1408 bool jump_on_miss) {
1409 const intptr_t cid_start = range.cid_start;
1410 if (range.IsSingleCid()) {
1411 __ AddImmediateSetFlags(class_id_reg, class_id_reg, bias - cid_start);
1412 __ BranchIf(jump_on_miss ? NOT_EQUAL : EQUAL, label);
1413 bias = cid_start;
1414 } else {
1415 __ AddImmediate(class_id_reg, bias - cid_start);
1416 bias = cid_start;
1417 __ CompareImmediate(class_id_reg, range.Extent());
1418 __ BranchIf(jump_on_miss ? UNSIGNED_GREATER : UNSIGNED_LESS_EQUAL, label);
1419 }
1420 return bias;
1421}
1422
1423#undef __
1424#define __ assembler()->
1425
1426void FlowGraphCompiler::EmitMove(Location destination,
1427 Location source,
1428 TemporaryRegisterAllocator* allocator) {
1429 if (destination.Equals(source)) return;
1430
1431 if (source.IsRegister()) {
1432 if (destination.IsRegister()) {
1433 __ mov(destination.reg(), source.reg());
1434 } else {
1435 ASSERT(destination.IsStackSlot());
1436 const intptr_t dest_offset = destination.ToStackSlotOffset();
1437 __ StoreToOffset(source.reg(), destination.base_reg(), dest_offset);
1438 }
1439 } else if (source.IsStackSlot()) {
1440 if (destination.IsRegister()) {
1441 const intptr_t source_offset = source.ToStackSlotOffset();
1442 __ LoadFromOffset(destination.reg(), source.base_reg(), source_offset);
1443 } else if (destination.IsFpuRegister()) {
1444 const intptr_t src_offset = source.ToStackSlotOffset();
1445 VRegister dst = destination.fpu_reg();
1446 __ LoadDFromOffset(dst, source.base_reg(), src_offset);
1447 } else {
1448 ASSERT(destination.IsStackSlot());
1449 const intptr_t source_offset = source.ToStackSlotOffset();
1450 const intptr_t dest_offset = destination.ToStackSlotOffset();
1451 Register tmp = allocator->AllocateTemporary();
1452 __ LoadFromOffset(tmp, source.base_reg(), source_offset);
1453 __ StoreToOffset(tmp, destination.base_reg(), dest_offset);
1454 allocator->ReleaseTemporary();
1455 }
1456 } else if (source.IsFpuRegister()) {
1457 if (destination.IsFpuRegister()) {
1458 __ vmov(destination.fpu_reg(), source.fpu_reg());
1459 } else {
1460 if (destination.IsStackSlot() /*32-bit float*/ ||
1461 destination.IsDoubleStackSlot()) {
1462 const intptr_t dest_offset = destination.ToStackSlotOffset();
1463 VRegister src = source.fpu_reg();
1464 __ StoreDToOffset(src, destination.base_reg(), dest_offset);
1465 } else {
1466 ASSERT(destination.IsQuadStackSlot());
1467 const intptr_t dest_offset = destination.ToStackSlotOffset();
1468 __ StoreQToOffset(source.fpu_reg(), destination.base_reg(),
1469 dest_offset);
1470 }
1471 }
1472 } else if (source.IsDoubleStackSlot()) {
1473 if (destination.IsFpuRegister()) {
1474 const intptr_t source_offset = source.ToStackSlotOffset();
1475 const VRegister dst = destination.fpu_reg();
1476 __ LoadDFromOffset(dst, source.base_reg(), source_offset);
1477 } else {
1478 ASSERT(destination.IsDoubleStackSlot() ||
1479 destination.IsStackSlot() /*32-bit float*/);
1480 const intptr_t source_offset = source.ToStackSlotOffset();
1481 const intptr_t dest_offset = destination.ToStackSlotOffset();
1482 __ LoadDFromOffset(VTMP, source.base_reg(), source_offset);
1483 __ StoreDToOffset(VTMP, destination.base_reg(), dest_offset);
1484 }
1485 } else if (source.IsQuadStackSlot()) {
1486 if (destination.IsFpuRegister()) {
1487 const intptr_t source_offset = source.ToStackSlotOffset();
1488 __ LoadQFromOffset(destination.fpu_reg(), source.base_reg(),
1489 source_offset);
1490 } else {
1491 ASSERT(destination.IsQuadStackSlot());
1492 const intptr_t source_offset = source.ToStackSlotOffset();
1493 const intptr_t dest_offset = destination.ToStackSlotOffset();
1494 __ LoadQFromOffset(VTMP, source.base_reg(), source_offset);
1495 __ StoreQToOffset(VTMP, destination.base_reg(), dest_offset);
1496 }
1497 } else {
1498 ASSERT(source.IsConstant());
1499 if (destination.IsStackSlot()) {
1500 Register tmp = allocator->AllocateTemporary();
1501 source.constant_instruction()->EmitMoveToLocation(this, destination, tmp);
1502 allocator->ReleaseTemporary();
1503 } else {
1504 source.constant_instruction()->EmitMoveToLocation(this, destination);
1505 }
1506 }
1507}
1508
1509static OperandSize BytesToOperandSize(intptr_t bytes) {
1510 switch (bytes) {
1511 case 8:
1512 return OperandSize::kDoubleWord;
1513 case 4:
1514 return OperandSize::kWord;
1515 case 2:
1516 return OperandSize::kHalfword;
1517 case 1:
1518 return OperandSize::kByte;
1519 default:
1520 UNIMPLEMENTED();
1521 }
1522}
1523
1524void FlowGraphCompiler::EmitNativeMoveArchitecture(
1525 const compiler::ffi::NativeLocation& destination,
1526 const compiler::ffi::NativeLocation& source) {
1527 const auto& src_type = source.payload_type();
1528 const auto& dst_type = destination.payload_type();
1529 ASSERT(src_type.IsFloat() == dst_type.IsFloat());
1530 ASSERT(src_type.IsInt() == dst_type.IsInt());
1531 ASSERT(src_type.IsSigned() == dst_type.IsSigned());
1532 ASSERT(src_type.IsFundamental());
1533 ASSERT(dst_type.IsFundamental());
1534 const intptr_t src_size = src_type.SizeInBytes();
1535 const intptr_t dst_size = dst_type.SizeInBytes();
1536 const bool sign_or_zero_extend = dst_size > src_size;
1537
1538 if (source.IsRegisters()) {
1539 const auto& src = source.AsRegisters();
1540 ASSERT(src.num_regs() == 1);
1541 const auto src_reg = src.reg_at(0);
1542
1543 if (destination.IsRegisters()) {
1544 const auto& dst = destination.AsRegisters();
1545 ASSERT(dst.num_regs() == 1);
1546 const auto dst_reg = dst.reg_at(0);
1547 if (!sign_or_zero_extend) {
1548 switch (dst_size) {
1549 case 8:
1550 __ mov(dst_reg, src_reg);
1551 return;
1552 case 4:
1553 __ movw(dst_reg, src_reg);
1554 return;
1555 default:
1556 UNIMPLEMENTED();
1557 }
1558 } else {
1559 switch (src_type.AsFundamental().representation()) {
1560 case compiler::ffi::kInt8: // Sign extend operand.
1561 __ sxtb(dst_reg, src_reg);
1562 return;
1563 case compiler::ffi::kInt16:
1564 __ sxth(dst_reg, src_reg);
1565 return;
1566 case compiler::ffi::kUint8: // Zero extend operand.
1567 __ uxtb(dst_reg, src_reg);
1568 return;
1569 case compiler::ffi::kUint16:
1570 __ uxth(dst_reg, src_reg);
1571 return;
1572 default:
1573 // 32 to 64 bit is covered in IL by Representation conversions.
1574 UNIMPLEMENTED();
1575 }
1576 }
1577
1578 } else if (destination.IsFpuRegisters()) {
1579 // Fpu Registers should only contain doubles and registers only ints.
1580 UNIMPLEMENTED();
1581
1582 } else {
1583 ASSERT(destination.IsStack());
1584 const auto& dst = destination.AsStack();
1585 ASSERT(!sign_or_zero_extend);
1586 const OperandSize op_size = BytesToOperandSize(dst_size);
1587 __ StoreToOffset(src.reg_at(0), dst.base_register(),
1588 dst.offset_in_bytes(), op_size);
1589 }
1590
1591 } else if (source.IsFpuRegisters()) {
1592 const auto& src = source.AsFpuRegisters();
1593 // We have not implemented conversions here, use IL convert instructions.
1594 ASSERT(src_type.Equals(dst_type));
1595
1596 if (destination.IsRegisters()) {
1597 // Fpu Registers should only contain doubles and registers only ints.
1598 UNIMPLEMENTED();
1599
1600 } else if (destination.IsFpuRegisters()) {
1601 const auto& dst = destination.AsFpuRegisters();
1602 __ vmov(dst.fpu_reg(), src.fpu_reg());
1603
1604 } else {
1605 ASSERT(destination.IsStack());
1606 ASSERT(src_type.IsFloat());
1607 const auto& dst = destination.AsStack();
1608 switch (dst_size) {
1609 case 8:
1610 __ StoreDToOffset(src.fpu_reg(), dst.base_register(),
1611 dst.offset_in_bytes());
1612 return;
1613 case 4:
1614 __ StoreSToOffset(src.fpu_reg(), dst.base_register(),
1615 dst.offset_in_bytes());
1616 return;
1617 default:
1618 UNREACHABLE();
1619 }
1620 }
1621
1622 } else {
1623 ASSERT(source.IsStack());
1624 const auto& src = source.AsStack();
1625 if (destination.IsRegisters()) {
1626 const auto& dst = destination.AsRegisters();
1627 ASSERT(dst.num_regs() == 1);
1628 const auto dst_reg = dst.reg_at(0);
1629 ASSERT(!sign_or_zero_extend);
1630 const OperandSize op_size = BytesToOperandSize(dst_size);
1631 __ LoadFromOffset(dst_reg, src.base_register(), src.offset_in_bytes(),
1632 op_size);
1633
1634 } else if (destination.IsFpuRegisters()) {
1635 ASSERT(src_type.Equals(dst_type));
1636 ASSERT(src_type.IsFloat());
1637 const auto& dst = destination.AsFpuRegisters();
1638 switch (src_size) {
1639 case 8:
1640 __ LoadDFromOffset(dst.fpu_reg(), src.base_register(),
1641 src.offset_in_bytes());
1642 return;
1643 case 4:
1644 __ LoadSFromOffset(dst.fpu_reg(), src.base_register(),
1645 src.offset_in_bytes());
1646 return;
1647 default:
1648 UNIMPLEMENTED();
1649 }
1650
1651 } else {
1652 ASSERT(destination.IsStack());
1653 UNREACHABLE();
1654 }
1655 }
1656}
1657
1658void FlowGraphCompiler::LoadBSSEntry(BSS::Relocation relocation,
1659 Register dst,
1660 Register tmp) {
1661 compiler::Label skip_reloc;
1662 __ b(&skip_reloc);
1663 InsertBSSRelocation(relocation);
1664 __ Bind(&skip_reloc);
1665
1666 __ adr(tmp, compiler::Immediate(-compiler::target::kWordSize));
1667
1668 // tmp holds the address of the relocation.
1669 __ ldr(dst, compiler::Address(tmp));
1670
1671 // dst holds the relocation itself: tmp - bss_start.
1672 // tmp = tmp + (bss_start - tmp) = bss_start
1673 __ add(tmp, tmp, compiler::Operand(dst));
1674
1675 // tmp holds the start of the BSS section.
1676 // Load the "get-thread" routine: *bss_start.
1677 __ ldr(dst, compiler::Address(tmp));
1678}
1679
1680#undef __
1681#define __ compiler_->assembler()->
1682
1683void ParallelMoveResolver::EmitSwap(int index) {
1684 MoveOperands* move = moves_[index];
1685 const Location source = move->src();
1686 const Location destination = move->dest();
1687
1688 if (source.IsRegister() && destination.IsRegister()) {
1689 ASSERT(source.reg() != TMP);
1690 ASSERT(destination.reg() != TMP);
1691 __ mov(TMP, source.reg());
1692 __ mov(source.reg(), destination.reg());
1693 __ mov(destination.reg(), TMP);
1694 } else if (source.IsRegister() && destination.IsStackSlot()) {
1695 Exchange(source.reg(), destination.base_reg(),
1696 destination.ToStackSlotOffset());
1697 } else if (source.IsStackSlot() && destination.IsRegister()) {
1698 Exchange(destination.reg(), source.base_reg(), source.ToStackSlotOffset());
1699 } else if (source.IsStackSlot() && destination.IsStackSlot()) {
1700 Exchange(source.base_reg(), source.ToStackSlotOffset(),
1701 destination.base_reg(), destination.ToStackSlotOffset());
1702 } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
1703 const VRegister dst = destination.fpu_reg();
1704 const VRegister src = source.fpu_reg();
1705 __ vmov(VTMP, src);
1706 __ vmov(src, dst);
1707 __ vmov(dst, VTMP);
1708 } else if (source.IsFpuRegister() || destination.IsFpuRegister()) {
1709 ASSERT(destination.IsDoubleStackSlot() || destination.IsQuadStackSlot() ||
1710 source.IsDoubleStackSlot() || source.IsQuadStackSlot());
1711 bool double_width =
1712 destination.IsDoubleStackSlot() || source.IsDoubleStackSlot();
1713 VRegister reg =
1714 source.IsFpuRegister() ? source.fpu_reg() : destination.fpu_reg();
1715 Register base_reg =
1716 source.IsFpuRegister() ? destination.base_reg() : source.base_reg();
1717 const intptr_t slot_offset = source.IsFpuRegister()
1718 ? destination.ToStackSlotOffset()
1719 : source.ToStackSlotOffset();
1720
1721 if (double_width) {
1722 __ LoadDFromOffset(VTMP, base_reg, slot_offset);
1723 __ StoreDToOffset(reg, base_reg, slot_offset);
1724 __ fmovdd(reg, VTMP);
1725 } else {
1726 __ LoadQFromOffset(VTMP, base_reg, slot_offset);
1727 __ StoreQToOffset(reg, base_reg, slot_offset);
1728 __ vmov(reg, VTMP);
1729 }
1730 } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) {
1731 const intptr_t source_offset = source.ToStackSlotOffset();
1732 const intptr_t dest_offset = destination.ToStackSlotOffset();
1733
1734 ScratchFpuRegisterScope ensure_scratch(this, kNoFpuRegister);
1735 VRegister scratch = ensure_scratch.reg();
1736 __ LoadDFromOffset(VTMP, source.base_reg(), source_offset);
1737 __ LoadDFromOffset(scratch, destination.base_reg(), dest_offset);
1738 __ StoreDToOffset(VTMP, destination.base_reg(), dest_offset);
1739 __ StoreDToOffset(scratch, source.base_reg(), source_offset);
1740 } else if (source.IsQuadStackSlot() && destination.IsQuadStackSlot()) {
1741 const intptr_t source_offset = source.ToStackSlotOffset();
1742 const intptr_t dest_offset = destination.ToStackSlotOffset();
1743
1744 ScratchFpuRegisterScope ensure_scratch(this, kNoFpuRegister);
1745 VRegister scratch = ensure_scratch.reg();
1746 __ LoadQFromOffset(VTMP, source.base_reg(), source_offset);
1747 __ LoadQFromOffset(scratch, destination.base_reg(), dest_offset);
1748 __ StoreQToOffset(VTMP, destination.base_reg(), dest_offset);
1749 __ StoreQToOffset(scratch, source.base_reg(), source_offset);
1750 } else {
1751 UNREACHABLE();
1752 }
1753
1754 // The swap of source and destination has executed a move from source to
1755 // destination.
1756 move->Eliminate();
1757
1758 // Any unperformed (including pending) move with a source of either
1759 // this move's source or destination needs to have their source
1760 // changed to reflect the state of affairs after the swap.
1761 for (int i = 0; i < moves_.length(); ++i) {
1762 const MoveOperands& other_move = *moves_[i];
1763 if (other_move.Blocks(source)) {
1764 moves_[i]->set_src(destination);
1765 } else if (other_move.Blocks(destination)) {
1766 moves_[i]->set_src(source);
1767 }
1768 }
1769}
1770
1771void ParallelMoveResolver::MoveMemoryToMemory(const compiler::Address& dst,
1772 const compiler::Address& src) {
1773 UNREACHABLE();
1774}
1775
1776// Do not call or implement this function. Instead, use the form below that
1777// uses an offset from the frame pointer instead of an Address.
1778void ParallelMoveResolver::Exchange(Register reg,
1779 const compiler::Address& mem) {
1780 UNREACHABLE();
1781}
1782
1783// Do not call or implement this function. Instead, use the form below that
1784// uses offsets from the frame pointer instead of Addresses.
1785void ParallelMoveResolver::Exchange(const compiler::Address& mem1,
1786 const compiler::Address& mem2) {
1787 UNREACHABLE();
1788}
1789
1790void ParallelMoveResolver::Exchange(Register reg,
1791 Register base_reg,
1792 intptr_t stack_offset) {
1793 ScratchRegisterScope tmp(this, reg);
1794 __ mov(tmp.reg(), reg);
1795 __ LoadFromOffset(reg, base_reg, stack_offset);
1796 __ StoreToOffset(tmp.reg(), base_reg, stack_offset);
1797}
1798
1799void ParallelMoveResolver::Exchange(Register base_reg1,
1800 intptr_t stack_offset1,
1801 Register base_reg2,
1802 intptr_t stack_offset2) {
1803 ScratchRegisterScope tmp1(this, kNoRegister);
1804 ScratchRegisterScope tmp2(this, tmp1.reg());
1805 __ LoadFromOffset(tmp1.reg(), base_reg1, stack_offset1);
1806 __ LoadFromOffset(tmp2.reg(), base_reg2, stack_offset2);
1807 __ StoreToOffset(tmp1.reg(), base_reg2, stack_offset2);
1808 __ StoreToOffset(tmp2.reg(), base_reg1, stack_offset1);
1809}
1810
1811void ParallelMoveResolver::SpillScratch(Register reg) {
1812 __ Push(reg);
1813}
1814
1815void ParallelMoveResolver::RestoreScratch(Register reg) {
1816 __ Pop(reg);
1817}
1818
1819void ParallelMoveResolver::SpillFpuScratch(FpuRegister reg) {
1820 __ PushDouble(reg);
1821}
1822
1823void ParallelMoveResolver::RestoreFpuScratch(FpuRegister reg) {
1824 __ PopDouble(reg);
1825}
1826
1827#undef __
1828
1829} // namespace dart
1830
1831#endif // defined(TARGET_ARCH_ARM64)
1832