1// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file
2// for details. All rights reserved. Use of this source code is governed by a
3// BSD-style license that can be found in the LICENSE file.
4
5#include "vm/globals.h" // Needed here to get TARGET_ARCH_X64.
6#if defined(TARGET_ARCH_X64)
7
8#include "vm/compiler/backend/flow_graph_compiler.h"
9
10#include "vm/compiler/api/type_check_mode.h"
11#include "vm/compiler/backend/il_printer.h"
12#include "vm/compiler/backend/locations.h"
13#include "vm/compiler/ffi/native_location.h"
14#include "vm/compiler/jit/compiler.h"
15#include "vm/dart_entry.h"
16#include "vm/deopt_instructions.h"
17#include "vm/dispatch_table.h"
18#include "vm/instructions.h"
19#include "vm/object_store.h"
20#include "vm/parser.h"
21#include "vm/stack_frame.h"
22#include "vm/stub_code.h"
23#include "vm/symbols.h"
24
25namespace dart {
26
27DEFINE_FLAG(bool, trap_on_deoptimization, false, "Trap on deoptimization.");
28DEFINE_FLAG(bool, unbox_mints, true, "Optimize 64-bit integer arithmetic.");
29DECLARE_FLAG(bool, enable_simd_inline);
30
31void FlowGraphCompiler::ArchSpecificInitialization() {
32 if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
33 auto object_store = isolate()->object_store();
34
35 const auto& stub =
36 Code::ZoneHandle(object_store->write_barrier_wrappers_stub());
37 if (CanPcRelativeCall(stub)) {
38 assembler_->generate_invoke_write_barrier_wrapper_ = [&](Register reg) {
39 const intptr_t offset_into_target =
40 Thread::WriteBarrierWrappersOffsetForRegister(reg);
41 assembler_->GenerateUnRelocatedPcRelativeCall(offset_into_target);
42 AddPcRelativeCallStubTarget(stub);
43 };
44 }
45
46 const auto& array_stub =
47 Code::ZoneHandle(object_store->array_write_barrier_stub());
48 if (CanPcRelativeCall(stub)) {
49 assembler_->generate_invoke_array_write_barrier_ = [&]() {
50 assembler_->GenerateUnRelocatedPcRelativeCall();
51 AddPcRelativeCallStubTarget(array_stub);
52 };
53 }
54 }
55}
56
57FlowGraphCompiler::~FlowGraphCompiler() {
58 // BlockInfos are zone-allocated, so their destructors are not called.
59 // Verify the labels explicitly here.
60 for (int i = 0; i < block_info_.length(); ++i) {
61 ASSERT(!block_info_[i]->jump_label()->IsLinked());
62 ASSERT(!block_info_[i]->jump_label()->HasNear());
63 }
64}
65
66bool FlowGraphCompiler::SupportsUnboxedDoubles() {
67 return true;
68}
69
70bool FlowGraphCompiler::SupportsUnboxedInt64() {
71 return FLAG_unbox_mints;
72}
73
74bool FlowGraphCompiler::SupportsUnboxedSimd128() {
75 return FLAG_enable_simd_inline;
76}
77
78bool FlowGraphCompiler::SupportsHardwareDivision() {
79 return true;
80}
81
82bool FlowGraphCompiler::CanConvertInt64ToDouble() {
83 return true;
84}
85
86void FlowGraphCompiler::EnterIntrinsicMode() {
87 ASSERT(!intrinsic_mode());
88 intrinsic_mode_ = true;
89 ASSERT(!assembler()->constant_pool_allowed());
90}
91
92void FlowGraphCompiler::ExitIntrinsicMode() {
93 ASSERT(intrinsic_mode());
94 intrinsic_mode_ = false;
95}
96
97TypedDataPtr CompilerDeoptInfo::CreateDeoptInfo(FlowGraphCompiler* compiler,
98 DeoptInfoBuilder* builder,
99 const Array& deopt_table) {
100 if (deopt_env_ == NULL) {
101 ++builder->current_info_number_;
102 return TypedData::null();
103 }
104
105 intptr_t stack_height = compiler->StackSize();
106 AllocateIncomingParametersRecursive(deopt_env_, &stack_height);
107
108 intptr_t slot_ix = 0;
109 Environment* current = deopt_env_;
110
111 // Emit all kMaterializeObject instructions describing objects to be
112 // materialized on the deoptimization as a prefix to the deoptimization info.
113 EmitMaterializations(deopt_env_, builder);
114
115 // The real frame starts here.
116 builder->MarkFrameStart();
117
118 Zone* zone = compiler->zone();
119
120 builder->AddPp(current->function(), slot_ix++);
121 builder->AddPcMarker(Function::ZoneHandle(zone), slot_ix++);
122 builder->AddCallerFp(slot_ix++);
123 builder->AddReturnAddress(current->function(), deopt_id(), slot_ix++);
124
125 // Emit all values that are needed for materialization as a part of the
126 // expression stack for the bottom-most frame. This guarantees that GC
127 // will be able to find them during materialization.
128 slot_ix = builder->EmitMaterializationArguments(slot_ix);
129
130 // For the innermost environment, set outgoing arguments and the locals.
131 for (intptr_t i = current->Length() - 1;
132 i >= current->fixed_parameter_count(); i--) {
133 builder->AddCopy(current->ValueAt(i), current->LocationAt(i), slot_ix++);
134 }
135
136 Environment* previous = current;
137 current = current->outer();
138 while (current != NULL) {
139 builder->AddPp(current->function(), slot_ix++);
140 builder->AddPcMarker(previous->function(), slot_ix++);
141 builder->AddCallerFp(slot_ix++);
142
143 // For any outer environment the deopt id is that of the call instruction
144 // which is recorded in the outer environment.
145 builder->AddReturnAddress(current->function(),
146 DeoptId::ToDeoptAfter(current->deopt_id()),
147 slot_ix++);
148
149 // The values of outgoing arguments can be changed from the inlined call so
150 // we must read them from the previous environment.
151 for (intptr_t i = previous->fixed_parameter_count() - 1; i >= 0; i--) {
152 builder->AddCopy(previous->ValueAt(i), previous->LocationAt(i),
153 slot_ix++);
154 }
155
156 // Set the locals, note that outgoing arguments are not in the environment.
157 for (intptr_t i = current->Length() - 1;
158 i >= current->fixed_parameter_count(); i--) {
159 builder->AddCopy(current->ValueAt(i), current->LocationAt(i), slot_ix++);
160 }
161
162 // Iterate on the outer environment.
163 previous = current;
164 current = current->outer();
165 }
166 // The previous pointer is now the outermost environment.
167 ASSERT(previous != NULL);
168
169 // Set slots for the outermost environment.
170 builder->AddCallerPp(slot_ix++);
171 builder->AddPcMarker(previous->function(), slot_ix++);
172 builder->AddCallerFp(slot_ix++);
173 builder->AddCallerPc(slot_ix++);
174
175 // For the outermost environment, set the incoming arguments.
176 for (intptr_t i = previous->fixed_parameter_count() - 1; i >= 0; i--) {
177 builder->AddCopy(previous->ValueAt(i), previous->LocationAt(i), slot_ix++);
178 }
179
180 return builder->CreateDeoptInfo(deopt_table);
181}
182
183void CompilerDeoptInfoWithStub::GenerateCode(FlowGraphCompiler* compiler,
184 intptr_t stub_ix) {
185 // Calls do not need stubs, they share a deoptimization trampoline.
186 ASSERT(reason() != ICData::kDeoptAtCall);
187 compiler::Assembler* assembler = compiler->assembler();
188#define __ assembler->
189 __ Comment("%s", Name());
190 __ Bind(entry_label());
191 if (FLAG_trap_on_deoptimization) {
192 __ int3();
193 }
194
195 ASSERT(deopt_env() != NULL);
196 __ call(compiler::Address(THR, Thread::deoptimize_entry_offset()));
197 set_pc_offset(assembler->CodeSize());
198 __ int3();
199#undef __
200}
201
202#define __ assembler()->
203
204// Fall through if bool_register contains null.
205void FlowGraphCompiler::GenerateBoolToJump(Register bool_register,
206 compiler::Label* is_true,
207 compiler::Label* is_false) {
208 compiler::Label fall_through;
209 __ CompareObject(bool_register, Object::null_object());
210 __ j(EQUAL, &fall_through, compiler::Assembler::kNearJump);
211 BranchLabels labels = {is_true, is_false, &fall_through};
212 Condition true_condition =
213 EmitBoolTest(bool_register, labels, /*invert=*/false);
214 ASSERT(true_condition != kInvalidCondition);
215 __ j(true_condition, is_true);
216 __ jmp(is_false);
217 __ Bind(&fall_through);
218}
219
220// Call stub to perform subtype test using a cache (see
221// stub_code_x64.cc:GenerateSubtypeNTestCacheStub)
222//
223// Inputs:
224// - RAX : instance to test against.
225// - RDX : instantiator type arguments (if necessary).
226// - RCX : function type arguments (if necessary).
227//
228// Preserves RAX/RCX/RDX.
229SubtypeTestCachePtr FlowGraphCompiler::GenerateCallSubtypeTestStub(
230 TypeTestStubKind test_kind,
231 Register instance_reg,
232 Register instantiator_type_arguments_reg,
233 Register function_type_arguments_reg,
234 Register temp_reg,
235 compiler::Label* is_instance_lbl,
236 compiler::Label* is_not_instance_lbl) {
237 ASSERT(temp_reg == kNoRegister);
238 const SubtypeTestCache& type_test_cache =
239 SubtypeTestCache::ZoneHandle(zone(), SubtypeTestCache::New());
240 __ LoadUniqueObject(R9, type_test_cache);
241 if (test_kind == kTestTypeOneArg) {
242 ASSERT(instantiator_type_arguments_reg == kNoRegister);
243 ASSERT(function_type_arguments_reg == kNoRegister);
244 __ Call(StubCode::Subtype1TestCache());
245 } else if (test_kind == kTestTypeTwoArgs) {
246 ASSERT(instantiator_type_arguments_reg == kNoRegister);
247 ASSERT(function_type_arguments_reg == kNoRegister);
248 __ Call(StubCode::Subtype2TestCache());
249 } else if (test_kind == kTestTypeFourArgs) {
250 ASSERT(instantiator_type_arguments_reg ==
251 TypeTestABI::kInstantiatorTypeArgumentsReg);
252 ASSERT(function_type_arguments_reg ==
253 TypeTestABI::kFunctionTypeArgumentsReg);
254 __ Call(StubCode::Subtype4TestCache());
255 } else if (test_kind == kTestTypeSixArgs) {
256 ASSERT(instantiator_type_arguments_reg ==
257 TypeTestABI::kInstantiatorTypeArgumentsReg);
258 ASSERT(function_type_arguments_reg ==
259 TypeTestABI::kFunctionTypeArgumentsReg);
260 __ Call(StubCode::Subtype6TestCache());
261 } else {
262 UNREACHABLE();
263 }
264 // Result is in R8: null -> not found, otherwise Bool::True or Bool::False.
265 GenerateBoolToJump(R8, is_instance_lbl, is_not_instance_lbl);
266 return type_test_cache.raw();
267}
268
269// Jumps to labels 'is_instance' or 'is_not_instance' respectively, if
270// type test is conclusive, otherwise fallthrough if a type test could not
271// be completed.
272// RAX: instance (must survive).
273// Clobbers R10.
274SubtypeTestCachePtr
275FlowGraphCompiler::GenerateInstantiatedTypeWithArgumentsTest(
276 TokenPosition token_pos,
277 const AbstractType& type,
278 compiler::Label* is_instance_lbl,
279 compiler::Label* is_not_instance_lbl) {
280 __ Comment("InstantiatedTypeWithArgumentsTest");
281 ASSERT(type.IsInstantiated());
282 ASSERT(!type.IsFunctionType());
283 const Class& type_class = Class::ZoneHandle(zone(), type.type_class());
284 ASSERT(type_class.NumTypeArguments() > 0);
285 const Type& smi_type = Type::Handle(zone(), Type::SmiType());
286 const bool smi_is_ok = smi_type.IsSubtypeOf(type, Heap::kOld);
287 __ testq(TypeTestABI::kInstanceReg, compiler::Immediate(kSmiTagMask));
288 if (smi_is_ok) {
289 // Fast case for type = FutureOr<int/num/top-type>.
290 __ j(ZERO, is_instance_lbl);
291 } else {
292 __ j(ZERO, is_not_instance_lbl);
293 }
294
295 const intptr_t num_type_args = type_class.NumTypeArguments();
296 const intptr_t num_type_params = type_class.NumTypeParameters();
297 const intptr_t from_index = num_type_args - num_type_params;
298 const TypeArguments& type_arguments =
299 TypeArguments::ZoneHandle(zone(), type.arguments());
300 const bool is_raw_type = type_arguments.IsNull() ||
301 type_arguments.IsRaw(from_index, num_type_params);
302 if (is_raw_type) {
303 const Register kClassIdReg = R10;
304 // dynamic type argument, check only classes.
305 __ LoadClassId(kClassIdReg, TypeTestABI::kInstanceReg);
306 __ cmpl(kClassIdReg, compiler::Immediate(type_class.id()));
307 __ j(EQUAL, is_instance_lbl);
308 // List is a very common case.
309 if (IsListClass(type_class)) {
310 GenerateListTypeCheck(kClassIdReg, is_instance_lbl);
311 }
312 return GenerateSubtype1TestCacheLookup(
313 token_pos, type_class, is_instance_lbl, is_not_instance_lbl);
314 }
315 // If one type argument only, check if type argument is a top type.
316 if (type_arguments.Length() == 1) {
317 const AbstractType& tp_argument =
318 AbstractType::ZoneHandle(zone(), type_arguments.TypeAt(0));
319 if (tp_argument.IsTopTypeForSubtyping()) {
320 // Instance class test only necessary.
321 return GenerateSubtype1TestCacheLookup(
322 token_pos, type_class, is_instance_lbl, is_not_instance_lbl);
323 }
324 }
325
326 // Regular subtype test cache involving instance's type arguments.
327 const Register kInstantiatorTypeArgumentsReg = kNoRegister;
328 const Register kFunctionTypeArgumentsReg = kNoRegister;
329 const Register kTempReg = kNoRegister;
330 return GenerateCallSubtypeTestStub(
331 kTestTypeTwoArgs, TypeTestABI::kInstanceReg,
332 kInstantiatorTypeArgumentsReg, kFunctionTypeArgumentsReg, kTempReg,
333 is_instance_lbl, is_not_instance_lbl);
334}
335
336void FlowGraphCompiler::CheckClassIds(Register class_id_reg,
337 const GrowableArray<intptr_t>& class_ids,
338 compiler::Label* is_equal_lbl,
339 compiler::Label* is_not_equal_lbl) {
340 for (intptr_t i = 0; i < class_ids.length(); i++) {
341 __ cmpl(class_id_reg, compiler::Immediate(class_ids[i]));
342 __ j(EQUAL, is_equal_lbl);
343 }
344 __ jmp(is_not_equal_lbl);
345}
346
347// Testing against an instantiated type with no arguments, without
348// SubtypeTestCache
349//
350// Inputs:
351// - RAX : instance to test against
352//
353// Preserves RAX/RCX/RDX.
354//
355// Returns true if there is a fallthrough.
356bool FlowGraphCompiler::GenerateInstantiatedTypeNoArgumentsTest(
357 TokenPosition token_pos,
358 const AbstractType& type,
359 compiler::Label* is_instance_lbl,
360 compiler::Label* is_not_instance_lbl) {
361 __ Comment("InstantiatedTypeNoArgumentsTest");
362 ASSERT(type.IsInstantiated());
363 ASSERT(!type.IsFunctionType());
364 const Class& type_class = Class::Handle(zone(), type.type_class());
365 ASSERT(type_class.NumTypeArguments() == 0);
366
367 __ testq(TypeTestABI::kInstanceReg, compiler::Immediate(kSmiTagMask));
368 // If instance is Smi, check directly.
369 const Class& smi_class = Class::Handle(zone(), Smi::Class());
370 if (Class::IsSubtypeOf(smi_class, Object::null_type_arguments(),
371 Nullability::kNonNullable, type, Heap::kOld)) {
372 // Fast case for type = int/num/top-type.
373 __ j(ZERO, is_instance_lbl);
374 } else {
375 __ j(ZERO, is_not_instance_lbl);
376 }
377 const Register kClassIdReg = R10;
378 __ LoadClassId(kClassIdReg, TypeTestABI::kInstanceReg);
379 // Bool interface can be implemented only by core class Bool.
380 if (type.IsBoolType()) {
381 __ cmpl(kClassIdReg, compiler::Immediate(kBoolCid));
382 __ j(EQUAL, is_instance_lbl);
383 __ jmp(is_not_instance_lbl);
384 return false;
385 }
386 // Custom checking for numbers (Smi, Mint and Double).
387 // Note that instance is not Smi (checked above).
388 if (type.IsNumberType() || type.IsIntType() || type.IsDoubleType()) {
389 GenerateNumberTypeCheck(kClassIdReg, type, is_instance_lbl,
390 is_not_instance_lbl);
391 return false;
392 }
393 if (type.IsStringType()) {
394 GenerateStringTypeCheck(kClassIdReg, is_instance_lbl, is_not_instance_lbl);
395 return false;
396 }
397 if (type.IsDartFunctionType()) {
398 // Check if instance is a closure.
399 __ cmpq(kClassIdReg, compiler::Immediate(kClosureCid));
400 __ j(EQUAL, is_instance_lbl);
401 return true;
402 }
403
404 // Fast case for cid-range based checks.
405 // Warning: This code destroys the contents of [kClassIdReg].
406 if (GenerateSubtypeRangeCheck(kClassIdReg, type_class, is_instance_lbl)) {
407 return false;
408 }
409
410 // Otherwise fallthrough, result non-conclusive.
411 return true;
412}
413
414// Uses SubtypeTestCache to store instance class and result.
415// Immediate class test already done.
416//
417// Inputs:
418// RAX : instance to test against.
419//
420// Preserves RAX/RCX/RDX.
421//
422// TODO(srdjan): Implement a quicker subtype check, as type test
423// arrays can grow too high, but they may be useful when optimizing
424// code (type-feedback).
425SubtypeTestCachePtr FlowGraphCompiler::GenerateSubtype1TestCacheLookup(
426 TokenPosition token_pos,
427 const Class& type_class,
428 compiler::Label* is_instance_lbl,
429 compiler::Label* is_not_instance_lbl) {
430 __ Comment("Subtype1TestCacheLookup");
431#if defined(DEBUG)
432 compiler::Label ok;
433 __ BranchIfNotSmi(TypeTestABI::kInstanceReg, &ok);
434 __ Breakpoint();
435 __ Bind(&ok);
436#endif
437 __ LoadClassId(TMP, TypeTestABI::kInstanceReg);
438 __ LoadClassById(R10, TMP);
439 // R10: instance class.
440 // Check immediate superclass equality. If type_class is Object, then testing
441 // supertype may yield a wrong result for Null in NNBD strong mode (because
442 // Null also extends Object).
443 if (!type_class.IsObjectClass() || !Isolate::Current()->null_safety()) {
444 __ movq(R13, compiler::FieldAddress(R10, Class::super_type_offset()));
445 __ movq(R13, compiler::FieldAddress(R13, Type::type_class_id_offset()));
446 __ CompareImmediate(R13,
447 compiler::Immediate(Smi::RawValue(type_class.id())));
448 __ j(EQUAL, is_instance_lbl);
449 }
450
451 const Register kInstantiatorTypeArgumentsReg = kNoRegister;
452 const Register kFunctionTypeArgumentsReg = kNoRegister;
453 const Register kTempReg = kNoRegister;
454 return GenerateCallSubtypeTestStub(kTestTypeOneArg, TypeTestABI::kInstanceReg,
455 kInstantiatorTypeArgumentsReg,
456 kFunctionTypeArgumentsReg, kTempReg,
457 is_instance_lbl, is_not_instance_lbl);
458}
459
460// Generates inlined check if 'type' is a type parameter or type itself
461//
462// Inputs:
463// - RAX : instance to test against.
464// - RDX : instantiator type arguments (if necessary).
465// - RCX : function type arguments (if necessary).
466//
467// Preserves RAX/RCX/RDX.
468SubtypeTestCachePtr FlowGraphCompiler::GenerateUninstantiatedTypeTest(
469 TokenPosition token_pos,
470 const AbstractType& type,
471 compiler::Label* is_instance_lbl,
472 compiler::Label* is_not_instance_lbl) {
473 const Register kTempReg = kNoRegister;
474 __ Comment("UninstantiatedTypeTest");
475 ASSERT(!type.IsInstantiated());
476 ASSERT(!type.IsFunctionType());
477 // Skip check if destination is a dynamic type.
478 if (type.IsTypeParameter()) {
479 const TypeParameter& type_param = TypeParameter::Cast(type);
480
481 const Register kTypeArgumentsReg =
482 type_param.IsClassTypeParameter()
483 ? TypeTestABI::kInstantiatorTypeArgumentsReg
484 : TypeTestABI::kFunctionTypeArgumentsReg;
485 // Check if type arguments are null, i.e. equivalent to vector of dynamic.
486 __ CompareObject(kTypeArgumentsReg, Object::null_object());
487 __ j(EQUAL, is_instance_lbl);
488 __ movq(RDI, compiler::FieldAddress(
489 kTypeArgumentsReg,
490 TypeArguments::type_at_offset(type_param.index())));
491 // RDI: Concrete type of type.
492 // Check if type argument is dynamic, Object?, or void.
493 __ CompareObject(RDI, Object::dynamic_type());
494 __ j(EQUAL, is_instance_lbl);
495 __ CompareObject(
496 RDI, Type::ZoneHandle(
497 zone(), isolate()->object_store()->nullable_object_type()));
498 __ j(EQUAL, is_instance_lbl);
499 __ CompareObject(RDI, Object::void_type());
500 __ j(EQUAL, is_instance_lbl);
501
502 // For Smi check quickly against int and num interfaces.
503 compiler::Label not_smi;
504 __ testq(RAX, compiler::Immediate(kSmiTagMask)); // Value is Smi?
505 __ j(NOT_ZERO, &not_smi, compiler::Assembler::kNearJump);
506 __ CompareObject(RDI, Type::ZoneHandle(zone(), Type::IntType()));
507 __ j(EQUAL, is_instance_lbl);
508 __ CompareObject(RDI, Type::ZoneHandle(zone(), Type::Number()));
509 __ j(EQUAL, is_instance_lbl);
510 // Smi can be handled by type test cache.
511 __ Bind(&not_smi);
512
513 const auto test_kind = GetTypeTestStubKindForTypeParameter(type_param);
514 const SubtypeTestCache& type_test_cache = SubtypeTestCache::ZoneHandle(
515 zone(), GenerateCallSubtypeTestStub(
516 test_kind, TypeTestABI::kInstanceReg,
517 TypeTestABI::kInstantiatorTypeArgumentsReg,
518 TypeTestABI::kFunctionTypeArgumentsReg, kTempReg,
519 is_instance_lbl, is_not_instance_lbl));
520 return type_test_cache.raw();
521 }
522 if (type.IsType()) {
523 // Smi is FutureOr<T>, when T is a top type or int or num.
524 if (!type.IsFutureOrType()) {
525 __ testq(TypeTestABI::kInstanceReg,
526 compiler::Immediate(kSmiTagMask)); // Is instance Smi?
527 __ j(ZERO, is_not_instance_lbl);
528 }
529 // Uninstantiated type class is known at compile time, but the type
530 // arguments are determined at runtime by the instantiator(s).
531 return GenerateCallSubtypeTestStub(
532 kTestTypeFourArgs, TypeTestABI::kInstanceReg,
533 TypeTestABI::kInstantiatorTypeArgumentsReg,
534 TypeTestABI::kFunctionTypeArgumentsReg, kTempReg, is_instance_lbl,
535 is_not_instance_lbl);
536 }
537 return SubtypeTestCache::null();
538}
539
540// Generates function type check.
541//
542// See [GenerateUninstantiatedTypeTest] for calling convention.
543SubtypeTestCachePtr FlowGraphCompiler::GenerateFunctionTypeTest(
544 TokenPosition token_pos,
545 const AbstractType& type,
546 compiler::Label* is_instance_lbl,
547 compiler::Label* is_not_instance_lbl) {
548 const Register kTempReg = kNoRegister;
549 __ Comment("FunctionTypeTest");
550
551 __ testq(TypeTestABI::kInstanceReg, compiler::Immediate(kSmiTagMask));
552 __ j(ZERO, is_not_instance_lbl);
553 return GenerateCallSubtypeTestStub(
554 kTestTypeSixArgs, TypeTestABI::kInstanceReg,
555 TypeTestABI::kInstantiatorTypeArgumentsReg,
556 TypeTestABI::kFunctionTypeArgumentsReg, kTempReg, is_instance_lbl,
557 is_not_instance_lbl);
558}
559
560// Inputs:
561// - RAX : instance to test against.
562// - RDX : instantiator type arguments.
563// - RCX : function type arguments.
564//
565// Preserves RAX/RCX/RDX.
566//
567// Note that this inlined code must be followed by the runtime_call code, as it
568// may fall through to it. Otherwise, this inline code will jump to the label
569// is_instance or to the label is_not_instance.
570SubtypeTestCachePtr FlowGraphCompiler::GenerateInlineInstanceof(
571 TokenPosition token_pos,
572 const AbstractType& type,
573 compiler::Label* is_instance_lbl,
574 compiler::Label* is_not_instance_lbl) {
575 __ Comment("InlineInstanceof");
576
577 if (type.IsFunctionType()) {
578 return GenerateFunctionTypeTest(token_pos, type, is_instance_lbl,
579 is_not_instance_lbl);
580 }
581
582 if (type.IsInstantiated()) {
583 const Class& type_class = Class::ZoneHandle(zone(), type.type_class());
584 // A class equality check is only applicable with a dst type (not a
585 // function type) of a non-parameterized class or with a raw dst type of
586 // a parameterized class.
587 if (type_class.NumTypeArguments() > 0) {
588 return GenerateInstantiatedTypeWithArgumentsTest(
589 token_pos, type, is_instance_lbl, is_not_instance_lbl);
590 // Fall through to runtime call.
591 }
592 const bool has_fall_through = GenerateInstantiatedTypeNoArgumentsTest(
593 token_pos, type, is_instance_lbl, is_not_instance_lbl);
594 if (has_fall_through) {
595 // If test non-conclusive so far, try the inlined type-test cache.
596 // 'type' is known at compile time.
597 return GenerateSubtype1TestCacheLookup(
598 token_pos, type_class, is_instance_lbl, is_not_instance_lbl);
599 } else {
600 return SubtypeTestCache::null();
601 }
602 }
603 return GenerateUninstantiatedTypeTest(token_pos, type, is_instance_lbl,
604 is_not_instance_lbl);
605}
606
607// If instanceof type test cannot be performed successfully at compile time and
608// therefore eliminated, optimize it by adding inlined tests for:
609// - Null -> see comment below.
610// - Smi -> compile time subtype check (only if dst class is not parameterized).
611// - Class equality (only if class is not parameterized).
612// Inputs:
613// - RAX: object.
614// - RDX: instantiator type arguments or raw_null.
615// - RCX: function type arguments or raw_null.
616// Returns:
617// - true or false in RAX.
618void FlowGraphCompiler::GenerateInstanceOf(TokenPosition token_pos,
619 intptr_t deopt_id,
620 const AbstractType& type,
621 LocationSummary* locs) {
622 ASSERT(type.IsFinalized());
623 ASSERT(!type.IsTopTypeForInstanceOf()); // Already checked.
624
625 compiler::Label is_instance, is_not_instance;
626 // 'null' is an instance of Null, Object*, Never*, void, and dynamic.
627 // In addition, 'null' is an instance of any nullable type.
628 // It is also an instance of FutureOr<T> if it is an instance of T.
629 const AbstractType& unwrapped_type =
630 AbstractType::Handle(type.UnwrapFutureOr());
631 if (!unwrapped_type.IsTypeParameter() || unwrapped_type.IsNullable()) {
632 // Only nullable type parameter remains nullable after instantiation.
633 // See NullIsInstanceOf().
634 __ CompareObject(TypeTestABI::kInstanceReg, Object::null_object());
635 __ j(EQUAL, (unwrapped_type.IsNullable() ||
636 (unwrapped_type.IsLegacy() && unwrapped_type.IsNeverType()))
637 ? &is_instance
638 : &is_not_instance);
639 }
640
641 // Generate inline instanceof test.
642 SubtypeTestCache& test_cache = SubtypeTestCache::ZoneHandle(zone());
643 // The registers RAX, RCX, RDX are preserved across the call.
644 test_cache =
645 GenerateInlineInstanceof(token_pos, type, &is_instance, &is_not_instance);
646
647 // test_cache is null if there is no fall-through.
648 compiler::Label done;
649 if (!test_cache.IsNull()) {
650 // Generate Runtime call.
651 __ LoadUniqueObject(TypeTestABI::kDstTypeReg, type);
652 __ LoadUniqueObject(TypeTestABI::kSubtypeTestCacheReg, test_cache);
653 GenerateStubCall(token_pos, StubCode::InstanceOf(),
654 /*kind=*/PcDescriptorsLayout::kOther, locs, deopt_id);
655 __ jmp(&done, compiler::Assembler::kNearJump);
656 }
657 __ Bind(&is_not_instance);
658 __ LoadObject(RAX, Bool::Get(false));
659 __ jmp(&done, compiler::Assembler::kNearJump);
660
661 __ Bind(&is_instance);
662 __ LoadObject(RAX, Bool::Get(true));
663 __ Bind(&done);
664}
665
666// Optimize assignable type check by adding inlined tests for:
667// - NULL -> return NULL.
668// - Smi -> compile time subtype check (only if dst class is not parameterized).
669// - Class equality (only if class is not parameterized).
670// Inputs:
671// - RAX: object.
672// - RBX: destination type (if non-constant).
673// - RDX: instantiator type arguments or raw_null.
674// - RCX: function type arguments or raw_null.
675// Returns:
676// - object in RAX for successful assignable check (or throws TypeError).
677// Performance notes: positive checks must be quick, negative checks can be slow
678// as they throw an exception.
679void FlowGraphCompiler::GenerateAssertAssignable(CompileType* receiver_type,
680 TokenPosition token_pos,
681 intptr_t deopt_id,
682 const String& dst_name,
683 LocationSummary* locs) {
684 ASSERT(!token_pos.IsClassifying());
685 ASSERT(CheckAssertAssignableTypeTestingABILocations(*locs));
686
687 compiler::Label is_assignable, runtime_call;
688
689 // Generate inline type check, linking to runtime call if not assignable.
690 SubtypeTestCache& test_cache = SubtypeTestCache::ZoneHandle(zone());
691
692 if (locs->in(1).IsConstant()) {
693 const auto& dst_type = AbstractType::Cast(locs->in(1).constant());
694 ASSERT(dst_type.IsFinalized());
695
696 if (dst_type.IsTopTypeForSubtyping()) return; // No code needed.
697
698 if (ShouldUseTypeTestingStubFor(is_optimizing(), dst_type)) {
699 GenerateAssertAssignableViaTypeTestingStub(receiver_type, token_pos,
700 deopt_id, dst_name, locs);
701 return;
702 }
703
704 if (Instance::NullIsAssignableTo(dst_type)) {
705 __ CompareObject(TypeTestABI::kInstanceReg, Object::null_object());
706 __ j(EQUAL, &is_assignable);
707 }
708
709 // The registers RAX, RCX, RDX are preserved across the call.
710 test_cache = GenerateInlineInstanceof(token_pos, dst_type, &is_assignable,
711 &runtime_call);
712
713 } else {
714 // TODO(dartbug.com/40813): Handle setting up the non-constant case.
715 UNREACHABLE();
716 }
717
718 __ Bind(&runtime_call);
719 __ PushObject(Object::null_object()); // Make room for the result.
720 __ pushq(TypeTestABI::kInstanceReg); // Push the source object.
721 // Push the destination type.
722 if (locs->in(1).IsConstant()) {
723 __ PushObject(locs->in(1).constant());
724 } else {
725 // TODO(dartbug.com/40813): Handle setting up the non-constant case.
726 UNREACHABLE();
727 }
728 __ pushq(TypeTestABI::kInstantiatorTypeArgumentsReg);
729 __ pushq(TypeTestABI::kFunctionTypeArgumentsReg);
730 __ PushObject(dst_name); // Push the name of the destination.
731 __ LoadUniqueObject(RAX, test_cache);
732 __ pushq(RAX);
733 __ PushImmediate(compiler::Immediate(Smi::RawValue(kTypeCheckFromInline)));
734 GenerateRuntimeCall(token_pos, deopt_id, kTypeCheckRuntimeEntry, 7, locs);
735 // Pop the parameters supplied to the runtime entry. The result of the
736 // type check runtime call is the checked value.
737 __ Drop(7);
738 __ popq(TypeTestABI::kInstanceReg);
739 __ Bind(&is_assignable);
740}
741
742void FlowGraphCompiler::GenerateAssertAssignableViaTypeTestingStub(
743 CompileType* receiver_type,
744 TokenPosition token_pos,
745 intptr_t deopt_id,
746 const String& dst_name,
747 LocationSummary* locs) {
748 ASSERT(CheckAssertAssignableTypeTestingABILocations(*locs));
749 // We must have a constant dst_type for generating a call to the stub.
750 ASSERT(locs->in(1).IsConstant());
751 const auto& dst_type = AbstractType::Cast(locs->in(1).constant());
752
753 // If the dst_type is instantiated we know the target TTS stub at
754 // compile-time and can therefore use a pc-relative call.
755 const bool use_pc_relative_call =
756 dst_type.IsInstantiated() && CanPcRelativeCall(dst_type);
757 const Register kScratchReg =
758 dst_type.IsTypeParameter() ? RSI : TypeTestABI::kDstTypeReg;
759
760 const Register kRegToCall = use_pc_relative_call ? kNoRegister : kScratchReg;
761
762 compiler::Label done;
763
764 GenerateAssertAssignableViaTypeTestingStub(receiver_type, dst_type, dst_name,
765 kRegToCall, kScratchReg, &done);
766
767 // We use 2 consecutive entries in the pool for the subtype cache and the
768 // destination name. The second entry, namely [dst_name] seems to be unused,
769 // but it will be used by the code throwing a TypeError if the type test fails
770 // (see runtime/vm/runtime_entry.cc:TypeCheck). It will use pattern matching
771 // on the call site to find out at which pool index the destination name is
772 // located.
773 const intptr_t sub_type_cache_index = __ object_pool_builder().AddObject(
774 Object::null_object(), compiler::ObjectPoolBuilderEntry::kPatchable);
775 const intptr_t sub_type_cache_offset =
776 ObjectPool::element_offset(sub_type_cache_index) - kHeapObjectTag;
777 const intptr_t dst_name_index = __ object_pool_builder().AddObject(
778 dst_name, compiler::ObjectPoolBuilderEntry::kPatchable);
779 ASSERT((sub_type_cache_index + 1) == dst_name_index);
780 ASSERT(__ constant_pool_allowed());
781
782 __ movq(TypeTestABI::kSubtypeTestCacheReg,
783 compiler::Address(PP, sub_type_cache_offset));
784 if (use_pc_relative_call) {
785 __ GenerateUnRelocatedPcRelativeCall();
786 AddPcRelativeTTSCallTypeTarget(dst_type);
787 } else {
788 __ call(compiler::FieldAddress(
789 kRegToCall, AbstractType::type_test_stub_entry_point_offset()));
790 }
791 EmitCallsiteMetadata(token_pos, deopt_id, PcDescriptorsLayout::kOther, locs);
792 __ Bind(&done);
793}
794
795void FlowGraphCompiler::EmitInstructionEpilogue(Instruction* instr) {
796 if (is_optimizing()) {
797 return;
798 }
799 Definition* defn = instr->AsDefinition();
800 if ((defn != NULL) && defn->HasTemp()) {
801 Location value = defn->locs()->out(0);
802 if (value.IsRegister()) {
803 __ pushq(value.reg());
804 } else if (value.IsConstant()) {
805 __ PushObject(value.constant());
806 } else {
807 ASSERT(value.IsStackSlot());
808 __ pushq(LocationToStackSlotAddress(value));
809 }
810 }
811}
812
813void FlowGraphCompiler::GenerateMethodExtractorIntrinsic(
814 const Function& extracted_method,
815 intptr_t type_arguments_field_offset) {
816 // No frame has been setup here.
817 ASSERT(!__ constant_pool_allowed());
818 ASSERT(extracted_method.IsZoneHandle());
819
820 const Code& build_method_extractor = Code::ZoneHandle(
821 isolate()->object_store()->build_method_extractor_code());
822 ASSERT(!build_method_extractor.IsNull());
823
824 const intptr_t stub_index = __ object_pool_builder().AddObject(
825 build_method_extractor, compiler::ObjectPoolBuilderEntry::kNotPatchable);
826 const intptr_t function_index = __ object_pool_builder().AddObject(
827 extracted_method, compiler::ObjectPoolBuilderEntry::kNotPatchable);
828
829 // We use a custom pool register to preserve caller PP.
830 Register kPoolReg = RAX;
831
832 // RBX = extracted function
833 // RDX = offset of type argument vector (or 0 if class is not generic)
834 if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
835 kPoolReg = PP;
836 } else {
837 __ movq(kPoolReg,
838 compiler::FieldAddress(CODE_REG, Code::object_pool_offset()));
839 }
840 __ movq(RDX, compiler::Immediate(type_arguments_field_offset));
841 __ movq(RBX, compiler::FieldAddress(
842 kPoolReg, ObjectPool::element_offset(function_index)));
843 __ movq(CODE_REG, compiler::FieldAddress(
844 kPoolReg, ObjectPool::element_offset(stub_index)));
845 __ jmp(compiler::FieldAddress(
846 CODE_REG, Code::entry_point_offset(Code::EntryKind::kUnchecked)));
847}
848
849// NOTE: If the entry code shape changes, ReturnAddressLocator in profiler.cc
850// needs to be updated to match.
851void FlowGraphCompiler::EmitFrameEntry() {
852 if (flow_graph().IsCompiledForOsr()) {
853 const intptr_t extra_slots = ExtraStackSlotsOnOsrEntry();
854 ASSERT(extra_slots >= 0);
855 __ EnterOsrFrame(extra_slots * kWordSize);
856 } else {
857 const Function& function = parsed_function().function();
858 if (CanOptimizeFunction() && function.IsOptimizable() &&
859 (!is_optimizing() || may_reoptimize())) {
860 __ Comment("Invocation Count Check");
861 const Register function_reg = RDI;
862 __ movq(function_reg,
863 compiler::FieldAddress(CODE_REG, Code::owner_offset()));
864
865 // Reoptimization of an optimized function is triggered by counting in
866 // IC stubs, but not at the entry of the function.
867 if (!is_optimizing()) {
868 __ incl(compiler::FieldAddress(function_reg,
869 Function::usage_counter_offset()));
870 }
871 __ cmpl(compiler::FieldAddress(function_reg,
872 Function::usage_counter_offset()),
873 compiler::Immediate(GetOptimizationThreshold()));
874 ASSERT(function_reg == RDI);
875 compiler::Label dont_optimize;
876 __ j(LESS, &dont_optimize, compiler::Assembler::kNearJump);
877 __ jmp(compiler::Address(THR, Thread::optimize_entry_offset()));
878 __ Bind(&dont_optimize);
879 }
880 ASSERT(StackSize() >= 0);
881 __ Comment("Enter frame");
882 __ EnterDartFrame(StackSize() * kWordSize);
883 }
884}
885
886void FlowGraphCompiler::EmitPrologue() {
887 BeginCodeSourceRange();
888
889 EmitFrameEntry();
890 ASSERT(assembler()->constant_pool_allowed());
891
892 // In unoptimized code, initialize (non-argument) stack allocated slots.
893 if (!is_optimizing()) {
894 const int num_locals = parsed_function().num_stack_locals();
895
896 intptr_t args_desc_slot = -1;
897 if (parsed_function().has_arg_desc_var()) {
898 args_desc_slot = compiler::target::frame_layout.FrameSlotForVariable(
899 parsed_function().arg_desc_var());
900 }
901
902 __ Comment("Initialize spill slots");
903 if (num_locals > 1 || (num_locals == 1 && args_desc_slot == -1)) {
904 __ LoadObject(RAX, Object::null_object());
905 }
906 for (intptr_t i = 0; i < num_locals; ++i) {
907 const intptr_t slot_index =
908 compiler::target::frame_layout.FrameSlotForVariableIndex(-i);
909 Register value_reg = slot_index == args_desc_slot ? ARGS_DESC_REG : RAX;
910 __ movq(compiler::Address(RBP, slot_index * kWordSize), value_reg);
911 }
912 }
913
914 EndCodeSourceRange(TokenPosition::kDartCodePrologue);
915}
916
917void FlowGraphCompiler::CompileGraph() {
918 InitCompiler();
919
920 // We have multiple entrypoints functionality which moved the frame
921 // setup into the [FunctionEntryInstr] (which will set the constant pool
922 // allowed bit to true). Despite this we still have to set the
923 // constant pool allowed bit to true here as well, because we can generate
924 // code for [CatchEntryInstr]s, which need the pool.
925 __ set_constant_pool_allowed(true);
926
927 ASSERT(!block_order().is_empty());
928 VisitBlocks();
929
930#if defined(DEBUG)
931 __ int3();
932#endif
933
934 if (!skip_body_compilation()) {
935 ASSERT(assembler()->constant_pool_allowed());
936 GenerateDeferredCode();
937 }
938
939 for (intptr_t i = 0; i < indirect_gotos_.length(); ++i) {
940 indirect_gotos_[i]->ComputeOffsetTable(this);
941 }
942}
943
944void FlowGraphCompiler::EmitCallToStub(const Code& stub) {
945 ASSERT(!stub.IsNull());
946 if (CanPcRelativeCall(stub)) {
947 __ GenerateUnRelocatedPcRelativeCall();
948 AddPcRelativeCallStubTarget(stub);
949 } else {
950 __ Call(stub);
951 AddStubCallTarget(stub);
952 }
953}
954
955void FlowGraphCompiler::EmitTailCallToStub(const Code& stub) {
956 ASSERT(!stub.IsNull());
957 if (CanPcRelativeCall(stub)) {
958 __ LeaveDartFrame();
959 __ GenerateUnRelocatedPcRelativeTailCall();
960 AddPcRelativeTailCallStubTarget(stub);
961#if defined(DEBUG)
962 __ Breakpoint();
963#endif
964 } else {
965 __ LoadObject(CODE_REG, stub);
966 __ LeaveDartFrame();
967 __ jmp(compiler::FieldAddress(
968 CODE_REG, compiler::target::Code::entry_point_offset()));
969 AddStubCallTarget(stub);
970 }
971}
972
973void FlowGraphCompiler::GeneratePatchableCall(TokenPosition token_pos,
974 const Code& stub,
975 PcDescriptorsLayout::Kind kind,
976 LocationSummary* locs) {
977 __ CallPatchable(stub);
978 EmitCallsiteMetadata(token_pos, DeoptId::kNone, kind, locs);
979}
980
981void FlowGraphCompiler::GenerateDartCall(intptr_t deopt_id,
982 TokenPosition token_pos,
983 const Code& stub,
984 PcDescriptorsLayout::Kind kind,
985 LocationSummary* locs,
986 Code::EntryKind entry_kind) {
987 ASSERT(CanCallDart());
988 __ CallPatchable(stub, entry_kind);
989 EmitCallsiteMetadata(token_pos, deopt_id, kind, locs);
990}
991
992void FlowGraphCompiler::GenerateStaticDartCall(intptr_t deopt_id,
993 TokenPosition token_pos,
994 PcDescriptorsLayout::Kind kind,
995 LocationSummary* locs,
996 const Function& target,
997 Code::EntryKind entry_kind) {
998 ASSERT(CanCallDart());
999 ASSERT(is_optimizing());
1000 if (CanPcRelativeCall(target)) {
1001 __ GenerateUnRelocatedPcRelativeCall();
1002 AddPcRelativeCallTarget(target, entry_kind);
1003 EmitCallsiteMetadata(token_pos, deopt_id, kind, locs);
1004 } else {
1005 // Call sites to the same target can share object pool entries. These
1006 // call sites are never patched for breakpoints: the function is deoptimized
1007 // and the unoptimized code with IC calls for static calls is patched
1008 // instead.
1009 const auto& stub_entry = StubCode::CallStaticFunction();
1010 __ CallWithEquivalence(stub_entry, target, entry_kind);
1011 EmitCallsiteMetadata(token_pos, deopt_id, kind, locs);
1012 AddStaticCallTarget(target, entry_kind);
1013 }
1014}
1015
1016void FlowGraphCompiler::GenerateRuntimeCall(TokenPosition token_pos,
1017 intptr_t deopt_id,
1018 const RuntimeEntry& entry,
1019 intptr_t argument_count,
1020 LocationSummary* locs) {
1021 __ CallRuntime(entry, argument_count);
1022 EmitCallsiteMetadata(token_pos, deopt_id, PcDescriptorsLayout::kOther, locs);
1023}
1024
1025void FlowGraphCompiler::EmitUnoptimizedStaticCall(intptr_t size_with_type_args,
1026 intptr_t deopt_id,
1027 TokenPosition token_pos,
1028 LocationSummary* locs,
1029 const ICData& ic_data,
1030 Code::EntryKind entry_kind) {
1031 ASSERT(CanCallDart());
1032 const Code& stub =
1033 StubCode::UnoptimizedStaticCallEntry(ic_data.NumArgsTested());
1034 __ LoadObject(RBX, ic_data);
1035 GenerateDartCall(deopt_id, token_pos, stub,
1036 PcDescriptorsLayout::kUnoptStaticCall, locs, entry_kind);
1037 __ Drop(size_with_type_args, RCX);
1038}
1039
1040void FlowGraphCompiler::EmitEdgeCounter(intptr_t edge_id) {
1041 // We do not check for overflow when incrementing the edge counter. The
1042 // function should normally be optimized long before the counter can
1043 // overflow; and though we do not reset the counters when we optimize or
1044 // deoptimize, there is a bound on the number of
1045 // optimization/deoptimization cycles we will attempt.
1046 ASSERT(!edge_counters_array_.IsNull());
1047 ASSERT(assembler_->constant_pool_allowed());
1048 __ Comment("Edge counter");
1049 __ LoadObject(RAX, edge_counters_array_);
1050 __ IncrementSmiField(
1051 compiler::FieldAddress(RAX, Array::element_offset(edge_id)), 1);
1052}
1053
1054void FlowGraphCompiler::EmitOptimizedInstanceCall(const Code& stub,
1055 const ICData& ic_data,
1056 intptr_t deopt_id,
1057 TokenPosition token_pos,
1058 LocationSummary* locs,
1059 Code::EntryKind entry_kind) {
1060 ASSERT(CanCallDart());
1061 ASSERT(Array::Handle(zone(), ic_data.arguments_descriptor()).Length() > 0);
1062 // Each ICData propagated from unoptimized to optimized code contains the
1063 // function that corresponds to the Dart function of that IC call. Due
1064 // to inlining in optimized code, that function may not correspond to the
1065 // top-level function (parsed_function().function()) which could be
1066 // reoptimized and which counter needs to be incremented.
1067 // Pass the function explicitly, it is used in IC stub.
1068 __ LoadObject(RDI, parsed_function().function());
1069 // Load receiver into RDX.
1070 __ movq(RDX, compiler::Address(
1071 RSP, (ic_data.SizeWithoutTypeArgs() - 1) * kWordSize));
1072 __ LoadUniqueObject(RBX, ic_data);
1073 GenerateDartCall(deopt_id, token_pos, stub, PcDescriptorsLayout::kIcCall,
1074 locs, entry_kind);
1075 __ Drop(ic_data.SizeWithTypeArgs(), RCX);
1076}
1077
1078void FlowGraphCompiler::EmitInstanceCallJIT(const Code& stub,
1079 const ICData& ic_data,
1080 intptr_t deopt_id,
1081 TokenPosition token_pos,
1082 LocationSummary* locs,
1083 Code::EntryKind entry_kind) {
1084 ASSERT(CanCallDart());
1085 ASSERT(entry_kind == Code::EntryKind::kNormal ||
1086 entry_kind == Code::EntryKind::kUnchecked);
1087 ASSERT(Array::Handle(zone(), ic_data.arguments_descriptor()).Length() > 0);
1088 // Load receiver into RDX.
1089 __ movq(RDX, compiler::Address(
1090 RSP, (ic_data.SizeWithoutTypeArgs() - 1) * kWordSize));
1091 __ LoadUniqueObject(RBX, ic_data);
1092 __ LoadUniqueObject(CODE_REG, stub);
1093 const intptr_t entry_point_offset =
1094 entry_kind == Code::EntryKind::kNormal
1095 ? Code::entry_point_offset(Code::EntryKind::kMonomorphic)
1096 : Code::entry_point_offset(Code::EntryKind::kMonomorphicUnchecked);
1097 __ call(compiler::FieldAddress(CODE_REG, entry_point_offset));
1098 EmitCallsiteMetadata(token_pos, deopt_id, PcDescriptorsLayout::kIcCall, locs);
1099 __ Drop(ic_data.SizeWithTypeArgs(), RCX);
1100}
1101
1102void FlowGraphCompiler::EmitMegamorphicInstanceCall(
1103 const String& name,
1104 const Array& arguments_descriptor,
1105 intptr_t deopt_id,
1106 TokenPosition token_pos,
1107 LocationSummary* locs,
1108 intptr_t try_index,
1109 intptr_t slow_path_argument_count) {
1110 ASSERT(CanCallDart());
1111 ASSERT(!arguments_descriptor.IsNull() && (arguments_descriptor.Length() > 0));
1112 const ArgumentsDescriptor args_desc(arguments_descriptor);
1113 const MegamorphicCache& cache = MegamorphicCache::ZoneHandle(
1114 zone(),
1115 MegamorphicCacheTable::Lookup(thread(), name, arguments_descriptor));
1116 __ Comment("MegamorphicCall");
1117 // Load receiver into RDX.
1118 __ movq(RDX, compiler::Address(RSP, (args_desc.Count() - 1) * kWordSize));
1119
1120 // Use same code pattern as instance call so it can be parsed by code patcher.
1121 if (FLAG_precompiled_mode) {
1122 if (FLAG_use_bare_instructions) {
1123 // The AOT runtime will replace the slot in the object pool with the
1124 // entrypoint address - see clustered_snapshot.cc.
1125 __ LoadUniqueObject(RCX, StubCode::MegamorphicCall());
1126 } else {
1127 __ LoadUniqueObject(CODE_REG, StubCode::MegamorphicCall());
1128 __ movq(RCX, compiler::FieldAddress(CODE_REG,
1129 Code::entry_point_offset(
1130 Code::EntryKind::kMonomorphic)));
1131 }
1132 __ LoadUniqueObject(RBX, cache);
1133 __ call(RCX);
1134 } else {
1135 __ LoadUniqueObject(RBX, cache);
1136 __ LoadUniqueObject(CODE_REG, StubCode::MegamorphicCall());
1137 __ call(compiler::FieldAddress(
1138 CODE_REG, Code::entry_point_offset(Code::EntryKind::kMonomorphic)));
1139 }
1140
1141 RecordSafepoint(locs, slow_path_argument_count);
1142 const intptr_t deopt_id_after = DeoptId::ToDeoptAfter(deopt_id);
1143 if (FLAG_precompiled_mode) {
1144 // Megamorphic calls may occur in slow path stubs.
1145 // If valid use try_index argument.
1146 if (try_index == kInvalidTryIndex) {
1147 try_index = CurrentTryIndex();
1148 }
1149 AddDescriptor(PcDescriptorsLayout::kOther, assembler()->CodeSize(),
1150 DeoptId::kNone, token_pos, try_index);
1151 } else if (is_optimizing()) {
1152 AddCurrentDescriptor(PcDescriptorsLayout::kOther, DeoptId::kNone,
1153 token_pos);
1154 AddDeoptIndexAtCall(deopt_id_after);
1155 } else {
1156 AddCurrentDescriptor(PcDescriptorsLayout::kOther, DeoptId::kNone,
1157 token_pos);
1158 // Add deoptimization continuation point after the call and before the
1159 // arguments are removed.
1160 AddCurrentDescriptor(PcDescriptorsLayout::kDeopt, deopt_id_after,
1161 token_pos);
1162 }
1163 RecordCatchEntryMoves(pending_deoptimization_env_, try_index);
1164 __ Drop(args_desc.SizeWithTypeArgs(), RCX);
1165}
1166
1167void FlowGraphCompiler::EmitInstanceCallAOT(const ICData& ic_data,
1168 intptr_t deopt_id,
1169 TokenPosition token_pos,
1170 LocationSummary* locs,
1171 Code::EntryKind entry_kind,
1172 bool receiver_can_be_smi) {
1173 ASSERT(CanCallDart());
1174 ASSERT(entry_kind == Code::EntryKind::kNormal ||
1175 entry_kind == Code::EntryKind::kUnchecked);
1176 ASSERT(ic_data.NumArgsTested() == 1);
1177 const Code& initial_stub = StubCode::SwitchableCallMiss();
1178 const char* switchable_call_mode = "smiable";
1179 if (!receiver_can_be_smi) {
1180 switchable_call_mode = "non-smi";
1181 ic_data.set_receiver_cannot_be_smi(true);
1182 }
1183 const UnlinkedCall& data =
1184 UnlinkedCall::ZoneHandle(zone(), ic_data.AsUnlinkedCall());
1185
1186 __ Comment("InstanceCallAOT (%s)", switchable_call_mode);
1187 __ movq(RDX, compiler::Address(
1188 RSP, (ic_data.SizeWithoutTypeArgs() - 1) * kWordSize));
1189 if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
1190 // The AOT runtime will replace the slot in the object pool with the
1191 // entrypoint address - see clustered_snapshot.cc.
1192 __ LoadUniqueObject(RCX, initial_stub);
1193 } else {
1194 const intptr_t entry_point_offset =
1195 entry_kind == Code::EntryKind::kNormal
1196 ? Code::entry_point_offset(Code::EntryKind::kMonomorphic)
1197 : Code::entry_point_offset(Code::EntryKind::kMonomorphicUnchecked);
1198 __ LoadUniqueObject(CODE_REG, initial_stub);
1199 __ movq(RCX, compiler::FieldAddress(CODE_REG, entry_point_offset));
1200 }
1201 __ LoadUniqueObject(RBX, data);
1202 __ call(RCX);
1203
1204 EmitCallsiteMetadata(token_pos, deopt_id, PcDescriptorsLayout::kOther, locs);
1205 __ Drop(ic_data.SizeWithTypeArgs(), RCX);
1206}
1207
1208void FlowGraphCompiler::EmitOptimizedStaticCall(
1209 const Function& function,
1210 const Array& arguments_descriptor,
1211 intptr_t size_with_type_args,
1212 intptr_t deopt_id,
1213 TokenPosition token_pos,
1214 LocationSummary* locs,
1215 Code::EntryKind entry_kind) {
1216 ASSERT(CanCallDart());
1217 ASSERT(!function.IsClosureFunction());
1218 if (function.HasOptionalParameters() || function.IsGeneric()) {
1219 __ LoadObject(R10, arguments_descriptor);
1220 } else {
1221 if (!(FLAG_precompiled_mode && FLAG_use_bare_instructions)) {
1222 __ xorl(R10, R10); // GC safe smi zero because of stub.
1223 }
1224 }
1225 // Do not use the code from the function, but let the code be patched so that
1226 // we can record the outgoing edges to other code.
1227 GenerateStaticDartCall(deopt_id, token_pos, PcDescriptorsLayout::kOther, locs,
1228 function, entry_kind);
1229 __ Drop(size_with_type_args, RCX);
1230}
1231
1232void FlowGraphCompiler::EmitDispatchTableCall(
1233 Register cid_reg,
1234 int32_t selector_offset,
1235 const Array& arguments_descriptor) {
1236 ASSERT(CanCallDart());
1237 const Register table_reg = RAX;
1238 ASSERT(cid_reg != table_reg);
1239 ASSERT(cid_reg != ARGS_DESC_REG);
1240 if (!arguments_descriptor.IsNull()) {
1241 __ LoadObject(ARGS_DESC_REG, arguments_descriptor);
1242 }
1243 const intptr_t offset = (selector_offset - DispatchTable::OriginElement()) *
1244 compiler::target::kWordSize;
1245 __ LoadDispatchTable(table_reg);
1246 __ call(compiler::Address(table_reg, cid_reg, TIMES_8, offset));
1247}
1248
1249Condition FlowGraphCompiler::EmitEqualityRegConstCompare(
1250 Register reg,
1251 const Object& obj,
1252 bool needs_number_check,
1253 TokenPosition token_pos,
1254 intptr_t deopt_id) {
1255 ASSERT(!needs_number_check || (!obj.IsMint() && !obj.IsDouble()));
1256
1257 if (obj.IsSmi() && (Smi::Cast(obj).Value() == 0)) {
1258 ASSERT(!needs_number_check);
1259 __ testq(reg, reg);
1260 return EQUAL;
1261 }
1262
1263 if (needs_number_check) {
1264 __ pushq(reg);
1265 __ PushObject(obj);
1266 if (is_optimizing()) {
1267 __ CallPatchable(StubCode::OptimizedIdenticalWithNumberCheck());
1268 } else {
1269 __ CallPatchable(StubCode::UnoptimizedIdenticalWithNumberCheck());
1270 }
1271 AddCurrentDescriptor(PcDescriptorsLayout::kRuntimeCall, deopt_id,
1272 token_pos);
1273 // Stub returns result in flags (result of a cmpq, we need ZF computed).
1274 __ popq(reg); // Discard constant.
1275 __ popq(reg); // Restore 'reg'.
1276 } else {
1277 __ CompareObject(reg, obj);
1278 }
1279 return EQUAL;
1280}
1281
1282Condition FlowGraphCompiler::EmitEqualityRegRegCompare(Register left,
1283 Register right,
1284 bool needs_number_check,
1285 TokenPosition token_pos,
1286 intptr_t deopt_id) {
1287 if (needs_number_check) {
1288 __ pushq(left);
1289 __ pushq(right);
1290 if (is_optimizing()) {
1291 __ CallPatchable(StubCode::OptimizedIdenticalWithNumberCheck());
1292 } else {
1293 __ CallPatchable(StubCode::UnoptimizedIdenticalWithNumberCheck());
1294 }
1295 AddCurrentDescriptor(PcDescriptorsLayout::kRuntimeCall, deopt_id,
1296 token_pos);
1297 // Stub returns result in flags (result of a cmpq, we need ZF computed).
1298 __ popq(right);
1299 __ popq(left);
1300 } else {
1301 __ CompareRegisters(left, right);
1302 }
1303 return EQUAL;
1304}
1305
1306Condition FlowGraphCompiler::EmitBoolTest(Register value,
1307 BranchLabels labels,
1308 bool invert) {
1309 __ Comment("BoolTest");
1310 __ testq(value, compiler::Immediate(
1311 compiler::target::ObjectAlignment::kBoolValueMask));
1312 return invert ? NOT_EQUAL : EQUAL;
1313}
1314
1315// This function must be in sync with FlowGraphCompiler::RecordSafepoint and
1316// FlowGraphCompiler::SlowPathEnvironmentFor.
1317void FlowGraphCompiler::SaveLiveRegisters(LocationSummary* locs) {
1318#if defined(DEBUG)
1319 locs->CheckWritableInputs();
1320 ClobberDeadTempRegisters(locs);
1321#endif
1322
1323 // TODO(vegorov): avoid saving non-volatile registers.
1324 __ PushRegisters(locs->live_registers()->cpu_registers(),
1325 locs->live_registers()->fpu_registers());
1326}
1327
1328void FlowGraphCompiler::RestoreLiveRegisters(LocationSummary* locs) {
1329 __ PopRegisters(locs->live_registers()->cpu_registers(),
1330 locs->live_registers()->fpu_registers());
1331}
1332
1333#if defined(DEBUG)
1334void FlowGraphCompiler::ClobberDeadTempRegisters(LocationSummary* locs) {
1335 // Clobber temporaries that have not been manually preserved.
1336 for (intptr_t i = 0; i < locs->temp_count(); ++i) {
1337 Location tmp = locs->temp(i);
1338 // TODO(zerny): clobber non-live temporary FPU registers.
1339 if (tmp.IsRegister() &&
1340 !locs->live_registers()->ContainsRegister(tmp.reg())) {
1341 __ movq(tmp.reg(), compiler::Immediate(0xf7));
1342 }
1343 }
1344}
1345#endif
1346
1347Register FlowGraphCompiler::EmitTestCidRegister() {
1348 return RDI;
1349}
1350
1351void FlowGraphCompiler::EmitTestAndCallLoadReceiver(
1352 intptr_t count_without_type_args,
1353 const Array& arguments_descriptor) {
1354 __ Comment("EmitTestAndCall");
1355 // Load receiver into RAX.
1356 __ movq(RAX,
1357 compiler::Address(RSP, (count_without_type_args - 1) * kWordSize));
1358 __ LoadObject(R10, arguments_descriptor);
1359}
1360
1361void FlowGraphCompiler::EmitTestAndCallSmiBranch(compiler::Label* label,
1362 bool if_smi) {
1363 __ testq(RAX, compiler::Immediate(kSmiTagMask));
1364 // Jump if receiver is (not) Smi.
1365 __ j(if_smi ? ZERO : NOT_ZERO, label);
1366}
1367
1368void FlowGraphCompiler::EmitTestAndCallLoadCid(Register class_id_reg) {
1369 ASSERT(class_id_reg != RAX);
1370 __ LoadClassId(class_id_reg, RAX);
1371}
1372
1373#undef __
1374#define __ assembler->
1375
1376int FlowGraphCompiler::EmitTestAndCallCheckCid(compiler::Assembler* assembler,
1377 compiler::Label* label,
1378 Register class_id_reg,
1379 const CidRangeValue& range,
1380 int bias,
1381 bool jump_on_miss) {
1382 // Note of WARNING: Due to smaller instruction encoding we use the 32-bit
1383 // instructions on x64, which means the compare instruction has to be
1384 // 32-bit (since the subtraction instruction is as well).
1385 intptr_t cid_start = range.cid_start;
1386 if (range.IsSingleCid()) {
1387 __ cmpl(class_id_reg, compiler::Immediate(cid_start - bias));
1388 __ BranchIf(jump_on_miss ? NOT_EQUAL : EQUAL, label);
1389 } else {
1390 __ addl(class_id_reg, compiler::Immediate(bias - cid_start));
1391 bias = cid_start;
1392 __ cmpl(class_id_reg, compiler::Immediate(range.Extent()));
1393 __ BranchIf(jump_on_miss ? UNSIGNED_GREATER : UNSIGNED_LESS_EQUAL, label);
1394 }
1395 return bias;
1396}
1397
1398#undef __
1399#define __ assembler()->
1400
1401void FlowGraphCompiler::EmitMove(Location destination,
1402 Location source,
1403 TemporaryRegisterAllocator* tmp) {
1404 if (destination.Equals(source)) return;
1405
1406 if (source.IsRegister()) {
1407 if (destination.IsRegister()) {
1408 __ movq(destination.reg(), source.reg());
1409 } else {
1410 ASSERT(destination.IsStackSlot());
1411 __ movq(LocationToStackSlotAddress(destination), source.reg());
1412 }
1413 } else if (source.IsStackSlot()) {
1414 if (destination.IsRegister()) {
1415 __ movq(destination.reg(), LocationToStackSlotAddress(source));
1416 } else if (destination.IsFpuRegister()) {
1417 // 32-bit float
1418 __ movq(TMP, LocationToStackSlotAddress(source));
1419 __ movq(destination.fpu_reg(), TMP);
1420 } else {
1421 ASSERT(destination.IsStackSlot());
1422 __ MoveMemoryToMemory(LocationToStackSlotAddress(destination),
1423 LocationToStackSlotAddress(source));
1424 }
1425 } else if (source.IsFpuRegister()) {
1426 if (destination.IsFpuRegister()) {
1427 // Optimization manual recommends using MOVAPS for register
1428 // to register moves.
1429 __ movaps(destination.fpu_reg(), source.fpu_reg());
1430 } else {
1431 if (destination.IsDoubleStackSlot()) {
1432 __ movsd(LocationToStackSlotAddress(destination), source.fpu_reg());
1433 } else {
1434 ASSERT(destination.IsQuadStackSlot());
1435 __ movups(LocationToStackSlotAddress(destination), source.fpu_reg());
1436 }
1437 }
1438 } else if (source.IsDoubleStackSlot()) {
1439 if (destination.IsFpuRegister()) {
1440 __ movsd(destination.fpu_reg(), LocationToStackSlotAddress(source));
1441 } else {
1442 ASSERT(destination.IsDoubleStackSlot() ||
1443 destination.IsStackSlot() /*32-bit float*/);
1444 __ movsd(FpuTMP, LocationToStackSlotAddress(source));
1445 __ movsd(LocationToStackSlotAddress(destination), FpuTMP);
1446 }
1447 } else if (source.IsQuadStackSlot()) {
1448 if (destination.IsFpuRegister()) {
1449 __ movups(destination.fpu_reg(), LocationToStackSlotAddress(source));
1450 } else {
1451 ASSERT(destination.IsQuadStackSlot());
1452 __ movups(FpuTMP, LocationToStackSlotAddress(source));
1453 __ movups(LocationToStackSlotAddress(destination), FpuTMP);
1454 }
1455 } else {
1456 ASSERT(!source.IsInvalid());
1457 ASSERT(source.IsConstant());
1458 if (destination.IsFpuRegister() || destination.IsDoubleStackSlot()) {
1459 Register scratch = tmp->AllocateTemporary();
1460 source.constant_instruction()->EmitMoveToLocation(this, destination,
1461 scratch);
1462 tmp->ReleaseTemporary();
1463 } else {
1464 source.constant_instruction()->EmitMoveToLocation(this, destination);
1465 }
1466 }
1467}
1468
1469void FlowGraphCompiler::EmitNativeMoveArchitecture(
1470 const compiler::ffi::NativeLocation& destination,
1471 const compiler::ffi::NativeLocation& source) {
1472 const auto& src_type = source.payload_type();
1473 const auto& dst_type = destination.payload_type();
1474 ASSERT(src_type.IsFloat() == dst_type.IsFloat());
1475 ASSERT(src_type.IsInt() == dst_type.IsInt());
1476 ASSERT(src_type.IsSigned() == dst_type.IsSigned());
1477 ASSERT(src_type.IsFundamental());
1478 ASSERT(dst_type.IsFundamental());
1479 const intptr_t src_size = src_type.SizeInBytes();
1480 const intptr_t dst_size = dst_type.SizeInBytes();
1481 const bool sign_or_zero_extend = dst_size > src_size;
1482
1483 if (source.IsRegisters()) {
1484 const auto& src = source.AsRegisters();
1485 ASSERT(src.num_regs() == 1);
1486 const auto src_reg = src.reg_at(0);
1487
1488 if (destination.IsRegisters()) {
1489 const auto& dst = destination.AsRegisters();
1490 ASSERT(dst.num_regs() == 1);
1491 const auto dst_reg = dst.reg_at(0);
1492 if (!sign_or_zero_extend) {
1493 switch (dst_size) {
1494 case 8:
1495 __ movq(dst_reg, src_reg);
1496 return;
1497 case 4:
1498 __ movl(dst_reg, src_reg);
1499 return;
1500 default:
1501 UNIMPLEMENTED();
1502 }
1503 } else {
1504 switch (src_type.AsFundamental().representation()) {
1505 case compiler::ffi::kInt8: // Sign extend operand.
1506 __ movsxb(dst_reg, src_reg);
1507 return;
1508 case compiler::ffi::kInt16:
1509 __ movsxw(dst_reg, src_reg);
1510 return;
1511 case compiler::ffi::kUint8: // Zero extend operand.
1512 __ movzxb(dst_reg, src_reg);
1513 return;
1514 case compiler::ffi::kUint16:
1515 __ movzxw(dst_reg, src_reg);
1516 return;
1517 default:
1518 // 32 to 64 bit is covered in IL by Representation conversions.
1519 UNIMPLEMENTED();
1520 }
1521 }
1522
1523 } else if (destination.IsFpuRegisters()) {
1524 // Fpu Registers should only contain doubles and registers only ints.
1525 UNIMPLEMENTED();
1526
1527 } else {
1528 ASSERT(destination.IsStack());
1529 const auto& dst = destination.AsStack();
1530 const auto dst_addr = NativeLocationToStackSlotAddress(dst);
1531 ASSERT(!sign_or_zero_extend);
1532 switch (dst_size) {
1533 case 8:
1534 __ movq(dst_addr, src_reg);
1535 return;
1536 case 4:
1537 __ movl(dst_addr, src_reg);
1538 return;
1539 case 2:
1540 __ movw(dst_addr, src_reg);
1541 return;
1542 case 1:
1543 __ movb(dst_addr, src_reg);
1544 return;
1545 default:
1546 UNREACHABLE();
1547 }
1548 }
1549
1550 } else if (source.IsFpuRegisters()) {
1551 const auto& src = source.AsFpuRegisters();
1552 // We have not implemented conversions here, use IL convert instructions.
1553 ASSERT(src_type.Equals(dst_type));
1554
1555 if (destination.IsRegisters()) {
1556 // Fpu Registers should only contain doubles and registers only ints.
1557 UNIMPLEMENTED();
1558
1559 } else if (destination.IsFpuRegisters()) {
1560 const auto& dst = destination.AsFpuRegisters();
1561 // Optimization manual recommends using MOVAPS for register
1562 // to register moves.
1563 __ movaps(dst.fpu_reg(), src.fpu_reg());
1564
1565 } else {
1566 ASSERT(destination.IsStack());
1567 ASSERT(src_type.IsFloat());
1568 const auto& dst = destination.AsStack();
1569 const auto dst_addr = NativeLocationToStackSlotAddress(dst);
1570 switch (dst_size) {
1571 case 8:
1572 __ movsd(dst_addr, src.fpu_reg());
1573 return;
1574 case 4:
1575 __ movss(dst_addr, src.fpu_reg());
1576 return;
1577 default:
1578 UNREACHABLE();
1579 }
1580 }
1581
1582 } else {
1583 ASSERT(source.IsStack());
1584 const auto& src = source.AsStack();
1585 const auto src_addr = NativeLocationToStackSlotAddress(src);
1586 if (destination.IsRegisters()) {
1587 const auto& dst = destination.AsRegisters();
1588 ASSERT(dst.num_regs() == 1);
1589 const auto dst_reg = dst.reg_at(0);
1590 if (!sign_or_zero_extend) {
1591 switch (dst_size) {
1592 case 8:
1593 __ movq(dst_reg, src_addr);
1594 return;
1595 case 4:
1596 __ movl(dst_reg, src_addr);
1597 return;
1598 default:
1599 UNIMPLEMENTED();
1600 }
1601 } else {
1602 switch (src_type.AsFundamental().representation()) {
1603 case compiler::ffi::kInt8: // Sign extend operand.
1604 __ movsxb(dst_reg, src_addr);
1605 return;
1606 case compiler::ffi::kInt16:
1607 __ movsxw(dst_reg, src_addr);
1608 return;
1609 case compiler::ffi::kUint8: // Zero extend operand.
1610 __ movzxb(dst_reg, src_addr);
1611 return;
1612 case compiler::ffi::kUint16:
1613 __ movzxw(dst_reg, src_addr);
1614 return;
1615 default:
1616 // 32 to 64 bit is covered in IL by Representation conversions.
1617 UNIMPLEMENTED();
1618 }
1619 }
1620
1621 } else if (destination.IsFpuRegisters()) {
1622 ASSERT(src_type.Equals(dst_type));
1623 ASSERT(src_type.IsFloat());
1624 const auto& dst = destination.AsFpuRegisters();
1625 switch (dst_size) {
1626 case 8:
1627 __ movsd(dst.fpu_reg(), src_addr);
1628 return;
1629 case 4:
1630 __ movss(dst.fpu_reg(), src_addr);
1631 return;
1632 default:
1633 UNREACHABLE();
1634 }
1635
1636 } else {
1637 ASSERT(destination.IsStack());
1638 UNREACHABLE();
1639 }
1640 }
1641}
1642
1643void FlowGraphCompiler::LoadBSSEntry(BSS::Relocation relocation,
1644 Register dst,
1645 Register tmp) {
1646 compiler::Label skip_reloc;
1647 __ jmp(&skip_reloc);
1648 InsertBSSRelocation(relocation);
1649 const intptr_t reloc_end = __ CodeSize();
1650 __ Bind(&skip_reloc);
1651
1652 const intptr_t kLeaqLength = 7;
1653 __ leaq(dst, compiler::Address::AddressRIPRelative(
1654 -kLeaqLength - compiler::target::kWordSize));
1655 ASSERT((__ CodeSize() - reloc_end) == kLeaqLength);
1656
1657 // dst holds the address of the relocation.
1658 __ movq(tmp, compiler::Address(dst, 0));
1659
1660 // tmp holds the relocation itself: dst - bss_start.
1661 // dst = dst + (bss_start - dst) = bss_start
1662 __ addq(dst, tmp);
1663
1664 // dst holds the start of the BSS section.
1665 // Load the routine.
1666 __ movq(dst, compiler::Address(dst, 0));
1667}
1668
1669#undef __
1670#define __ compiler_->assembler()->
1671
1672void ParallelMoveResolver::EmitSwap(int index) {
1673 MoveOperands* move = moves_[index];
1674 const Location source = move->src();
1675 const Location destination = move->dest();
1676
1677 if (source.IsRegister() && destination.IsRegister()) {
1678 __ xchgq(destination.reg(), source.reg());
1679 } else if (source.IsRegister() && destination.IsStackSlot()) {
1680 Exchange(source.reg(), LocationToStackSlotAddress(destination));
1681 } else if (source.IsStackSlot() && destination.IsRegister()) {
1682 Exchange(destination.reg(), LocationToStackSlotAddress(source));
1683 } else if (source.IsStackSlot() && destination.IsStackSlot()) {
1684 Exchange(LocationToStackSlotAddress(destination),
1685 LocationToStackSlotAddress(source));
1686 } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
1687 __ movaps(FpuTMP, source.fpu_reg());
1688 __ movaps(source.fpu_reg(), destination.fpu_reg());
1689 __ movaps(destination.fpu_reg(), FpuTMP);
1690 } else if (source.IsFpuRegister() || destination.IsFpuRegister()) {
1691 ASSERT(destination.IsDoubleStackSlot() || destination.IsQuadStackSlot() ||
1692 source.IsDoubleStackSlot() || source.IsQuadStackSlot());
1693 bool double_width =
1694 destination.IsDoubleStackSlot() || source.IsDoubleStackSlot();
1695 XmmRegister reg =
1696 source.IsFpuRegister() ? source.fpu_reg() : destination.fpu_reg();
1697 compiler::Address slot_address =
1698 source.IsFpuRegister() ? LocationToStackSlotAddress(destination)
1699 : LocationToStackSlotAddress(source);
1700
1701 if (double_width) {
1702 __ movsd(FpuTMP, slot_address);
1703 __ movsd(slot_address, reg);
1704 } else {
1705 __ movups(FpuTMP, slot_address);
1706 __ movups(slot_address, reg);
1707 }
1708 __ movaps(reg, FpuTMP);
1709 } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) {
1710 const compiler::Address& source_slot_address =
1711 LocationToStackSlotAddress(source);
1712 const compiler::Address& destination_slot_address =
1713 LocationToStackSlotAddress(destination);
1714
1715 ScratchFpuRegisterScope ensure_scratch(this, FpuTMP);
1716 __ movsd(FpuTMP, source_slot_address);
1717 __ movsd(ensure_scratch.reg(), destination_slot_address);
1718 __ movsd(destination_slot_address, FpuTMP);
1719 __ movsd(source_slot_address, ensure_scratch.reg());
1720 } else if (source.IsQuadStackSlot() && destination.IsQuadStackSlot()) {
1721 const compiler::Address& source_slot_address =
1722 LocationToStackSlotAddress(source);
1723 const compiler::Address& destination_slot_address =
1724 LocationToStackSlotAddress(destination);
1725
1726 ScratchFpuRegisterScope ensure_scratch(this, FpuTMP);
1727 __ movups(FpuTMP, source_slot_address);
1728 __ movups(ensure_scratch.reg(), destination_slot_address);
1729 __ movups(destination_slot_address, FpuTMP);
1730 __ movups(source_slot_address, ensure_scratch.reg());
1731 } else {
1732 UNREACHABLE();
1733 }
1734
1735 // The swap of source and destination has executed a move from source to
1736 // destination.
1737 move->Eliminate();
1738
1739 // Any unperformed (including pending) move with a source of either
1740 // this move's source or destination needs to have their source
1741 // changed to reflect the state of affairs after the swap.
1742 for (int i = 0; i < moves_.length(); ++i) {
1743 const MoveOperands& other_move = *moves_[i];
1744 if (other_move.Blocks(source)) {
1745 moves_[i]->set_src(destination);
1746 } else if (other_move.Blocks(destination)) {
1747 moves_[i]->set_src(source);
1748 }
1749 }
1750}
1751
1752void ParallelMoveResolver::MoveMemoryToMemory(const compiler::Address& dst,
1753 const compiler::Address& src) {
1754 __ MoveMemoryToMemory(dst, src);
1755}
1756
1757void ParallelMoveResolver::Exchange(Register reg,
1758 const compiler::Address& mem) {
1759 __ Exchange(reg, mem);
1760}
1761
1762void ParallelMoveResolver::Exchange(const compiler::Address& mem1,
1763 const compiler::Address& mem2) {
1764 __ Exchange(mem1, mem2);
1765}
1766
1767void ParallelMoveResolver::Exchange(Register reg,
1768 Register base_reg,
1769 intptr_t stack_offset) {
1770 UNREACHABLE();
1771}
1772
1773void ParallelMoveResolver::Exchange(Register base_reg1,
1774 intptr_t stack_offset1,
1775 Register base_reg2,
1776 intptr_t stack_offset2) {
1777 UNREACHABLE();
1778}
1779
1780void ParallelMoveResolver::SpillScratch(Register reg) {
1781 __ pushq(reg);
1782}
1783
1784void ParallelMoveResolver::RestoreScratch(Register reg) {
1785 __ popq(reg);
1786}
1787
1788void ParallelMoveResolver::SpillFpuScratch(FpuRegister reg) {
1789 __ AddImmediate(RSP, compiler::Immediate(-kFpuRegisterSize));
1790 __ movups(compiler::Address(RSP, 0), reg);
1791}
1792
1793void ParallelMoveResolver::RestoreFpuScratch(FpuRegister reg) {
1794 __ movups(reg, compiler::Address(RSP, 0));
1795 __ AddImmediate(RSP, compiler::Immediate(kFpuRegisterSize));
1796}
1797
1798#undef __
1799
1800} // namespace dart
1801
1802#endif // defined(TARGET_ARCH_X64)
1803