1// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file
2// for details. All rights reserved. Use of this source code is governed by a
3// BSD-style license that can be found in the LICENSE file.
4
5#include "vm/globals.h" // Needed here to get TARGET_ARCH_X64.
6#if defined(TARGET_ARCH_X64)
7
8#include "vm/compiler/backend/il.h"
9
10#include "vm/compiler/assembler/assembler.h"
11#include "vm/compiler/backend/flow_graph.h"
12#include "vm/compiler/backend/flow_graph_compiler.h"
13#include "vm/compiler/backend/locations.h"
14#include "vm/compiler/backend/locations_helpers.h"
15#include "vm/compiler/backend/range_analysis.h"
16#include "vm/compiler/ffi/native_calling_convention.h"
17#include "vm/compiler/jit/compiler.h"
18#include "vm/dart_entry.h"
19#include "vm/instructions.h"
20#include "vm/object_store.h"
21#include "vm/parser.h"
22#include "vm/stack_frame.h"
23#include "vm/stub_code.h"
24#include "vm/symbols.h"
25#include "vm/type_testing_stubs.h"
26
27#define __ compiler->assembler()->
28#define Z (compiler->zone())
29
30namespace dart {
31
32// Generic summary for call instructions that have all arguments pushed
33// on the stack and return the result in a fixed register RAX (or XMM0 if
34// the return type is double).
35LocationSummary* Instruction::MakeCallSummary(Zone* zone,
36 const Instruction* instr,
37 LocationSummary* locs) {
38 ASSERT(locs == nullptr || locs->always_calls());
39 LocationSummary* result =
40 ((locs == nullptr)
41 ? (new (zone) LocationSummary(zone, 0, 0, LocationSummary::kCall))
42 : locs);
43 const auto representation = instr->representation();
44 switch (representation) {
45 case kTagged:
46 case kUnboxedInt64:
47 result->set_out(
48 0, Location::RegisterLocation(CallingConventions::kReturnReg));
49 break;
50 case kUnboxedDouble:
51 result->set_out(
52 0, Location::FpuRegisterLocation(CallingConventions::kReturnFpuReg));
53 break;
54 default:
55 UNREACHABLE();
56 break;
57 }
58 return result;
59}
60
61LocationSummary* LoadIndexedUnsafeInstr::MakeLocationSummary(Zone* zone,
62 bool opt) const {
63 const intptr_t kNumInputs = 1;
64 const intptr_t kNumTemps = 0;
65 LocationSummary* locs = new (zone)
66 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
67
68 locs->set_in(0, Location::RequiresRegister());
69 switch (representation()) {
70 case kTagged:
71 case kUnboxedInt64:
72 locs->set_out(0, Location::RequiresRegister());
73 break;
74 case kUnboxedDouble:
75 locs->set_out(0, Location::RequiresFpuRegister());
76 break;
77 default:
78 UNREACHABLE();
79 break;
80 }
81 return locs;
82}
83
84void LoadIndexedUnsafeInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
85 ASSERT(RequiredInputRepresentation(0) == kTagged); // It is a Smi.
86 ASSERT(kSmiTag == 0);
87 ASSERT(kSmiTagSize == 1);
88
89 const Register index = locs()->in(0).reg();
90
91 switch (representation()) {
92 case kTagged:
93 case kUnboxedInt64: {
94 const auto out = locs()->out(0).reg();
95 __ movq(out, compiler::Address(base_reg(), index, TIMES_4, offset()));
96 break;
97 }
98 case kUnboxedDouble: {
99 const auto out = locs()->out(0).fpu_reg();
100 __ movsd(out, compiler::Address(base_reg(), index, TIMES_4, offset()));
101 break;
102 }
103 default:
104 UNREACHABLE();
105 break;
106 }
107}
108
109DEFINE_BACKEND(StoreIndexedUnsafe,
110 (NoLocation, Register index, Register value)) {
111 ASSERT(instr->RequiredInputRepresentation(
112 StoreIndexedUnsafeInstr::kIndexPos) == kTagged); // It is a Smi.
113 __ movq(compiler::Address(instr->base_reg(), index, TIMES_4, instr->offset()),
114 value);
115
116 ASSERT(kSmiTag == 0);
117 ASSERT(kSmiTagSize == 1);
118}
119
120DEFINE_BACKEND(TailCall, (NoLocation, Fixed<Register, ARGS_DESC_REG>)) {
121 compiler->EmitTailCallToStub(instr->code());
122
123 // Even though the TailCallInstr will be the last instruction in a basic
124 // block, the flow graph compiler will emit native code for other blocks after
125 // the one containing this instruction and needs to be able to use the pool.
126 // (The `LeaveDartFrame` above disables usages of the pool.)
127 __ set_constant_pool_allowed(true);
128}
129
130LocationSummary* MemoryCopyInstr::MakeLocationSummary(Zone* zone,
131 bool opt) const {
132 const intptr_t kNumInputs = 5;
133 const intptr_t kNumTemps = 0;
134 LocationSummary* locs = new (zone)
135 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
136 locs->set_in(kSrcPos, Location::RegisterLocation(RSI));
137 locs->set_in(kDestPos, Location::RegisterLocation(RDI));
138 locs->set_in(kSrcStartPos, Location::WritableRegister());
139 locs->set_in(kDestStartPos, Location::WritableRegister());
140 locs->set_in(kLengthPos, Location::RegisterLocation(RCX));
141 return locs;
142}
143
144void MemoryCopyInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
145 const Register src_start_reg = locs()->in(kSrcStartPos).reg();
146 const Register dest_start_reg = locs()->in(kDestStartPos).reg();
147
148 EmitComputeStartPointer(compiler, src_cid_, src_start(), RSI, src_start_reg);
149 EmitComputeStartPointer(compiler, dest_cid_, dest_start(), RDI,
150 dest_start_reg);
151 if (element_size_ <= 8) {
152 __ SmiUntag(RCX);
153 }
154 switch (element_size_) {
155 case 1:
156 __ rep_movsb();
157 break;
158 case 2:
159 __ rep_movsw();
160 break;
161 case 4:
162 __ rep_movsl();
163 break;
164 case 8:
165 case 16:
166 __ rep_movsq();
167 break;
168 }
169}
170
171void MemoryCopyInstr::EmitComputeStartPointer(FlowGraphCompiler* compiler,
172 classid_t array_cid,
173 Value* start,
174 Register array_reg,
175 Register start_reg) {
176 intptr_t offset;
177 if (IsTypedDataBaseClassId(array_cid)) {
178 __ movq(
179 array_reg,
180 compiler::FieldAddress(
181 array_reg, compiler::target::TypedDataBase::data_field_offset()));
182 offset = 0;
183 } else {
184 switch (array_cid) {
185 case kOneByteStringCid:
186 offset =
187 compiler::target::OneByteString::data_offset() - kHeapObjectTag;
188 break;
189 case kTwoByteStringCid:
190 offset =
191 compiler::target::TwoByteString::data_offset() - kHeapObjectTag;
192 break;
193 case kExternalOneByteStringCid:
194 __ movq(array_reg,
195 compiler::FieldAddress(array_reg,
196 compiler::target::ExternalOneByteString::
197 external_data_offset()));
198 offset = 0;
199 break;
200 case kExternalTwoByteStringCid:
201 __ movq(array_reg,
202 compiler::FieldAddress(array_reg,
203 compiler::target::ExternalTwoByteString::
204 external_data_offset()));
205 offset = 0;
206 break;
207 default:
208 UNREACHABLE();
209 break;
210 }
211 }
212 ScaleFactor scale;
213 switch (element_size_) {
214 case 1:
215 __ SmiUntag(start_reg);
216 scale = TIMES_1;
217 break;
218 case 2:
219 scale = TIMES_1;
220 break;
221 case 4:
222 scale = TIMES_2;
223 break;
224 case 8:
225 scale = TIMES_4;
226 break;
227 case 16:
228 scale = TIMES_8;
229 break;
230 default:
231 UNREACHABLE();
232 break;
233 }
234 __ leaq(array_reg, compiler::Address(array_reg, start_reg, scale, offset));
235}
236
237LocationSummary* PushArgumentInstr::MakeLocationSummary(Zone* zone,
238 bool opt) const {
239 const intptr_t kNumInputs = 1;
240 const intptr_t kNumTemps = 0;
241 LocationSummary* locs = new (zone)
242 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
243 if (representation() == kUnboxedDouble) {
244 locs->set_in(0, Location::RequiresFpuRegister());
245 } else if (representation() == kUnboxedInt64) {
246 locs->set_in(0, Location::RequiresRegister());
247 } else {
248 locs->set_in(0, LocationAnyOrConstant(value()));
249 }
250 return locs;
251}
252
253void PushArgumentInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
254 // In SSA mode, we need an explicit push. Nothing to do in non-SSA mode
255 // where arguments are pushed by their definitions.
256 if (compiler->is_optimizing()) {
257 Location value = locs()->in(0);
258 if (value.IsRegister()) {
259 __ pushq(value.reg());
260 } else if (value.IsConstant()) {
261 __ PushObject(value.constant());
262 } else if (value.IsFpuRegister()) {
263 __ AddImmediate(RSP, compiler::Immediate(-kDoubleSize));
264 __ movsd(compiler::Address(RSP, 0), value.fpu_reg());
265 } else {
266 ASSERT(value.IsStackSlot());
267 __ pushq(LocationToStackSlotAddress(value));
268 }
269 }
270}
271
272LocationSummary* ReturnInstr::MakeLocationSummary(Zone* zone, bool opt) const {
273 const intptr_t kNumInputs = 1;
274 const intptr_t kNumTemps = 0;
275 LocationSummary* locs = new (zone)
276 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
277 switch (representation()) {
278 case kTagged:
279 case kUnboxedInt64:
280 locs->set_in(0,
281 Location::RegisterLocation(CallingConventions::kReturnReg));
282 break;
283 case kUnboxedDouble:
284 locs->set_in(
285 0, Location::FpuRegisterLocation(CallingConventions::kReturnFpuReg));
286 break;
287 default:
288 UNREACHABLE();
289 break;
290 }
291 return locs;
292}
293
294// Attempt optimized compilation at return instruction instead of at the entry.
295// The entry needs to be patchable, no inlined objects are allowed in the area
296// that will be overwritten by the patch instruction: a jump).
297void ReturnInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
298 if (locs()->in(0).IsRegister()) {
299 const Register result = locs()->in(0).reg();
300 ASSERT(result == CallingConventions::kReturnReg);
301 } else {
302 ASSERT(locs()->in(0).IsFpuRegister());
303 const FpuRegister result = locs()->in(0).fpu_reg();
304 ASSERT(result == CallingConventions::kReturnFpuReg);
305 }
306
307 if (compiler->intrinsic_mode()) {
308 // Intrinsics don't have a frame.
309 __ ret();
310 return;
311 }
312
313#if defined(DEBUG)
314 __ Comment("Stack Check");
315 compiler::Label done;
316 const intptr_t fp_sp_dist =
317 (compiler::target::frame_layout.first_local_from_fp + 1 -
318 compiler->StackSize()) *
319 kWordSize;
320 ASSERT(fp_sp_dist <= 0);
321 __ movq(RDI, RSP);
322 __ subq(RDI, RBP);
323 __ CompareImmediate(RDI, compiler::Immediate(fp_sp_dist));
324 __ j(EQUAL, &done, compiler::Assembler::kNearJump);
325 __ int3();
326 __ Bind(&done);
327#endif
328 ASSERT(__ constant_pool_allowed());
329 if (yield_index() != PcDescriptorsLayout::kInvalidYieldIndex) {
330 compiler->EmitYieldPositionMetadata(token_pos(), yield_index());
331 }
332 __ LeaveDartFrame(); // Disallows constant pool use.
333 __ ret();
334 // This ReturnInstr may be emitted out of order by the optimizer. The next
335 // block may be a target expecting a properly set constant pool pointer.
336 __ set_constant_pool_allowed(true);
337}
338
339void NativeReturnInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
340 EmitReturnMoves(compiler);
341
342 __ LeaveDartFrame();
343
344 // Pop dummy return address.
345 __ popq(TMP);
346
347 // Anything besides the return register.
348 const Register vm_tag_reg = RBX;
349 const Register old_exit_frame_reg = RCX;
350 const Register old_exit_through_ffi_reg = RDI;
351
352 __ popq(old_exit_frame_reg);
353
354 __ popq(old_exit_through_ffi_reg);
355
356 // Restore top_resource.
357 __ popq(TMP);
358 __ movq(
359 compiler::Address(THR, compiler::target::Thread::top_resource_offset()),
360 TMP);
361
362 __ popq(vm_tag_reg);
363
364 // If we were called by a trampoline, it will enter the safepoint on our
365 // behalf.
366 __ TransitionGeneratedToNative(
367 vm_tag_reg, old_exit_frame_reg, old_exit_through_ffi_reg,
368 /*enter_safepoint=*/!NativeCallbackTrampolines::Enabled());
369
370 // Restore C++ ABI callee-saved registers.
371 __ PopRegisters(CallingConventions::kCalleeSaveCpuRegisters,
372 CallingConventions::kCalleeSaveXmmRegisters);
373
374#if defined(TARGET_OS_FUCHSIA)
375 UNREACHABLE(); // Fuchsia does not allow dart:ffi.
376#elif defined(USING_SHADOW_CALL_STACK)
377#error Unimplemented
378#endif
379
380 // Leave the entry frame.
381 __ LeaveFrame();
382
383 // Leave the dummy frame holding the pushed arguments.
384 __ LeaveFrame();
385
386 __ ret();
387
388 // For following blocks.
389 __ set_constant_pool_allowed(true);
390}
391
392// Detect pattern when one value is zero and another is a power of 2.
393static bool IsPowerOfTwoKind(intptr_t v1, intptr_t v2) {
394 return (Utils::IsPowerOfTwo(v1) && (v2 == 0)) ||
395 (Utils::IsPowerOfTwo(v2) && (v1 == 0));
396}
397
398LocationSummary* IfThenElseInstr::MakeLocationSummary(Zone* zone,
399 bool opt) const {
400 comparison()->InitializeLocationSummary(zone, opt);
401 // TODO(dartbug.com/30952) support convertion of Register to corresponding
402 // least significant byte register (e.g. RAX -> AL, RSI -> SIL, r15 -> r15b).
403 comparison()->locs()->set_out(0, Location::RegisterLocation(RDX));
404 return comparison()->locs();
405}
406
407void IfThenElseInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
408 ASSERT(locs()->out(0).reg() == RDX);
409
410 // Clear upper part of the out register. We are going to use setcc on it
411 // which is a byte move.
412 __ xorq(RDX, RDX);
413
414 // Emit comparison code. This must not overwrite the result register.
415 // IfThenElseInstr::Supports() should prevent EmitComparisonCode from using
416 // the labels or returning an invalid condition.
417 BranchLabels labels = {NULL, NULL, NULL};
418 Condition true_condition = comparison()->EmitComparisonCode(compiler, labels);
419 ASSERT(true_condition != kInvalidCondition);
420
421 const bool is_power_of_two_kind = IsPowerOfTwoKind(if_true_, if_false_);
422
423 intptr_t true_value = if_true_;
424 intptr_t false_value = if_false_;
425
426 if (is_power_of_two_kind) {
427 if (true_value == 0) {
428 // We need to have zero in RDX on true_condition.
429 true_condition = InvertCondition(true_condition);
430 }
431 } else {
432 if (true_value == 0) {
433 // Swap values so that false_value is zero.
434 intptr_t temp = true_value;
435 true_value = false_value;
436 false_value = temp;
437 } else {
438 true_condition = InvertCondition(true_condition);
439 }
440 }
441
442 __ setcc(true_condition, DL);
443
444 if (is_power_of_two_kind) {
445 const intptr_t shift =
446 Utils::ShiftForPowerOfTwo(Utils::Maximum(true_value, false_value));
447 __ shlq(RDX, compiler::Immediate(shift + kSmiTagSize));
448 } else {
449 __ decq(RDX);
450 __ AndImmediate(RDX, compiler::Immediate(Smi::RawValue(true_value) -
451 Smi::RawValue(false_value)));
452 if (false_value != 0) {
453 __ AddImmediate(RDX, compiler::Immediate(Smi::RawValue(false_value)));
454 }
455 }
456}
457
458LocationSummary* LoadLocalInstr::MakeLocationSummary(Zone* zone,
459 bool opt) const {
460 const intptr_t kNumInputs = 0;
461 const intptr_t stack_index =
462 compiler::target::frame_layout.FrameSlotForVariable(&local());
463 return LocationSummary::Make(zone, kNumInputs,
464 Location::StackSlot(stack_index, FPREG),
465 LocationSummary::kNoCall);
466}
467
468void LoadLocalInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
469 ASSERT(!compiler->is_optimizing());
470 // Nothing to do.
471}
472
473LocationSummary* StoreLocalInstr::MakeLocationSummary(Zone* zone,
474 bool opt) const {
475 const intptr_t kNumInputs = 1;
476 return LocationSummary::Make(zone, kNumInputs, Location::SameAsFirstInput(),
477 LocationSummary::kNoCall);
478}
479
480void StoreLocalInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
481 Register value = locs()->in(0).reg();
482 Register result = locs()->out(0).reg();
483 ASSERT(result == value); // Assert that register assignment is correct.
484 __ movq(compiler::Address(
485 RBP, compiler::target::FrameOffsetInBytesForVariable(&local())),
486 value);
487}
488
489LocationSummary* ConstantInstr::MakeLocationSummary(Zone* zone,
490 bool opt) const {
491 const intptr_t kNumInputs = 0;
492 return LocationSummary::Make(zone, kNumInputs,
493 compiler::Assembler::IsSafe(value())
494 ? Location::Constant(this)
495 : Location::RequiresRegister(),
496 LocationSummary::kNoCall);
497}
498
499void ConstantInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
500 // The register allocator drops constant definitions that have no uses.
501 Location out = locs()->out(0);
502 ASSERT(out.IsRegister() || out.IsConstant() || out.IsInvalid());
503 if (out.IsRegister()) {
504 Register result = out.reg();
505 __ LoadObject(result, value());
506 }
507}
508
509void ConstantInstr::EmitMoveToLocation(FlowGraphCompiler* compiler,
510 const Location& destination,
511 Register tmp) {
512 if (destination.IsRegister()) {
513 if (representation() == kUnboxedInt32 ||
514 representation() == kUnboxedInt64) {
515 const int64_t value = Integer::Cast(value_).AsInt64Value();
516 if (value == 0) {
517 __ xorl(destination.reg(), destination.reg());
518 } else {
519 __ movq(destination.reg(), compiler::Immediate(value));
520 }
521 } else {
522 ASSERT(representation() == kTagged);
523 __ LoadObject(destination.reg(), value_);
524 }
525 } else if (destination.IsFpuRegister()) {
526 if (Utils::DoublesBitEqual(Double::Cast(value_).value(), 0.0)) {
527 __ xorps(destination.fpu_reg(), destination.fpu_reg());
528 } else {
529 ASSERT(tmp != kNoRegister);
530 __ LoadObject(tmp, value_);
531 __ movsd(destination.fpu_reg(),
532 compiler::FieldAddress(tmp, Double::value_offset()));
533 }
534 } else if (destination.IsDoubleStackSlot()) {
535 if (Utils::DoublesBitEqual(Double::Cast(value_).value(), 0.0)) {
536 __ xorps(FpuTMP, FpuTMP);
537 } else {
538 ASSERT(tmp != kNoRegister);
539 __ LoadObject(tmp, value_);
540 __ movsd(FpuTMP, compiler::FieldAddress(tmp, Double::value_offset()));
541 }
542 __ movsd(LocationToStackSlotAddress(destination), FpuTMP);
543 } else {
544 ASSERT(destination.IsStackSlot());
545 if (representation() == kUnboxedInt32 ||
546 representation() == kUnboxedInt64) {
547 const int64_t value = Integer::Cast(value_).AsInt64Value();
548 __ movq(LocationToStackSlotAddress(destination),
549 compiler::Immediate(value));
550 } else {
551 ASSERT(representation() == kTagged);
552 __ StoreObject(LocationToStackSlotAddress(destination), value_);
553 }
554 }
555}
556
557LocationSummary* UnboxedConstantInstr::MakeLocationSummary(Zone* zone,
558 bool opt) const {
559 const intptr_t kNumInputs = 0;
560 const intptr_t kNumTemps = IsUnboxedSignedIntegerConstant() ? 0 : 1;
561 LocationSummary* locs = new (zone)
562 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
563 switch (representation()) {
564 case kUnboxedDouble:
565 locs->set_out(0, Location::RequiresFpuRegister());
566 locs->set_temp(0, Location::RequiresRegister());
567 break;
568 case kUnboxedInt32:
569 case kUnboxedInt64:
570 locs->set_out(0, Location::RequiresRegister());
571 break;
572 default:
573 UNREACHABLE();
574 break;
575 }
576 return locs;
577}
578
579void UnboxedConstantInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
580 // The register allocator drops constant definitions that have no uses.
581 if (!locs()->out(0).IsInvalid()) {
582 const Register scratch =
583 IsUnboxedSignedIntegerConstant() ? kNoRegister : locs()->temp(0).reg();
584 EmitMoveToLocation(compiler, locs()->out(0), scratch);
585 }
586}
587
588LocationSummary* AssertAssignableInstr::MakeLocationSummary(Zone* zone,
589 bool opt) const {
590 auto const dst_type_loc =
591 LocationFixedRegisterOrConstant(dst_type(), TypeTestABI::kDstTypeReg);
592
593 // When using a type testing stub, we want to prevent spilling of the
594 // function/instantiator type argument vectors, since stub preserves them. So
595 // we make this a `kNoCall` summary, even though most other registers can be
596 // modified by the stub. To tell the register allocator about it, we reserve
597 // all the other registers as temporary registers.
598 // TODO(http://dartbug.com/32788): Simplify this.
599 const bool using_stub = dst_type_loc.IsConstant() &&
600 FlowGraphCompiler::ShouldUseTypeTestingStubFor(
601 opt, AbstractType::Cast(dst_type_loc.constant()));
602
603 const intptr_t kNonChangeableInputRegs =
604 (1 << TypeTestABI::kInstanceReg) |
605 ((dst_type_loc.IsRegister() ? 1 : 0) << TypeTestABI::kDstTypeReg) |
606 (1 << TypeTestABI::kInstantiatorTypeArgumentsReg) |
607 (1 << TypeTestABI::kFunctionTypeArgumentsReg);
608
609 const intptr_t kNumInputs = 4;
610
611 // We invoke a stub that can potentially clobber any CPU register
612 // but can only clobber FPU registers on the slow path when
613 // entering runtime. Preserve all FPU registers that are
614 // not guarateed to be preserved by the ABI.
615 const intptr_t kCpuRegistersToPreserve =
616 kDartAvailableCpuRegs & ~kNonChangeableInputRegs;
617 const intptr_t kFpuRegistersToPreserve =
618 CallingConventions::kVolatileXmmRegisters & ~(1 << FpuTMP);
619
620 const intptr_t kNumTemps =
621 using_stub ? (Utils::CountOneBits64(kCpuRegistersToPreserve) +
622 Utils::CountOneBits64(kFpuRegistersToPreserve))
623 : 0;
624
625 LocationSummary* summary = new (zone) LocationSummary(
626 zone, kNumInputs, kNumTemps,
627 using_stub ? LocationSummary::kCallCalleeSafe : LocationSummary::kCall);
628 summary->set_in(0, Location::RegisterLocation(TypeTestABI::kInstanceReg));
629 summary->set_in(1, dst_type_loc);
630 summary->set_in(2, Location::RegisterLocation(
631 TypeTestABI::kInstantiatorTypeArgumentsReg));
632 summary->set_in(
633 3, Location::RegisterLocation(TypeTestABI::kFunctionTypeArgumentsReg));
634 summary->set_out(0, Location::SameAsFirstInput());
635
636 if (using_stub) {
637 // Let's reserve all registers except for the input ones.
638 intptr_t next_temp = 0;
639 for (intptr_t i = 0; i < kNumberOfCpuRegisters; ++i) {
640 const bool should_preserve = ((1 << i) & kCpuRegistersToPreserve) != 0;
641 if (should_preserve) {
642 summary->set_temp(next_temp++,
643 Location::RegisterLocation(static_cast<Register>(i)));
644 }
645 }
646
647 for (intptr_t i = 0; i < kNumberOfFpuRegisters; i++) {
648 const bool should_preserve = ((1 << i) & kFpuRegistersToPreserve) != 0;
649 if (should_preserve) {
650 summary->set_temp(next_temp++, Location::FpuRegisterLocation(
651 static_cast<FpuRegister>(i)));
652 }
653 }
654 }
655
656 return summary;
657}
658
659static Condition TokenKindToIntCondition(Token::Kind kind) {
660 switch (kind) {
661 case Token::kEQ:
662 return EQUAL;
663 case Token::kNE:
664 return NOT_EQUAL;
665 case Token::kLT:
666 return LESS;
667 case Token::kGT:
668 return GREATER;
669 case Token::kLTE:
670 return LESS_EQUAL;
671 case Token::kGTE:
672 return GREATER_EQUAL;
673 default:
674 UNREACHABLE();
675 return OVERFLOW;
676 }
677}
678
679LocationSummary* EqualityCompareInstr::MakeLocationSummary(Zone* zone,
680 bool opt) const {
681 const intptr_t kNumInputs = 2;
682 if (operation_cid() == kDoubleCid) {
683 const intptr_t kNumTemps = 0;
684 LocationSummary* locs = new (zone)
685 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
686 locs->set_in(0, Location::RequiresFpuRegister());
687 locs->set_in(1, Location::RequiresFpuRegister());
688 locs->set_out(0, Location::RequiresRegister());
689 return locs;
690 }
691 if (operation_cid() == kSmiCid || operation_cid() == kMintCid) {
692 const intptr_t kNumTemps = 0;
693 LocationSummary* locs = new (zone)
694 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
695 locs->set_in(0, LocationRegisterOrConstant(left()));
696 // Only one input can be a constant operand. The case of two constant
697 // operands should be handled by constant propagation.
698 // Only right can be a stack slot.
699 locs->set_in(1, locs->in(0).IsConstant()
700 ? Location::RequiresRegister()
701 : LocationRegisterOrConstant(right()));
702 locs->set_out(0, Location::RequiresRegister());
703 return locs;
704 }
705 UNREACHABLE();
706 return NULL;
707}
708
709static void LoadValueCid(FlowGraphCompiler* compiler,
710 Register value_cid_reg,
711 Register value_reg,
712 compiler::Label* value_is_smi = NULL) {
713 compiler::Label done;
714 if (value_is_smi == NULL) {
715 __ LoadImmediate(value_cid_reg, compiler::Immediate(kSmiCid));
716 }
717 __ testq(value_reg, compiler::Immediate(kSmiTagMask));
718 if (value_is_smi == NULL) {
719 __ j(ZERO, &done, compiler::Assembler::kNearJump);
720 } else {
721 __ j(ZERO, value_is_smi);
722 }
723 __ LoadClassId(value_cid_reg, value_reg);
724 __ Bind(&done);
725}
726
727static Condition FlipCondition(Condition condition) {
728 switch (condition) {
729 case EQUAL:
730 return EQUAL;
731 case NOT_EQUAL:
732 return NOT_EQUAL;
733 case LESS:
734 return GREATER;
735 case LESS_EQUAL:
736 return GREATER_EQUAL;
737 case GREATER:
738 return LESS;
739 case GREATER_EQUAL:
740 return LESS_EQUAL;
741 case BELOW:
742 return ABOVE;
743 case BELOW_EQUAL:
744 return ABOVE_EQUAL;
745 case ABOVE:
746 return BELOW;
747 case ABOVE_EQUAL:
748 return BELOW_EQUAL;
749 default:
750 UNIMPLEMENTED();
751 return EQUAL;
752 }
753}
754
755static void EmitBranchOnCondition(FlowGraphCompiler* compiler,
756 Condition true_condition,
757 BranchLabels labels) {
758 if (labels.fall_through == labels.false_label) {
759 // If the next block is the false successor, fall through to it.
760 __ j(true_condition, labels.true_label);
761 } else {
762 // If the next block is not the false successor, branch to it.
763 Condition false_condition = InvertCondition(true_condition);
764 __ j(false_condition, labels.false_label);
765
766 // Fall through or jump to the true successor.
767 if (labels.fall_through != labels.true_label) {
768 __ jmp(labels.true_label);
769 }
770 }
771}
772
773static Condition EmitInt64ComparisonOp(FlowGraphCompiler* compiler,
774 const LocationSummary& locs,
775 Token::Kind kind) {
776 Location left = locs.in(0);
777 Location right = locs.in(1);
778 ASSERT(!left.IsConstant() || !right.IsConstant());
779
780 Condition true_condition = TokenKindToIntCondition(kind);
781 if (left.IsConstant() || right.IsConstant()) {
782 // Ensure constant is on the right.
783 ConstantInstr* constant = NULL;
784 if (left.IsConstant()) {
785 constant = left.constant_instruction();
786 Location tmp = right;
787 right = left;
788 left = tmp;
789 true_condition = FlipCondition(true_condition);
790 } else {
791 constant = right.constant_instruction();
792 }
793
794 if (constant->IsUnboxedSignedIntegerConstant()) {
795 __ cmpq(left.reg(),
796 compiler::Immediate(
797 constant->GetUnboxedSignedIntegerConstantValue()));
798 } else {
799 ASSERT(constant->representation() == kTagged);
800 __ CompareObject(left.reg(), right.constant());
801 }
802 } else if (right.IsStackSlot()) {
803 __ cmpq(left.reg(), LocationToStackSlotAddress(right));
804 } else {
805 __ cmpq(left.reg(), right.reg());
806 }
807 return true_condition;
808}
809
810static Condition TokenKindToDoubleCondition(Token::Kind kind) {
811 switch (kind) {
812 case Token::kEQ:
813 return EQUAL;
814 case Token::kNE:
815 return NOT_EQUAL;
816 case Token::kLT:
817 return BELOW;
818 case Token::kGT:
819 return ABOVE;
820 case Token::kLTE:
821 return BELOW_EQUAL;
822 case Token::kGTE:
823 return ABOVE_EQUAL;
824 default:
825 UNREACHABLE();
826 return OVERFLOW;
827 }
828}
829
830static Condition EmitDoubleComparisonOp(FlowGraphCompiler* compiler,
831 const LocationSummary& locs,
832 Token::Kind kind,
833 BranchLabels labels) {
834 XmmRegister left = locs.in(0).fpu_reg();
835 XmmRegister right = locs.in(1).fpu_reg();
836
837 __ comisd(left, right);
838
839 Condition true_condition = TokenKindToDoubleCondition(kind);
840 compiler::Label* nan_result =
841 (true_condition == NOT_EQUAL) ? labels.true_label : labels.false_label;
842 __ j(PARITY_EVEN, nan_result);
843 return true_condition;
844}
845
846Condition EqualityCompareInstr::EmitComparisonCode(FlowGraphCompiler* compiler,
847 BranchLabels labels) {
848 if ((operation_cid() == kSmiCid) || (operation_cid() == kMintCid)) {
849 return EmitInt64ComparisonOp(compiler, *locs(), kind());
850 } else {
851 ASSERT(operation_cid() == kDoubleCid);
852 return EmitDoubleComparisonOp(compiler, *locs(), kind(), labels);
853 }
854}
855
856void ComparisonInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
857 compiler::Label is_true, is_false;
858 BranchLabels labels = {&is_true, &is_false, &is_false};
859 Condition true_condition = EmitComparisonCode(compiler, labels);
860 if (true_condition != kInvalidCondition) {
861 EmitBranchOnCondition(compiler, true_condition, labels);
862 }
863
864 Register result = locs()->out(0).reg();
865 compiler::Label done;
866 __ Bind(&is_false);
867 __ LoadObject(result, Bool::False());
868 __ jmp(&done);
869 __ Bind(&is_true);
870 __ LoadObject(result, Bool::True());
871 __ Bind(&done);
872}
873
874void ComparisonInstr::EmitBranchCode(FlowGraphCompiler* compiler,
875 BranchInstr* branch) {
876 BranchLabels labels = compiler->CreateBranchLabels(branch);
877 Condition true_condition = EmitComparisonCode(compiler, labels);
878 if (true_condition != kInvalidCondition) {
879 EmitBranchOnCondition(compiler, true_condition, labels);
880 }
881}
882
883LocationSummary* TestSmiInstr::MakeLocationSummary(Zone* zone, bool opt) const {
884 const intptr_t kNumInputs = 2;
885 const intptr_t kNumTemps = 0;
886 LocationSummary* locs = new (zone)
887 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
888 locs->set_in(0, Location::RequiresRegister());
889 // Only one input can be a constant operand. The case of two constant
890 // operands should be handled by constant propagation.
891 locs->set_in(1, LocationRegisterOrConstant(right()));
892 return locs;
893}
894
895Condition TestSmiInstr::EmitComparisonCode(FlowGraphCompiler* compiler,
896 BranchLabels labels) {
897 Register left_reg = locs()->in(0).reg();
898 Location right = locs()->in(1);
899 if (right.IsConstant()) {
900 ASSERT(right.constant().IsSmi());
901 const int64_t imm = static_cast<int64_t>(right.constant().raw());
902 __ TestImmediate(left_reg, compiler::Immediate(imm));
903 } else {
904 __ testq(left_reg, right.reg());
905 }
906 Condition true_condition = (kind() == Token::kNE) ? NOT_ZERO : ZERO;
907 return true_condition;
908}
909
910LocationSummary* TestCidsInstr::MakeLocationSummary(Zone* zone,
911 bool opt) const {
912 const intptr_t kNumInputs = 1;
913 const intptr_t kNumTemps = 1;
914 LocationSummary* locs = new (zone)
915 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
916 locs->set_in(0, Location::RequiresRegister());
917 locs->set_temp(0, Location::RequiresRegister());
918 locs->set_out(0, Location::RequiresRegister());
919 return locs;
920}
921
922Condition TestCidsInstr::EmitComparisonCode(FlowGraphCompiler* compiler,
923 BranchLabels labels) {
924 ASSERT((kind() == Token::kIS) || (kind() == Token::kISNOT));
925 Register val_reg = locs()->in(0).reg();
926 Register cid_reg = locs()->temp(0).reg();
927
928 compiler::Label* deopt =
929 CanDeoptimize()
930 ? compiler->AddDeoptStub(deopt_id(), ICData::kDeoptTestCids,
931 licm_hoisted_ ? ICData::kHoisted : 0)
932 : NULL;
933
934 const intptr_t true_result = (kind() == Token::kIS) ? 1 : 0;
935 const ZoneGrowableArray<intptr_t>& data = cid_results();
936 ASSERT(data[0] == kSmiCid);
937 bool result = data[1] == true_result;
938 __ testq(val_reg, compiler::Immediate(kSmiTagMask));
939 __ j(ZERO, result ? labels.true_label : labels.false_label);
940 __ LoadClassId(cid_reg, val_reg);
941 for (intptr_t i = 2; i < data.length(); i += 2) {
942 const intptr_t test_cid = data[i];
943 ASSERT(test_cid != kSmiCid);
944 result = data[i + 1] == true_result;
945 __ cmpq(cid_reg, compiler::Immediate(test_cid));
946 __ j(EQUAL, result ? labels.true_label : labels.false_label);
947 }
948 // No match found, deoptimize or default action.
949 if (deopt == NULL) {
950 // If the cid is not in the list, jump to the opposite label from the cids
951 // that are in the list. These must be all the same (see asserts in the
952 // constructor).
953 compiler::Label* target = result ? labels.false_label : labels.true_label;
954 if (target != labels.fall_through) {
955 __ jmp(target);
956 }
957 } else {
958 __ jmp(deopt);
959 }
960 // Dummy result as this method already did the jump, there's no need
961 // for the caller to branch on a condition.
962 return kInvalidCondition;
963}
964
965LocationSummary* RelationalOpInstr::MakeLocationSummary(Zone* zone,
966 bool opt) const {
967 const intptr_t kNumInputs = 2;
968 const intptr_t kNumTemps = 0;
969 if (operation_cid() == kDoubleCid) {
970 LocationSummary* summary = new (zone)
971 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
972 summary->set_in(0, Location::RequiresFpuRegister());
973 summary->set_in(1, Location::RequiresFpuRegister());
974 summary->set_out(0, Location::RequiresRegister());
975 return summary;
976 }
977 if (operation_cid() == kSmiCid || operation_cid() == kMintCid) {
978 LocationSummary* summary = new (zone)
979 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
980 summary->set_in(0, LocationRegisterOrConstant(left()));
981 // Only one input can be a constant operand. The case of two constant
982 // operands should be handled by constant propagation.
983 summary->set_in(1, summary->in(0).IsConstant()
984 ? Location::RequiresRegister()
985 : LocationRegisterOrConstant(right()));
986 summary->set_out(0, Location::RequiresRegister());
987 return summary;
988 }
989 UNREACHABLE();
990 return NULL;
991}
992
993Condition RelationalOpInstr::EmitComparisonCode(FlowGraphCompiler* compiler,
994 BranchLabels labels) {
995 if (operation_cid() == kSmiCid || operation_cid() == kMintCid) {
996 return EmitInt64ComparisonOp(compiler, *locs(), kind());
997 } else {
998 ASSERT(operation_cid() == kDoubleCid);
999 return EmitDoubleComparisonOp(compiler, *locs(), kind(), labels);
1000 }
1001}
1002
1003void NativeCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
1004 SetupNative();
1005 Register result = locs()->out(0).reg();
1006 const intptr_t argc_tag = NativeArguments::ComputeArgcTag(function());
1007
1008 // All arguments are already @RSP due to preceding PushArgument()s.
1009 ASSERT(ArgumentCount() ==
1010 function().NumParameters() + (function().IsGeneric() ? 1 : 0));
1011
1012 // Push the result place holder initialized to NULL.
1013 __ PushObject(Object::null_object());
1014
1015 // Pass a pointer to the first argument in RAX.
1016 __ leaq(RAX, compiler::Address(RSP, ArgumentCount() * kWordSize));
1017
1018 __ LoadImmediate(R10, compiler::Immediate(argc_tag));
1019 const Code* stub;
1020 if (link_lazily()) {
1021 stub = &StubCode::CallBootstrapNative();
1022 compiler::ExternalLabel label(NativeEntry::LinkNativeCallEntry());
1023 __ LoadNativeEntry(RBX, &label,
1024 compiler::ObjectPoolBuilderEntry::kPatchable);
1025 compiler->GeneratePatchableCall(token_pos(), *stub,
1026 PcDescriptorsLayout::kOther, locs());
1027 } else {
1028 if (is_bootstrap_native()) {
1029 stub = &StubCode::CallBootstrapNative();
1030 } else if (is_auto_scope()) {
1031 stub = &StubCode::CallAutoScopeNative();
1032 } else {
1033 stub = &StubCode::CallNoScopeNative();
1034 }
1035 const compiler::ExternalLabel label(
1036 reinterpret_cast<uword>(native_c_function()));
1037 __ LoadNativeEntry(RBX, &label,
1038 compiler::ObjectPoolBuilderEntry::kNotPatchable);
1039 compiler->GenerateStubCall(token_pos(), *stub, PcDescriptorsLayout::kOther,
1040 locs());
1041 }
1042 __ popq(result);
1043
1044 __ Drop(ArgumentCount()); // Drop the arguments.
1045}
1046
1047void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
1048 const Register saved_fp = locs()->temp(0).reg();
1049 const Register target_address = locs()->in(TargetAddressIndex()).reg();
1050
1051 // Save frame pointer because we're going to update it when we enter the exit
1052 // frame.
1053 __ movq(saved_fp, FPREG);
1054
1055 // Make a space to put the return address.
1056 __ pushq(compiler::Immediate(0));
1057
1058 // We need to create a dummy "exit frame". It will share the same pool pointer
1059 // but have a null code object.
1060 __ LoadObject(CODE_REG, Object::null_object());
1061 __ set_constant_pool_allowed(false);
1062 __ EnterDartFrame(marshaller_.StackTopInBytes(), PP);
1063
1064 // Align frame before entering C++ world.
1065 if (OS::ActivationFrameAlignment() > 1) {
1066 __ andq(SPREG, compiler::Immediate(~(OS::ActivationFrameAlignment() - 1)));
1067 }
1068
1069 EmitParamMoves(compiler);
1070
1071 // We need to copy a dummy return address up into the dummy stack frame so the
1072 // stack walker will know which safepoint to use. RIP points to the *next*
1073 // instruction, so 'AddressRIPRelative' loads the address of the following
1074 // 'movq'.
1075 __ leaq(TMP, compiler::Address::AddressRIPRelative(0));
1076 compiler->EmitCallsiteMetadata(TokenPosition::kNoSource, deopt_id(),
1077 PcDescriptorsLayout::Kind::kOther, locs());
1078 __ movq(compiler::Address(FPREG, kSavedCallerPcSlotFromFp * kWordSize), TMP);
1079
1080 if (CanExecuteGeneratedCodeInSafepoint()) {
1081 // Update information in the thread object and enter a safepoint.
1082 __ movq(TMP,
1083 compiler::Immediate(compiler::target::Thread::exit_through_ffi()));
1084 __ TransitionGeneratedToNative(target_address, FPREG, TMP,
1085 /*enter_safepoint=*/true);
1086
1087 __ CallCFunction(target_address);
1088
1089 // Update information in the thread object and leave the safepoint.
1090 __ TransitionNativeToGenerated(/*leave_safepoint=*/true);
1091 } else {
1092 // We cannot trust that this code will be executable within a safepoint.
1093 // Therefore we delegate the responsibility of entering/exiting the
1094 // safepoint to a stub which in the VM isolate's heap, which will never lose
1095 // execute permission.
1096 __ movq(TMP,
1097 compiler::Address(
1098 THR, compiler::target::Thread::
1099 call_native_through_safepoint_entry_point_offset()));
1100
1101 // Calls RBX within a safepoint.
1102 ASSERT(saved_fp == RBX);
1103 __ movq(RBX, target_address);
1104 __ call(TMP);
1105 }
1106
1107 EmitReturnMoves(compiler);
1108
1109 // Although PP is a callee-saved register, it may have been moved by the GC.
1110 __ LeaveDartFrame(compiler::kRestoreCallerPP);
1111
1112 // Restore the global object pool after returning from runtime (old space is
1113 // moving, so the GOP could have been relocated).
1114 if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
1115 __ movq(PP, compiler::Address(THR, Thread::global_object_pool_offset()));
1116 }
1117
1118 __ set_constant_pool_allowed(true);
1119
1120 // Instead of returning to the "fake" return address, we just pop it.
1121 __ popq(TMP);
1122}
1123
1124void NativeEntryInstr::SaveArgument(
1125 FlowGraphCompiler* compiler,
1126 const compiler::ffi::NativeLocation& nloc) const {
1127 if (nloc.IsStack()) return;
1128
1129 if (nloc.IsRegisters()) {
1130 const auto& regs_loc = nloc.AsRegisters();
1131 ASSERT(regs_loc.num_regs() == 1);
1132 __ pushq(regs_loc.reg_at(0));
1133 } else if (nloc.IsFpuRegisters()) {
1134 // TODO(dartbug.com/40469): Reduce code size.
1135 __ movq(TMP, nloc.AsFpuRegisters().fpu_reg());
1136 __ pushq(TMP);
1137 } else {
1138 UNREACHABLE();
1139 }
1140}
1141
1142void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
1143 __ Bind(compiler->GetJumpLabel(this));
1144
1145 // Create a dummy frame holding the pushed arguments. This simplifies
1146 // NativeReturnInstr::EmitNativeCode.
1147 __ EnterFrame(0);
1148
1149#if defined(TARGET_OS_FUCHSIA)
1150 UNREACHABLE(); // Fuchsia does not allow dart:ffi.
1151#elif defined(USING_SHADOW_CALL_STACK)
1152#error Unimplemented
1153#endif
1154
1155 // Save the argument registers, in reverse order.
1156 for (intptr_t i = marshaller_.num_args(); i-- > 0;) {
1157 SaveArgument(compiler, marshaller_.Location(i));
1158 }
1159
1160 // Enter the entry frame. Push a dummy return address for consistency with
1161 // EnterFrame on ARM(64).
1162 __ PushImmediate(compiler::Immediate(0));
1163 __ EnterFrame(0);
1164
1165 // Save a space for the code object.
1166 __ PushImmediate(compiler::Immediate(0));
1167
1168 // InvokeDartCodeStub saves the arguments descriptor here. We don't have one,
1169 // but we need to follow the same frame layout for the stack walker.
1170 __ PushImmediate(compiler::Immediate(0));
1171
1172 // Save ABI callee-saved registers.
1173 __ PushRegisters(CallingConventions::kCalleeSaveCpuRegisters,
1174 CallingConventions::kCalleeSaveXmmRegisters);
1175
1176 // Load the address of DLRT_GetThreadForNativeCallback without using Thread.
1177 if (FLAG_precompiled_mode) {
1178 compiler->LoadBSSEntry(BSS::Relocation::DRT_GetThreadForNativeCallback, RAX,
1179 RCX);
1180 } else if (!NativeCallbackTrampolines::Enabled()) {
1181 // In JIT mode, we can just paste the address of the runtime entry into the
1182 // generated code directly. This is not a problem since we don't save
1183 // callbacks into JIT snapshots.
1184 __ movq(RAX, compiler::Immediate(reinterpret_cast<intptr_t>(
1185 DLRT_GetThreadForNativeCallback)));
1186 }
1187
1188 // Create another frame to align the frame before continuing in "native" code.
1189 // If we were called by a trampoline, it has already loaded the thread.
1190 if (!NativeCallbackTrampolines::Enabled()) {
1191 __ EnterFrame(0);
1192 __ ReserveAlignedFrameSpace(0);
1193
1194 COMPILE_ASSERT(RAX != CallingConventions::kArg1Reg);
1195 __ movq(CallingConventions::kArg1Reg, compiler::Immediate(callback_id_));
1196 __ CallCFunction(RAX);
1197 __ movq(THR, RAX);
1198
1199 __ LeaveFrame();
1200 }
1201
1202 // Save the current VMTag on the stack.
1203 __ movq(RAX, compiler::Assembler::VMTagAddress());
1204 __ pushq(RAX);
1205
1206 // Save top resource.
1207 __ pushq(
1208 compiler::Address(THR, compiler::target::Thread::top_resource_offset()));
1209 __ movq(
1210 compiler::Address(THR, compiler::target::Thread::top_resource_offset()),
1211 compiler::Immediate(0));
1212
1213 __ pushq(compiler::Address(
1214 THR, compiler::target::Thread::exit_through_ffi_offset()));
1215
1216 // Save top exit frame info. Stack walker expects it to be here.
1217 __ pushq(compiler::Address(
1218 THR, compiler::target::Thread::top_exit_frame_info_offset()));
1219
1220 // In debug mode, verify that we've pushed the top exit frame info at the
1221 // correct offset from FP.
1222 __ EmitEntryFrameVerification();
1223
1224 // Either DLRT_GetThreadForNativeCallback or the callback trampoline (caller)
1225 // will leave the safepoint for us.
1226 __ TransitionNativeToGenerated(/*exit_safepoint=*/false);
1227
1228 // Load the code object.
1229 __ movq(RAX, compiler::Address(
1230 THR, compiler::target::Thread::callback_code_offset()));
1231 __ movq(RAX, compiler::FieldAddress(
1232 RAX, compiler::target::GrowableObjectArray::data_offset()));
1233 __ movq(CODE_REG, compiler::FieldAddress(
1234 RAX, compiler::target::Array::data_offset() +
1235 callback_id_ * compiler::target::kWordSize));
1236
1237 // Put the code object in the reserved slot.
1238 __ movq(compiler::Address(FPREG,
1239 kPcMarkerSlotFromFp * compiler::target::kWordSize),
1240 CODE_REG);
1241
1242 if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
1243 __ movq(PP,
1244 compiler::Address(
1245 THR, compiler::target::Thread::global_object_pool_offset()));
1246 } else {
1247 __ xorq(PP, PP); // GC-safe value into PP.
1248 }
1249
1250 // Load a GC-safe value for arguments descriptor (unused but tagged).
1251 __ xorq(ARGS_DESC_REG, ARGS_DESC_REG);
1252
1253 // Push a dummy return address which suggests that we are inside of
1254 // InvokeDartCodeStub. This is how the stack walker detects an entry frame.
1255 __ movq(RAX,
1256 compiler::Address(
1257 THR, compiler::target::Thread::invoke_dart_code_stub_offset()));
1258 __ pushq(compiler::FieldAddress(
1259 RAX, compiler::target::Code::entry_point_offset()));
1260
1261 // Continue with Dart frame setup.
1262 FunctionEntryInstr::EmitNativeCode(compiler);
1263}
1264
1265static bool CanBeImmediateIndex(Value* index, intptr_t cid) {
1266 if (!index->definition()->IsConstant()) return false;
1267 const Object& constant = index->definition()->AsConstant()->value();
1268 if (!constant.IsSmi()) return false;
1269 const Smi& smi_const = Smi::Cast(constant);
1270 const intptr_t scale = Instance::ElementSizeFor(cid);
1271 const intptr_t data_offset = Instance::DataOffsetFor(cid);
1272 const int64_t disp = smi_const.AsInt64Value() * scale + data_offset;
1273 return Utils::IsInt(32, disp);
1274}
1275
1276LocationSummary* OneByteStringFromCharCodeInstr::MakeLocationSummary(
1277 Zone* zone,
1278 bool opt) const {
1279 const intptr_t kNumInputs = 1;
1280 // TODO(fschneider): Allow immediate operands for the char code.
1281 return LocationSummary::Make(zone, kNumInputs, Location::RequiresRegister(),
1282 LocationSummary::kNoCall);
1283}
1284
1285void OneByteStringFromCharCodeInstr::EmitNativeCode(
1286 FlowGraphCompiler* compiler) {
1287 ASSERT(compiler->is_optimizing());
1288 Register char_code = locs()->in(0).reg();
1289 Register result = locs()->out(0).reg();
1290
1291 __ movq(result,
1292 compiler::Address(THR, Thread::predefined_symbols_address_offset()));
1293 __ movq(result,
1294 compiler::Address(result, char_code,
1295 TIMES_HALF_WORD_SIZE, // Char code is a smi.
1296 Symbols::kNullCharCodeSymbolOffset * kWordSize));
1297}
1298
1299LocationSummary* StringToCharCodeInstr::MakeLocationSummary(Zone* zone,
1300 bool opt) const {
1301 const intptr_t kNumInputs = 1;
1302 return LocationSummary::Make(zone, kNumInputs, Location::RequiresRegister(),
1303 LocationSummary::kNoCall);
1304}
1305
1306void StringToCharCodeInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
1307 ASSERT(cid_ == kOneByteStringCid);
1308 Register str = locs()->in(0).reg();
1309 Register result = locs()->out(0).reg();
1310 compiler::Label is_one, done;
1311 __ movq(result, compiler::FieldAddress(str, String::length_offset()));
1312 __ cmpq(result, compiler::Immediate(Smi::RawValue(1)));
1313 __ j(EQUAL, &is_one, compiler::Assembler::kNearJump);
1314 __ movq(result, compiler::Immediate(Smi::RawValue(-1)));
1315 __ jmp(&done);
1316 __ Bind(&is_one);
1317 __ movzxb(result, compiler::FieldAddress(str, OneByteString::data_offset()));
1318 __ SmiTag(result);
1319 __ Bind(&done);
1320}
1321
1322LocationSummary* StringInterpolateInstr::MakeLocationSummary(Zone* zone,
1323 bool opt) const {
1324 const intptr_t kNumInputs = 1;
1325 const intptr_t kNumTemps = 0;
1326 LocationSummary* summary = new (zone)
1327 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kCall);
1328 summary->set_in(0, Location::RegisterLocation(RAX));
1329 summary->set_out(0, Location::RegisterLocation(RAX));
1330 return summary;
1331}
1332
1333void StringInterpolateInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
1334 Register array = locs()->in(0).reg();
1335 __ pushq(array);
1336 const int kTypeArgsLen = 0;
1337 const int kNumberOfArguments = 1;
1338 constexpr int kSizeOfArguments = 1;
1339 const Array& kNoArgumentNames = Object::null_array();
1340 ArgumentsInfo args_info(kTypeArgsLen, kNumberOfArguments, kSizeOfArguments,
1341 kNoArgumentNames);
1342 compiler->GenerateStaticCall(deopt_id(), token_pos(), CallFunction(),
1343 args_info, locs(), ICData::Handle(),
1344 ICData::kStatic);
1345 ASSERT(locs()->out(0).reg() == RAX);
1346}
1347
1348LocationSummary* Utf8ScanInstr::MakeLocationSummary(Zone* zone,
1349 bool opt) const {
1350 const intptr_t kNumInputs = 5;
1351 const intptr_t kNumTemps = 1;
1352 LocationSummary* summary = new (zone)
1353 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
1354 summary->set_in(0, Location::Any()); // decoder
1355 summary->set_in(1, Location::WritableRegister()); // bytes
1356 summary->set_in(2, Location::WritableRegister()); // start
1357 summary->set_in(3, Location::WritableRegister()); // end
1358 summary->set_in(4, Location::RequiresRegister()); // table
1359 summary->set_temp(0, Location::RequiresRegister());
1360 summary->set_out(0, Location::RequiresRegister());
1361 return summary;
1362}
1363
1364void Utf8ScanInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
1365 const Register bytes_reg = locs()->in(1).reg();
1366 const Register start_reg = locs()->in(2).reg();
1367 const Register end_reg = locs()->in(3).reg();
1368 const Register table_reg = locs()->in(4).reg();
1369 const Register size_reg = locs()->out(0).reg();
1370
1371 const Register bytes_ptr_reg = start_reg;
1372 const Register bytes_end_reg = end_reg;
1373 const Register bytes_end_minus_16_reg = bytes_reg;
1374 const Register flags_reg = locs()->temp(0).reg();
1375 const Register temp_reg = TMP;
1376 const XmmRegister vector_reg = FpuTMP;
1377
1378 static const intptr_t kSizeMask = 0x03;
1379 static const intptr_t kFlagsMask = 0x3C;
1380
1381 compiler::Label scan_ascii, ascii_loop, ascii_loop_in, nonascii_loop;
1382 compiler::Label rest, rest_loop, rest_loop_in, done;
1383
1384 // Address of input bytes.
1385 __ movq(bytes_reg,
1386 compiler::FieldAddress(
1387 bytes_reg, compiler::target::TypedDataBase::data_field_offset()));
1388
1389 // Pointers to start, end and end-16.
1390 __ leaq(bytes_ptr_reg, compiler::Address(bytes_reg, start_reg, TIMES_1, 0));
1391 __ leaq(bytes_end_reg, compiler::Address(bytes_reg, end_reg, TIMES_1, 0));
1392 __ leaq(bytes_end_minus_16_reg, compiler::Address(bytes_end_reg, -16));
1393
1394 // Initialize size and flags.
1395 __ xorq(size_reg, size_reg);
1396 __ xorq(flags_reg, flags_reg);
1397
1398 __ jmp(&scan_ascii, compiler::Assembler::kNearJump);
1399
1400 // Loop scanning through ASCII bytes one 16-byte vector at a time.
1401 // While scanning, the size register contains the size as it was at the start
1402 // of the current block of ASCII bytes, minus the address of the start of the
1403 // block. After the block, the end address of the block is added to update the
1404 // size to include the bytes in the block.
1405 __ Bind(&ascii_loop);
1406 __ addq(bytes_ptr_reg, compiler::Immediate(16));
1407 __ Bind(&ascii_loop_in);
1408
1409 // Exit vectorized loop when there are less than 16 bytes left.
1410 __ cmpq(bytes_ptr_reg, bytes_end_minus_16_reg);
1411 __ j(UNSIGNED_GREATER, &rest, compiler::Assembler::kNearJump);
1412
1413 // Find next non-ASCII byte within the next 16 bytes.
1414 // Note: In principle, we should use MOVDQU here, since the loaded value is
1415 // used as input to an integer instruction. In practice, according to Agner
1416 // Fog, there is no penalty for using the wrong kind of load.
1417 __ movups(vector_reg, compiler::Address(bytes_ptr_reg, 0));
1418 __ pmovmskb(temp_reg, vector_reg);
1419 __ bsfq(temp_reg, temp_reg);
1420 __ j(EQUAL, &ascii_loop, compiler::Assembler::kNearJump);
1421
1422 // Point to non-ASCII byte and update size.
1423 __ addq(bytes_ptr_reg, temp_reg);
1424 __ addq(size_reg, bytes_ptr_reg);
1425
1426 // Read first non-ASCII byte.
1427 __ movzxb(temp_reg, compiler::Address(bytes_ptr_reg, 0));
1428
1429 // Loop over block of non-ASCII bytes.
1430 __ Bind(&nonascii_loop);
1431 __ addq(bytes_ptr_reg, compiler::Immediate(1));
1432
1433 // Update size and flags based on byte value.
1434 __ movzxb(temp_reg, compiler::FieldAddress(
1435 table_reg, temp_reg, TIMES_1,
1436 compiler::target::OneByteString::data_offset()));
1437 __ orq(flags_reg, temp_reg);
1438 __ andq(temp_reg, compiler::Immediate(kSizeMask));
1439 __ addq(size_reg, temp_reg);
1440
1441 // Stop if end is reached.
1442 __ cmpq(bytes_ptr_reg, bytes_end_reg);
1443 __ j(UNSIGNED_GREATER_EQUAL, &done, compiler::Assembler::kNearJump);
1444
1445 // Go to ASCII scan if next byte is ASCII, otherwise loop.
1446 __ movzxb(temp_reg, compiler::Address(bytes_ptr_reg, 0));
1447 __ testq(temp_reg, compiler::Immediate(0x80));
1448 __ j(NOT_EQUAL, &nonascii_loop, compiler::Assembler::kNearJump);
1449
1450 // Enter the ASCII scanning loop.
1451 __ Bind(&scan_ascii);
1452 __ subq(size_reg, bytes_ptr_reg);
1453 __ jmp(&ascii_loop_in);
1454
1455 // Less than 16 bytes left. Process the remaining bytes individually.
1456 __ Bind(&rest);
1457
1458 // Update size after ASCII scanning loop.
1459 __ addq(size_reg, bytes_ptr_reg);
1460 __ jmp(&rest_loop_in, compiler::Assembler::kNearJump);
1461
1462 __ Bind(&rest_loop);
1463
1464 // Read byte and increment pointer.
1465 __ movzxb(temp_reg, compiler::Address(bytes_ptr_reg, 0));
1466 __ addq(bytes_ptr_reg, compiler::Immediate(1));
1467
1468 // Update size and flags based on byte value.
1469 __ movzxb(temp_reg, compiler::FieldAddress(
1470 table_reg, temp_reg, TIMES_1,
1471 compiler::target::OneByteString::data_offset()));
1472 __ orq(flags_reg, temp_reg);
1473 __ andq(temp_reg, compiler::Immediate(kSizeMask));
1474 __ addq(size_reg, temp_reg);
1475
1476 // Stop if end is reached.
1477 __ Bind(&rest_loop_in);
1478 __ cmpq(bytes_ptr_reg, bytes_end_reg);
1479 __ j(UNSIGNED_LESS, &rest_loop, compiler::Assembler::kNearJump);
1480 __ Bind(&done);
1481
1482 // Write flags to field.
1483 __ andq(flags_reg, compiler::Immediate(kFlagsMask));
1484 if (!IsScanFlagsUnboxed()) {
1485 __ SmiTag(flags_reg);
1486 }
1487 Register decoder_reg;
1488 const Location decoder_location = locs()->in(0);
1489 if (decoder_location.IsStackSlot()) {
1490 __ movq(temp_reg, LocationToStackSlotAddress(decoder_location));
1491 decoder_reg = temp_reg;
1492 } else {
1493 decoder_reg = decoder_location.reg();
1494 }
1495 const auto scan_flags_field_offset = scan_flags_field_.offset_in_bytes();
1496 __ orq(compiler::FieldAddress(decoder_reg, scan_flags_field_offset),
1497 flags_reg);
1498}
1499
1500LocationSummary* LoadUntaggedInstr::MakeLocationSummary(Zone* zone,
1501 bool opt) const {
1502 const intptr_t kNumInputs = 1;
1503 return LocationSummary::Make(zone, kNumInputs, Location::RequiresRegister(),
1504 LocationSummary::kNoCall);
1505}
1506
1507void LoadUntaggedInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
1508 Register obj = locs()->in(0).reg();
1509 Register result = locs()->out(0).reg();
1510 if (object()->definition()->representation() == kUntagged) {
1511 __ movq(result, compiler::Address(obj, offset()));
1512 } else {
1513 ASSERT(object()->definition()->representation() == kTagged);
1514 __ movq(result, compiler::FieldAddress(obj, offset()));
1515 }
1516}
1517
1518DEFINE_BACKEND(StoreUntagged, (NoLocation, Register obj, Register value)) {
1519 __ movq(compiler::Address(obj, instr->offset_from_tagged()), value);
1520}
1521
1522class BoxAllocationSlowPath : public TemplateSlowPathCode<Instruction> {
1523 public:
1524 BoxAllocationSlowPath(Instruction* instruction,
1525 const Class& cls,
1526 Register result)
1527 : TemplateSlowPathCode(instruction), cls_(cls), result_(result) {}
1528
1529 virtual void EmitNativeCode(FlowGraphCompiler* compiler) {
1530 if (compiler::Assembler::EmittingComments()) {
1531 __ Comment("%s slow path allocation of %s", instruction()->DebugName(),
1532 String::Handle(cls_.ScrubbedName()).ToCString());
1533 }
1534 __ Bind(entry_label());
1535 const Code& stub = Code::ZoneHandle(
1536 compiler->zone(), StubCode::GetAllocationStubForClass(cls_));
1537
1538 LocationSummary* locs = instruction()->locs();
1539
1540 locs->live_registers()->Remove(Location::RegisterLocation(result_));
1541
1542 compiler->SaveLiveRegisters(locs);
1543 compiler->GenerateStubCall(TokenPosition::kNoSource, // No token position.
1544 stub, PcDescriptorsLayout::kOther, locs);
1545 __ MoveRegister(result_, RAX);
1546 compiler->RestoreLiveRegisters(locs);
1547 __ jmp(exit_label());
1548 }
1549
1550 static void Allocate(FlowGraphCompiler* compiler,
1551 Instruction* instruction,
1552 const Class& cls,
1553 Register result,
1554 Register temp) {
1555 if (compiler->intrinsic_mode()) {
1556 __ TryAllocate(cls, compiler->intrinsic_slow_path_label(),
1557 compiler::Assembler::kFarJump, result, temp);
1558 } else {
1559 BoxAllocationSlowPath* slow_path =
1560 new BoxAllocationSlowPath(instruction, cls, result);
1561 compiler->AddSlowPathCode(slow_path);
1562
1563 __ TryAllocate(cls, slow_path->entry_label(),
1564 compiler::Assembler::kFarJump, result, temp);
1565 __ Bind(slow_path->exit_label());
1566 }
1567 }
1568
1569 private:
1570 const Class& cls_;
1571 const Register result_;
1572};
1573
1574Representation LoadIndexedInstr::representation() const {
1575 switch (class_id_) {
1576 case kArrayCid:
1577 case kImmutableArrayCid:
1578 return kTagged;
1579 case kOneByteStringCid:
1580 case kTwoByteStringCid:
1581 case kTypedDataInt8ArrayCid:
1582 case kTypedDataInt16ArrayCid:
1583 case kTypedDataUint8ArrayCid:
1584 case kTypedDataUint8ClampedArrayCid:
1585 case kTypedDataUint16ArrayCid:
1586 case kExternalOneByteStringCid:
1587 case kExternalTwoByteStringCid:
1588 case kExternalTypedDataUint8ArrayCid:
1589 case kExternalTypedDataUint8ClampedArrayCid:
1590 return kUnboxedIntPtr;
1591 case kTypedDataInt32ArrayCid:
1592 return kUnboxedInt32;
1593 case kTypedDataUint32ArrayCid:
1594 return kUnboxedUint32;
1595 case kTypedDataInt64ArrayCid:
1596 case kTypedDataUint64ArrayCid:
1597 return kUnboxedInt64;
1598 case kTypedDataFloat32ArrayCid:
1599 case kTypedDataFloat64ArrayCid:
1600 return kUnboxedDouble;
1601 case kTypedDataInt32x4ArrayCid:
1602 return kUnboxedInt32x4;
1603 case kTypedDataFloat32x4ArrayCid:
1604 return kUnboxedFloat32x4;
1605 case kTypedDataFloat64x2ArrayCid:
1606 return kUnboxedFloat64x2;
1607 default:
1608 UNIMPLEMENTED();
1609 return kTagged;
1610 }
1611}
1612
1613LocationSummary* LoadIndexedInstr::MakeLocationSummary(Zone* zone,
1614 bool opt) const {
1615 const intptr_t kNumInputs = 2;
1616 const intptr_t kNumTemps = 0;
1617 LocationSummary* locs = new (zone)
1618 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
1619 locs->set_in(0, Location::RequiresRegister());
1620 // For tagged index with index_scale=1 as well as untagged index with
1621 // index_scale=16 we need a writable register due to assdressing mode
1622 // restrictions on X64.
1623 const bool need_writable_index_register =
1624 (index_scale() == 1 && !index_unboxed_) ||
1625 (index_scale() == 16 && index_unboxed_);
1626 locs->set_in(
1627 1, CanBeImmediateIndex(index(), class_id())
1628 ? Location::Constant(index()->definition()->AsConstant())
1629 : (need_writable_index_register ? Location::WritableRegister()
1630 : Location::RequiresRegister()));
1631 if ((representation() == kUnboxedDouble) ||
1632 (representation() == kUnboxedFloat32x4) ||
1633 (representation() == kUnboxedInt32x4) ||
1634 (representation() == kUnboxedFloat64x2)) {
1635 locs->set_out(0, Location::RequiresFpuRegister());
1636 } else {
1637 locs->set_out(0, Location::RequiresRegister());
1638 }
1639 return locs;
1640}
1641
1642void LoadIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
1643 // The array register points to the backing store for external arrays.
1644 const Register array = locs()->in(0).reg();
1645 const Location index = locs()->in(1);
1646
1647 intptr_t index_scale = index_scale_;
1648 if (index.IsRegister()) {
1649 if (index_scale == 1 && !index_unboxed_) {
1650 __ SmiUntag(index.reg());
1651 } else if (index_scale == 16 && index_unboxed_) {
1652 // X64 does not support addressing mode using TIMES_16.
1653 __ SmiTag(index.reg());
1654 index_scale >>= 1;
1655 }
1656 } else {
1657 ASSERT(index.IsConstant());
1658 }
1659
1660 compiler::Address element_address =
1661 index.IsRegister() ? compiler::Assembler::ElementAddressForRegIndex(
1662 IsExternal(), class_id(), index_scale,
1663 index_unboxed_, array, index.reg())
1664 : compiler::Assembler::ElementAddressForIntIndex(
1665 IsExternal(), class_id(), index_scale, array,
1666 Smi::Cast(index.constant()).Value());
1667
1668 if (representation() == kUnboxedDouble ||
1669 representation() == kUnboxedFloat32x4 ||
1670 representation() == kUnboxedInt32x4 ||
1671 representation() == kUnboxedFloat64x2) {
1672 XmmRegister result = locs()->out(0).fpu_reg();
1673 if (class_id() == kTypedDataFloat32ArrayCid) {
1674 // Load single precision float.
1675 __ movss(result, element_address);
1676 } else if (class_id() == kTypedDataFloat64ArrayCid) {
1677 __ movsd(result, element_address);
1678 } else {
1679 ASSERT((class_id() == kTypedDataInt32x4ArrayCid) ||
1680 (class_id() == kTypedDataFloat32x4ArrayCid) ||
1681 (class_id() == kTypedDataFloat64x2ArrayCid));
1682 __ movups(result, element_address);
1683 }
1684 return;
1685 }
1686
1687 Register result = locs()->out(0).reg();
1688 switch (class_id()) {
1689 case kTypedDataInt32ArrayCid:
1690 ASSERT(representation() == kUnboxedInt32);
1691 __ movsxd(result, element_address);
1692 break;
1693 case kTypedDataUint32ArrayCid:
1694 ASSERT(representation() == kUnboxedUint32);
1695 __ movl(result, element_address);
1696 break;
1697 case kTypedDataInt64ArrayCid:
1698 case kTypedDataUint64ArrayCid:
1699 ASSERT(representation() == kUnboxedInt64);
1700 __ movq(result, element_address);
1701 break;
1702 case kTypedDataInt8ArrayCid:
1703 ASSERT(representation() == kUnboxedIntPtr);
1704 __ movsxb(result, element_address);
1705 break;
1706 case kTypedDataUint8ArrayCid:
1707 case kTypedDataUint8ClampedArrayCid:
1708 case kExternalTypedDataUint8ArrayCid:
1709 case kExternalTypedDataUint8ClampedArrayCid:
1710 case kOneByteStringCid:
1711 case kExternalOneByteStringCid:
1712 ASSERT(representation() == kUnboxedIntPtr);
1713 __ movzxb(result, element_address);
1714 break;
1715 case kTypedDataInt16ArrayCid:
1716 ASSERT(representation() == kUnboxedIntPtr);
1717 __ movsxw(result, element_address);
1718 break;
1719 case kTypedDataUint16ArrayCid:
1720 case kTwoByteStringCid:
1721 case kExternalTwoByteStringCid:
1722 ASSERT(representation() == kUnboxedIntPtr);
1723 __ movzxw(result, element_address);
1724 break;
1725 default:
1726 ASSERT(representation() == kTagged);
1727 ASSERT((class_id() == kArrayCid) || (class_id() == kImmutableArrayCid));
1728 __ movq(result, element_address);
1729 break;
1730 }
1731}
1732
1733LocationSummary* LoadCodeUnitsInstr::MakeLocationSummary(Zone* zone,
1734 bool opt) const {
1735 const intptr_t kNumInputs = 2;
1736 const intptr_t kNumTemps = 0;
1737 LocationSummary* summary = new (zone)
1738 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
1739 summary->set_in(0, Location::RequiresRegister());
1740 // The smi index is either untagged (element size == 1), or it is left smi
1741 // tagged (for all element sizes > 1).
1742 summary->set_in(1, index_scale() == 1 ? Location::WritableRegister()
1743 : Location::RequiresRegister());
1744 summary->set_out(0, Location::RequiresRegister());
1745 return summary;
1746}
1747
1748void LoadCodeUnitsInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
1749 // The string register points to the backing store for external strings.
1750 const Register str = locs()->in(0).reg();
1751 const Location index = locs()->in(1);
1752
1753 compiler::Address element_address =
1754 compiler::Assembler::ElementAddressForRegIndex(
1755 IsExternal(), class_id(), index_scale(), /*index_unboxed=*/false, str,
1756 index.reg());
1757
1758 if ((index_scale() == 1)) {
1759 __ SmiUntag(index.reg());
1760 }
1761 Register result = locs()->out(0).reg();
1762 switch (class_id()) {
1763 case kOneByteStringCid:
1764 case kExternalOneByteStringCid:
1765 switch (element_count()) {
1766 case 1:
1767 __ movzxb(result, element_address);
1768 break;
1769 case 2:
1770 __ movzxw(result, element_address);
1771 break;
1772 case 4:
1773 __ movl(result, element_address);
1774 break;
1775 default:
1776 UNREACHABLE();
1777 }
1778 __ SmiTag(result);
1779 break;
1780 case kTwoByteStringCid:
1781 case kExternalTwoByteStringCid:
1782 switch (element_count()) {
1783 case 1:
1784 __ movzxw(result, element_address);
1785 break;
1786 case 2:
1787 __ movl(result, element_address);
1788 break;
1789 default:
1790 UNREACHABLE();
1791 }
1792 __ SmiTag(result);
1793 break;
1794 default:
1795 UNREACHABLE();
1796 break;
1797 }
1798}
1799
1800Representation StoreIndexedInstr::RequiredInputRepresentation(
1801 intptr_t idx) const {
1802 if (idx == 0) return kNoRepresentation;
1803 if (idx == 1) {
1804 if (index_unboxed_) {
1805 return kUnboxedInt64;
1806 } else {
1807 return kTagged; // Index is a smi.
1808 }
1809 }
1810 ASSERT(idx == 2);
1811 switch (class_id_) {
1812 case kArrayCid:
1813 return kTagged;
1814 case kOneByteStringCid:
1815 case kTwoByteStringCid:
1816 case kTypedDataInt8ArrayCid:
1817 case kTypedDataInt16ArrayCid:
1818 case kTypedDataUint8ArrayCid:
1819 case kTypedDataUint8ClampedArrayCid:
1820 case kTypedDataUint16ArrayCid:
1821 case kExternalTypedDataUint8ArrayCid:
1822 case kExternalTypedDataUint8ClampedArrayCid:
1823 return kUnboxedIntPtr;
1824 case kTypedDataInt32ArrayCid:
1825 return kUnboxedInt32;
1826 case kTypedDataUint32ArrayCid:
1827 return kUnboxedUint32;
1828 case kTypedDataInt64ArrayCid:
1829 case kTypedDataUint64ArrayCid:
1830 return kUnboxedInt64;
1831 case kTypedDataFloat32ArrayCid:
1832 case kTypedDataFloat64ArrayCid:
1833 return kUnboxedDouble;
1834 case kTypedDataFloat32x4ArrayCid:
1835 return kUnboxedFloat32x4;
1836 case kTypedDataInt32x4ArrayCid:
1837 return kUnboxedInt32x4;
1838 case kTypedDataFloat64x2ArrayCid:
1839 return kUnboxedFloat64x2;
1840 default:
1841 UNIMPLEMENTED();
1842 return kTagged;
1843 }
1844}
1845
1846LocationSummary* StoreIndexedInstr::MakeLocationSummary(Zone* zone,
1847 bool opt) const {
1848 const intptr_t kNumInputs = 3;
1849 const intptr_t kNumTemps =
1850 class_id() == kArrayCid && ShouldEmitStoreBarrier() ? 1 : 0;
1851 LocationSummary* locs = new (zone)
1852 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
1853 locs->set_in(0, Location::RequiresRegister());
1854 // For tagged index with index_scale=1 as well as untagged index with
1855 // index_scale=16 we need a writable register due to assdressing mode
1856 // restrictions on X64.
1857 const bool need_writable_index_register =
1858 (index_scale() == 1 && !index_unboxed_) ||
1859 (index_scale() == 16 && index_unboxed_);
1860 locs->set_in(
1861 1, CanBeImmediateIndex(index(), class_id())
1862 ? Location::Constant(index()->definition()->AsConstant())
1863 : (need_writable_index_register ? Location::WritableRegister()
1864 : Location::RequiresRegister()));
1865 switch (class_id()) {
1866 case kArrayCid:
1867 locs->set_in(2, ShouldEmitStoreBarrier()
1868 ? Location::RegisterLocation(kWriteBarrierValueReg)
1869 : LocationRegisterOrConstant(value()));
1870 if (ShouldEmitStoreBarrier()) {
1871 locs->set_in(0, Location::RegisterLocation(kWriteBarrierObjectReg));
1872 locs->set_temp(0, Location::RegisterLocation(kWriteBarrierSlotReg));
1873 }
1874 break;
1875 case kExternalTypedDataUint8ArrayCid:
1876 case kExternalTypedDataUint8ClampedArrayCid:
1877 case kTypedDataInt8ArrayCid:
1878 case kTypedDataUint8ArrayCid:
1879 case kTypedDataUint8ClampedArrayCid:
1880 case kOneByteStringCid:
1881 case kTwoByteStringCid:
1882 // TODO(fschneider): Add location constraint for byte registers (RAX,
1883 // RBX, RCX, RDX) instead of using a fixed register.
1884 locs->set_in(2, LocationFixedRegisterOrSmiConstant(value(), RAX));
1885 break;
1886 case kTypedDataInt16ArrayCid:
1887 case kTypedDataUint16ArrayCid:
1888 // Writable register because the value must be untagged before storing.
1889 locs->set_in(2, Location::WritableRegister());
1890 break;
1891 case kTypedDataInt32ArrayCid:
1892 case kTypedDataUint32ArrayCid:
1893 case kTypedDataInt64ArrayCid:
1894 case kTypedDataUint64ArrayCid:
1895 locs->set_in(2, Location::RequiresRegister());
1896 break;
1897 case kTypedDataFloat32ArrayCid:
1898 case kTypedDataFloat64ArrayCid:
1899 // TODO(srdjan): Support Float64 constants.
1900 locs->set_in(2, Location::RequiresFpuRegister());
1901 break;
1902 case kTypedDataInt32x4ArrayCid:
1903 case kTypedDataFloat64x2ArrayCid:
1904 case kTypedDataFloat32x4ArrayCid:
1905 locs->set_in(2, Location::RequiresFpuRegister());
1906 break;
1907 default:
1908 UNREACHABLE();
1909 return NULL;
1910 }
1911 return locs;
1912}
1913
1914void StoreIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
1915 // The array register points to the backing store for external arrays.
1916 const Register array = locs()->in(0).reg();
1917 const Location index = locs()->in(1);
1918
1919 intptr_t index_scale = index_scale_;
1920 if (index.IsRegister()) {
1921 if (index_scale == 1 && !index_unboxed_) {
1922 __ SmiUntag(index.reg());
1923 } else if (index_scale == 16 && index_unboxed_) {
1924 // X64 does not support addressing mode using TIMES_16.
1925 __ SmiTag(index.reg());
1926 index_scale >>= 1;
1927 }
1928 } else {
1929 ASSERT(index.IsConstant());
1930 }
1931
1932 compiler::Address element_address =
1933 index.IsRegister() ? compiler::Assembler::ElementAddressForRegIndex(
1934 IsExternal(), class_id(), index_scale,
1935 index_unboxed_, array, index.reg())
1936 : compiler::Assembler::ElementAddressForIntIndex(
1937 IsExternal(), class_id(), index_scale, array,
1938 Smi::Cast(index.constant()).Value());
1939
1940 switch (class_id()) {
1941 case kArrayCid:
1942 if (ShouldEmitStoreBarrier()) {
1943 Register value = locs()->in(2).reg();
1944 Register slot = locs()->temp(0).reg();
1945 __ leaq(slot, element_address);
1946 __ StoreIntoArray(array, slot, value, CanValueBeSmi());
1947 } else if (locs()->in(2).IsConstant()) {
1948 const Object& constant = locs()->in(2).constant();
1949 __ StoreIntoObjectNoBarrier(array, element_address, constant);
1950 } else {
1951 Register value = locs()->in(2).reg();
1952 __ StoreIntoObjectNoBarrier(array, element_address, value);
1953 }
1954 break;
1955 case kOneByteStringCid:
1956 case kTypedDataInt8ArrayCid:
1957 case kTypedDataUint8ArrayCid:
1958 case kExternalTypedDataUint8ArrayCid:
1959 ASSERT(RequiredInputRepresentation(2) == kUnboxedIntPtr);
1960 if (locs()->in(2).IsConstant()) {
1961 const Smi& constant = Smi::Cast(locs()->in(2).constant());
1962 __ movb(element_address,
1963 compiler::Immediate(static_cast<int8_t>(constant.Value())));
1964 } else {
1965 ASSERT(locs()->in(2).reg() == RAX);
1966 __ movb(element_address, RAX);
1967 }
1968 break;
1969 case kTypedDataUint8ClampedArrayCid:
1970 case kExternalTypedDataUint8ClampedArrayCid: {
1971 ASSERT(RequiredInputRepresentation(2) == kUnboxedIntPtr);
1972 if (locs()->in(2).IsConstant()) {
1973 const Smi& constant = Smi::Cast(locs()->in(2).constant());
1974 intptr_t value = constant.Value();
1975 // Clamp to 0x0 or 0xFF respectively.
1976 if (value > 0xFF) {
1977 value = 0xFF;
1978 } else if (value < 0) {
1979 value = 0;
1980 }
1981 __ movb(element_address,
1982 compiler::Immediate(static_cast<int8_t>(value)));
1983 } else {
1984 ASSERT(locs()->in(2).reg() == RAX);
1985 compiler::Label store_value, store_0xff;
1986 __ CompareImmediate(RAX, compiler::Immediate(0xFF));
1987 __ j(BELOW_EQUAL, &store_value, compiler::Assembler::kNearJump);
1988 // Clamp to 0x0 or 0xFF respectively.
1989 __ j(GREATER, &store_0xff);
1990 __ xorq(RAX, RAX);
1991 __ jmp(&store_value, compiler::Assembler::kNearJump);
1992 __ Bind(&store_0xff);
1993 __ LoadImmediate(RAX, compiler::Immediate(0xFF));
1994 __ Bind(&store_value);
1995 __ movb(element_address, RAX);
1996 }
1997 break;
1998 }
1999 case kTwoByteStringCid:
2000 case kTypedDataInt16ArrayCid:
2001 case kTypedDataUint16ArrayCid: {
2002 ASSERT(RequiredInputRepresentation(2) == kUnboxedIntPtr);
2003 Register value = locs()->in(2).reg();
2004 __ movw(element_address, value);
2005 break;
2006 }
2007 case kTypedDataInt32ArrayCid:
2008 case kTypedDataUint32ArrayCid: {
2009 Register value = locs()->in(2).reg();
2010 __ movl(element_address, value);
2011 break;
2012 }
2013 case kTypedDataInt64ArrayCid:
2014 case kTypedDataUint64ArrayCid: {
2015 Register value = locs()->in(2).reg();
2016 __ movq(element_address, value);
2017 break;
2018 }
2019 case kTypedDataFloat32ArrayCid:
2020 __ movss(element_address, locs()->in(2).fpu_reg());
2021 break;
2022 case kTypedDataFloat64ArrayCid:
2023 __ movsd(element_address, locs()->in(2).fpu_reg());
2024 break;
2025 case kTypedDataInt32x4ArrayCid:
2026 case kTypedDataFloat64x2ArrayCid:
2027 case kTypedDataFloat32x4ArrayCid:
2028 __ movups(element_address, locs()->in(2).fpu_reg());
2029 break;
2030 default:
2031 UNREACHABLE();
2032 }
2033}
2034
2035LocationSummary* GuardFieldClassInstr::MakeLocationSummary(Zone* zone,
2036 bool opt) const {
2037 const intptr_t kNumInputs = 1;
2038
2039 const intptr_t value_cid = value()->Type()->ToCid();
2040 const intptr_t field_cid = field().guarded_cid();
2041
2042 const bool emit_full_guard = !opt || (field_cid == kIllegalCid);
2043 const bool needs_value_cid_temp_reg =
2044 (value_cid == kDynamicCid) && (emit_full_guard || (field_cid != kSmiCid));
2045 const bool needs_field_temp_reg = emit_full_guard;
2046
2047 intptr_t num_temps = 0;
2048 if (needs_value_cid_temp_reg) {
2049 num_temps++;
2050 }
2051 if (needs_field_temp_reg) {
2052 num_temps++;
2053 }
2054
2055 LocationSummary* summary = new (zone)
2056 LocationSummary(zone, kNumInputs, num_temps, LocationSummary::kNoCall);
2057 summary->set_in(0, Location::RequiresRegister());
2058
2059 for (intptr_t i = 0; i < num_temps; i++) {
2060 summary->set_temp(i, Location::RequiresRegister());
2061 }
2062
2063 return summary;
2064}
2065
2066void GuardFieldClassInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
2067 ASSERT(compiler::target::ObjectLayout::kClassIdTagSize == 16);
2068 ASSERT(sizeof(FieldLayout::guarded_cid_) == 2);
2069 ASSERT(sizeof(FieldLayout::is_nullable_) == 2);
2070
2071 const intptr_t value_cid = value()->Type()->ToCid();
2072 const intptr_t field_cid = field().guarded_cid();
2073 const intptr_t nullability = field().is_nullable() ? kNullCid : kIllegalCid;
2074
2075 if (field_cid == kDynamicCid) {
2076 return; // Nothing to emit.
2077 }
2078
2079 const bool emit_full_guard =
2080 !compiler->is_optimizing() || (field_cid == kIllegalCid);
2081
2082 const bool needs_value_cid_temp_reg =
2083 (value_cid == kDynamicCid) && (emit_full_guard || (field_cid != kSmiCid));
2084
2085 const bool needs_field_temp_reg = emit_full_guard;
2086
2087 const Register value_reg = locs()->in(0).reg();
2088
2089 const Register value_cid_reg =
2090 needs_value_cid_temp_reg ? locs()->temp(0).reg() : kNoRegister;
2091
2092 const Register field_reg = needs_field_temp_reg
2093 ? locs()->temp(locs()->temp_count() - 1).reg()
2094 : kNoRegister;
2095
2096 compiler::Label ok, fail_label;
2097
2098 compiler::Label* deopt = NULL;
2099 if (compiler->is_optimizing()) {
2100 deopt = compiler->AddDeoptStub(deopt_id(), ICData::kDeoptGuardField);
2101 }
2102
2103 compiler::Label* fail = (deopt != NULL) ? deopt : &fail_label;
2104
2105 if (emit_full_guard) {
2106 __ LoadObject(field_reg, Field::ZoneHandle(field().Original()));
2107
2108 compiler::FieldAddress field_cid_operand(field_reg,
2109 Field::guarded_cid_offset());
2110 compiler::FieldAddress field_nullability_operand(
2111 field_reg, Field::is_nullable_offset());
2112
2113 if (value_cid == kDynamicCid) {
2114 LoadValueCid(compiler, value_cid_reg, value_reg);
2115
2116 __ cmpw(value_cid_reg, field_cid_operand);
2117 __ j(EQUAL, &ok);
2118 __ cmpw(value_cid_reg, field_nullability_operand);
2119 } else if (value_cid == kNullCid) {
2120 __ cmpw(field_nullability_operand, compiler::Immediate(value_cid));
2121 } else {
2122 __ cmpw(field_cid_operand, compiler::Immediate(value_cid));
2123 }
2124 __ j(EQUAL, &ok);
2125
2126 // Check if the tracked state of the guarded field can be initialized
2127 // inline. If the field needs length check or requires type arguments and
2128 // class hierarchy processing for exactness tracking then we fall through
2129 // into runtime which is responsible for computing offset of the length
2130 // field based on the class id.
2131 const bool is_complicated_field =
2132 field().needs_length_check() ||
2133 field().static_type_exactness_state().IsUninitialized();
2134 if (!is_complicated_field) {
2135 // Uninitialized field can be handled inline. Check if the
2136 // field is still unitialized.
2137 __ cmpw(field_cid_operand, compiler::Immediate(kIllegalCid));
2138 __ j(NOT_EQUAL, fail);
2139
2140 if (value_cid == kDynamicCid) {
2141 __ movw(field_cid_operand, value_cid_reg);
2142 __ movw(field_nullability_operand, value_cid_reg);
2143 } else {
2144 ASSERT(field_reg != kNoRegister);
2145 __ movw(field_cid_operand, compiler::Immediate(value_cid));
2146 __ movw(field_nullability_operand, compiler::Immediate(value_cid));
2147 }
2148
2149 __ jmp(&ok);
2150 }
2151
2152 if (deopt == NULL) {
2153 ASSERT(!compiler->is_optimizing());
2154 __ Bind(fail);
2155
2156 __ cmpw(compiler::FieldAddress(field_reg, Field::guarded_cid_offset()),
2157 compiler::Immediate(kDynamicCid));
2158 __ j(EQUAL, &ok);
2159
2160 __ pushq(field_reg);
2161 __ pushq(value_reg);
2162 __ CallRuntime(kUpdateFieldCidRuntimeEntry, 2);
2163 __ Drop(2); // Drop the field and the value.
2164 } else {
2165 __ jmp(fail);
2166 }
2167 } else {
2168 ASSERT(compiler->is_optimizing());
2169 ASSERT(deopt != NULL);
2170
2171 // Field guard class has been initialized and is known.
2172 if (value_cid == kDynamicCid) {
2173 // Value's class id is not known.
2174 __ testq(value_reg, compiler::Immediate(kSmiTagMask));
2175
2176 if (field_cid != kSmiCid) {
2177 __ j(ZERO, fail);
2178 __ LoadClassId(value_cid_reg, value_reg);
2179 __ CompareImmediate(value_cid_reg, compiler::Immediate(field_cid));
2180 }
2181
2182 if (field().is_nullable() && (field_cid != kNullCid)) {
2183 __ j(EQUAL, &ok);
2184 __ CompareObject(value_reg, Object::null_object());
2185 }
2186
2187 __ j(NOT_EQUAL, fail);
2188 } else if (value_cid == field_cid) {
2189 // This would normaly be caught by Canonicalize, but RemoveRedefinitions
2190 // may sometimes produce the situation after the last Canonicalize pass.
2191 } else {
2192 // Both value's and field's class id is known.
2193 ASSERT(value_cid != nullability);
2194 __ jmp(fail);
2195 }
2196 }
2197 __ Bind(&ok);
2198}
2199
2200LocationSummary* GuardFieldLengthInstr::MakeLocationSummary(Zone* zone,
2201 bool opt) const {
2202 const intptr_t kNumInputs = 1;
2203 if (!opt || (field().guarded_list_length() == Field::kUnknownFixedLength)) {
2204 const intptr_t kNumTemps = 3;
2205 LocationSummary* summary = new (zone)
2206 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
2207 summary->set_in(0, Location::RequiresRegister());
2208 // We need temporaries for field object, length offset and expected length.
2209 summary->set_temp(0, Location::RequiresRegister());
2210 summary->set_temp(1, Location::RequiresRegister());
2211 summary->set_temp(2, Location::RequiresRegister());
2212 return summary;
2213 } else {
2214 LocationSummary* summary = new (zone)
2215 LocationSummary(zone, kNumInputs, 0, LocationSummary::kNoCall);
2216 summary->set_in(0, Location::RequiresRegister());
2217 return summary;
2218 }
2219 UNREACHABLE();
2220}
2221
2222void GuardFieldLengthInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
2223 if (field().guarded_list_length() == Field::kNoFixedLength) {
2224 return; // Nothing to emit.
2225 }
2226
2227 compiler::Label* deopt =
2228 compiler->is_optimizing()
2229 ? compiler->AddDeoptStub(deopt_id(), ICData::kDeoptGuardField)
2230 : NULL;
2231
2232 const Register value_reg = locs()->in(0).reg();
2233
2234 if (!compiler->is_optimizing() ||
2235 (field().guarded_list_length() == Field::kUnknownFixedLength)) {
2236 const Register field_reg = locs()->temp(0).reg();
2237 const Register offset_reg = locs()->temp(1).reg();
2238 const Register length_reg = locs()->temp(2).reg();
2239
2240 compiler::Label ok;
2241
2242 __ LoadObject(field_reg, Field::ZoneHandle(field().Original()));
2243
2244 __ movsxb(
2245 offset_reg,
2246 compiler::FieldAddress(
2247 field_reg, Field::guarded_list_length_in_object_offset_offset()));
2248 __ movq(length_reg, compiler::FieldAddress(
2249 field_reg, Field::guarded_list_length_offset()));
2250
2251 __ cmpq(offset_reg, compiler::Immediate(0));
2252 __ j(NEGATIVE, &ok);
2253
2254 // Load the length from the value. GuardFieldClass already verified that
2255 // value's class matches guarded class id of the field.
2256 // offset_reg contains offset already corrected by -kHeapObjectTag that is
2257 // why we use Address instead of FieldAddress.
2258 __ cmpq(length_reg, compiler::Address(value_reg, offset_reg, TIMES_1, 0));
2259
2260 if (deopt == NULL) {
2261 __ j(EQUAL, &ok);
2262
2263 __ pushq(field_reg);
2264 __ pushq(value_reg);
2265 __ CallRuntime(kUpdateFieldCidRuntimeEntry, 2);
2266 __ Drop(2); // Drop the field and the value.
2267 } else {
2268 __ j(NOT_EQUAL, deopt);
2269 }
2270
2271 __ Bind(&ok);
2272 } else {
2273 ASSERT(compiler->is_optimizing());
2274 ASSERT(field().guarded_list_length() >= 0);
2275 ASSERT(field().guarded_list_length_in_object_offset() !=
2276 Field::kUnknownLengthOffset);
2277
2278 __ CompareImmediate(
2279 compiler::FieldAddress(value_reg,
2280 field().guarded_list_length_in_object_offset()),
2281 compiler::Immediate(Smi::RawValue(field().guarded_list_length())));
2282 __ j(NOT_EQUAL, deopt);
2283 }
2284}
2285
2286LocationSummary* GuardFieldTypeInstr::MakeLocationSummary(Zone* zone,
2287 bool opt) const {
2288 const intptr_t kNumInputs = 1;
2289 const intptr_t kNumTemps = 1;
2290 LocationSummary* summary = new (zone)
2291 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
2292 summary->set_in(0, Location::RequiresRegister());
2293 summary->set_temp(0, Location::RequiresRegister());
2294 return summary;
2295}
2296
2297void GuardFieldTypeInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
2298 // Should never emit GuardFieldType for fields that are marked as NotTracking.
2299 ASSERT(field().static_type_exactness_state().IsTracking());
2300 if (!field().static_type_exactness_state().NeedsFieldGuard()) {
2301 // Nothing to do: we only need to perform checks for trivially invariant
2302 // fields. If optimizing Canonicalize pass should have removed
2303 // this instruction.
2304 return;
2305 }
2306
2307 compiler::Label* deopt =
2308 compiler->is_optimizing()
2309 ? compiler->AddDeoptStub(deopt_id(), ICData::kDeoptGuardField)
2310 : NULL;
2311
2312 compiler::Label ok;
2313
2314 const Register value_reg = locs()->in(0).reg();
2315 const Register temp = locs()->temp(0).reg();
2316
2317 // Skip null values for nullable fields.
2318 if (!compiler->is_optimizing() || field().is_nullable()) {
2319 __ CompareObject(value_reg, Object::Handle());
2320 __ j(EQUAL, &ok);
2321 }
2322
2323 // Get the state.
2324 const Field& original =
2325 Field::ZoneHandle(compiler->zone(), field().Original());
2326 __ LoadObject(temp, original);
2327 __ movsxb(temp, compiler::FieldAddress(
2328 temp, Field::static_type_exactness_state_offset()));
2329
2330 if (!compiler->is_optimizing()) {
2331 // Check if field requires checking (it is in unitialized or trivially
2332 // exact state).
2333 __ cmpq(temp,
2334 compiler::Immediate(StaticTypeExactnessState::kUninitialized));
2335 __ j(LESS, &ok);
2336 }
2337
2338 compiler::Label call_runtime;
2339 if (field().static_type_exactness_state().IsUninitialized()) {
2340 // Can't initialize the field state inline in optimized code.
2341 __ cmpq(temp,
2342 compiler::Immediate(StaticTypeExactnessState::kUninitialized));
2343 __ j(EQUAL, compiler->is_optimizing() ? deopt : &call_runtime);
2344 }
2345
2346 // At this point temp is known to be type arguments offset in words.
2347 __ movq(temp, compiler::FieldAddress(value_reg, temp, TIMES_8, 0));
2348 __ CompareObject(temp, TypeArguments::ZoneHandle(
2349 compiler->zone(),
2350 AbstractType::Handle(field().type()).arguments()));
2351 if (deopt != nullptr) {
2352 __ j(NOT_EQUAL, deopt);
2353 } else {
2354 __ j(EQUAL, &ok);
2355
2356 __ Bind(&call_runtime);
2357 __ PushObject(original);
2358 __ pushq(value_reg);
2359 __ CallRuntime(kUpdateFieldCidRuntimeEntry, 2);
2360 __ Drop(2);
2361 }
2362
2363 __ Bind(&ok);
2364}
2365
2366LocationSummary* StoreInstanceFieldInstr::MakeLocationSummary(Zone* zone,
2367 bool opt) const {
2368 const intptr_t kNumInputs = 2;
2369 const intptr_t kNumTemps = (IsUnboxedStore() && opt)
2370 ? (FLAG_precompiled_mode ? 0 : 2)
2371 : (IsPotentialUnboxedStore() ? 3 : 0);
2372 LocationSummary* summary = new (zone)
2373 LocationSummary(zone, kNumInputs, kNumTemps,
2374 (!FLAG_precompiled_mode &&
2375 ((IsUnboxedStore() && opt && is_initialization()) ||
2376 IsPotentialUnboxedStore()))
2377 ? LocationSummary::kCallOnSlowPath
2378 : LocationSummary::kNoCall);
2379
2380 summary->set_in(0, Location::RequiresRegister());
2381 if (IsUnboxedStore() && opt) {
2382 if (slot().field().is_non_nullable_integer()) {
2383 ASSERT(FLAG_precompiled_mode);
2384 summary->set_in(1, Location::RequiresRegister());
2385 } else {
2386 summary->set_in(1, Location::RequiresFpuRegister());
2387 }
2388 if (!FLAG_precompiled_mode) {
2389 summary->set_temp(0, Location::RequiresRegister());
2390 summary->set_temp(1, Location::RequiresRegister());
2391 }
2392 } else if (IsPotentialUnboxedStore()) {
2393 summary->set_in(1, ShouldEmitStoreBarrier() ? Location::WritableRegister()
2394 : Location::RequiresRegister());
2395 summary->set_temp(0, Location::RequiresRegister());
2396 summary->set_temp(1, Location::RequiresRegister());
2397 summary->set_temp(2, opt ? Location::RequiresFpuRegister()
2398 : Location::FpuRegisterLocation(XMM1));
2399 } else {
2400 summary->set_in(1, ShouldEmitStoreBarrier()
2401 ? Location::RegisterLocation(kWriteBarrierValueReg)
2402 : LocationRegisterOrConstant(value()));
2403 }
2404 return summary;
2405}
2406
2407static void EnsureMutableBox(FlowGraphCompiler* compiler,
2408 StoreInstanceFieldInstr* instruction,
2409 Register box_reg,
2410 const Class& cls,
2411 Register instance_reg,
2412 intptr_t offset,
2413 Register temp) {
2414 compiler::Label done;
2415 __ movq(box_reg, compiler::FieldAddress(instance_reg, offset));
2416 __ CompareObject(box_reg, Object::null_object());
2417 __ j(NOT_EQUAL, &done);
2418 BoxAllocationSlowPath::Allocate(compiler, instruction, cls, box_reg, temp);
2419 __ movq(temp, box_reg);
2420 __ StoreIntoObject(instance_reg, compiler::FieldAddress(instance_reg, offset),
2421 temp, compiler::Assembler::kValueIsNotSmi);
2422
2423 __ Bind(&done);
2424}
2425
2426void StoreInstanceFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
2427 ASSERT(compiler::target::ObjectLayout::kClassIdTagSize == 16);
2428 ASSERT(sizeof(FieldLayout::guarded_cid_) == 2);
2429 ASSERT(sizeof(FieldLayout::is_nullable_) == 2);
2430
2431 compiler::Label skip_store;
2432
2433 const Register instance_reg = locs()->in(0).reg();
2434 const intptr_t offset_in_bytes = OffsetInBytes();
2435 ASSERT(offset_in_bytes > 0); // Field is finalized and points after header.
2436
2437 if (IsUnboxedStore() && compiler->is_optimizing()) {
2438 if (slot().field().is_non_nullable_integer()) {
2439 const Register value = locs()->in(1).reg();
2440 __ Comment("UnboxedIntegerStoreInstanceFieldInstr");
2441 __ movq(compiler::FieldAddress(instance_reg, offset_in_bytes), value);
2442 return;
2443 }
2444
2445 XmmRegister value = locs()->in(1).fpu_reg();
2446 const intptr_t cid = slot().field().UnboxedFieldCid();
2447
2448 // Real unboxed field
2449 if (FLAG_precompiled_mode) {
2450 switch (cid) {
2451 case kDoubleCid:
2452 __ Comment("UnboxedDoubleStoreInstanceFieldInstr");
2453 __ movsd(compiler::FieldAddress(instance_reg, offset_in_bytes),
2454 value);
2455 return;
2456 case kFloat32x4Cid:
2457 __ Comment("UnboxedFloat32x4StoreInstanceFieldInstr");
2458 __ movups(compiler::FieldAddress(instance_reg, offset_in_bytes),
2459 value);
2460 return;
2461 case kFloat64x2Cid:
2462 __ Comment("UnboxedFloat64x2StoreInstanceFieldInstr");
2463 __ movups(compiler::FieldAddress(instance_reg, offset_in_bytes),
2464 value);
2465 return;
2466 default:
2467 UNREACHABLE();
2468 }
2469 }
2470
2471 Register temp = locs()->temp(0).reg();
2472 Register temp2 = locs()->temp(1).reg();
2473
2474 if (is_initialization()) {
2475 const Class* cls = NULL;
2476 switch (cid) {
2477 case kDoubleCid:
2478 cls = &compiler->double_class();
2479 break;
2480 case kFloat32x4Cid:
2481 cls = &compiler->float32x4_class();
2482 break;
2483 case kFloat64x2Cid:
2484 cls = &compiler->float64x2_class();
2485 break;
2486 default:
2487 UNREACHABLE();
2488 }
2489
2490 BoxAllocationSlowPath::Allocate(compiler, this, *cls, temp, temp2);
2491 __ movq(temp2, temp);
2492 __ StoreIntoObject(instance_reg,
2493 compiler::FieldAddress(instance_reg, offset_in_bytes),
2494 temp2, compiler::Assembler::kValueIsNotSmi);
2495 } else {
2496 __ movq(temp, compiler::FieldAddress(instance_reg, offset_in_bytes));
2497 }
2498 switch (cid) {
2499 case kDoubleCid:
2500 __ Comment("UnboxedDoubleStoreInstanceFieldInstr");
2501 __ movsd(compiler::FieldAddress(temp, Double::value_offset()), value);
2502 break;
2503 case kFloat32x4Cid:
2504 __ Comment("UnboxedFloat32x4StoreInstanceFieldInstr");
2505 __ movups(compiler::FieldAddress(temp, Float32x4::value_offset()),
2506 value);
2507 break;
2508 case kFloat64x2Cid:
2509 __ Comment("UnboxedFloat64x2StoreInstanceFieldInstr");
2510 __ movups(compiler::FieldAddress(temp, Float64x2::value_offset()),
2511 value);
2512 break;
2513 default:
2514 UNREACHABLE();
2515 }
2516 return;
2517 }
2518
2519 if (IsPotentialUnboxedStore()) {
2520 Register value_reg = locs()->in(1).reg();
2521 Register temp = locs()->temp(0).reg();
2522 Register temp2 = locs()->temp(1).reg();
2523 FpuRegister fpu_temp = locs()->temp(2).fpu_reg();
2524
2525 if (ShouldEmitStoreBarrier()) {
2526 // Value input is a writable register and should be manually preserved
2527 // across allocation slow-path.
2528 locs()->live_registers()->Add(locs()->in(1), kTagged);
2529 }
2530
2531 compiler::Label store_pointer;
2532 compiler::Label store_double;
2533 compiler::Label store_float32x4;
2534 compiler::Label store_float64x2;
2535
2536 __ LoadObject(temp, Field::ZoneHandle(Z, slot().field().Original()));
2537
2538 __ cmpw(compiler::FieldAddress(temp, Field::is_nullable_offset()),
2539 compiler::Immediate(kNullCid));
2540 __ j(EQUAL, &store_pointer);
2541
2542 __ movzxb(temp2, compiler::FieldAddress(temp, Field::kind_bits_offset()));
2543 __ testq(temp2, compiler::Immediate(1 << Field::kUnboxingCandidateBit));
2544 __ j(ZERO, &store_pointer);
2545
2546 __ cmpw(compiler::FieldAddress(temp, Field::guarded_cid_offset()),
2547 compiler::Immediate(kDoubleCid));
2548 __ j(EQUAL, &store_double);
2549
2550 __ cmpw(compiler::FieldAddress(temp, Field::guarded_cid_offset()),
2551 compiler::Immediate(kFloat32x4Cid));
2552 __ j(EQUAL, &store_float32x4);
2553
2554 __ cmpw(compiler::FieldAddress(temp, Field::guarded_cid_offset()),
2555 compiler::Immediate(kFloat64x2Cid));
2556 __ j(EQUAL, &store_float64x2);
2557
2558 // Fall through.
2559 __ jmp(&store_pointer);
2560
2561 if (!compiler->is_optimizing()) {
2562 locs()->live_registers()->Add(locs()->in(0));
2563 locs()->live_registers()->Add(locs()->in(1));
2564 }
2565
2566 {
2567 __ Bind(&store_double);
2568 EnsureMutableBox(compiler, this, temp, compiler->double_class(),
2569 instance_reg, offset_in_bytes, temp2);
2570 __ movsd(fpu_temp,
2571 compiler::FieldAddress(value_reg, Double::value_offset()));
2572 __ movsd(compiler::FieldAddress(temp, Double::value_offset()), fpu_temp);
2573 __ jmp(&skip_store);
2574 }
2575
2576 {
2577 __ Bind(&store_float32x4);
2578 EnsureMutableBox(compiler, this, temp, compiler->float32x4_class(),
2579 instance_reg, offset_in_bytes, temp2);
2580 __ movups(fpu_temp,
2581 compiler::FieldAddress(value_reg, Float32x4::value_offset()));
2582 __ movups(compiler::FieldAddress(temp, Float32x4::value_offset()),
2583 fpu_temp);
2584 __ jmp(&skip_store);
2585 }
2586
2587 {
2588 __ Bind(&store_float64x2);
2589 EnsureMutableBox(compiler, this, temp, compiler->float64x2_class(),
2590 instance_reg, offset_in_bytes, temp2);
2591 __ movups(fpu_temp,
2592 compiler::FieldAddress(value_reg, Float64x2::value_offset()));
2593 __ movups(compiler::FieldAddress(temp, Float64x2::value_offset()),
2594 fpu_temp);
2595 __ jmp(&skip_store);
2596 }
2597
2598 __ Bind(&store_pointer);
2599 }
2600
2601 if (ShouldEmitStoreBarrier()) {
2602 Register value_reg = locs()->in(1).reg();
2603 __ StoreIntoObject(instance_reg,
2604 compiler::FieldAddress(instance_reg, offset_in_bytes),
2605 value_reg, CanValueBeSmi());
2606 } else {
2607 if (locs()->in(1).IsConstant()) {
2608 __ StoreIntoObjectNoBarrier(
2609 instance_reg, compiler::FieldAddress(instance_reg, offset_in_bytes),
2610 locs()->in(1).constant());
2611 } else {
2612 Register value_reg = locs()->in(1).reg();
2613 __ StoreIntoObjectNoBarrier(
2614 instance_reg, compiler::FieldAddress(instance_reg, offset_in_bytes),
2615 value_reg);
2616 }
2617 }
2618 __ Bind(&skip_store);
2619}
2620
2621LocationSummary* StoreStaticFieldInstr::MakeLocationSummary(Zone* zone,
2622 bool opt) const {
2623 const intptr_t kNumInputs = 1;
2624 const intptr_t kNumTemps = 1;
2625 LocationSummary* locs = new (zone)
2626 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
2627 locs->set_in(0, Location::RegisterLocation(kWriteBarrierValueReg));
2628 locs->set_temp(0, Location::RequiresRegister());
2629 return locs;
2630}
2631
2632void StoreStaticFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
2633 Register value = locs()->in(0).reg();
2634 Register temp = locs()->temp(0).reg();
2635
2636 compiler->used_static_fields().Add(&field());
2637
2638 __ movq(temp,
2639 compiler::Address(
2640 THR, compiler::target::Thread::field_table_values_offset()));
2641 // Note: static fields ids won't be changed by hot-reload.
2642 __ movq(
2643 compiler::Address(temp, compiler::target::FieldTable::OffsetOf(field())),
2644 value);
2645}
2646
2647LocationSummary* InstanceOfInstr::MakeLocationSummary(Zone* zone,
2648 bool opt) const {
2649 const intptr_t kNumInputs = 3;
2650 const intptr_t kNumTemps = 0;
2651 LocationSummary* summary = new (zone)
2652 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kCall);
2653 summary->set_in(0, Location::RegisterLocation(TypeTestABI::kInstanceReg));
2654 summary->set_in(1, Location::RegisterLocation(
2655 TypeTestABI::kInstantiatorTypeArgumentsReg));
2656 summary->set_in(
2657 2, Location::RegisterLocation(TypeTestABI::kFunctionTypeArgumentsReg));
2658 summary->set_out(0, Location::RegisterLocation(RAX));
2659 return summary;
2660}
2661
2662void InstanceOfInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
2663 ASSERT(locs()->in(0).reg() == TypeTestABI::kInstanceReg);
2664 ASSERT(locs()->in(1).reg() == TypeTestABI::kInstantiatorTypeArgumentsReg);
2665 ASSERT(locs()->in(2).reg() == TypeTestABI::kFunctionTypeArgumentsReg);
2666
2667 compiler->GenerateInstanceOf(token_pos(), deopt_id(), type(), locs());
2668 ASSERT(locs()->out(0).reg() == RAX);
2669}
2670
2671// TODO(srdjan): In case of constant inputs make CreateArray kNoCall and
2672// use slow path stub.
2673LocationSummary* CreateArrayInstr::MakeLocationSummary(Zone* zone,
2674 bool opt) const {
2675 const intptr_t kNumInputs = 2;
2676 const intptr_t kNumTemps = 0;
2677 LocationSummary* locs = new (zone)
2678 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kCall);
2679 locs->set_in(0, Location::RegisterLocation(RBX));
2680 locs->set_in(1, Location::RegisterLocation(R10));
2681 locs->set_out(0, Location::RegisterLocation(RAX));
2682 return locs;
2683}
2684
2685// Inlines array allocation for known constant values.
2686static void InlineArrayAllocation(FlowGraphCompiler* compiler,
2687 intptr_t num_elements,
2688 compiler::Label* slow_path,
2689 compiler::Label* done) {
2690 const int kInlineArraySize = 12; // Same as kInlineInstanceSize.
2691 const Register kLengthReg = R10;
2692 const Register kElemTypeReg = RBX;
2693 const intptr_t instance_size = Array::InstanceSize(num_elements);
2694
2695 __ TryAllocateArray(kArrayCid, instance_size, slow_path,
2696 compiler::Assembler::kFarJump,
2697 RAX, // instance
2698 RCX, // end address
2699 R13); // temp
2700
2701 // RAX: new object start as a tagged pointer.
2702 // Store the type argument field.
2703 __ StoreIntoObjectNoBarrier(
2704 RAX, compiler::FieldAddress(RAX, Array::type_arguments_offset()),
2705 kElemTypeReg);
2706
2707 // Set the length field.
2708 __ StoreIntoObjectNoBarrier(
2709 RAX, compiler::FieldAddress(RAX, Array::length_offset()), kLengthReg);
2710
2711 // Initialize all array elements to raw_null.
2712 // RAX: new object start as a tagged pointer.
2713 // RCX: new object end address.
2714 // RDI: iterator which initially points to the start of the variable
2715 // data area to be initialized.
2716 if (num_elements > 0) {
2717 const intptr_t array_size = instance_size - sizeof(ArrayLayout);
2718 __ LoadObject(R12, Object::null_object());
2719 __ leaq(RDI, compiler::FieldAddress(RAX, sizeof(ArrayLayout)));
2720 if (array_size < (kInlineArraySize * kWordSize)) {
2721 intptr_t current_offset = 0;
2722 while (current_offset < array_size) {
2723 __ StoreIntoObjectNoBarrier(RAX, compiler::Address(RDI, current_offset),
2724 R12);
2725 current_offset += kWordSize;
2726 }
2727 } else {
2728 compiler::Label init_loop;
2729 __ Bind(&init_loop);
2730 __ StoreIntoObjectNoBarrier(RAX, compiler::Address(RDI, 0), R12);
2731 __ addq(RDI, compiler::Immediate(kWordSize));
2732 __ cmpq(RDI, RCX);
2733 __ j(BELOW, &init_loop, compiler::Assembler::kNearJump);
2734 }
2735 }
2736 __ jmp(done, compiler::Assembler::kNearJump);
2737}
2738
2739void CreateArrayInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
2740 TypeUsageInfo* type_usage_info = compiler->thread()->type_usage_info();
2741 if (type_usage_info != nullptr) {
2742 const Class& list_class = Class::Handle(
2743 compiler->thread()->isolate()->class_table()->At(kArrayCid));
2744 RegisterTypeArgumentsUse(compiler->function(), type_usage_info, list_class,
2745 element_type()->definition());
2746 }
2747
2748 // Allocate the array. R10 = length, RBX = element type.
2749 const Register kLengthReg = R10;
2750 const Register kElemTypeReg = RBX;
2751 const Register kResultReg = RAX;
2752 ASSERT(locs()->in(0).reg() == kElemTypeReg);
2753 ASSERT(locs()->in(1).reg() == kLengthReg);
2754
2755 compiler::Label slow_path, done;
2756 if (compiler->is_optimizing() && !FLAG_precompiled_mode &&
2757 num_elements()->BindsToConstant() &&
2758 num_elements()->BoundConstant().IsSmi()) {
2759 const intptr_t length = Smi::Cast(num_elements()->BoundConstant()).Value();
2760 if (Array::IsValidLength(length)) {
2761 InlineArrayAllocation(compiler, length, &slow_path, &done);
2762 }
2763 }
2764
2765 __ Bind(&slow_path);
2766 auto object_store = compiler->isolate()->object_store();
2767 const auto& allocate_array_stub =
2768 Code::ZoneHandle(compiler->zone(), object_store->allocate_array_stub());
2769 compiler->GenerateStubCall(token_pos(), allocate_array_stub,
2770 PcDescriptorsLayout::kOther, locs(), deopt_id());
2771 __ Bind(&done);
2772 ASSERT(locs()->out(0).reg() == kResultReg);
2773}
2774
2775LocationSummary* LoadFieldInstr::MakeLocationSummary(Zone* zone,
2776 bool opt) const {
2777 const intptr_t kNumInputs = 1;
2778 const intptr_t kNumTemps = (IsUnboxedLoad() && opt)
2779 ? (FLAG_precompiled_mode ? 0 : 1)
2780 : (IsPotentialUnboxedLoad() ? 2 : 0);
2781 const auto contains_call =
2782 (IsUnboxedLoad() && opt)
2783 ? LocationSummary::kNoCall
2784 : (IsPotentialUnboxedLoad()
2785 ? LocationSummary::kCallOnSlowPath
2786 : (calls_initializer() ? LocationSummary::kCall
2787 : LocationSummary::kNoCall));
2788
2789 LocationSummary* locs =
2790 new (zone) LocationSummary(zone, kNumInputs, kNumTemps, contains_call);
2791
2792 locs->set_in(0, calls_initializer() ? Location::RegisterLocation(
2793 InitInstanceFieldABI::kInstanceReg)
2794 : Location::RequiresRegister());
2795
2796 if (IsUnboxedLoad() && opt) {
2797 ASSERT(!calls_initializer());
2798 if (!FLAG_precompiled_mode) {
2799 locs->set_temp(0, Location::RequiresRegister());
2800 }
2801 if (slot().field().is_non_nullable_integer()) {
2802 ASSERT(FLAG_precompiled_mode);
2803 locs->set_out(0, Location::RequiresRegister());
2804 } else {
2805 locs->set_out(0, Location::RequiresFpuRegister());
2806 }
2807 } else if (IsPotentialUnboxedLoad()) {
2808 ASSERT(!calls_initializer());
2809 locs->set_temp(0, opt ? Location::RequiresFpuRegister()
2810 : Location::FpuRegisterLocation(XMM1));
2811 locs->set_temp(1, Location::RequiresRegister());
2812 locs->set_out(0, Location::RequiresRegister());
2813 } else if (calls_initializer()) {
2814 locs->set_out(0,
2815 Location::RegisterLocation(InitInstanceFieldABI::kResultReg));
2816 } else {
2817 locs->set_out(0, Location::RequiresRegister());
2818 }
2819 return locs;
2820}
2821
2822void LoadFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
2823 ASSERT(compiler::target::ObjectLayout::kClassIdTagSize == 16);
2824 ASSERT(sizeof(FieldLayout::guarded_cid_) == 2);
2825 ASSERT(sizeof(FieldLayout::is_nullable_) == 2);
2826
2827 Register instance_reg = locs()->in(0).reg();
2828 if (IsUnboxedLoad() && compiler->is_optimizing()) {
2829 ASSERT(!calls_initializer());
2830 if (slot().field().is_non_nullable_integer()) {
2831 const Register result = locs()->out(0).reg();
2832 __ Comment("UnboxedIntegerLoadFieldInstr");
2833 __ movq(result, compiler::FieldAddress(instance_reg, OffsetInBytes()));
2834 return;
2835 }
2836
2837 XmmRegister result = locs()->out(0).fpu_reg();
2838 const intptr_t cid = slot().field().UnboxedFieldCid();
2839
2840 // Real unboxed field
2841 if (FLAG_precompiled_mode) {
2842 switch (cid) {
2843 case kDoubleCid:
2844 __ Comment("UnboxedDoubleLoadFieldInstr");
2845 __ movsd(result,
2846 compiler::FieldAddress(instance_reg, OffsetInBytes()));
2847 break;
2848 case kFloat32x4Cid:
2849 __ Comment("UnboxedFloat32x4LoadFieldInstr");
2850 __ movups(result,
2851 compiler::FieldAddress(instance_reg, OffsetInBytes()));
2852 break;
2853 case kFloat64x2Cid:
2854 __ Comment("UnboxedFloat64x2LoadFieldInstr");
2855 __ movups(result,
2856 compiler::FieldAddress(instance_reg, OffsetInBytes()));
2857 break;
2858 default:
2859 UNREACHABLE();
2860 }
2861 return;
2862 }
2863
2864 Register temp = locs()->temp(0).reg();
2865 __ movq(temp, compiler::FieldAddress(instance_reg, OffsetInBytes()));
2866 switch (cid) {
2867 case kDoubleCid:
2868 __ Comment("UnboxedDoubleLoadFieldInstr");
2869 __ movsd(result, compiler::FieldAddress(temp, Double::value_offset()));
2870 break;
2871 case kFloat32x4Cid:
2872 __ Comment("UnboxedFloat32x4LoadFieldInstr");
2873 __ movups(result,
2874 compiler::FieldAddress(temp, Float32x4::value_offset()));
2875 break;
2876 case kFloat64x2Cid:
2877 __ Comment("UnboxedFloat64x2LoadFieldInstr");
2878 __ movups(result,
2879 compiler::FieldAddress(temp, Float64x2::value_offset()));
2880 break;
2881 default:
2882 UNREACHABLE();
2883 }
2884 return;
2885 }
2886
2887 compiler::Label done;
2888 Register result = locs()->out(0).reg();
2889 if (IsPotentialUnboxedLoad()) {
2890 ASSERT(!calls_initializer());
2891 Register temp = locs()->temp(1).reg();
2892 XmmRegister value = locs()->temp(0).fpu_reg();
2893
2894 compiler::Label load_pointer;
2895 compiler::Label load_double;
2896 compiler::Label load_float32x4;
2897 compiler::Label load_float64x2;
2898
2899 __ LoadObject(result, Field::ZoneHandle(slot().field().Original()));
2900
2901 compiler::FieldAddress field_cid_operand(result,
2902 Field::guarded_cid_offset());
2903 compiler::FieldAddress field_nullability_operand(
2904 result, Field::is_nullable_offset());
2905
2906 __ cmpw(field_nullability_operand, compiler::Immediate(kNullCid));
2907 __ j(EQUAL, &load_pointer);
2908
2909 __ cmpw(field_cid_operand, compiler::Immediate(kDoubleCid));
2910 __ j(EQUAL, &load_double);
2911
2912 __ cmpw(field_cid_operand, compiler::Immediate(kFloat32x4Cid));
2913 __ j(EQUAL, &load_float32x4);
2914
2915 __ cmpw(field_cid_operand, compiler::Immediate(kFloat64x2Cid));
2916 __ j(EQUAL, &load_float64x2);
2917
2918 // Fall through.
2919 __ jmp(&load_pointer);
2920
2921 if (!compiler->is_optimizing()) {
2922 locs()->live_registers()->Add(locs()->in(0));
2923 }
2924
2925 {
2926 __ Bind(&load_double);
2927 BoxAllocationSlowPath::Allocate(compiler, this, compiler->double_class(),
2928 result, temp);
2929 __ movq(temp, compiler::FieldAddress(instance_reg, OffsetInBytes()));
2930 __ movsd(value, compiler::FieldAddress(temp, Double::value_offset()));
2931 __ movsd(compiler::FieldAddress(result, Double::value_offset()), value);
2932 __ jmp(&done);
2933 }
2934
2935 {
2936 __ Bind(&load_float32x4);
2937 BoxAllocationSlowPath::Allocate(
2938 compiler, this, compiler->float32x4_class(), result, temp);
2939 __ movq(temp, compiler::FieldAddress(instance_reg, OffsetInBytes()));
2940 __ movups(value, compiler::FieldAddress(temp, Float32x4::value_offset()));
2941 __ movups(compiler::FieldAddress(result, Float32x4::value_offset()),
2942 value);
2943 __ jmp(&done);
2944 }
2945
2946 {
2947 __ Bind(&load_float64x2);
2948 BoxAllocationSlowPath::Allocate(
2949 compiler, this, compiler->float64x2_class(), result, temp);
2950 __ movq(temp, compiler::FieldAddress(instance_reg, OffsetInBytes()));
2951 __ movups(value, compiler::FieldAddress(temp, Float64x2::value_offset()));
2952 __ movups(compiler::FieldAddress(result, Float64x2::value_offset()),
2953 value);
2954 __ jmp(&done);
2955 }
2956
2957 __ Bind(&load_pointer);
2958 }
2959
2960 __ movq(result, compiler::FieldAddress(instance_reg, OffsetInBytes()));
2961
2962 if (calls_initializer()) {
2963 EmitNativeCodeForInitializerCall(compiler);
2964 }
2965
2966 __ Bind(&done);
2967}
2968
2969LocationSummary* InstantiateTypeInstr::MakeLocationSummary(Zone* zone,
2970 bool opt) const {
2971 const intptr_t kNumInputs = 2;
2972 const intptr_t kNumTemps = 0;
2973 LocationSummary* locs = new (zone)
2974 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kCall);
2975 locs->set_in(0, Location::RegisterLocation(
2976 InstantiationABI::kInstantiatorTypeArgumentsReg));
2977 locs->set_in(1, Location::RegisterLocation(
2978 InstantiationABI::kFunctionTypeArgumentsReg));
2979 locs->set_out(0,
2980 Location::RegisterLocation(InstantiationABI::kResultTypeReg));
2981 return locs;
2982}
2983
2984void InstantiateTypeInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
2985 Register instantiator_type_args_reg = locs()->in(0).reg();
2986 Register function_type_args_reg = locs()->in(1).reg();
2987 Register result_reg = locs()->out(0).reg();
2988
2989 // 'instantiator_type_args_reg' is a TypeArguments object (or null).
2990 // 'function_type_args_reg' is a TypeArguments object (or null).
2991 // A runtime call to instantiate the type is required.
2992 __ PushObject(Object::null_object()); // Make room for the result.
2993 __ PushObject(type());
2994 __ pushq(instantiator_type_args_reg); // Push instantiator type arguments.
2995 __ pushq(function_type_args_reg); // Push function type arguments.
2996 compiler->GenerateRuntimeCall(token_pos(), deopt_id(),
2997 kInstantiateTypeRuntimeEntry, 3, locs());
2998 __ Drop(3); // Drop 2 type vectors, and uninstantiated type.
2999 __ popq(result_reg); // Pop instantiated type.
3000}
3001
3002LocationSummary* InstantiateTypeArgumentsInstr::MakeLocationSummary(
3003 Zone* zone,
3004 bool opt) const {
3005 const intptr_t kNumInputs = 2;
3006 const intptr_t kNumTemps = 0;
3007 LocationSummary* locs = new (zone)
3008 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kCall);
3009 locs->set_in(0, Location::RegisterLocation(
3010 InstantiationABI::kInstantiatorTypeArgumentsReg));
3011 locs->set_in(1, Location::RegisterLocation(
3012 InstantiationABI::kFunctionTypeArgumentsReg));
3013 locs->set_out(
3014 0, Location::RegisterLocation(InstantiationABI::kResultTypeArgumentsReg));
3015 return locs;
3016}
3017
3018void InstantiateTypeArgumentsInstr::EmitNativeCode(
3019 FlowGraphCompiler* compiler) {
3020 Register instantiator_type_args_reg = locs()->in(0).reg();
3021 Register function_type_args_reg = locs()->in(1).reg();
3022 Register result_reg = locs()->out(0).reg();
3023
3024 // 'instantiator_type_args_reg' is a TypeArguments object (or null).
3025 // 'function_type_args_reg' is a TypeArguments object (or null).
3026
3027 // If both the instantiator and function type arguments are null and if the
3028 // type argument vector instantiated from null becomes a vector of dynamic,
3029 // then use null as the type arguments.
3030 compiler::Label type_arguments_instantiated;
3031 const intptr_t len = type_arguments().Length();
3032 const bool can_function_type_args_be_null =
3033 function_type_arguments()->CanBe(Object::null_object());
3034 if (type_arguments().IsRawWhenInstantiatedFromRaw(len) &&
3035 can_function_type_args_be_null) {
3036 compiler::Label non_null_type_args;
3037 ASSERT(result_reg != instantiator_type_args_reg &&
3038 result_reg != function_type_args_reg);
3039 __ LoadObject(result_reg, Object::null_object());
3040 __ cmpq(instantiator_type_args_reg, result_reg);
3041 if (!function_type_arguments()->BindsToConstant()) {
3042 __ j(NOT_EQUAL, &non_null_type_args, compiler::Assembler::kNearJump);
3043 __ cmpq(function_type_args_reg, result_reg);
3044 }
3045 __ j(EQUAL, &type_arguments_instantiated, compiler::Assembler::kNearJump);
3046 __ Bind(&non_null_type_args);
3047 }
3048 __ LoadObject(InstantiationABI::kUninstantiatedTypeArgumentsReg,
3049 type_arguments());
3050 compiler->GenerateStubCall(token_pos(), GetStub(),
3051 PcDescriptorsLayout::kOther, locs());
3052 __ Bind(&type_arguments_instantiated);
3053}
3054
3055LocationSummary* AllocateUninitializedContextInstr::MakeLocationSummary(
3056 Zone* zone,
3057 bool opt) const {
3058 ASSERT(opt);
3059 const intptr_t kNumInputs = 0;
3060 const intptr_t kNumTemps = 2;
3061 LocationSummary* locs = new (zone) LocationSummary(
3062 zone, kNumInputs, kNumTemps, LocationSummary::kCallOnSlowPath);
3063 locs->set_temp(0, Location::RegisterLocation(R10));
3064 locs->set_temp(1, Location::RegisterLocation(R13));
3065 locs->set_out(0, Location::RegisterLocation(RAX));
3066 return locs;
3067}
3068
3069class AllocateContextSlowPath
3070 : public TemplateSlowPathCode<AllocateUninitializedContextInstr> {
3071 public:
3072 explicit AllocateContextSlowPath(
3073 AllocateUninitializedContextInstr* instruction)
3074 : TemplateSlowPathCode(instruction) {}
3075
3076 virtual void EmitNativeCode(FlowGraphCompiler* compiler) {
3077 __ Comment("AllocateContextSlowPath");
3078 __ Bind(entry_label());
3079
3080 LocationSummary* locs = instruction()->locs();
3081 locs->live_registers()->Remove(locs->out(0));
3082
3083 compiler->SaveLiveRegisters(locs);
3084
3085 auto object_store = compiler->isolate()->object_store();
3086 const auto& allocate_context_stub = Code::ZoneHandle(
3087 compiler->zone(), object_store->allocate_context_stub());
3088
3089 __ LoadImmediate(
3090 R10, compiler::Immediate(instruction()->num_context_variables()));
3091 compiler->GenerateStubCall(instruction()->token_pos(),
3092 allocate_context_stub,
3093 PcDescriptorsLayout::kOther, locs);
3094 ASSERT(instruction()->locs()->out(0).reg() == RAX);
3095 compiler->RestoreLiveRegisters(instruction()->locs());
3096 __ jmp(exit_label());
3097 }
3098};
3099
3100void AllocateUninitializedContextInstr::EmitNativeCode(
3101 FlowGraphCompiler* compiler) {
3102 ASSERT(compiler->is_optimizing());
3103 Register temp = locs()->temp(0).reg();
3104 Register result = locs()->out(0).reg();
3105 // Try allocate the object.
3106 AllocateContextSlowPath* slow_path = new AllocateContextSlowPath(this);
3107 compiler->AddSlowPathCode(slow_path);
3108 intptr_t instance_size = Context::InstanceSize(num_context_variables());
3109
3110 __ TryAllocateArray(kContextCid, instance_size, slow_path->entry_label(),
3111 compiler::Assembler::kFarJump,
3112 result, // instance
3113 temp, // end address
3114 locs()->temp(1).reg());
3115
3116 // Setup up number of context variables field.
3117 __ movq(compiler::FieldAddress(result, Context::num_variables_offset()),
3118 compiler::Immediate(num_context_variables()));
3119
3120 __ Bind(slow_path->exit_label());
3121}
3122
3123LocationSummary* AllocateContextInstr::MakeLocationSummary(Zone* zone,
3124 bool opt) const {
3125 const intptr_t kNumInputs = 0;
3126 const intptr_t kNumTemps = 1;
3127 LocationSummary* locs = new (zone)
3128 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kCall);
3129 locs->set_temp(0, Location::RegisterLocation(R10));
3130 locs->set_out(0, Location::RegisterLocation(RAX));
3131 return locs;
3132}
3133
3134void AllocateContextInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
3135 ASSERT(locs()->temp(0).reg() == R10);
3136 ASSERT(locs()->out(0).reg() == RAX);
3137
3138 auto object_store = compiler->isolate()->object_store();
3139 const auto& allocate_context_stub =
3140 Code::ZoneHandle(compiler->zone(), object_store->allocate_context_stub());
3141
3142 __ LoadImmediate(R10, compiler::Immediate(num_context_variables()));
3143 compiler->GenerateStubCall(token_pos(), allocate_context_stub,
3144 PcDescriptorsLayout::kOther, locs());
3145}
3146
3147LocationSummary* CloneContextInstr::MakeLocationSummary(Zone* zone,
3148 bool opt) const {
3149 const intptr_t kNumInputs = 1;
3150 const intptr_t kNumTemps = 0;
3151 LocationSummary* locs = new (zone)
3152 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kCall);
3153 locs->set_in(0, Location::RegisterLocation(R9));
3154 locs->set_out(0, Location::RegisterLocation(RAX));
3155 return locs;
3156}
3157
3158void CloneContextInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
3159 ASSERT(locs()->in(0).reg() == R9);
3160 ASSERT(locs()->out(0).reg() == RAX);
3161
3162 auto object_store = compiler->isolate()->object_store();
3163 const auto& clone_context_stub =
3164 Code::ZoneHandle(compiler->zone(), object_store->clone_context_stub());
3165 compiler->GenerateStubCall(token_pos(), clone_context_stub,
3166 /*kind=*/PcDescriptorsLayout::kOther, locs());
3167}
3168
3169LocationSummary* CatchBlockEntryInstr::MakeLocationSummary(Zone* zone,
3170 bool opt) const {
3171 UNREACHABLE();
3172 return NULL;
3173}
3174
3175void CatchBlockEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
3176 __ Bind(compiler->GetJumpLabel(this));
3177 compiler->AddExceptionHandler(
3178 catch_try_index(), try_index(), compiler->assembler()->CodeSize(),
3179 is_generated(), catch_handler_types_, needs_stacktrace());
3180 if (!FLAG_precompiled_mode) {
3181 // On lazy deoptimization we patch the optimized code here to enter the
3182 // deoptimization stub.
3183 const intptr_t deopt_id = DeoptId::ToDeoptAfter(GetDeoptId());
3184 if (compiler->is_optimizing()) {
3185 compiler->AddDeoptIndexAtCall(deopt_id);
3186 } else {
3187 compiler->AddCurrentDescriptor(PcDescriptorsLayout::kDeopt, deopt_id,
3188 TokenPosition::kNoSource);
3189 }
3190 }
3191 if (HasParallelMove()) {
3192 compiler->parallel_move_resolver()->EmitNativeCode(parallel_move());
3193 }
3194
3195 // Restore RSP from RBP as we are coming from a throw and the code for
3196 // popping arguments has not been run.
3197 const intptr_t fp_sp_dist =
3198 (compiler::target::frame_layout.first_local_from_fp + 1 -
3199 compiler->StackSize()) *
3200 kWordSize;
3201 ASSERT(fp_sp_dist <= 0);
3202 __ leaq(RSP, compiler::Address(RBP, fp_sp_dist));
3203
3204 if (!compiler->is_optimizing()) {
3205 if (raw_exception_var_ != nullptr) {
3206 __ movq(compiler::Address(RBP,
3207 compiler::target::FrameOffsetInBytesForVariable(
3208 raw_exception_var_)),
3209 kExceptionObjectReg);
3210 }
3211 if (raw_stacktrace_var_ != nullptr) {
3212 __ movq(compiler::Address(RBP,
3213 compiler::target::FrameOffsetInBytesForVariable(
3214 raw_stacktrace_var_)),
3215 kStackTraceObjectReg);
3216 }
3217 }
3218}
3219
3220LocationSummary* CheckStackOverflowInstr::MakeLocationSummary(Zone* zone,
3221 bool opt) const {
3222 const intptr_t kNumInputs = 0;
3223 const intptr_t kNumTemps = 1;
3224 const bool using_shared_stub = UseSharedSlowPathStub(opt);
3225 LocationSummary* summary = new (zone)
3226 LocationSummary(zone, kNumInputs, kNumTemps,
3227 using_shared_stub ? LocationSummary::kCallOnSharedSlowPath
3228 : LocationSummary::kCallOnSlowPath);
3229 summary->set_temp(0, Location::RequiresRegister());
3230 return summary;
3231}
3232
3233class CheckStackOverflowSlowPath
3234 : public TemplateSlowPathCode<CheckStackOverflowInstr> {
3235 public:
3236 static constexpr intptr_t kNumSlowPathArgs = 0;
3237
3238 explicit CheckStackOverflowSlowPath(CheckStackOverflowInstr* instruction)
3239 : TemplateSlowPathCode(instruction) {}
3240
3241 virtual void EmitNativeCode(FlowGraphCompiler* compiler) {
3242 if (compiler->isolate()->use_osr() && osr_entry_label()->IsLinked()) {
3243 __ Comment("CheckStackOverflowSlowPathOsr");
3244 __ Bind(osr_entry_label());
3245 __ movq(compiler::Address(THR, Thread::stack_overflow_flags_offset()),
3246 compiler::Immediate(Thread::kOsrRequest));
3247 }
3248 __ Comment("CheckStackOverflowSlowPath");
3249 __ Bind(entry_label());
3250 const bool using_shared_stub =
3251 instruction()->locs()->call_on_shared_slow_path();
3252 if (!using_shared_stub) {
3253 compiler->SaveLiveRegisters(instruction()->locs());
3254 }
3255 // pending_deoptimization_env_ is needed to generate a runtime call that
3256 // may throw an exception.
3257 ASSERT(compiler->pending_deoptimization_env_ == NULL);
3258 Environment* env =
3259 compiler->SlowPathEnvironmentFor(instruction(), kNumSlowPathArgs);
3260 compiler->pending_deoptimization_env_ = env;
3261
3262 if (using_shared_stub) {
3263 const uword entry_point_offset =
3264 Thread::stack_overflow_shared_stub_entry_point_offset(
3265 instruction()->locs()->live_registers()->FpuRegisterCount() > 0);
3266 __ call(compiler::Address(THR, entry_point_offset));
3267 compiler->RecordSafepoint(instruction()->locs(), kNumSlowPathArgs);
3268 compiler->RecordCatchEntryMoves();
3269 compiler->AddDescriptor(
3270 PcDescriptorsLayout::kOther, compiler->assembler()->CodeSize(),
3271 instruction()->deopt_id(), instruction()->token_pos(),
3272 compiler->CurrentTryIndex());
3273 } else {
3274 compiler->GenerateRuntimeCall(
3275 instruction()->token_pos(), instruction()->deopt_id(),
3276 kStackOverflowRuntimeEntry, kNumSlowPathArgs, instruction()->locs());
3277 }
3278
3279 if (compiler->isolate()->use_osr() && !compiler->is_optimizing() &&
3280 instruction()->in_loop()) {
3281 // In unoptimized code, record loop stack checks as possible OSR entries.
3282 compiler->AddCurrentDescriptor(PcDescriptorsLayout::kOsrEntry,
3283 instruction()->deopt_id(),
3284 TokenPosition::kNoSource);
3285 }
3286 compiler->pending_deoptimization_env_ = NULL;
3287 if (!using_shared_stub) {
3288 compiler->RestoreLiveRegisters(instruction()->locs());
3289 }
3290 __ jmp(exit_label());
3291 }
3292
3293 compiler::Label* osr_entry_label() {
3294 ASSERT(Isolate::Current()->use_osr());
3295 return &osr_entry_label_;
3296 }
3297
3298 private:
3299 compiler::Label osr_entry_label_;
3300};
3301
3302void CheckStackOverflowInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
3303 CheckStackOverflowSlowPath* slow_path = new CheckStackOverflowSlowPath(this);
3304 compiler->AddSlowPathCode(slow_path);
3305
3306 Register temp = locs()->temp(0).reg();
3307 // Generate stack overflow check.
3308 __ cmpq(RSP, compiler::Address(THR, Thread::stack_limit_offset()));
3309 __ j(BELOW_EQUAL, slow_path->entry_label());
3310 if (compiler->CanOSRFunction() && in_loop()) {
3311 // In unoptimized code check the usage counter to trigger OSR at loop
3312 // stack checks. Use progressively higher thresholds for more deeply
3313 // nested loops to attempt to hit outer loops with OSR when possible.
3314 __ LoadObject(temp, compiler->parsed_function().function());
3315 int32_t threshold =
3316 FLAG_optimization_counter_threshold * (loop_depth() + 1);
3317 __ incl(compiler::FieldAddress(temp, Function::usage_counter_offset()));
3318 __ cmpl(compiler::FieldAddress(temp, Function::usage_counter_offset()),
3319 compiler::Immediate(threshold));
3320 __ j(GREATER_EQUAL, slow_path->osr_entry_label());
3321 }
3322 if (compiler->ForceSlowPathForStackOverflow()) {
3323 __ jmp(slow_path->entry_label());
3324 }
3325 __ Bind(slow_path->exit_label());
3326}
3327
3328static void EmitSmiShiftLeft(FlowGraphCompiler* compiler,
3329 BinarySmiOpInstr* shift_left) {
3330 const LocationSummary& locs = *shift_left->locs();
3331 Register left = locs.in(0).reg();
3332 Register result = locs.out(0).reg();
3333 ASSERT(left == result);
3334 compiler::Label* deopt =
3335 shift_left->CanDeoptimize()
3336 ? compiler->AddDeoptStub(shift_left->deopt_id(),
3337 ICData::kDeoptBinarySmiOp)
3338 : NULL;
3339 if (locs.in(1).IsConstant()) {
3340 const Object& constant = locs.in(1).constant();
3341 ASSERT(constant.IsSmi());
3342 // shlq operation masks the count to 6 bits.
3343 const intptr_t kCountLimit = 0x3F;
3344 const intptr_t value = Smi::Cast(constant).Value();
3345 ASSERT((0 < value) && (value < kCountLimit));
3346 if (shift_left->can_overflow()) {
3347 if (value == 1) {
3348 // Use overflow flag.
3349 __ shlq(left, compiler::Immediate(1));
3350 __ j(OVERFLOW, deopt);
3351 return;
3352 }
3353 // Check for overflow.
3354 Register temp = locs.temp(0).reg();
3355 __ movq(temp, left);
3356 __ shlq(left, compiler::Immediate(value));
3357 __ sarq(left, compiler::Immediate(value));
3358 __ cmpq(left, temp);
3359 __ j(NOT_EQUAL, deopt); // Overflow.
3360 }
3361 // Shift for result now we know there is no overflow.
3362 __ shlq(left, compiler::Immediate(value));
3363 return;
3364 }
3365
3366 // Right (locs.in(1)) is not constant.
3367 Register right = locs.in(1).reg();
3368 Range* right_range = shift_left->right_range();
3369 if (shift_left->left()->BindsToConstant() && shift_left->can_overflow()) {
3370 // TODO(srdjan): Implement code below for is_truncating().
3371 // If left is constant, we know the maximal allowed size for right.
3372 const Object& obj = shift_left->left()->BoundConstant();
3373 if (obj.IsSmi()) {
3374 const intptr_t left_int = Smi::Cast(obj).Value();
3375 if (left_int == 0) {
3376 __ CompareImmediate(right, compiler::Immediate(0));
3377 __ j(NEGATIVE, deopt);
3378 return;
3379 }
3380 const intptr_t max_right = kSmiBits - Utils::HighestBit(left_int);
3381 const bool right_needs_check =
3382 !RangeUtils::IsWithin(right_range, 0, max_right - 1);
3383 if (right_needs_check) {
3384 __ CompareImmediate(
3385 right,
3386 compiler::Immediate(static_cast<int64_t>(Smi::New(max_right))));
3387 __ j(ABOVE_EQUAL, deopt);
3388 }
3389 __ SmiUntag(right);
3390 __ shlq(left, right);
3391 }
3392 return;
3393 }
3394
3395 const bool right_needs_check =
3396 !RangeUtils::IsWithin(right_range, 0, (Smi::kBits - 1));
3397 ASSERT(right == RCX); // Count must be in RCX
3398 if (!shift_left->can_overflow()) {
3399 if (right_needs_check) {
3400 const bool right_may_be_negative =
3401 (right_range == NULL) || !right_range->IsPositive();
3402 if (right_may_be_negative) {
3403 ASSERT(shift_left->CanDeoptimize());
3404 __ CompareImmediate(right, compiler::Immediate(0));
3405 __ j(NEGATIVE, deopt);
3406 }
3407 compiler::Label done, is_not_zero;
3408 __ CompareImmediate(
3409 right,
3410 compiler::Immediate(static_cast<int64_t>(Smi::New(Smi::kBits))));
3411 __ j(BELOW, &is_not_zero, compiler::Assembler::kNearJump);
3412 __ xorq(left, left);
3413 __ jmp(&done, compiler::Assembler::kNearJump);
3414 __ Bind(&is_not_zero);
3415 __ SmiUntag(right);
3416 __ shlq(left, right);
3417 __ Bind(&done);
3418 } else {
3419 __ SmiUntag(right);
3420 __ shlq(left, right);
3421 }
3422 } else {
3423 if (right_needs_check) {
3424 ASSERT(shift_left->CanDeoptimize());
3425 __ CompareImmediate(
3426 right,
3427 compiler::Immediate(static_cast<int64_t>(Smi::New(Smi::kBits))));
3428 __ j(ABOVE_EQUAL, deopt);
3429 }
3430 // Left is not a constant.
3431 Register temp = locs.temp(0).reg();
3432 // Check if count too large for handling it inlined.
3433 __ movq(temp, left);
3434 __ SmiUntag(right);
3435 // Overflow test (preserve temp and right);
3436 __ shlq(left, right);
3437 __ sarq(left, right);
3438 __ cmpq(left, temp);
3439 __ j(NOT_EQUAL, deopt); // Overflow.
3440 // Shift for result now we know there is no overflow.
3441 __ shlq(left, right);
3442 }
3443}
3444
3445class CheckedSmiSlowPath : public TemplateSlowPathCode<CheckedSmiOpInstr> {
3446 public:
3447 CheckedSmiSlowPath(CheckedSmiOpInstr* instruction, intptr_t try_index)
3448 : TemplateSlowPathCode(instruction), try_index_(try_index) {}
3449
3450 static constexpr intptr_t kNumSlowPathArgs = 2;
3451
3452 virtual void EmitNativeCode(FlowGraphCompiler* compiler) {
3453 if (compiler::Assembler::EmittingComments()) {
3454 __ Comment("slow path smi operation");
3455 }
3456 __ Bind(entry_label());
3457 LocationSummary* locs = instruction()->locs();
3458 Register result = locs->out(0).reg();
3459 locs->live_registers()->Remove(Location::RegisterLocation(result));
3460
3461 compiler->SaveLiveRegisters(locs);
3462 if (instruction()->env() != NULL) {
3463 Environment* env =
3464 compiler->SlowPathEnvironmentFor(instruction(), kNumSlowPathArgs);
3465 compiler->pending_deoptimization_env_ = env;
3466 }
3467 __ pushq(locs->in(0).reg());
3468 __ pushq(locs->in(1).reg());
3469 const auto& selector = String::Handle(instruction()->call()->Selector());
3470 const auto& arguments_descriptor =
3471 Array::Handle(ArgumentsDescriptor::NewBoxed(
3472 /*type_args_len=*/0, /*num_arguments=*/2));
3473 compiler->EmitMegamorphicInstanceCall(
3474 selector, arguments_descriptor, instruction()->call()->deopt_id(),
3475 instruction()->token_pos(), locs, try_index_, kNumSlowPathArgs);
3476 __ MoveRegister(result, RAX);
3477 compiler->RestoreLiveRegisters(locs);
3478 __ jmp(exit_label());
3479 compiler->pending_deoptimization_env_ = NULL;
3480 }
3481
3482 private:
3483 intptr_t try_index_;
3484};
3485
3486LocationSummary* CheckedSmiOpInstr::MakeLocationSummary(Zone* zone,
3487 bool opt) const {
3488 bool is_shift = (op_kind() == Token::kSHL) || (op_kind() == Token::kSHR);
3489 const intptr_t kNumInputs = 2;
3490 const intptr_t kNumTemps = is_shift ? 1 : 0;
3491 LocationSummary* summary = new (zone) LocationSummary(
3492 zone, kNumInputs, kNumTemps, LocationSummary::kCallOnSlowPath);
3493 summary->set_in(0, Location::RequiresRegister());
3494 summary->set_in(1, Location::RequiresRegister());
3495 switch (op_kind()) {
3496 case Token::kADD:
3497 case Token::kSUB:
3498 case Token::kMUL:
3499 case Token::kSHL:
3500 case Token::kSHR:
3501 summary->set_out(0, Location::RequiresRegister());
3502 break;
3503 case Token::kBIT_OR:
3504 case Token::kBIT_AND:
3505 case Token::kBIT_XOR:
3506 summary->set_out(0, Location::SameAsFirstInput());
3507 break;
3508 default:
3509 UNIMPLEMENTED();
3510 }
3511 if (is_shift) {
3512 summary->set_temp(0, Location::RegisterLocation(RCX));
3513 }
3514 return summary;
3515}
3516
3517void CheckedSmiOpInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
3518 CheckedSmiSlowPath* slow_path =
3519 new CheckedSmiSlowPath(this, compiler->CurrentTryIndex());
3520 compiler->AddSlowPathCode(slow_path);
3521 // Test operands if necessary.
3522
3523 intptr_t left_cid = left()->Type()->ToCid();
3524 intptr_t right_cid = right()->Type()->ToCid();
3525 Register left = locs()->in(0).reg();
3526 Register right = locs()->in(1).reg();
3527 if (this->left()->definition() == this->right()->definition()) {
3528 __ testq(left, compiler::Immediate(kSmiTagMask));
3529 } else if (left_cid == kSmiCid) {
3530 __ testq(right, compiler::Immediate(kSmiTagMask));
3531 } else if (right_cid == kSmiCid) {
3532 __ testq(left, compiler::Immediate(kSmiTagMask));
3533 } else {
3534 __ movq(TMP, left);
3535 __ orq(TMP, right);
3536 __ testq(TMP, compiler::Immediate(kSmiTagMask));
3537 }
3538 __ j(NOT_ZERO, slow_path->entry_label());
3539 Register result = locs()->out(0).reg();
3540 switch (op_kind()) {
3541 case Token::kADD:
3542 __ movq(result, left);
3543 __ addq(result, right);
3544 __ j(OVERFLOW, slow_path->entry_label());
3545 break;
3546 case Token::kSUB:
3547 __ movq(result, left);
3548 __ subq(result, right);
3549 __ j(OVERFLOW, slow_path->entry_label());
3550 break;
3551 case Token::kMUL:
3552 __ movq(result, left);
3553 __ SmiUntag(result);
3554 __ imulq(result, right);
3555 __ j(OVERFLOW, slow_path->entry_label());
3556 break;
3557 case Token::kBIT_OR:
3558 ASSERT(left == result);
3559 __ orq(result, right);
3560 break;
3561 case Token::kBIT_AND:
3562 ASSERT(left == result);
3563 __ andq(result, right);
3564 break;
3565 case Token::kBIT_XOR:
3566 ASSERT(left == result);
3567 __ xorq(result, right);
3568 break;
3569 case Token::kSHL:
3570 ASSERT(result != right);
3571 ASSERT(locs()->temp(0).reg() == RCX);
3572 __ cmpq(right, compiler::Immediate(Smi::RawValue(Smi::kBits)));
3573 __ j(ABOVE_EQUAL, slow_path->entry_label());
3574
3575 __ movq(RCX, right);
3576 __ SmiUntag(RCX);
3577 __ movq(result, left);
3578 __ shlq(result, RCX);
3579 __ movq(TMP, result);
3580 __ sarq(TMP, RCX);
3581 __ cmpq(TMP, left);
3582 __ j(NOT_EQUAL, slow_path->entry_label());
3583 break;
3584 case Token::kSHR: {
3585 compiler::Label shift_count_ok;
3586 ASSERT(result != right);
3587 ASSERT(locs()->temp(0).reg() == RCX);
3588 __ cmpq(right, compiler::Immediate(Smi::RawValue(Smi::kBits)));
3589 __ j(ABOVE_EQUAL, slow_path->entry_label());
3590
3591 __ movq(RCX, right);
3592 __ SmiUntag(RCX);
3593 __ movq(result, left);
3594 __ SmiUntag(result);
3595 __ sarq(result, RCX);
3596 __ SmiTag(result);
3597 break;
3598 }
3599 default:
3600 UNIMPLEMENTED();
3601 }
3602 __ Bind(slow_path->exit_label());
3603}
3604
3605class CheckedSmiComparisonSlowPath
3606 : public TemplateSlowPathCode<CheckedSmiComparisonInstr> {
3607 public:
3608 static constexpr intptr_t kNumSlowPathArgs = 2;
3609
3610 CheckedSmiComparisonSlowPath(CheckedSmiComparisonInstr* instruction,
3611 Environment* env,
3612 intptr_t try_index,
3613 BranchLabels labels,
3614 bool merged)
3615 : TemplateSlowPathCode(instruction),
3616 try_index_(try_index),
3617 labels_(labels),
3618 merged_(merged),
3619 env_(env) {
3620 // The environment must either come from the comparison or the environment
3621 // was cleared from the comparison (and moved to a branch).
3622 ASSERT(env == instruction->env() ||
3623 (merged && instruction->env() == nullptr));
3624 }
3625
3626 virtual void EmitNativeCode(FlowGraphCompiler* compiler) {
3627 if (compiler::Assembler::EmittingComments()) {
3628 __ Comment("slow path smi comparison");
3629 }
3630 __ Bind(entry_label());
3631 LocationSummary* locs = instruction()->locs();
3632 Register result = merged_ ? locs->temp(0).reg() : locs->out(0).reg();
3633 locs->live_registers()->Remove(Location::RegisterLocation(result));
3634
3635 compiler->SaveLiveRegisters(locs);
3636 if (env_ != nullptr) {
3637 compiler->pending_deoptimization_env_ =
3638 compiler->SlowPathEnvironmentFor(env_, locs, kNumSlowPathArgs);
3639 }
3640 __ pushq(locs->in(0).reg());
3641 __ pushq(locs->in(1).reg());
3642
3643 const auto& selector = String::Handle(instruction()->call()->Selector());
3644 const auto& arguments_descriptor =
3645 Array::Handle(ArgumentsDescriptor::NewBoxed(
3646 /*type_args_len=*/0, /*num_arguments=*/2));
3647
3648 compiler->EmitMegamorphicInstanceCall(
3649 selector, arguments_descriptor, instruction()->call()->deopt_id(),
3650 instruction()->token_pos(), locs, try_index_, kNumSlowPathArgs);
3651 __ MoveRegister(result, RAX);
3652 compiler->RestoreLiveRegisters(locs);
3653 compiler->pending_deoptimization_env_ = nullptr;
3654 if (merged_) {
3655 __ CompareObject(result, Bool::True());
3656 __ j(EQUAL, instruction()->is_negated() ? labels_.false_label
3657 : labels_.true_label);
3658 __ jmp(instruction()->is_negated() ? labels_.true_label
3659 : labels_.false_label);
3660 ASSERT(exit_label()->IsUnused());
3661 } else {
3662 ASSERT(!instruction()->is_negated());
3663 __ jmp(exit_label());
3664 }
3665 }
3666
3667 private:
3668 intptr_t try_index_;
3669 BranchLabels labels_;
3670 bool merged_;
3671 Environment* env_;
3672};
3673
3674LocationSummary* CheckedSmiComparisonInstr::MakeLocationSummary(
3675 Zone* zone,
3676 bool opt) const {
3677 const intptr_t kNumInputs = 2;
3678 const intptr_t kNumTemps = 1;
3679 LocationSummary* summary = new (zone) LocationSummary(
3680 zone, kNumInputs, kNumTemps, LocationSummary::kCallOnSlowPath);
3681 summary->set_in(0, Location::RequiresRegister());
3682 summary->set_in(1, Location::RequiresRegister());
3683 summary->set_temp(0, Location::RequiresRegister());
3684 summary->set_out(0, Location::RequiresRegister());
3685 return summary;
3686}
3687
3688Condition CheckedSmiComparisonInstr::EmitComparisonCode(
3689 FlowGraphCompiler* compiler,
3690 BranchLabels labels) {
3691 return EmitInt64ComparisonOp(compiler, *locs(), kind());
3692}
3693
3694#define EMIT_SMI_CHECK \
3695 intptr_t left_cid = left()->Type()->ToCid(); \
3696 intptr_t right_cid = right()->Type()->ToCid(); \
3697 Register left = locs()->in(0).reg(); \
3698 Register right = locs()->in(1).reg(); \
3699 if (this->left()->definition() == this->right()->definition()) { \
3700 __ testq(left, compiler::Immediate(kSmiTagMask)); \
3701 } else if (left_cid == kSmiCid) { \
3702 __ testq(right, compiler::Immediate(kSmiTagMask)); \
3703 } else if (right_cid == kSmiCid) { \
3704 __ testq(left, compiler::Immediate(kSmiTagMask)); \
3705 } else { \
3706 __ movq(TMP, left); \
3707 __ orq(TMP, right); \
3708 __ testq(TMP, compiler::Immediate(kSmiTagMask)); \
3709 } \
3710 __ j(NOT_ZERO, slow_path->entry_label())
3711
3712void CheckedSmiComparisonInstr::EmitBranchCode(FlowGraphCompiler* compiler,
3713 BranchInstr* branch) {
3714 BranchLabels labels = compiler->CreateBranchLabels(branch);
3715 CheckedSmiComparisonSlowPath* slow_path = new CheckedSmiComparisonSlowPath(
3716 this, branch->env(), compiler->CurrentTryIndex(), labels,
3717 /* merged = */ true);
3718 compiler->AddSlowPathCode(slow_path);
3719 EMIT_SMI_CHECK;
3720 Condition true_condition = EmitComparisonCode(compiler, labels);
3721 ASSERT(true_condition != kInvalidCondition);
3722 EmitBranchOnCondition(compiler, true_condition, labels);
3723 // No need to bind slow_path->exit_label() as slow path exits through
3724 // true/false branch labels.
3725}
3726
3727void CheckedSmiComparisonInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
3728 // Zone-allocate labels to pass them to slow-path which outlives local scope.
3729 compiler::Label* true_label = new (Z) compiler::Label();
3730 compiler::Label* false_label = new (Z) compiler::Label();
3731 compiler::Label done;
3732 BranchLabels labels = {true_label, false_label, false_label};
3733 // In case of negated comparison result of a slow path call should be negated.
3734 // For this purpose, 'merged' slow path is generated: it tests
3735 // result of a call and jumps directly to true or false label.
3736 CheckedSmiComparisonSlowPath* slow_path = new CheckedSmiComparisonSlowPath(
3737 this, env(), compiler->CurrentTryIndex(), labels,
3738 /* merged = */ is_negated());
3739 compiler->AddSlowPathCode(slow_path);
3740 EMIT_SMI_CHECK;
3741 Condition true_condition = EmitComparisonCode(compiler, labels);
3742 ASSERT(true_condition != kInvalidCondition);
3743 EmitBranchOnCondition(compiler, true_condition, labels);
3744 Register result = locs()->out(0).reg();
3745 __ Bind(false_label);
3746 __ LoadObject(result, Bool::False());
3747 __ jmp(&done);
3748 __ Bind(true_label);
3749 __ LoadObject(result, Bool::True());
3750 __ Bind(&done);
3751 // In case of negated comparison slow path exits through true/false labels.
3752 if (!is_negated()) {
3753 __ Bind(slow_path->exit_label());
3754 }
3755}
3756
3757static bool CanBeImmediate(const Object& constant) {
3758 return constant.IsSmi() &&
3759 compiler::Immediate(static_cast<int64_t>(constant.raw())).is_int32();
3760}
3761
3762static bool IsSmiValue(const Object& constant, intptr_t value) {
3763 return constant.IsSmi() && (Smi::Cast(constant).Value() == value);
3764}
3765
3766LocationSummary* BinarySmiOpInstr::MakeLocationSummary(Zone* zone,
3767 bool opt) const {
3768 const intptr_t kNumInputs = 2;
3769
3770 ConstantInstr* right_constant = right()->definition()->AsConstant();
3771 if ((right_constant != NULL) && (op_kind() != Token::kTRUNCDIV) &&
3772 (op_kind() != Token::kSHL) && (op_kind() != Token::kMUL) &&
3773 (op_kind() != Token::kMOD) && CanBeImmediate(right_constant->value())) {
3774 const intptr_t kNumTemps = 0;
3775 LocationSummary* summary = new (zone)
3776 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
3777 summary->set_in(0, Location::RequiresRegister());
3778 summary->set_in(1, Location::Constant(right_constant));
3779 summary->set_out(0, Location::SameAsFirstInput());
3780 return summary;
3781 }
3782
3783 if (op_kind() == Token::kTRUNCDIV) {
3784 const intptr_t kNumTemps = 1;
3785 LocationSummary* summary = new (zone)
3786 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
3787 if (RightIsPowerOfTwoConstant()) {
3788 summary->set_in(0, Location::RequiresRegister());
3789 ConstantInstr* right_constant = right()->definition()->AsConstant();
3790 summary->set_in(1, Location::Constant(right_constant));
3791 summary->set_temp(0, Location::RequiresRegister());
3792 summary->set_out(0, Location::SameAsFirstInput());
3793 } else {
3794 // Both inputs must be writable because they will be untagged.
3795 summary->set_in(0, Location::RegisterLocation(RAX));
3796 summary->set_in(1, Location::WritableRegister());
3797 summary->set_out(0, Location::SameAsFirstInput());
3798 // Will be used for sign extension and division.
3799 summary->set_temp(0, Location::RegisterLocation(RDX));
3800 }
3801 return summary;
3802 } else if (op_kind() == Token::kMOD) {
3803 const intptr_t kNumTemps = 1;
3804 LocationSummary* summary = new (zone)
3805 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
3806 // Both inputs must be writable because they will be untagged.
3807 summary->set_in(0, Location::RegisterLocation(RDX));
3808 summary->set_in(1, Location::WritableRegister());
3809 summary->set_out(0, Location::SameAsFirstInput());
3810 // Will be used for sign extension and division.
3811 summary->set_temp(0, Location::RegisterLocation(RAX));
3812 return summary;
3813 } else if (op_kind() == Token::kSHR) {
3814 const intptr_t kNumTemps = 0;
3815 LocationSummary* summary = new (zone)
3816 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
3817 summary->set_in(0, Location::RequiresRegister());
3818 summary->set_in(1, LocationFixedRegisterOrSmiConstant(right(), RCX));
3819 summary->set_out(0, Location::SameAsFirstInput());
3820 return summary;
3821 } else if (op_kind() == Token::kSHL) {
3822 // Shift-by-1 overflow checking can use flags, otherwise we need a temp.
3823 const bool shiftBy1 =
3824 (right_constant != NULL) && IsSmiValue(right_constant->value(), 1);
3825 const intptr_t kNumTemps = (can_overflow() && !shiftBy1) ? 1 : 0;
3826 LocationSummary* summary = new (zone)
3827 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
3828 summary->set_in(0, Location::RequiresRegister());
3829 summary->set_in(1, LocationFixedRegisterOrSmiConstant(right(), RCX));
3830 if (kNumTemps == 1) {
3831 summary->set_temp(0, Location::RequiresRegister());
3832 }
3833 summary->set_out(0, Location::SameAsFirstInput());
3834 return summary;
3835 } else {
3836 const intptr_t kNumTemps = 0;
3837 LocationSummary* summary = new (zone)
3838 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
3839 summary->set_in(0, Location::RequiresRegister());
3840 ConstantInstr* constant = right()->definition()->AsConstant();
3841 if (constant != NULL) {
3842 summary->set_in(1, LocationRegisterOrSmiConstant(right()));
3843 } else {
3844 summary->set_in(1, Location::PrefersRegister());
3845 }
3846 summary->set_out(0, Location::SameAsFirstInput());
3847 return summary;
3848 }
3849}
3850
3851void BinarySmiOpInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
3852 if (op_kind() == Token::kSHL) {
3853 EmitSmiShiftLeft(compiler, this);
3854 return;
3855 }
3856
3857 Register left = locs()->in(0).reg();
3858 Register result = locs()->out(0).reg();
3859 ASSERT(left == result);
3860 compiler::Label* deopt = NULL;
3861 if (CanDeoptimize()) {
3862 deopt = compiler->AddDeoptStub(deopt_id(), ICData::kDeoptBinarySmiOp);
3863 }
3864
3865 if (locs()->in(1).IsConstant()) {
3866 const Object& constant = locs()->in(1).constant();
3867 ASSERT(constant.IsSmi());
3868 const int64_t imm = static_cast<int64_t>(constant.raw());
3869 switch (op_kind()) {
3870 case Token::kADD: {
3871 __ AddImmediate(left, compiler::Immediate(imm));
3872 if (deopt != NULL) __ j(OVERFLOW, deopt);
3873 break;
3874 }
3875 case Token::kSUB: {
3876 __ SubImmediate(left, compiler::Immediate(imm));
3877 if (deopt != NULL) __ j(OVERFLOW, deopt);
3878 break;
3879 }
3880 case Token::kMUL: {
3881 // Keep left value tagged and untag right value.
3882 const intptr_t value = Smi::Cast(constant).Value();
3883 __ MulImmediate(left, compiler::Immediate(value));
3884 if (deopt != NULL) __ j(OVERFLOW, deopt);
3885 break;
3886 }
3887 case Token::kTRUNCDIV: {
3888 const intptr_t value = Smi::Cast(constant).Value();
3889 ASSERT(value != kIntptrMin);
3890 ASSERT(Utils::IsPowerOfTwo(Utils::Abs(value)));
3891 const intptr_t shift_count =
3892 Utils::ShiftForPowerOfTwo(Utils::Abs(value)) + kSmiTagSize;
3893 ASSERT(kSmiTagSize == 1);
3894 Register temp = locs()->temp(0).reg();
3895 __ movq(temp, left);
3896 __ sarq(temp, compiler::Immediate(63));
3897 ASSERT(shift_count > 1); // 1, -1 case handled above.
3898 __ shrq(temp, compiler::Immediate(64 - shift_count));
3899 __ addq(left, temp);
3900 ASSERT(shift_count > 0);
3901 __ sarq(left, compiler::Immediate(shift_count));
3902 if (value < 0) {
3903 __ negq(left);
3904 }
3905 __ SmiTag(left);
3906 break;
3907 }
3908 case Token::kBIT_AND: {
3909 // No overflow check.
3910 __ AndImmediate(left, compiler::Immediate(imm));
3911 break;
3912 }
3913 case Token::kBIT_OR: {
3914 // No overflow check.
3915 __ OrImmediate(left, compiler::Immediate(imm));
3916 break;
3917 }
3918 case Token::kBIT_XOR: {
3919 // No overflow check.
3920 __ XorImmediate(left, compiler::Immediate(imm));
3921 break;
3922 }
3923
3924 case Token::kSHR: {
3925 // sarq operation masks the count to 6 bits.
3926 const intptr_t kCountLimit = 0x3F;
3927 const intptr_t value = Smi::Cast(constant).Value();
3928 __ sarq(left, compiler::Immediate(
3929 Utils::Minimum(value + kSmiTagSize, kCountLimit)));
3930 __ SmiTag(left);
3931 break;
3932 }
3933
3934 default:
3935 UNREACHABLE();
3936 break;
3937 }
3938 return;
3939 } // locs()->in(1).IsConstant().
3940
3941 if (locs()->in(1).IsStackSlot()) {
3942 const compiler::Address& right = LocationToStackSlotAddress(locs()->in(1));
3943 switch (op_kind()) {
3944 case Token::kADD: {
3945 __ addq(left, right);
3946 if (deopt != NULL) __ j(OVERFLOW, deopt);
3947 break;
3948 }
3949 case Token::kSUB: {
3950 __ subq(left, right);
3951 if (deopt != NULL) __ j(OVERFLOW, deopt);
3952 break;
3953 }
3954 case Token::kMUL: {
3955 __ SmiUntag(left);
3956 __ imulq(left, right);
3957 if (deopt != NULL) __ j(OVERFLOW, deopt);
3958 break;
3959 }
3960 case Token::kBIT_AND: {
3961 // No overflow check.
3962 __ andq(left, right);
3963 break;
3964 }
3965 case Token::kBIT_OR: {
3966 // No overflow check.
3967 __ orq(left, right);
3968 break;
3969 }
3970 case Token::kBIT_XOR: {
3971 // No overflow check.
3972 __ xorq(left, right);
3973 break;
3974 }
3975 default:
3976 UNREACHABLE();
3977 break;
3978 }
3979 return;
3980 } // locs()->in(1).IsStackSlot().
3981
3982 // if locs()->in(1).IsRegister.
3983 Register right = locs()->in(1).reg();
3984 switch (op_kind()) {
3985 case Token::kADD: {
3986 __ addq(left, right);
3987 if (deopt != NULL) __ j(OVERFLOW, deopt);
3988 break;
3989 }
3990 case Token::kSUB: {
3991 __ subq(left, right);
3992 if (deopt != NULL) __ j(OVERFLOW, deopt);
3993 break;
3994 }
3995 case Token::kMUL: {
3996 __ SmiUntag(left);
3997 __ imulq(left, right);
3998 if (deopt != NULL) __ j(OVERFLOW, deopt);
3999 break;
4000 }
4001 case Token::kBIT_AND: {
4002 // No overflow check.
4003 __ andq(left, right);
4004 break;
4005 }
4006 case Token::kBIT_OR: {
4007 // No overflow check.
4008 __ orq(left, right);
4009 break;
4010 }
4011 case Token::kBIT_XOR: {
4012 // No overflow check.
4013 __ xorq(left, right);
4014 break;
4015 }
4016 case Token::kTRUNCDIV: {
4017 compiler::Label not_32bit, done;
4018
4019 Register temp = locs()->temp(0).reg();
4020 ASSERT(left == RAX);
4021 ASSERT((right != RDX) && (right != RAX));
4022 ASSERT(temp == RDX);
4023 ASSERT(result == RAX);
4024 if (RangeUtils::CanBeZero(right_range())) {
4025 // Handle divide by zero in runtime.
4026 __ testq(right, right);
4027 __ j(ZERO, deopt);
4028 }
4029 // Check if both operands fit into 32bits as idiv with 64bit operands
4030 // requires twice as many cycles and has much higher latency.
4031 // We are checking this before untagging them to avoid corner case
4032 // dividing INT_MAX by -1 that raises exception because quotient is
4033 // too large for 32bit register.
4034 __ movsxd(temp, left);
4035 __ cmpq(temp, left);
4036 __ j(NOT_EQUAL, &not_32bit);
4037 __ movsxd(temp, right);
4038 __ cmpq(temp, right);
4039 __ j(NOT_EQUAL, &not_32bit);
4040
4041 // Both operands are 31bit smis. Divide using 32bit idiv.
4042 __ SmiUntag(left);
4043 __ SmiUntag(right);
4044 __ cdq();
4045 __ idivl(right);
4046 __ movsxd(result, result);
4047 __ jmp(&done);
4048
4049 // Divide using 64bit idiv.
4050 __ Bind(&not_32bit);
4051 __ SmiUntag(left);
4052 __ SmiUntag(right);
4053 __ cqo(); // Sign extend RAX -> RDX:RAX.
4054 __ idivq(right); // RAX: quotient, RDX: remainder.
4055 if (RangeUtils::Overlaps(right_range(), -1, -1)) {
4056 // Check the corner case of dividing the 'MIN_SMI' with -1, in which
4057 // case we cannot tag the result.
4058 __ CompareImmediate(result, compiler::Immediate(0x4000000000000000));
4059 __ j(EQUAL, deopt);
4060 }
4061 __ Bind(&done);
4062 __ SmiTag(result);
4063 break;
4064 }
4065 case Token::kMOD: {
4066 compiler::Label not_32bit, div_done;
4067
4068 Register temp = locs()->temp(0).reg();
4069 ASSERT(left == RDX);
4070 ASSERT((right != RDX) && (right != RAX));
4071 ASSERT(temp == RAX);
4072 ASSERT(result == RDX);
4073 if (RangeUtils::CanBeZero(right_range())) {
4074 // Handle divide by zero in runtime.
4075 __ testq(right, right);
4076 __ j(ZERO, deopt);
4077 }
4078 // Check if both operands fit into 32bits as idiv with 64bit operands
4079 // requires twice as many cycles and has much higher latency.
4080 // We are checking this before untagging them to avoid corner case
4081 // dividing INT_MAX by -1 that raises exception because quotient is
4082 // too large for 32bit register.
4083 __ movsxd(temp, left);
4084 __ cmpq(temp, left);
4085 __ j(NOT_EQUAL, &not_32bit);
4086 __ movsxd(temp, right);
4087 __ cmpq(temp, right);
4088 __ j(NOT_EQUAL, &not_32bit);
4089 // Both operands are 31bit smis. Divide using 32bit idiv.
4090 __ SmiUntag(left);
4091 __ SmiUntag(right);
4092 __ movq(RAX, RDX);
4093 __ cdq();
4094 __ idivl(right);
4095 __ movsxd(result, result);
4096 __ jmp(&div_done);
4097
4098 // Divide using 64bit idiv.
4099 __ Bind(&not_32bit);
4100 __ SmiUntag(left);
4101 __ SmiUntag(right);
4102 __ movq(RAX, RDX);
4103 __ cqo(); // Sign extend RAX -> RDX:RAX.
4104 __ idivq(right); // RAX: quotient, RDX: remainder.
4105 __ Bind(&div_done);
4106 // res = left % right;
4107 // if (res < 0) {
4108 // if (right < 0) {
4109 // res = res - right;
4110 // } else {
4111 // res = res + right;
4112 // }
4113 // }
4114 compiler::Label all_done;
4115 __ cmpq(result, compiler::Immediate(0));
4116 __ j(GREATER_EQUAL, &all_done, compiler::Assembler::kNearJump);
4117 // Result is negative, adjust it.
4118 if (RangeUtils::Overlaps(right_range(), -1, 1)) {
4119 compiler::Label subtract;
4120 __ cmpq(right, compiler::Immediate(0));
4121 __ j(LESS, &subtract, compiler::Assembler::kNearJump);
4122 __ addq(result, right);
4123 __ jmp(&all_done, compiler::Assembler::kNearJump);
4124 __ Bind(&subtract);
4125 __ subq(result, right);
4126 } else if (right_range()->IsPositive()) {
4127 // Right is positive.
4128 __ addq(result, right);
4129 } else {
4130 // Right is negative.
4131 __ subq(result, right);
4132 }
4133 __ Bind(&all_done);
4134 __ SmiTag(result);
4135 break;
4136 }
4137 case Token::kSHR: {
4138 if (CanDeoptimize()) {
4139 __ CompareImmediate(right, compiler::Immediate(0));
4140 __ j(LESS, deopt);
4141 }
4142 __ SmiUntag(right);
4143 // sarq operation masks the count to 6 bits.
4144 const intptr_t kCountLimit = 0x3F;
4145 if (!RangeUtils::OnlyLessThanOrEqualTo(right_range(), kCountLimit)) {
4146 __ CompareImmediate(right, compiler::Immediate(kCountLimit));
4147 compiler::Label count_ok;
4148 __ j(LESS, &count_ok, compiler::Assembler::kNearJump);
4149 __ LoadImmediate(right, compiler::Immediate(kCountLimit));
4150 __ Bind(&count_ok);
4151 }
4152 ASSERT(right == RCX); // Count must be in RCX
4153 __ SmiUntag(left);
4154 __ sarq(left, right);
4155 __ SmiTag(left);
4156 break;
4157 }
4158 case Token::kDIV: {
4159 // Dispatches to 'Double./'.
4160 // TODO(srdjan): Implement as conversion to double and double division.
4161 UNREACHABLE();
4162 break;
4163 }
4164 case Token::kOR:
4165 case Token::kAND: {
4166 // Flow graph builder has dissected this operation to guarantee correct
4167 // behavior (short-circuit evaluation).
4168 UNREACHABLE();
4169 break;
4170 }
4171 default:
4172 UNREACHABLE();
4173 break;
4174 }
4175}
4176
4177LocationSummary* CheckEitherNonSmiInstr::MakeLocationSummary(Zone* zone,
4178 bool opt) const {
4179 intptr_t left_cid = left()->Type()->ToCid();
4180 intptr_t right_cid = right()->Type()->ToCid();
4181 ASSERT((left_cid != kDoubleCid) && (right_cid != kDoubleCid));
4182 const intptr_t kNumInputs = 2;
4183 const bool need_temp = (left()->definition() != right()->definition()) &&
4184 (left_cid != kSmiCid) && (right_cid != kSmiCid);
4185 const intptr_t kNumTemps = need_temp ? 1 : 0;
4186 LocationSummary* summary = new (zone)
4187 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
4188 summary->set_in(0, Location::RequiresRegister());
4189 summary->set_in(1, Location::RequiresRegister());
4190 if (need_temp) summary->set_temp(0, Location::RequiresRegister());
4191 return summary;
4192}
4193
4194void CheckEitherNonSmiInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
4195 compiler::Label* deopt =
4196 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptBinaryDoubleOp,
4197 licm_hoisted_ ? ICData::kHoisted : 0);
4198 intptr_t left_cid = left()->Type()->ToCid();
4199 intptr_t right_cid = right()->Type()->ToCid();
4200 Register left = locs()->in(0).reg();
4201 Register right = locs()->in(1).reg();
4202 if (this->left()->definition() == this->right()->definition()) {
4203 __ testq(left, compiler::Immediate(kSmiTagMask));
4204 } else if (left_cid == kSmiCid) {
4205 __ testq(right, compiler::Immediate(kSmiTagMask));
4206 } else if (right_cid == kSmiCid) {
4207 __ testq(left, compiler::Immediate(kSmiTagMask));
4208 } else {
4209 Register temp = locs()->temp(0).reg();
4210 __ movq(temp, left);
4211 __ orq(temp, right);
4212 __ testq(temp, compiler::Immediate(kSmiTagMask));
4213 }
4214 __ j(ZERO, deopt);
4215}
4216
4217LocationSummary* BoxInstr::MakeLocationSummary(Zone* zone, bool opt) const {
4218 const intptr_t kNumInputs = 1;
4219 const intptr_t kNumTemps = 1;
4220 LocationSummary* summary = new (zone) LocationSummary(
4221 zone, kNumInputs, kNumTemps, LocationSummary::kCallOnSlowPath);
4222 summary->set_in(0, Location::RequiresFpuRegister());
4223 summary->set_temp(0, Location::RequiresRegister());
4224 summary->set_out(0, Location::RequiresRegister());
4225 return summary;
4226}
4227
4228void BoxInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
4229 Register out_reg = locs()->out(0).reg();
4230 Register temp = locs()->temp(0).reg();
4231 XmmRegister value = locs()->in(0).fpu_reg();
4232
4233 BoxAllocationSlowPath::Allocate(compiler, this,
4234 compiler->BoxClassFor(from_representation()),
4235 out_reg, temp);
4236
4237 switch (from_representation()) {
4238 case kUnboxedDouble:
4239 __ movsd(compiler::FieldAddress(out_reg, ValueOffset()), value);
4240 break;
4241 case kUnboxedFloat: {
4242 __ cvtss2sd(FpuTMP, value);
4243 __ movsd(compiler::FieldAddress(out_reg, ValueOffset()), FpuTMP);
4244 break;
4245 }
4246 case kUnboxedFloat32x4:
4247 case kUnboxedFloat64x2:
4248 case kUnboxedInt32x4:
4249 __ movups(compiler::FieldAddress(out_reg, ValueOffset()), value);
4250 break;
4251 default:
4252 UNREACHABLE();
4253 break;
4254 }
4255}
4256
4257LocationSummary* UnboxInstr::MakeLocationSummary(Zone* zone, bool opt) const {
4258 const intptr_t kNumInputs = 1;
4259 const intptr_t kNumTemps = 0;
4260 const bool needs_writable_input =
4261 (representation() != kUnboxedInt64) &&
4262 (value()->Type()->ToNullableCid() != BoxCid());
4263 LocationSummary* summary = new (zone)
4264 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
4265 summary->set_in(0, needs_writable_input ? Location::WritableRegister()
4266 : Location::RequiresRegister());
4267 if (representation() == kUnboxedInt64 || representation() == kUnboxedInt32) {
4268 summary->set_out(0, Location::SameAsFirstInput());
4269 } else {
4270 summary->set_out(0, Location::RequiresFpuRegister());
4271 }
4272 return summary;
4273}
4274
4275void UnboxInstr::EmitLoadFromBox(FlowGraphCompiler* compiler) {
4276 const Register box = locs()->in(0).reg();
4277
4278 switch (representation()) {
4279 case kUnboxedInt64: {
4280 const Register result = locs()->out(0).reg();
4281 __ movq(result, compiler::FieldAddress(box, ValueOffset()));
4282 break;
4283 }
4284
4285 case kUnboxedDouble: {
4286 const FpuRegister result = locs()->out(0).fpu_reg();
4287 __ movsd(result, compiler::FieldAddress(box, ValueOffset()));
4288 break;
4289 }
4290
4291 case kUnboxedFloat: {
4292 const FpuRegister result = locs()->out(0).fpu_reg();
4293 __ movsd(result, compiler::FieldAddress(box, ValueOffset()));
4294 __ cvtsd2ss(result, result);
4295 break;
4296 }
4297
4298 case kUnboxedFloat32x4:
4299 case kUnboxedFloat64x2:
4300 case kUnboxedInt32x4: {
4301 const FpuRegister result = locs()->out(0).fpu_reg();
4302 __ movups(result, compiler::FieldAddress(box, ValueOffset()));
4303 break;
4304 }
4305
4306 default:
4307 UNREACHABLE();
4308 break;
4309 }
4310}
4311
4312void UnboxInstr::EmitSmiConversion(FlowGraphCompiler* compiler) {
4313 const Register box = locs()->in(0).reg();
4314
4315 switch (representation()) {
4316 case kUnboxedInt64: {
4317 const Register result = locs()->out(0).reg();
4318 ASSERT(result == box);
4319 __ SmiUntag(box);
4320 break;
4321 }
4322
4323 case kUnboxedDouble: {
4324 const FpuRegister result = locs()->out(0).fpu_reg();
4325 __ SmiUntag(box);
4326 __ cvtsi2sdq(result, box);
4327 break;
4328 }
4329
4330 default:
4331 UNREACHABLE();
4332 break;
4333 }
4334}
4335
4336void UnboxInstr::EmitLoadInt32FromBoxOrSmi(FlowGraphCompiler* compiler) {
4337 const Register value = locs()->in(0).reg();
4338 const Register result = locs()->out(0).reg();
4339 ASSERT(value == result);
4340 compiler::Label done;
4341 __ SmiUntag(value);
4342 __ j(NOT_CARRY, &done, compiler::Assembler::kNearJump);
4343 __ movsxw(result, compiler::Address(value, TIMES_2, Mint::value_offset()));
4344 __ Bind(&done);
4345}
4346
4347void UnboxInstr::EmitLoadInt64FromBoxOrSmi(FlowGraphCompiler* compiler) {
4348 const Register value = locs()->in(0).reg();
4349 const Register result = locs()->out(0).reg();
4350 ASSERT(value == result);
4351 compiler::Label done;
4352 __ SmiUntag(value);
4353 __ j(NOT_CARRY, &done, compiler::Assembler::kNearJump);
4354 __ movq(value, compiler::Address(value, TIMES_2, Mint::value_offset()));
4355 __ Bind(&done);
4356}
4357
4358LocationSummary* UnboxInteger32Instr::MakeLocationSummary(Zone* zone,
4359 bool opt) const {
4360 const intptr_t kNumInputs = 1;
4361 const intptr_t kNumTemps = (!is_truncating() && CanDeoptimize()) ? 1 : 0;
4362 LocationSummary* summary = new (zone)
4363 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
4364 summary->set_in(0, Location::RequiresRegister());
4365 summary->set_out(0, Location::SameAsFirstInput());
4366 if (kNumTemps > 0) {
4367 summary->set_temp(0, Location::RequiresRegister());
4368 }
4369 return summary;
4370}
4371
4372void UnboxInteger32Instr::EmitNativeCode(FlowGraphCompiler* compiler) {
4373 const intptr_t value_cid = value()->Type()->ToCid();
4374 const Register value = locs()->in(0).reg();
4375 compiler::Label* deopt =
4376 CanDeoptimize()
4377 ? compiler->AddDeoptStub(GetDeoptId(), ICData::kDeoptUnboxInteger)
4378 : NULL;
4379 ASSERT(value == locs()->out(0).reg());
4380
4381 if (value_cid == kSmiCid) {
4382 __ SmiUntag(value);
4383 } else if (value_cid == kMintCid) {
4384 __ movq(value, compiler::FieldAddress(value, Mint::value_offset()));
4385 } else if (!CanDeoptimize()) {
4386 // Type information is not conclusive, but range analysis found
4387 // the value to be in int64 range. Therefore it must be a smi
4388 // or mint value.
4389 ASSERT(is_truncating());
4390 compiler::Label done;
4391 __ SmiUntag(value);
4392 __ j(NOT_CARRY, &done, compiler::Assembler::kNearJump);
4393 __ movq(value, compiler::Address(value, TIMES_2, Mint::value_offset()));
4394 __ Bind(&done);
4395 return;
4396 } else {
4397 compiler::Label done;
4398 // Optimistically untag value.
4399 __ SmiUntagOrCheckClass(value, kMintCid, &done);
4400 __ j(NOT_EQUAL, deopt);
4401 // Undo untagging by multiplying value with 2.
4402 __ movq(value, compiler::Address(value, TIMES_2, Mint::value_offset()));
4403 __ Bind(&done);
4404 }
4405
4406 // TODO(vegorov): as it is implemented right now truncating unboxing would
4407 // leave "garbage" in the higher word.
4408 if (!is_truncating() && (deopt != NULL)) {
4409 ASSERT(representation() == kUnboxedInt32);
4410 Register temp = locs()->temp(0).reg();
4411 __ movsxd(temp, value);
4412 __ cmpq(temp, value);
4413 __ j(NOT_EQUAL, deopt);
4414 }
4415}
4416
4417LocationSummary* BoxInteger32Instr::MakeLocationSummary(Zone* zone,
4418 bool opt) const {
4419 ASSERT((from_representation() == kUnboxedInt32) ||
4420 (from_representation() == kUnboxedUint32));
4421 const intptr_t kNumInputs = 1;
4422 const intptr_t kNumTemps = 0;
4423 LocationSummary* summary = new (zone)
4424 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
4425 summary->set_in(0, Location::RequiresRegister());
4426 summary->set_out(0, Location::RequiresRegister());
4427 return summary;
4428}
4429
4430void BoxInteger32Instr::EmitNativeCode(FlowGraphCompiler* compiler) {
4431 const Register value = locs()->in(0).reg();
4432 const Register out = locs()->out(0).reg();
4433 ASSERT(value != out);
4434
4435 ASSERT(kSmiTagSize == 1);
4436 if (from_representation() == kUnboxedInt32) {
4437 __ movsxd(out, value);
4438 } else {
4439 ASSERT(from_representation() == kUnboxedUint32);
4440 __ movl(out, value);
4441 }
4442 __ SmiTag(out);
4443}
4444
4445LocationSummary* BoxInt64Instr::MakeLocationSummary(Zone* zone,
4446 bool opt) const {
4447 const intptr_t kNumInputs = 1;
4448 const intptr_t kNumTemps = ValueFitsSmi() ? 0 : 1;
4449 // Shared slow path is used in BoxInt64Instr::EmitNativeCode in
4450 // FLAG_use_bare_instructions mode and only after VM isolate stubs where
4451 // replaced with isolate-specific stubs.
4452 auto object_store = Isolate::Current()->object_store();
4453 const bool stubs_in_vm_isolate =
4454 object_store->allocate_mint_with_fpu_regs_stub()
4455 ->ptr()
4456 ->InVMIsolateHeap() ||
4457 object_store->allocate_mint_without_fpu_regs_stub()
4458 ->ptr()
4459 ->InVMIsolateHeap();
4460 const bool shared_slow_path_call = SlowPathSharingSupported(opt) &&
4461 FLAG_use_bare_instructions &&
4462 !stubs_in_vm_isolate;
4463 LocationSummary* summary = new (zone) LocationSummary(
4464 zone, kNumInputs, kNumTemps,
4465 ValueFitsSmi()
4466 ? LocationSummary::kNoCall
4467 : ((shared_slow_path_call ? LocationSummary::kCallOnSharedSlowPath
4468 : LocationSummary::kCallOnSlowPath)));
4469 summary->set_in(0, Location::RequiresRegister());
4470 if (ValueFitsSmi()) {
4471 summary->set_out(0, Location::RequiresRegister());
4472 } else if (shared_slow_path_call) {
4473 summary->set_out(0,
4474 Location::RegisterLocation(AllocateMintABI::kResultReg));
4475 summary->set_temp(0, Location::RegisterLocation(AllocateMintABI::kTempReg));
4476 } else {
4477 summary->set_out(0, Location::RequiresRegister());
4478 summary->set_temp(0, Location::RequiresRegister());
4479 }
4480 return summary;
4481}
4482
4483void BoxInt64Instr::EmitNativeCode(FlowGraphCompiler* compiler) {
4484 const Register out = locs()->out(0).reg();
4485 const Register value = locs()->in(0).reg();
4486 __ MoveRegister(out, value);
4487 __ SmiTag(out);
4488 if (ValueFitsSmi()) {
4489 return;
4490 }
4491 // If the value doesn't fit in a smi, the tagging changes the sign,
4492 // which causes the overflow flag to be set.
4493 compiler::Label done;
4494 __ j(NO_OVERFLOW, &done);
4495
4496 const Register temp = locs()->temp(0).reg();
4497 if (compiler->intrinsic_mode()) {
4498 __ TryAllocate(compiler->mint_class(),
4499 compiler->intrinsic_slow_path_label(),
4500 /*near_jump=*/true, out, temp);
4501 } else if (locs()->call_on_shared_slow_path()) {
4502 auto object_store = compiler->isolate()->object_store();
4503 const bool live_fpu_regs = locs()->live_registers()->FpuRegisterCount() > 0;
4504 const auto& stub = Code::ZoneHandle(
4505 compiler->zone(),
4506 live_fpu_regs ? object_store->allocate_mint_with_fpu_regs_stub()
4507 : object_store->allocate_mint_without_fpu_regs_stub());
4508
4509 ASSERT(!locs()->live_registers()->ContainsRegister(
4510 AllocateMintABI::kResultReg));
4511 auto extended_env = compiler->SlowPathEnvironmentFor(this, 0);
4512 compiler->GenerateStubCall(token_pos(), stub, PcDescriptorsLayout::kOther,
4513 locs(), DeoptId::kNone, extended_env);
4514 } else {
4515 BoxAllocationSlowPath::Allocate(compiler, this, compiler->mint_class(), out,
4516 temp);
4517 }
4518
4519 __ movq(compiler::FieldAddress(out, Mint::value_offset()), value);
4520 __ Bind(&done);
4521}
4522
4523LocationSummary* BinaryDoubleOpInstr::MakeLocationSummary(Zone* zone,
4524 bool opt) const {
4525 const intptr_t kNumInputs = 2;
4526 const intptr_t kNumTemps = 0;
4527 LocationSummary* summary = new (zone)
4528 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
4529 summary->set_in(0, Location::RequiresFpuRegister());
4530 summary->set_in(1, Location::RequiresFpuRegister());
4531 summary->set_out(0, Location::SameAsFirstInput());
4532 return summary;
4533}
4534
4535void BinaryDoubleOpInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
4536 XmmRegister left = locs()->in(0).fpu_reg();
4537 XmmRegister right = locs()->in(1).fpu_reg();
4538
4539 ASSERT(locs()->out(0).fpu_reg() == left);
4540
4541 switch (op_kind()) {
4542 case Token::kADD:
4543 __ addsd(left, right);
4544 break;
4545 case Token::kSUB:
4546 __ subsd(left, right);
4547 break;
4548 case Token::kMUL:
4549 __ mulsd(left, right);
4550 break;
4551 case Token::kDIV:
4552 __ divsd(left, right);
4553 break;
4554 default:
4555 UNREACHABLE();
4556 }
4557}
4558
4559LocationSummary* DoubleTestOpInstr::MakeLocationSummary(Zone* zone,
4560 bool opt) const {
4561 const intptr_t kNumInputs = 1;
4562 const intptr_t kNumTemps =
4563 (op_kind() == MethodRecognizer::kDouble_getIsInfinite) ? 1 : 0;
4564 LocationSummary* summary = new (zone)
4565 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
4566 summary->set_in(0, Location::RequiresFpuRegister());
4567 if (op_kind() == MethodRecognizer::kDouble_getIsInfinite) {
4568 summary->set_temp(0, Location::RequiresRegister());
4569 }
4570 summary->set_out(0, Location::RequiresRegister());
4571 return summary;
4572}
4573
4574Condition DoubleTestOpInstr::EmitComparisonCode(FlowGraphCompiler* compiler,
4575 BranchLabels labels) {
4576 ASSERT(compiler->is_optimizing());
4577 const XmmRegister value = locs()->in(0).fpu_reg();
4578 const bool is_negated = kind() != Token::kEQ;
4579 if (op_kind() == MethodRecognizer::kDouble_getIsNaN) {
4580 compiler::Label is_nan;
4581 __ comisd(value, value);
4582 return is_negated ? PARITY_ODD : PARITY_EVEN;
4583 } else {
4584 ASSERT(op_kind() == MethodRecognizer::kDouble_getIsInfinite);
4585 const Register temp = locs()->temp(0).reg();
4586 __ AddImmediate(RSP, compiler::Immediate(-kDoubleSize));
4587 __ movsd(compiler::Address(RSP, 0), value);
4588 __ movq(temp, compiler::Address(RSP, 0));
4589 __ AddImmediate(RSP, compiler::Immediate(kDoubleSize));
4590 // Mask off the sign.
4591 __ AndImmediate(temp, compiler::Immediate(0x7FFFFFFFFFFFFFFFLL));
4592 // Compare with +infinity.
4593 __ CompareImmediate(temp, compiler::Immediate(0x7FF0000000000000LL));
4594 return is_negated ? NOT_EQUAL : EQUAL;
4595 }
4596}
4597
4598// SIMD
4599
4600#define DEFINE_EMIT(Name, Args) \
4601 static void Emit##Name(FlowGraphCompiler* compiler, SimdOpInstr* instr, \
4602 PP_APPLY(PP_UNPACK, Args))
4603
4604#define SIMD_OP_FLOAT_ARITH(V, Name, op) \
4605 V(Float32x4##Name, op##ps) \
4606 V(Float64x2##Name, op##pd)
4607
4608#define SIMD_OP_SIMPLE_BINARY(V) \
4609 SIMD_OP_FLOAT_ARITH(V, Add, add) \
4610 SIMD_OP_FLOAT_ARITH(V, Sub, sub) \
4611 SIMD_OP_FLOAT_ARITH(V, Mul, mul) \
4612 SIMD_OP_FLOAT_ARITH(V, Div, div) \
4613 SIMD_OP_FLOAT_ARITH(V, Min, min) \
4614 SIMD_OP_FLOAT_ARITH(V, Max, max) \
4615 V(Int32x4Add, addpl) \
4616 V(Int32x4Sub, subpl) \
4617 V(Int32x4BitAnd, andps) \
4618 V(Int32x4BitOr, orps) \
4619 V(Int32x4BitXor, xorps) \
4620 V(Float32x4Equal, cmppseq) \
4621 V(Float32x4NotEqual, cmppsneq) \
4622 V(Float32x4GreaterThan, cmppsnle) \
4623 V(Float32x4GreaterThanOrEqual, cmppsnlt) \
4624 V(Float32x4LessThan, cmppslt) \
4625 V(Float32x4LessThanOrEqual, cmppsle)
4626
4627DEFINE_EMIT(SimdBinaryOp,
4628 (SameAsFirstInput, XmmRegister left, XmmRegister right)) {
4629 switch (instr->kind()) {
4630#define EMIT(Name, op) \
4631 case SimdOpInstr::k##Name: \
4632 __ op(left, right); \
4633 break;
4634 SIMD_OP_SIMPLE_BINARY(EMIT)
4635#undef EMIT
4636 case SimdOpInstr::kFloat32x4Scale:
4637 __ cvtsd2ss(left, left);
4638 __ shufps(left, left, compiler::Immediate(0x00));
4639 __ mulps(left, right);
4640 break;
4641 case SimdOpInstr::kFloat32x4ShuffleMix:
4642 case SimdOpInstr::kInt32x4ShuffleMix:
4643 __ shufps(left, right, compiler::Immediate(instr->mask()));
4644 break;
4645 case SimdOpInstr::kFloat64x2FromDoubles:
4646 // shufpd mask 0x0 results in:
4647 // Lower 64-bits of left = Lower 64-bits of left.
4648 // Upper 64-bits of left = Lower 64-bits of right.
4649 __ shufpd(left, right, compiler::Immediate(0x0));
4650 break;
4651 case SimdOpInstr::kFloat64x2Scale:
4652 __ shufpd(right, right, compiler::Immediate(0x00));
4653 __ mulpd(left, right);
4654 break;
4655 case SimdOpInstr::kFloat64x2WithX:
4656 case SimdOpInstr::kFloat64x2WithY: {
4657 // TODO(dartbug.com/30949) avoid transfer through memory.
4658 COMPILE_ASSERT(SimdOpInstr::kFloat64x2WithY ==
4659 (SimdOpInstr::kFloat64x2WithX + 1));
4660 const intptr_t lane_index = instr->kind() - SimdOpInstr::kFloat64x2WithX;
4661 ASSERT(0 <= lane_index && lane_index < 2);
4662
4663 __ SubImmediate(RSP, compiler::Immediate(kSimd128Size));
4664 __ movups(compiler::Address(RSP, 0), left);
4665 __ movsd(compiler::Address(RSP, lane_index * kDoubleSize), right);
4666 __ movups(left, compiler::Address(RSP, 0));
4667 __ AddImmediate(RSP, compiler::Immediate(kSimd128Size));
4668 break;
4669 }
4670 case SimdOpInstr::kFloat32x4WithX:
4671 case SimdOpInstr::kFloat32x4WithY:
4672 case SimdOpInstr::kFloat32x4WithZ:
4673 case SimdOpInstr::kFloat32x4WithW: {
4674 // TODO(dartbug.com/30949) avoid transfer through memory. SSE4.1 has
4675 // insertps. SSE2 these instructions can be implemented via a combination
4676 // of shufps/movss/movlhps.
4677 COMPILE_ASSERT(
4678 SimdOpInstr::kFloat32x4WithY == (SimdOpInstr::kFloat32x4WithX + 1) &&
4679 SimdOpInstr::kFloat32x4WithZ == (SimdOpInstr::kFloat32x4WithX + 2) &&
4680 SimdOpInstr::kFloat32x4WithW == (SimdOpInstr::kFloat32x4WithX + 3));
4681 const intptr_t lane_index = instr->kind() - SimdOpInstr::kFloat32x4WithX;
4682 ASSERT(0 <= lane_index && lane_index < 4);
4683 __ cvtsd2ss(left, left);
4684 __ SubImmediate(RSP, compiler::Immediate(kSimd128Size));
4685 __ movups(compiler::Address(RSP, 0), right);
4686 __ movss(compiler::Address(RSP, lane_index * kFloatSize), left);
4687 __ movups(left, compiler::Address(RSP, 0));
4688 __ AddImmediate(RSP, compiler::Immediate(kSimd128Size));
4689 break;
4690 }
4691
4692 default:
4693 UNREACHABLE();
4694 }
4695}
4696
4697#define SIMD_OP_SIMPLE_UNARY(V) \
4698 SIMD_OP_FLOAT_ARITH(V, Sqrt, sqrt) \
4699 SIMD_OP_FLOAT_ARITH(V, Negate, negate) \
4700 SIMD_OP_FLOAT_ARITH(V, Abs, abs) \
4701 V(Float32x4Reciprocal, rcpps) \
4702 V(Float32x4ReciprocalSqrt, rsqrtps)
4703
4704DEFINE_EMIT(SimdUnaryOp, (SameAsFirstInput, XmmRegister value)) {
4705 // TODO(dartbug.com/30949) select better register constraints to avoid
4706 // redundant move of input into a different register.
4707 switch (instr->kind()) {
4708#define EMIT(Name, op) \
4709 case SimdOpInstr::k##Name: \
4710 __ op(value, value); \
4711 break;
4712 SIMD_OP_SIMPLE_UNARY(EMIT)
4713#undef EMIT
4714 case SimdOpInstr::kFloat32x4ShuffleX:
4715 // Shuffle not necessary.
4716 __ cvtss2sd(value, value);
4717 break;
4718 case SimdOpInstr::kFloat32x4ShuffleY:
4719 __ shufps(value, value, compiler::Immediate(0x55));
4720 __ cvtss2sd(value, value);
4721 break;
4722 case SimdOpInstr::kFloat32x4ShuffleZ:
4723 __ shufps(value, value, compiler::Immediate(0xAA));
4724 __ cvtss2sd(value, value);
4725 break;
4726 case SimdOpInstr::kFloat32x4ShuffleW:
4727 __ shufps(value, value, compiler::Immediate(0xFF));
4728 __ cvtss2sd(value, value);
4729 break;
4730 case SimdOpInstr::kFloat32x4Shuffle:
4731 case SimdOpInstr::kInt32x4Shuffle:
4732 __ shufps(value, value, compiler::Immediate(instr->mask()));
4733 break;
4734 case SimdOpInstr::kFloat32x4Splat:
4735 // Convert to Float32.
4736 __ cvtsd2ss(value, value);
4737 // Splat across all lanes.
4738 __ shufps(value, value, compiler::Immediate(0x00));
4739 break;
4740 case SimdOpInstr::kFloat32x4ToFloat64x2:
4741 __ cvtps2pd(value, value);
4742 break;
4743 case SimdOpInstr::kFloat64x2ToFloat32x4:
4744 __ cvtpd2ps(value, value);
4745 break;
4746 case SimdOpInstr::kInt32x4ToFloat32x4:
4747 case SimdOpInstr::kFloat32x4ToInt32x4:
4748 // TODO(dartbug.com/30949) these operations are essentially nop and should
4749 // not generate any code. They should be removed from the graph before
4750 // code generation.
4751 break;
4752 case SimdOpInstr::kFloat64x2GetX:
4753 // NOP.
4754 break;
4755 case SimdOpInstr::kFloat64x2GetY:
4756 __ shufpd(value, value, compiler::Immediate(0x33));
4757 break;
4758 case SimdOpInstr::kFloat64x2Splat:
4759 __ shufpd(value, value, compiler::Immediate(0x0));
4760 break;
4761 default:
4762 UNREACHABLE();
4763 break;
4764 }
4765}
4766
4767DEFINE_EMIT(SimdGetSignMask, (Register out, XmmRegister value)) {
4768 switch (instr->kind()) {
4769 case SimdOpInstr::kFloat32x4GetSignMask:
4770 case SimdOpInstr::kInt32x4GetSignMask:
4771 __ movmskps(out, value);
4772 break;
4773 case SimdOpInstr::kFloat64x2GetSignMask:
4774 __ movmskpd(out, value);
4775 break;
4776 default:
4777 UNREACHABLE();
4778 break;
4779 }
4780}
4781
4782DEFINE_EMIT(
4783 Float32x4FromDoubles,
4784 (SameAsFirstInput, XmmRegister v0, XmmRegister, XmmRegister, XmmRegister)) {
4785 // TODO(dartbug.com/30949) avoid transfer through memory. SSE4.1 has
4786 // insertps, with SSE2 this instruction can be implemented through unpcklps.
4787 const XmmRegister out = v0;
4788 __ SubImmediate(RSP, compiler::Immediate(kSimd128Size));
4789 for (intptr_t i = 0; i < 4; i++) {
4790 __ cvtsd2ss(out, instr->locs()->in(i).fpu_reg());
4791 __ movss(compiler::Address(RSP, i * kFloatSize), out);
4792 }
4793 __ movups(out, compiler::Address(RSP, 0));
4794 __ AddImmediate(RSP, compiler::Immediate(kSimd128Size));
4795}
4796
4797DEFINE_EMIT(Float32x4Zero, (XmmRegister value)) {
4798 __ xorps(value, value);
4799}
4800
4801DEFINE_EMIT(Float64x2Zero, (XmmRegister value)) {
4802 __ xorpd(value, value);
4803}
4804
4805DEFINE_EMIT(Float32x4Clamp,
4806 (SameAsFirstInput,
4807 XmmRegister value,
4808 XmmRegister lower,
4809 XmmRegister upper)) {
4810 __ minps(value, upper);
4811 __ maxps(value, lower);
4812}
4813
4814DEFINE_EMIT(Int32x4FromInts,
4815 (XmmRegister result, Register, Register, Register, Register)) {
4816 // TODO(dartbug.com/30949) avoid transfer through memory.
4817 __ SubImmediate(RSP, compiler::Immediate(kSimd128Size));
4818 for (intptr_t i = 0; i < 4; i++) {
4819 __ movl(compiler::Address(RSP, i * kInt32Size), instr->locs()->in(i).reg());
4820 }
4821 __ movups(result, compiler::Address(RSP, 0));
4822 __ AddImmediate(RSP, compiler::Immediate(kSimd128Size));
4823}
4824
4825DEFINE_EMIT(Int32x4FromBools,
4826 (XmmRegister result,
4827 Register,
4828 Register,
4829 Register,
4830 Register,
4831 Temp<Register> temp)) {
4832 // TODO(dartbug.com/30949) avoid transfer through memory.
4833 __ SubImmediate(RSP, compiler::Immediate(kSimd128Size));
4834 for (intptr_t i = 0; i < 4; i++) {
4835 compiler::Label done, load_false;
4836 __ xorq(temp, temp);
4837 __ CompareObject(instr->locs()->in(i).reg(), Bool::True());
4838 __ setcc(EQUAL, ByteRegisterOf(temp));
4839 __ negl(temp); // temp = input ? -1 : 0
4840 __ movl(compiler::Address(RSP, kInt32Size * i), temp);
4841 }
4842 __ movups(result, compiler::Address(RSP, 0));
4843 __ AddImmediate(RSP, compiler::Immediate(kSimd128Size));
4844}
4845
4846static void EmitToBoolean(FlowGraphCompiler* compiler, Register out) {
4847 ASSERT_BOOL_FALSE_FOLLOWS_BOOL_TRUE();
4848 __ testl(out, out);
4849 __ setcc(ZERO, ByteRegisterOf(out));
4850 __ movzxb(out, out);
4851 __ movq(out,
4852 compiler::Address(THR, out, TIMES_8, Thread::bool_true_offset()));
4853}
4854
4855DEFINE_EMIT(Int32x4GetFlagZorW,
4856 (Register out, XmmRegister value, Temp<XmmRegister> temp)) {
4857 __ movhlps(temp, value); // extract upper half.
4858 __ movq(out, temp);
4859 if (instr->kind() == SimdOpInstr::kInt32x4GetFlagW) {
4860 __ shrq(out, compiler::Immediate(32)); // extract upper 32bits.
4861 }
4862 EmitToBoolean(compiler, out);
4863}
4864
4865DEFINE_EMIT(Int32x4GetFlagXorY, (Register out, XmmRegister value)) {
4866 __ movq(out, value);
4867 if (instr->kind() == SimdOpInstr::kInt32x4GetFlagY) {
4868 __ shrq(out, compiler::Immediate(32)); // extract upper 32bits.
4869 }
4870 EmitToBoolean(compiler, out);
4871}
4872
4873DEFINE_EMIT(
4874 Int32x4WithFlag,
4875 (SameAsFirstInput, XmmRegister mask, Register flag, Temp<Register> temp)) {
4876 // TODO(dartbug.com/30949) avoid transfer through memory.
4877 COMPILE_ASSERT(
4878 SimdOpInstr::kInt32x4WithFlagY == (SimdOpInstr::kInt32x4WithFlagX + 1) &&
4879 SimdOpInstr::kInt32x4WithFlagZ == (SimdOpInstr::kInt32x4WithFlagX + 2) &&
4880 SimdOpInstr::kInt32x4WithFlagW == (SimdOpInstr::kInt32x4WithFlagX + 3));
4881 const intptr_t lane_index = instr->kind() - SimdOpInstr::kInt32x4WithFlagX;
4882 ASSERT(0 <= lane_index && lane_index < 4);
4883 __ SubImmediate(RSP, compiler::Immediate(kSimd128Size));
4884 __ movups(compiler::Address(RSP, 0), mask);
4885
4886 // temp = flag == true ? -1 : 0
4887 __ xorq(temp, temp);
4888 __ CompareObject(flag, Bool::True());
4889 __ setcc(EQUAL, ByteRegisterOf(temp));
4890 __ negl(temp);
4891
4892 __ movl(compiler::Address(RSP, lane_index * kInt32Size), temp);
4893 __ movups(mask, compiler::Address(RSP, 0));
4894 __ AddImmediate(RSP, compiler::Immediate(kSimd128Size));
4895}
4896
4897DEFINE_EMIT(Int32x4Select,
4898 (SameAsFirstInput,
4899 XmmRegister mask,
4900 XmmRegister trueValue,
4901 XmmRegister falseValue,
4902 Temp<XmmRegister> temp)) {
4903 // Copy mask.
4904 __ movaps(temp, mask);
4905 // Invert it.
4906 __ notps(temp, temp);
4907 // mask = mask & trueValue.
4908 __ andps(mask, trueValue);
4909 // temp = temp & falseValue.
4910 __ andps(temp, falseValue);
4911 // out = mask | temp.
4912 __ orps(mask, temp);
4913}
4914
4915// Map SimdOpInstr::Kind-s to corresponding emit functions. Uses the following
4916// format:
4917//
4918// CASE(OpA) CASE(OpB) ____(Emitter) - Emitter is used to emit OpA and OpB.
4919// SIMPLE(OpA) - Emitter with name OpA is used to emit OpA.
4920//
4921#define SIMD_OP_VARIANTS(CASE, ____, SIMPLE) \
4922 SIMD_OP_SIMPLE_BINARY(CASE) \
4923 CASE(Float32x4Scale) \
4924 CASE(Float32x4ShuffleMix) \
4925 CASE(Int32x4ShuffleMix) \
4926 CASE(Float64x2FromDoubles) \
4927 CASE(Float64x2Scale) \
4928 CASE(Float64x2WithX) \
4929 CASE(Float64x2WithY) \
4930 CASE(Float32x4WithX) \
4931 CASE(Float32x4WithY) \
4932 CASE(Float32x4WithZ) \
4933 CASE(Float32x4WithW) \
4934 ____(SimdBinaryOp) \
4935 SIMD_OP_SIMPLE_UNARY(CASE) \
4936 CASE(Float32x4ShuffleX) \
4937 CASE(Float32x4ShuffleY) \
4938 CASE(Float32x4ShuffleZ) \
4939 CASE(Float32x4ShuffleW) \
4940 CASE(Float32x4Shuffle) \
4941 CASE(Int32x4Shuffle) \
4942 CASE(Float32x4Splat) \
4943 CASE(Float32x4ToFloat64x2) \
4944 CASE(Float64x2ToFloat32x4) \
4945 CASE(Int32x4ToFloat32x4) \
4946 CASE(Float32x4ToInt32x4) \
4947 CASE(Float64x2GetX) \
4948 CASE(Float64x2GetY) \
4949 CASE(Float64x2Splat) \
4950 ____(SimdUnaryOp) \
4951 CASE(Float32x4GetSignMask) \
4952 CASE(Int32x4GetSignMask) \
4953 CASE(Float64x2GetSignMask) \
4954 ____(SimdGetSignMask) \
4955 SIMPLE(Float32x4FromDoubles) \
4956 SIMPLE(Int32x4FromInts) \
4957 SIMPLE(Int32x4FromBools) \
4958 SIMPLE(Float32x4Zero) \
4959 SIMPLE(Float64x2Zero) \
4960 SIMPLE(Float32x4Clamp) \
4961 CASE(Int32x4GetFlagX) \
4962 CASE(Int32x4GetFlagY) \
4963 ____(Int32x4GetFlagXorY) \
4964 CASE(Int32x4GetFlagZ) \
4965 CASE(Int32x4GetFlagW) \
4966 ____(Int32x4GetFlagZorW) \
4967 CASE(Int32x4WithFlagX) \
4968 CASE(Int32x4WithFlagY) \
4969 CASE(Int32x4WithFlagZ) \
4970 CASE(Int32x4WithFlagW) \
4971 ____(Int32x4WithFlag) \
4972 SIMPLE(Int32x4Select)
4973
4974LocationSummary* SimdOpInstr::MakeLocationSummary(Zone* zone, bool opt) const {
4975 switch (kind()) {
4976#define CASE(Name, ...) case k##Name:
4977#define EMIT(Name) \
4978 return MakeLocationSummaryFromEmitter(zone, this, &Emit##Name);
4979#define SIMPLE(Name) CASE(Name) EMIT(Name)
4980 SIMD_OP_VARIANTS(CASE, EMIT, SIMPLE)
4981#undef CASE
4982#undef EMIT
4983#undef SIMPLE
4984 case kIllegalSimdOp:
4985 break;
4986 }
4987 UNREACHABLE();
4988 return NULL;
4989}
4990
4991void SimdOpInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
4992 switch (kind()) {
4993#define CASE(Name, ...) case k##Name:
4994#define EMIT(Name) \
4995 InvokeEmitter(compiler, this, &Emit##Name); \
4996 break;
4997#define SIMPLE(Name) CASE(Name) EMIT(Name)
4998 SIMD_OP_VARIANTS(CASE, EMIT, SIMPLE)
4999#undef CASE
5000#undef EMIT
5001#undef SIMPLE
5002 case kIllegalSimdOp:
5003 UNREACHABLE();
5004 break;
5005 }
5006}
5007
5008#undef DEFINE_EMIT
5009
5010LocationSummary* MathUnaryInstr::MakeLocationSummary(Zone* zone,
5011 bool opt) const {
5012 ASSERT((kind() == MathUnaryInstr::kSqrt) ||
5013 (kind() == MathUnaryInstr::kDoubleSquare));
5014 const intptr_t kNumInputs = 1;
5015 const intptr_t kNumTemps = 0;
5016 LocationSummary* summary = new (zone)
5017 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
5018 summary->set_in(0, Location::RequiresFpuRegister());
5019 if (kind() == MathUnaryInstr::kDoubleSquare) {
5020 summary->set_out(0, Location::SameAsFirstInput());
5021 } else {
5022 summary->set_out(0, Location::RequiresFpuRegister());
5023 }
5024 return summary;
5025}
5026
5027void MathUnaryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5028 if (kind() == MathUnaryInstr::kSqrt) {
5029 __ sqrtsd(locs()->out(0).fpu_reg(), locs()->in(0).fpu_reg());
5030 } else if (kind() == MathUnaryInstr::kDoubleSquare) {
5031 XmmRegister value_reg = locs()->in(0).fpu_reg();
5032 __ mulsd(value_reg, value_reg);
5033 ASSERT(value_reg == locs()->out(0).fpu_reg());
5034 } else {
5035 UNREACHABLE();
5036 }
5037}
5038
5039LocationSummary* CaseInsensitiveCompareInstr::MakeLocationSummary(
5040 Zone* zone,
5041 bool opt) const {
5042 const intptr_t kNumTemps = 0;
5043 LocationSummary* summary = new (zone)
5044 LocationSummary(zone, InputCount(), kNumTemps, LocationSummary::kCall);
5045 summary->set_in(0, Location::RegisterLocation(CallingConventions::kArg1Reg));
5046 summary->set_in(1, Location::RegisterLocation(CallingConventions::kArg2Reg));
5047 summary->set_in(2, Location::RegisterLocation(CallingConventions::kArg3Reg));
5048 summary->set_in(3, Location::RegisterLocation(CallingConventions::kArg4Reg));
5049 summary->set_out(0, Location::RegisterLocation(RAX));
5050 return summary;
5051}
5052
5053void CaseInsensitiveCompareInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5054 // Save RSP. R13 is chosen because it is callee saved so we do not need to
5055 // back it up before calling into the runtime.
5056 static const Register kSavedSPReg = R13;
5057 __ movq(kSavedSPReg, RSP);
5058 __ ReserveAlignedFrameSpace(0);
5059
5060 // Call the function. Parameters are already in their correct spots.
5061 __ CallRuntime(TargetFunction(), TargetFunction().argument_count());
5062
5063 // Restore RSP.
5064 __ movq(RSP, kSavedSPReg);
5065}
5066
5067LocationSummary* UnarySmiOpInstr::MakeLocationSummary(Zone* zone,
5068 bool opt) const {
5069 const intptr_t kNumInputs = 1;
5070 return LocationSummary::Make(zone, kNumInputs, Location::SameAsFirstInput(),
5071 LocationSummary::kNoCall);
5072}
5073
5074void UnarySmiOpInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5075 Register value = locs()->in(0).reg();
5076 ASSERT(value == locs()->out(0).reg());
5077 switch (op_kind()) {
5078 case Token::kNEGATE: {
5079 compiler::Label* deopt =
5080 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptUnaryOp);
5081 __ negq(value);
5082 __ j(OVERFLOW, deopt);
5083 break;
5084 }
5085 case Token::kBIT_NOT:
5086 __ notq(value);
5087 // Remove inverted smi-tag.
5088 __ AndImmediate(value, compiler::Immediate(~kSmiTagMask));
5089 break;
5090 default:
5091 UNREACHABLE();
5092 }
5093}
5094
5095LocationSummary* UnaryDoubleOpInstr::MakeLocationSummary(Zone* zone,
5096 bool opt) const {
5097 const intptr_t kNumInputs = 1;
5098 const intptr_t kNumTemps = 0;
5099 LocationSummary* summary = new (zone)
5100 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
5101 summary->set_in(0, Location::RequiresFpuRegister());
5102 summary->set_out(0, Location::SameAsFirstInput());
5103 return summary;
5104}
5105
5106void UnaryDoubleOpInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5107 XmmRegister value = locs()->in(0).fpu_reg();
5108 ASSERT(locs()->out(0).fpu_reg() == value);
5109 __ DoubleNegate(value, value);
5110}
5111
5112LocationSummary* MathMinMaxInstr::MakeLocationSummary(Zone* zone,
5113 bool opt) const {
5114 if (result_cid() == kDoubleCid) {
5115 const intptr_t kNumInputs = 2;
5116 const intptr_t kNumTemps = 1;
5117 LocationSummary* summary = new (zone)
5118 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
5119 summary->set_in(0, Location::RequiresFpuRegister());
5120 summary->set_in(1, Location::RequiresFpuRegister());
5121 // Reuse the left register so that code can be made shorter.
5122 summary->set_out(0, Location::SameAsFirstInput());
5123 summary->set_temp(0, Location::RequiresRegister());
5124 return summary;
5125 }
5126 ASSERT(result_cid() == kSmiCid);
5127 const intptr_t kNumInputs = 2;
5128 const intptr_t kNumTemps = 0;
5129 LocationSummary* summary = new (zone)
5130 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
5131 summary->set_in(0, Location::RequiresRegister());
5132 summary->set_in(1, Location::RequiresRegister());
5133 // Reuse the left register so that code can be made shorter.
5134 summary->set_out(0, Location::SameAsFirstInput());
5135 return summary;
5136}
5137
5138void MathMinMaxInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5139 ASSERT((op_kind() == MethodRecognizer::kMathMin) ||
5140 (op_kind() == MethodRecognizer::kMathMax));
5141 const bool is_min = op_kind() == MethodRecognizer::kMathMin;
5142 if (result_cid() == kDoubleCid) {
5143 compiler::Label done, returns_nan, are_equal;
5144 XmmRegister left = locs()->in(0).fpu_reg();
5145 XmmRegister right = locs()->in(1).fpu_reg();
5146 XmmRegister result = locs()->out(0).fpu_reg();
5147 Register temp = locs()->temp(0).reg();
5148 __ comisd(left, right);
5149 __ j(PARITY_EVEN, &returns_nan, compiler::Assembler::kNearJump);
5150 __ j(EQUAL, &are_equal, compiler::Assembler::kNearJump);
5151 const Condition double_condition =
5152 is_min ? TokenKindToDoubleCondition(Token::kLT)
5153 : TokenKindToDoubleCondition(Token::kGT);
5154 ASSERT(left == result);
5155 __ j(double_condition, &done, compiler::Assembler::kNearJump);
5156 __ movsd(result, right);
5157 __ jmp(&done, compiler::Assembler::kNearJump);
5158
5159 __ Bind(&returns_nan);
5160 __ movq(temp, compiler::Address(THR, Thread::double_nan_address_offset()));
5161 __ movsd(result, compiler::Address(temp, 0));
5162 __ jmp(&done, compiler::Assembler::kNearJump);
5163
5164 __ Bind(&are_equal);
5165 compiler::Label left_is_negative;
5166 // Check for negative zero: -0.0 is equal 0.0 but min or max must return
5167 // -0.0 or 0.0 respectively.
5168 // Check for negative left value (get the sign bit):
5169 // - min -> left is negative ? left : right.
5170 // - max -> left is negative ? right : left
5171 // Check the sign bit.
5172 __ movmskpd(temp, left);
5173 __ testq(temp, compiler::Immediate(1));
5174 if (is_min) {
5175 ASSERT(left == result);
5176 __ j(NOT_ZERO, &done,
5177 compiler::Assembler::kNearJump); // Negative -> return left.
5178 } else {
5179 ASSERT(left == result);
5180 __ j(ZERO, &done,
5181 compiler::Assembler::kNearJump); // Positive -> return left.
5182 }
5183 __ movsd(result, right);
5184 __ Bind(&done);
5185 return;
5186 }
5187
5188 ASSERT(result_cid() == kSmiCid);
5189 Register left = locs()->in(0).reg();
5190 Register right = locs()->in(1).reg();
5191 Register result = locs()->out(0).reg();
5192 __ cmpq(left, right);
5193 ASSERT(result == left);
5194 if (is_min) {
5195 __ cmovgeq(result, right);
5196 } else {
5197 __ cmovlq(result, right);
5198 }
5199}
5200
5201LocationSummary* Int32ToDoubleInstr::MakeLocationSummary(Zone* zone,
5202 bool opt) const {
5203 const intptr_t kNumInputs = 1;
5204 const intptr_t kNumTemps = 0;
5205 LocationSummary* result = new (zone)
5206 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
5207 result->set_in(0, Location::RequiresRegister());
5208 result->set_out(0, Location::RequiresFpuRegister());
5209 return result;
5210}
5211
5212void Int32ToDoubleInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5213 Register value = locs()->in(0).reg();
5214 FpuRegister result = locs()->out(0).fpu_reg();
5215 __ cvtsi2sdl(result, value);
5216}
5217
5218LocationSummary* SmiToDoubleInstr::MakeLocationSummary(Zone* zone,
5219 bool opt) const {
5220 const intptr_t kNumInputs = 1;
5221 const intptr_t kNumTemps = 0;
5222 LocationSummary* result = new (zone)
5223 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
5224 result->set_in(0, Location::WritableRegister());
5225 result->set_out(0, Location::RequiresFpuRegister());
5226 return result;
5227}
5228
5229void SmiToDoubleInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5230 Register value = locs()->in(0).reg();
5231 FpuRegister result = locs()->out(0).fpu_reg();
5232 __ SmiUntag(value);
5233 __ cvtsi2sdq(result, value);
5234}
5235
5236DEFINE_BACKEND(Int64ToDouble, (FpuRegister result, Register value)) {
5237 __ cvtsi2sdq(result, value);
5238}
5239
5240LocationSummary* DoubleToIntegerInstr::MakeLocationSummary(Zone* zone,
5241 bool opt) const {
5242 const intptr_t kNumInputs = 1;
5243 const intptr_t kNumTemps = 1;
5244 LocationSummary* result = new (zone)
5245 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kCall);
5246 result->set_in(0, Location::RegisterLocation(RCX));
5247 result->set_out(0, Location::RegisterLocation(RAX));
5248 result->set_temp(0, Location::RegisterLocation(RBX));
5249 return result;
5250}
5251
5252void DoubleToIntegerInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5253 Register result = locs()->out(0).reg();
5254 Register value_obj = locs()->in(0).reg();
5255 Register temp = locs()->temp(0).reg();
5256 XmmRegister value_double = FpuTMP;
5257 ASSERT(result == RAX);
5258 ASSERT(result != value_obj);
5259 ASSERT(result != temp);
5260 __ movsd(value_double,
5261 compiler::FieldAddress(value_obj, Double::value_offset()));
5262 __ cvttsd2siq(result, value_double);
5263 // Overflow is signalled with minint.
5264 compiler::Label do_call, done;
5265 // Check for overflow and that it fits into Smi.
5266 __ movq(temp, result);
5267 __ shlq(temp, compiler::Immediate(1));
5268 __ j(OVERFLOW, &do_call, compiler::Assembler::kNearJump);
5269 __ SmiTag(result);
5270 __ jmp(&done);
5271 __ Bind(&do_call);
5272 __ pushq(value_obj);
5273 ASSERT(instance_call()->HasICData());
5274 const ICData& ic_data = *instance_call()->ic_data();
5275 ASSERT(ic_data.NumberOfChecksIs(1));
5276 const Function& target = Function::ZoneHandle(ic_data.GetTargetAt(0));
5277 const int kTypeArgsLen = 0;
5278 const int kNumberOfArguments = 1;
5279 constexpr int kSizeOfArguments = 1;
5280 const Array& kNoArgumentNames = Object::null_array();
5281 ArgumentsInfo args_info(kTypeArgsLen, kNumberOfArguments, kSizeOfArguments,
5282 kNoArgumentNames);
5283 compiler->GenerateStaticCall(deopt_id(), instance_call()->token_pos(), target,
5284 args_info, locs(), ICData::Handle(),
5285 ICData::kStatic);
5286 __ Bind(&done);
5287}
5288
5289LocationSummary* DoubleToSmiInstr::MakeLocationSummary(Zone* zone,
5290 bool opt) const {
5291 const intptr_t kNumInputs = 1;
5292 const intptr_t kNumTemps = 1;
5293 LocationSummary* result = new (zone)
5294 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
5295 result->set_in(0, Location::RequiresFpuRegister());
5296 result->set_out(0, Location::RequiresRegister());
5297 result->set_temp(0, Location::RequiresRegister());
5298 return result;
5299}
5300
5301void DoubleToSmiInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5302 compiler::Label* deopt =
5303 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptDoubleToSmi);
5304 Register result = locs()->out(0).reg();
5305 XmmRegister value = locs()->in(0).fpu_reg();
5306 Register temp = locs()->temp(0).reg();
5307
5308 __ cvttsd2siq(result, value);
5309 // Overflow is signalled with minint.
5310 compiler::Label do_call, done;
5311 // Check for overflow and that it fits into Smi.
5312 __ movq(temp, result);
5313 __ shlq(temp, compiler::Immediate(1));
5314 __ j(OVERFLOW, deopt);
5315 __ SmiTag(result);
5316}
5317
5318LocationSummary* DoubleToDoubleInstr::MakeLocationSummary(Zone* zone,
5319 bool opt) const {
5320 const intptr_t kNumInputs = 1;
5321 const intptr_t kNumTemps = 0;
5322 LocationSummary* result = new (zone)
5323 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
5324 result->set_in(0, Location::RequiresFpuRegister());
5325 result->set_out(0, Location::RequiresFpuRegister());
5326 return result;
5327}
5328
5329void DoubleToDoubleInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5330 XmmRegister value = locs()->in(0).fpu_reg();
5331 XmmRegister result = locs()->out(0).fpu_reg();
5332 switch (recognized_kind()) {
5333 case MethodRecognizer::kDoubleTruncate:
5334 __ roundsd(result, value, compiler::Assembler::kRoundToZero);
5335 break;
5336 case MethodRecognizer::kDoubleFloor:
5337 __ roundsd(result, value, compiler::Assembler::kRoundDown);
5338 break;
5339 case MethodRecognizer::kDoubleCeil:
5340 __ roundsd(result, value, compiler::Assembler::kRoundUp);
5341 break;
5342 default:
5343 UNREACHABLE();
5344 }
5345}
5346
5347LocationSummary* DoubleToFloatInstr::MakeLocationSummary(Zone* zone,
5348 bool opt) const {
5349 const intptr_t kNumInputs = 1;
5350 const intptr_t kNumTemps = 0;
5351 LocationSummary* result = new (zone)
5352 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
5353 result->set_in(0, Location::RequiresFpuRegister());
5354 result->set_out(0, Location::SameAsFirstInput());
5355 return result;
5356}
5357
5358void DoubleToFloatInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5359 __ cvtsd2ss(locs()->out(0).fpu_reg(), locs()->in(0).fpu_reg());
5360}
5361
5362LocationSummary* FloatToDoubleInstr::MakeLocationSummary(Zone* zone,
5363 bool opt) const {
5364 const intptr_t kNumInputs = 1;
5365 const intptr_t kNumTemps = 0;
5366 LocationSummary* result = new (zone)
5367 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
5368 result->set_in(0, Location::RequiresFpuRegister());
5369 result->set_out(0, Location::SameAsFirstInput());
5370 return result;
5371}
5372
5373void FloatToDoubleInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5374 __ cvtss2sd(locs()->out(0).fpu_reg(), locs()->in(0).fpu_reg());
5375}
5376
5377LocationSummary* InvokeMathCFunctionInstr::MakeLocationSummary(Zone* zone,
5378 bool opt) const {
5379 // Calling convention on x64 uses XMM0 and XMM1 to pass the first two
5380 // double arguments and XMM0 to return the result.
5381 //
5382 // TODO(sjindel): allow XMM0 to be used. Requires refactoring InvokeDoublePow
5383 // to allow input 1/output register to be equal.
5384 ASSERT((InputCount() == 1) || (InputCount() == 2));
5385 const intptr_t kNumTemps =
5386 (recognized_kind() == MethodRecognizer::kMathDoublePow) ? 4 : 1;
5387 LocationSummary* result = new (zone)
5388 LocationSummary(zone, InputCount(), kNumTemps, LocationSummary::kCall);
5389 ASSERT(R13 != CALLEE_SAVED_TEMP);
5390 ASSERT(((1 << R13) & CallingConventions::kCalleeSaveCpuRegisters) != 0);
5391 result->set_temp(0, Location::RegisterLocation(R13));
5392 result->set_in(0, Location::FpuRegisterLocation(XMM2));
5393 if (InputCount() == 2) {
5394 result->set_in(1, Location::FpuRegisterLocation(XMM1));
5395 }
5396 if (recognized_kind() == MethodRecognizer::kMathDoublePow) {
5397 // Temp index 1.
5398 result->set_temp(1, Location::RegisterLocation(RAX));
5399 // Temp index 2.
5400 result->set_temp(2, Location::FpuRegisterLocation(XMM4));
5401 // Block XMM0 for the calling convention.
5402 result->set_temp(3, Location::FpuRegisterLocation(XMM0));
5403 }
5404 result->set_out(0, Location::FpuRegisterLocation(XMM3));
5405 return result;
5406}
5407
5408// Pseudo code:
5409// if (exponent == 0.0) return 1.0;
5410// // Speed up simple cases.
5411// if (exponent == 1.0) return base;
5412// if (exponent == 2.0) return base * base;
5413// if (exponent == 3.0) return base * base * base;
5414// if (base == 1.0) return 1.0;
5415// if (base.isNaN || exponent.isNaN) {
5416// return double.NAN;
5417// }
5418// if (base != -Infinity && exponent == 0.5) {
5419// if (base == 0.0) return 0.0;
5420// return sqrt(value);
5421// }
5422// TODO(srdjan): Move into a stub?
5423static void InvokeDoublePow(FlowGraphCompiler* compiler,
5424 InvokeMathCFunctionInstr* instr) {
5425 ASSERT(instr->recognized_kind() == MethodRecognizer::kMathDoublePow);
5426 const intptr_t kInputCount = 2;
5427 ASSERT(instr->InputCount() == kInputCount);
5428 LocationSummary* locs = instr->locs();
5429
5430 XmmRegister base = locs->in(0).fpu_reg();
5431 XmmRegister exp = locs->in(1).fpu_reg();
5432 XmmRegister result = locs->out(0).fpu_reg();
5433 Register temp = locs->temp(InvokeMathCFunctionInstr::kObjectTempIndex).reg();
5434 XmmRegister zero_temp =
5435 locs->temp(InvokeMathCFunctionInstr::kDoubleTempIndex).fpu_reg();
5436
5437 __ xorps(zero_temp, zero_temp);
5438 __ LoadObject(temp, Double::ZoneHandle(Double::NewCanonical(1)));
5439 __ movsd(result, compiler::FieldAddress(temp, Double::value_offset()));
5440
5441 compiler::Label check_base, skip_call;
5442 // exponent == 0.0 -> return 1.0;
5443 __ comisd(exp, zero_temp);
5444 __ j(PARITY_EVEN, &check_base, compiler::Assembler::kNearJump);
5445 __ j(EQUAL, &skip_call); // 'result' is 1.0.
5446
5447 // exponent == 1.0 ?
5448 __ comisd(exp, result);
5449 compiler::Label return_base;
5450 __ j(EQUAL, &return_base, compiler::Assembler::kNearJump);
5451
5452 // exponent == 2.0 ?
5453 __ LoadObject(temp, Double::ZoneHandle(Double::NewCanonical(2.0)));
5454 __ movsd(XMM0, compiler::FieldAddress(temp, Double::value_offset()));
5455 __ comisd(exp, XMM0);
5456 compiler::Label return_base_times_2;
5457 __ j(EQUAL, &return_base_times_2, compiler::Assembler::kNearJump);
5458
5459 // exponent == 3.0 ?
5460 __ LoadObject(temp, Double::ZoneHandle(Double::NewCanonical(3.0)));
5461 __ movsd(XMM0, compiler::FieldAddress(temp, Double::value_offset()));
5462 __ comisd(exp, XMM0);
5463 __ j(NOT_EQUAL, &check_base);
5464
5465 // Base times 3.
5466 __ movsd(result, base);
5467 __ mulsd(result, base);
5468 __ mulsd(result, base);
5469 __ jmp(&skip_call);
5470
5471 __ Bind(&return_base);
5472 __ movsd(result, base);
5473 __ jmp(&skip_call);
5474
5475 __ Bind(&return_base_times_2);
5476 __ movsd(result, base);
5477 __ mulsd(result, base);
5478 __ jmp(&skip_call);
5479
5480 __ Bind(&check_base);
5481 // Note: 'exp' could be NaN.
5482
5483 compiler::Label return_nan;
5484 // base == 1.0 -> return 1.0;
5485 __ comisd(base, result);
5486 __ j(PARITY_EVEN, &return_nan, compiler::Assembler::kNearJump);
5487 __ j(EQUAL, &skip_call, compiler::Assembler::kNearJump);
5488 // Note: 'base' could be NaN.
5489 __ comisd(exp, base);
5490 // Neither 'exp' nor 'base' is NaN.
5491 compiler::Label try_sqrt;
5492 __ j(PARITY_ODD, &try_sqrt, compiler::Assembler::kNearJump);
5493 // Return NaN.
5494 __ Bind(&return_nan);
5495 __ LoadObject(temp, Double::ZoneHandle(Double::NewCanonical(NAN)));
5496 __ movsd(result, compiler::FieldAddress(temp, Double::value_offset()));
5497 __ jmp(&skip_call);
5498
5499 compiler::Label do_pow, return_zero;
5500 __ Bind(&try_sqrt);
5501 // Before calling pow, check if we could use sqrt instead of pow.
5502 __ LoadObject(temp, Double::ZoneHandle(Double::NewCanonical(kNegInfinity)));
5503 __ movsd(result, compiler::FieldAddress(temp, Double::value_offset()));
5504 // base == -Infinity -> call pow;
5505 __ comisd(base, result);
5506 __ j(EQUAL, &do_pow, compiler::Assembler::kNearJump);
5507
5508 // exponent == 0.5 ?
5509 __ LoadObject(temp, Double::ZoneHandle(Double::NewCanonical(0.5)));
5510 __ movsd(result, compiler::FieldAddress(temp, Double::value_offset()));
5511 __ comisd(exp, result);
5512 __ j(NOT_EQUAL, &do_pow, compiler::Assembler::kNearJump);
5513
5514 // base == 0 -> return 0;
5515 __ comisd(base, zero_temp);
5516 __ j(EQUAL, &return_zero, compiler::Assembler::kNearJump);
5517
5518 __ sqrtsd(result, base);
5519 __ jmp(&skip_call, compiler::Assembler::kNearJump);
5520
5521 __ Bind(&return_zero);
5522 __ movsd(result, zero_temp);
5523 __ jmp(&skip_call);
5524
5525 __ Bind(&do_pow);
5526
5527 // Save RSP.
5528 __ movq(locs->temp(InvokeMathCFunctionInstr::kSavedSpTempIndex).reg(), RSP);
5529 __ ReserveAlignedFrameSpace(0);
5530 __ movaps(XMM0, locs->in(0).fpu_reg());
5531 ASSERT(locs->in(1).fpu_reg() == XMM1);
5532
5533 __ CallRuntime(instr->TargetFunction(), kInputCount);
5534 __ movaps(locs->out(0).fpu_reg(), XMM0);
5535 // Restore RSP.
5536 __ movq(RSP, locs->temp(InvokeMathCFunctionInstr::kSavedSpTempIndex).reg());
5537 __ Bind(&skip_call);
5538}
5539
5540void InvokeMathCFunctionInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5541 if (recognized_kind() == MethodRecognizer::kMathDoublePow) {
5542 InvokeDoublePow(compiler, this);
5543 return;
5544 }
5545 // Save RSP.
5546 __ movq(locs()->temp(kSavedSpTempIndex).reg(), RSP);
5547 __ ReserveAlignedFrameSpace(0);
5548 __ movaps(XMM0, locs()->in(0).fpu_reg());
5549 if (InputCount() == 2) {
5550 ASSERT(locs()->in(1).fpu_reg() == XMM1);
5551 }
5552
5553 __ CallRuntime(TargetFunction(), InputCount());
5554 __ movaps(locs()->out(0).fpu_reg(), XMM0);
5555 // Restore RSP.
5556 __ movq(RSP, locs()->temp(kSavedSpTempIndex).reg());
5557}
5558
5559LocationSummary* ExtractNthOutputInstr::MakeLocationSummary(Zone* zone,
5560 bool opt) const {
5561 // Only use this instruction in optimized code.
5562 ASSERT(opt);
5563 const intptr_t kNumInputs = 1;
5564 LocationSummary* summary =
5565 new (zone) LocationSummary(zone, kNumInputs, 0, LocationSummary::kNoCall);
5566 if (representation() == kUnboxedDouble) {
5567 if (index() == 0) {
5568 summary->set_in(
5569 0, Location::Pair(Location::RequiresFpuRegister(), Location::Any()));
5570 } else {
5571 ASSERT(index() == 1);
5572 summary->set_in(
5573 0, Location::Pair(Location::Any(), Location::RequiresFpuRegister()));
5574 }
5575 summary->set_out(0, Location::RequiresFpuRegister());
5576 } else {
5577 ASSERT(representation() == kTagged);
5578 if (index() == 0) {
5579 summary->set_in(
5580 0, Location::Pair(Location::RequiresRegister(), Location::Any()));
5581 } else {
5582 ASSERT(index() == 1);
5583 summary->set_in(
5584 0, Location::Pair(Location::Any(), Location::RequiresRegister()));
5585 }
5586 summary->set_out(0, Location::RequiresRegister());
5587 }
5588 return summary;
5589}
5590
5591void ExtractNthOutputInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5592 ASSERT(locs()->in(0).IsPairLocation());
5593 PairLocation* pair = locs()->in(0).AsPairLocation();
5594 Location in_loc = pair->At(index());
5595 if (representation() == kUnboxedDouble) {
5596 XmmRegister out = locs()->out(0).fpu_reg();
5597 XmmRegister in = in_loc.fpu_reg();
5598 __ movaps(out, in);
5599 } else {
5600 ASSERT(representation() == kTagged);
5601 Register out = locs()->out(0).reg();
5602 Register in = in_loc.reg();
5603 __ movq(out, in);
5604 }
5605}
5606
5607LocationSummary* TruncDivModInstr::MakeLocationSummary(Zone* zone,
5608 bool opt) const {
5609 const intptr_t kNumInputs = 2;
5610 const intptr_t kNumTemps = 0;
5611 LocationSummary* summary = new (zone)
5612 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
5613 // Both inputs must be writable because they will be untagged.
5614 summary->set_in(0, Location::RegisterLocation(RAX));
5615 summary->set_in(1, Location::WritableRegister());
5616 summary->set_out(0, Location::Pair(Location::RegisterLocation(RAX),
5617 Location::RegisterLocation(RDX)));
5618 return summary;
5619}
5620
5621void TruncDivModInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5622 ASSERT(CanDeoptimize());
5623 compiler::Label* deopt =
5624 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptBinarySmiOp);
5625 Register left = locs()->in(0).reg();
5626 Register right = locs()->in(1).reg();
5627 ASSERT(locs()->out(0).IsPairLocation());
5628 PairLocation* pair = locs()->out(0).AsPairLocation();
5629 Register result1 = pair->At(0).reg();
5630 Register result2 = pair->At(1).reg();
5631 compiler::Label not_32bit, done;
5632 Register temp = RDX;
5633 ASSERT(left == RAX);
5634 ASSERT((right != RDX) && (right != RAX));
5635 ASSERT(result1 == RAX);
5636 ASSERT(result2 == RDX);
5637 if (RangeUtils::CanBeZero(divisor_range())) {
5638 // Handle divide by zero in runtime.
5639 __ testq(right, right);
5640 __ j(ZERO, deopt);
5641 }
5642 // Check if both operands fit into 32bits as idiv with 64bit operands
5643 // requires twice as many cycles and has much higher latency.
5644 // We are checking this before untagging them to avoid corner case
5645 // dividing INT_MAX by -1 that raises exception because quotient is
5646 // too large for 32bit register.
5647 __ movsxd(temp, left);
5648 __ cmpq(temp, left);
5649 __ j(NOT_EQUAL, &not_32bit);
5650 __ movsxd(temp, right);
5651 __ cmpq(temp, right);
5652 __ j(NOT_EQUAL, &not_32bit);
5653
5654 // Both operands are 31bit smis. Divide using 32bit idiv.
5655 __ SmiUntag(left);
5656 __ SmiUntag(right);
5657 __ cdq();
5658 __ idivl(right);
5659 __ movsxd(RAX, RAX);
5660 __ movsxd(RDX, RDX);
5661 __ jmp(&done);
5662
5663 // Divide using 64bit idiv.
5664 __ Bind(&not_32bit);
5665 __ SmiUntag(left);
5666 __ SmiUntag(right);
5667 __ cqo(); // Sign extend RAX -> RDX:RAX.
5668 __ idivq(right); // RAX: quotient, RDX: remainder.
5669 // Check the corner case of dividing the 'MIN_SMI' with -1, in which
5670 // case we cannot tag the result.
5671 __ CompareImmediate(RAX, compiler::Immediate(0x4000000000000000));
5672 __ j(EQUAL, deopt);
5673 __ Bind(&done);
5674
5675 // Modulo correction (RDX).
5676 // res = left % right;
5677 // if (res < 0) {
5678 // if (right < 0) {
5679 // res = res - right;
5680 // } else {
5681 // res = res + right;
5682 // }
5683 // }
5684 compiler::Label all_done;
5685 __ cmpq(RDX, compiler::Immediate(0));
5686 __ j(GREATER_EQUAL, &all_done, compiler::Assembler::kNearJump);
5687 // Result is negative, adjust it.
5688 if ((divisor_range() == NULL) || divisor_range()->Overlaps(-1, 1)) {
5689 compiler::Label subtract;
5690 __ cmpq(right, compiler::Immediate(0));
5691 __ j(LESS, &subtract, compiler::Assembler::kNearJump);
5692 __ addq(RDX, right);
5693 __ jmp(&all_done, compiler::Assembler::kNearJump);
5694 __ Bind(&subtract);
5695 __ subq(RDX, right);
5696 } else if (divisor_range()->IsPositive()) {
5697 // Right is positive.
5698 __ addq(RDX, right);
5699 } else {
5700 // Right is negative.
5701 __ subq(RDX, right);
5702 }
5703 __ Bind(&all_done);
5704
5705 __ SmiTag(RAX);
5706 __ SmiTag(RDX);
5707 // Note that the result of an integer division/modulo of two
5708 // in-range arguments, cannot create out-of-range result.
5709}
5710
5711LocationSummary* BranchInstr::MakeLocationSummary(Zone* zone, bool opt) const {
5712 comparison()->InitializeLocationSummary(zone, opt);
5713 // Branches don't produce a result.
5714 comparison()->locs()->set_out(0, Location::NoLocation());
5715 return comparison()->locs();
5716}
5717
5718void BranchInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5719 comparison()->EmitBranchCode(compiler, this);
5720}
5721
5722LocationSummary* CheckClassInstr::MakeLocationSummary(Zone* zone,
5723 bool opt) const {
5724 const intptr_t kNumInputs = 1;
5725 const bool need_mask_temp = IsBitTest();
5726 const intptr_t kNumTemps = !IsNullCheck() ? (need_mask_temp ? 2 : 1) : 0;
5727 LocationSummary* summary = new (zone)
5728 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
5729 summary->set_in(0, Location::RequiresRegister());
5730 if (!IsNullCheck()) {
5731 summary->set_temp(0, Location::RequiresRegister());
5732 if (need_mask_temp) {
5733 summary->set_temp(1, Location::RequiresRegister());
5734 }
5735 }
5736 return summary;
5737}
5738
5739void CheckClassInstr::EmitNullCheck(FlowGraphCompiler* compiler,
5740 compiler::Label* deopt) {
5741 __ CompareObject(locs()->in(0).reg(), Object::null_object());
5742 Condition cond = IsDeoptIfNull() ? EQUAL : NOT_EQUAL;
5743 __ j(cond, deopt);
5744}
5745
5746void CheckClassInstr::EmitBitTest(FlowGraphCompiler* compiler,
5747 intptr_t min,
5748 intptr_t max,
5749 intptr_t mask,
5750 compiler::Label* deopt) {
5751 Register biased_cid = locs()->temp(0).reg();
5752 __ subq(biased_cid, compiler::Immediate(min));
5753 __ cmpq(biased_cid, compiler::Immediate(max - min));
5754 __ j(ABOVE, deopt);
5755
5756 Register mask_reg = locs()->temp(1).reg();
5757 __ movq(mask_reg, compiler::Immediate(mask));
5758 __ btq(mask_reg, biased_cid);
5759 __ j(NOT_CARRY, deopt);
5760}
5761
5762int CheckClassInstr::EmitCheckCid(FlowGraphCompiler* compiler,
5763 int bias,
5764 intptr_t cid_start,
5765 intptr_t cid_end,
5766 bool is_last,
5767 compiler::Label* is_ok,
5768 compiler::Label* deopt,
5769 bool use_near_jump) {
5770 Register biased_cid = locs()->temp(0).reg();
5771 Condition no_match, match;
5772 if (cid_start == cid_end) {
5773 __ cmpl(biased_cid, compiler::Immediate(cid_start - bias));
5774 no_match = NOT_EQUAL;
5775 match = EQUAL;
5776 } else {
5777 // For class ID ranges use a subtract followed by an unsigned
5778 // comparison to check both ends of the ranges with one comparison.
5779 __ addl(biased_cid, compiler::Immediate(bias - cid_start));
5780 bias = cid_start;
5781 __ cmpl(biased_cid, compiler::Immediate(cid_end - cid_start));
5782 no_match = ABOVE;
5783 match = BELOW_EQUAL;
5784 }
5785
5786 if (is_last) {
5787 __ j(no_match, deopt);
5788 } else {
5789 if (use_near_jump) {
5790 __ j(match, is_ok, compiler::Assembler::kNearJump);
5791 } else {
5792 __ j(match, is_ok);
5793 }
5794 }
5795 return bias;
5796}
5797
5798LocationSummary* CheckSmiInstr::MakeLocationSummary(Zone* zone,
5799 bool opt) const {
5800 const intptr_t kNumInputs = 1;
5801 const intptr_t kNumTemps = 0;
5802 LocationSummary* summary = new (zone)
5803 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
5804 summary->set_in(0, Location::RequiresRegister());
5805 return summary;
5806}
5807
5808void CheckSmiInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5809 Register value = locs()->in(0).reg();
5810 compiler::Label* deopt = compiler->AddDeoptStub(
5811 deopt_id(), ICData::kDeoptCheckSmi, licm_hoisted_ ? ICData::kHoisted : 0);
5812 __ BranchIfNotSmi(value, deopt);
5813}
5814
5815void CheckNullInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5816 ThrowErrorSlowPathCode* slow_path =
5817 new NullErrorSlowPath(this, compiler->CurrentTryIndex());
5818 compiler->AddSlowPathCode(slow_path);
5819
5820 Register value_reg = locs()->in(0).reg();
5821 // TODO(dartbug.com/30480): Consider passing `null` literal as an argument
5822 // in order to be able to allocate it on register.
5823 __ CompareObject(value_reg, Object::null_object());
5824 __ BranchIf(EQUAL, slow_path->entry_label());
5825}
5826
5827LocationSummary* CheckClassIdInstr::MakeLocationSummary(Zone* zone,
5828 bool opt) const {
5829 const intptr_t kNumInputs = 1;
5830 const intptr_t kNumTemps = 0;
5831 LocationSummary* summary = new (zone)
5832 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
5833 summary->set_in(0, cids_.IsSingleCid() ? Location::RequiresRegister()
5834 : Location::WritableRegister());
5835 return summary;
5836}
5837
5838void CheckClassIdInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5839 Register value = locs()->in(0).reg();
5840 compiler::Label* deopt =
5841 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptCheckClass);
5842 if (cids_.IsSingleCid()) {
5843 __ CompareImmediate(value,
5844 compiler::Immediate(Smi::RawValue(cids_.cid_start)));
5845 __ j(NOT_ZERO, deopt);
5846 } else {
5847 __ AddImmediate(value,
5848 compiler::Immediate(-Smi::RawValue(cids_.cid_start)));
5849 __ cmpq(value, compiler::Immediate(Smi::RawValue(cids_.Extent())));
5850 __ j(ABOVE, deopt);
5851 }
5852}
5853
5854LocationSummary* CheckConditionInstr::MakeLocationSummary(Zone* zone,
5855 bool opt) const {
5856 comparison()->InitializeLocationSummary(zone, opt);
5857 comparison()->locs()->set_out(0, Location::NoLocation());
5858 return comparison()->locs();
5859}
5860
5861void CheckConditionInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5862 compiler::Label if_true;
5863 compiler::Label* if_false =
5864 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptUnknown);
5865 BranchLabels labels = {&if_true, if_false, &if_true};
5866 Condition true_condition = comparison()->EmitComparisonCode(compiler, labels);
5867 if (true_condition != kInvalidCondition) {
5868 __ j(InvertCondition(true_condition), if_false);
5869 }
5870 __ Bind(&if_true);
5871}
5872
5873LocationSummary* CheckArrayBoundInstr::MakeLocationSummary(Zone* zone,
5874 bool opt) const {
5875 const intptr_t kNumInputs = 2;
5876 const intptr_t kNumTemps = 0;
5877 LocationSummary* locs = new (zone)
5878 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
5879 locs->set_in(kLengthPos, LocationRegisterOrSmiConstant(length()));
5880 locs->set_in(kIndexPos, LocationRegisterOrSmiConstant(index()));
5881 return locs;
5882}
5883
5884void CheckArrayBoundInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
5885 uint32_t flags = generalized_ ? ICData::kGeneralized : 0;
5886 flags |= licm_hoisted_ ? ICData::kHoisted : 0;
5887 compiler::Label* deopt =
5888 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptCheckArrayBound, flags);
5889
5890 Location length_loc = locs()->in(kLengthPos);
5891 Location index_loc = locs()->in(kIndexPos);
5892
5893 if (length_loc.IsConstant() && index_loc.IsConstant()) {
5894 ASSERT((Smi::Cast(length_loc.constant()).Value() <=
5895 Smi::Cast(index_loc.constant()).Value()) ||
5896 (Smi::Cast(index_loc.constant()).Value() < 0));
5897 // Unconditionally deoptimize for constant bounds checks because they
5898 // only occur only when index is out-of-bounds.
5899 __ jmp(deopt);
5900 return;
5901 }
5902
5903 const intptr_t index_cid = index()->Type()->ToCid();
5904 if (index_loc.IsConstant()) {
5905 Register length = length_loc.reg();
5906 const Smi& index = Smi::Cast(index_loc.constant());
5907 __ CompareImmediate(length,
5908 compiler::Immediate(static_cast<int64_t>(index.raw())));
5909 __ j(BELOW_EQUAL, deopt);
5910 } else if (length_loc.IsConstant()) {
5911 const Smi& length = Smi::Cast(length_loc.constant());
5912 Register index = index_loc.reg();
5913 if (index_cid != kSmiCid) {
5914 __ BranchIfNotSmi(index, deopt);
5915 }
5916 if (length.Value() == Smi::kMaxValue) {
5917 __ testq(index, index);
5918 __ j(NEGATIVE, deopt);
5919 } else {
5920 __ CompareImmediate(
5921 index, compiler::Immediate(static_cast<int64_t>(length.raw())));
5922 __ j(ABOVE_EQUAL, deopt);
5923 }
5924 } else {
5925 Register length = length_loc.reg();
5926 Register index = index_loc.reg();
5927 if (index_cid != kSmiCid) {
5928 __ BranchIfNotSmi(index, deopt);
5929 }
5930 __ cmpq(index, length);
5931 __ j(ABOVE_EQUAL, deopt);
5932 }
5933}
5934
5935class Int64DivideSlowPath : public ThrowErrorSlowPathCode {
5936 public:
5937 static const intptr_t kNumberOfArguments = 0;
5938
5939 Int64DivideSlowPath(BinaryInt64OpInstr* instruction,
5940 Register divisor,
5941 Range* divisor_range,
5942 intptr_t try_index)
5943 : ThrowErrorSlowPathCode(instruction,
5944 kIntegerDivisionByZeroExceptionRuntimeEntry,
5945 kNumberOfArguments,
5946 try_index),
5947 is_mod_(instruction->op_kind() == Token::kMOD),
5948 divisor_(divisor),
5949 divisor_range_(divisor_range),
5950 div_by_minus_one_label_(),
5951 adjust_sign_label_() {}
5952
5953 void EmitNativeCode(FlowGraphCompiler* compiler) override {
5954 // Handle modulo/division by zero, if needed. Use superclass code.
5955 if (has_divide_by_zero()) {
5956 ThrowErrorSlowPathCode::EmitNativeCode(compiler);
5957 } else {
5958 __ Bind(entry_label()); // not used, but keeps destructor happy
5959 if (compiler::Assembler::EmittingComments()) {
5960 __ Comment("slow path %s operation (no throw)", name());
5961 }
5962 }
5963 // Handle modulo/division by minus one, if needed.
5964 // Note: an exact -1 divisor is best optimized prior to codegen.
5965 if (has_divide_by_minus_one()) {
5966 __ Bind(div_by_minus_one_label());
5967 if (is_mod_) {
5968 __ xorq(RDX, RDX); // x % -1 = 0
5969 } else {
5970 __ negq(RAX); // x / -1 = -x
5971 }
5972 __ jmp(exit_label());
5973 }
5974 // Adjust modulo for negative sign, optimized for known ranges.
5975 // if (divisor < 0)
5976 // out -= divisor;
5977 // else
5978 // out += divisor;
5979 if (has_adjust_sign()) {
5980 __ Bind(adjust_sign_label());
5981 if (RangeUtils::Overlaps(divisor_range_, -1, 1)) {
5982 // General case.
5983 compiler::Label subtract;
5984 __ testq(divisor_, divisor_);
5985 __ j(LESS, &subtract, compiler::Assembler::kNearJump);
5986 __ addq(RDX, divisor_);
5987 __ jmp(exit_label());
5988 __ Bind(&subtract);
5989 __ subq(RDX, divisor_);
5990 } else if (divisor_range_->IsPositive()) {
5991 // Always positive.
5992 __ addq(RDX, divisor_);
5993 } else {
5994 // Always negative.
5995 __ subq(RDX, divisor_);
5996 }
5997 __ jmp(exit_label());
5998 }
5999 }
6000
6001 const char* name() override { return "int64 divide"; }
6002
6003 bool has_divide_by_zero() { return RangeUtils::CanBeZero(divisor_range_); }
6004
6005 bool has_divide_by_minus_one() {
6006 return RangeUtils::Overlaps(divisor_range_, -1, -1);
6007 }
6008
6009 bool has_adjust_sign() { return is_mod_; }
6010
6011 bool is_needed() {
6012 return has_divide_by_zero() || has_divide_by_minus_one() ||
6013 has_adjust_sign();
6014 }
6015
6016 compiler::Label* div_by_minus_one_label() {
6017 ASSERT(has_divide_by_minus_one());
6018 return &div_by_minus_one_label_;
6019 }
6020
6021 compiler::Label* adjust_sign_label() {
6022 ASSERT(has_adjust_sign());
6023 return &adjust_sign_label_;
6024 }
6025
6026 private:
6027 bool is_mod_;
6028 Register divisor_;
6029 Range* divisor_range_;
6030 compiler::Label div_by_minus_one_label_;
6031 compiler::Label adjust_sign_label_;
6032};
6033
6034static void EmitInt64ModTruncDiv(FlowGraphCompiler* compiler,
6035 BinaryInt64OpInstr* instruction,
6036 Token::Kind op_kind,
6037 Register left,
6038 Register right,
6039 Register tmp,
6040 Register out) {
6041 ASSERT(op_kind == Token::kMOD || op_kind == Token::kTRUNCDIV);
6042
6043 // Special case 64-bit div/mod by compile-time constant. Note that various
6044 // special constants (such as powers of two) should have been optimized
6045 // earlier in the pipeline. Div or mod by zero falls into general code
6046 // to implement the exception.
6047 if (auto c = instruction->right()->definition()->AsConstant()) {
6048 if (c->value().IsInteger()) {
6049 const int64_t divisor = Integer::Cast(c->value()).AsInt64Value();
6050 if (divisor <= -2 || divisor >= 2) {
6051 // For x DIV c or x MOD c: use magic operations.
6052 compiler::Label pos;
6053 int64_t magic = 0;
6054 int64_t shift = 0;
6055 Utils::CalculateMagicAndShiftForDivRem(divisor, &magic, &shift);
6056 // RDX:RAX = magic * numerator.
6057 ASSERT(left == RAX);
6058 __ MoveRegister(TMP, RAX); // save numerator
6059 __ LoadImmediate(RAX, compiler::Immediate(magic));
6060 __ imulq(TMP);
6061 // RDX +/-= numerator.
6062 if (divisor > 0 && magic < 0) {
6063 __ addq(RDX, TMP);
6064 } else if (divisor < 0 && magic > 0) {
6065 __ subq(RDX, TMP);
6066 }
6067 // Shift if needed.
6068 if (shift != 0) {
6069 __ sarq(RDX, compiler::Immediate(shift));
6070 }
6071 // RDX += 1 if RDX < 0.
6072 __ movq(RAX, RDX);
6073 __ shrq(RDX, compiler::Immediate(63));
6074 __ addq(RDX, RAX);
6075 // Finalize DIV or MOD.
6076 if (op_kind == Token::kTRUNCDIV) {
6077 ASSERT(out == RAX && tmp == RDX);
6078 __ movq(RAX, RDX);
6079 } else {
6080 ASSERT(out == RDX && tmp == RAX);
6081 __ movq(RAX, TMP);
6082 __ LoadImmediate(TMP, compiler::Immediate(divisor));
6083 __ imulq(RDX, TMP);
6084 __ subq(RAX, RDX);
6085 // Compensate for Dart's Euclidean view of MOD.
6086 __ testq(RAX, RAX);
6087 __ j(GREATER_EQUAL, &pos);
6088 if (divisor > 0) {
6089 __ addq(RAX, TMP);
6090 } else {
6091 __ subq(RAX, TMP);
6092 }
6093 __ Bind(&pos);
6094 __ movq(RDX, RAX);
6095 }
6096 return;
6097 }
6098 }
6099 }
6100
6101 // Prepare a slow path.
6102 Range* right_range = instruction->right()->definition()->range();
6103 Int64DivideSlowPath* slow_path = new (Z) Int64DivideSlowPath(
6104 instruction, right, right_range, compiler->CurrentTryIndex());
6105
6106 // Handle modulo/division by zero exception on slow path.
6107 if (slow_path->has_divide_by_zero()) {
6108 __ testq(right, right);
6109 __ j(EQUAL, slow_path->entry_label());
6110 }
6111
6112 // Handle modulo/division by minus one explicitly on slow path
6113 // (to avoid arithmetic exception on 0x8000000000000000 / -1).
6114 if (slow_path->has_divide_by_minus_one()) {
6115 __ cmpq(right, compiler::Immediate(-1));
6116 __ j(EQUAL, slow_path->div_by_minus_one_label());
6117 }
6118
6119 // Perform actual operation
6120 // out = left % right
6121 // or
6122 // out = left / right.
6123 //
6124 // Note that since 64-bit division requires twice as many cycles
6125 // and has much higher latency compared to the 32-bit division,
6126 // even for this non-speculative 64-bit path we add a "fast path".
6127 // Integers are untagged at this stage, so testing if sign extending
6128 // the lower half of each operand equals the full operand, effectively
6129 // tests if the values fit in 32-bit operands (and the slightly
6130 // dangerous division by -1 has been handled above already).
6131 ASSERT(left == RAX);
6132 ASSERT(right != RDX); // available at this stage
6133 compiler::Label div_64;
6134 compiler::Label div_merge;
6135 __ movsxd(RDX, left);
6136 __ cmpq(RDX, left);
6137 __ j(NOT_EQUAL, &div_64, compiler::Assembler::kNearJump);
6138 __ movsxd(RDX, right);
6139 __ cmpq(RDX, right);
6140 __ j(NOT_EQUAL, &div_64, compiler::Assembler::kNearJump);
6141 __ cdq(); // sign-ext eax into edx:eax
6142 __ idivl(right); // quotient eax, remainder edx
6143 __ movsxd(out, out);
6144 __ jmp(&div_merge, compiler::Assembler::kNearJump);
6145 __ Bind(&div_64);
6146 __ cqo(); // sign-ext rax into rdx:rax
6147 __ idivq(right); // quotient rax, remainder rdx
6148 __ Bind(&div_merge);
6149 if (op_kind == Token::kMOD) {
6150 ASSERT(out == RDX);
6151 ASSERT(tmp == RAX);
6152 // For the % operator, again the idiv instruction does
6153 // not quite do what we want. Adjust for sign on slow path.
6154 __ testq(out, out);
6155 __ j(LESS, slow_path->adjust_sign_label());
6156 } else {
6157 ASSERT(out == RAX);
6158 ASSERT(tmp == RDX);
6159 }
6160
6161 if (slow_path->is_needed()) {
6162 __ Bind(slow_path->exit_label());
6163 compiler->AddSlowPathCode(slow_path);
6164 }
6165}
6166
6167template <typename OperandType>
6168static void EmitInt64Arithmetic(FlowGraphCompiler* compiler,
6169 Token::Kind op_kind,
6170 Register left,
6171 const OperandType& right) {
6172 switch (op_kind) {
6173 case Token::kADD:
6174 __ addq(left, right);
6175 break;
6176 case Token::kSUB:
6177 __ subq(left, right);
6178 break;
6179 case Token::kBIT_AND:
6180 __ andq(left, right);
6181 break;
6182 case Token::kBIT_OR:
6183 __ orq(left, right);
6184 break;
6185 case Token::kBIT_XOR:
6186 __ xorq(left, right);
6187 break;
6188 case Token::kMUL:
6189 __ imulq(left, right);
6190 break;
6191 default:
6192 UNREACHABLE();
6193 }
6194}
6195
6196LocationSummary* BinaryInt64OpInstr::MakeLocationSummary(Zone* zone,
6197 bool opt) const {
6198 switch (op_kind()) {
6199 case Token::kMOD:
6200 case Token::kTRUNCDIV: {
6201 const intptr_t kNumInputs = 2;
6202 const intptr_t kNumTemps = 1;
6203 LocationSummary* summary = new (zone) LocationSummary(
6204 zone, kNumInputs, kNumTemps, LocationSummary::kCallOnSlowPath);
6205 summary->set_in(0, Location::RegisterLocation(RAX));
6206 summary->set_in(1, Location::RequiresRegister());
6207 // Intel uses rdx:rax with quotient rax and remainder rdx. Pick the
6208 // appropriate one for output and reserve the other as temp.
6209 summary->set_out(
6210 0, Location::RegisterLocation(op_kind() == Token::kMOD ? RDX : RAX));
6211 summary->set_temp(
6212 0, Location::RegisterLocation(op_kind() == Token::kMOD ? RAX : RDX));
6213 return summary;
6214 }
6215 default: {
6216 const intptr_t kNumInputs = 2;
6217 const intptr_t kNumTemps = 0;
6218 LocationSummary* summary = new (zone) LocationSummary(
6219 zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
6220 summary->set_in(0, Location::RequiresRegister());
6221 summary->set_in(1, LocationRegisterOrConstant(right()));
6222 summary->set_out(0, Location::SameAsFirstInput());
6223 return summary;
6224 }
6225 }
6226}
6227
6228void BinaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
6229 const Location left = locs()->in(0);
6230 const Location right = locs()->in(1);
6231 const Location out = locs()->out(0);
6232 ASSERT(!can_overflow());
6233 ASSERT(!CanDeoptimize());
6234
6235 if (op_kind() == Token::kMOD || op_kind() == Token::kTRUNCDIV) {
6236 const Location temp = locs()->temp(0);
6237 EmitInt64ModTruncDiv(compiler, this, op_kind(), left.reg(), right.reg(),
6238 temp.reg(), out.reg());
6239 } else if (right.IsConstant()) {
6240 ASSERT(out.reg() == left.reg());
6241 ConstantInstr* constant_instr = right.constant_instruction();
6242 const int64_t value =
6243 constant_instr->GetUnboxedSignedIntegerConstantValue();
6244 EmitInt64Arithmetic(compiler, op_kind(), left.reg(),
6245 compiler::Immediate(value));
6246 } else {
6247 ASSERT(out.reg() == left.reg());
6248 EmitInt64Arithmetic(compiler, op_kind(), left.reg(), right.reg());
6249 }
6250}
6251
6252LocationSummary* UnaryInt64OpInstr::MakeLocationSummary(Zone* zone,
6253 bool opt) const {
6254 const intptr_t kNumInputs = 1;
6255 const intptr_t kNumTemps = 0;
6256 LocationSummary* summary = new (zone)
6257 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
6258 summary->set_in(0, Location::RequiresRegister());
6259 summary->set_out(0, Location::SameAsFirstInput());
6260 return summary;
6261}
6262
6263void UnaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
6264 const Register left = locs()->in(0).reg();
6265 const Register out = locs()->out(0).reg();
6266 ASSERT(out == left);
6267 switch (op_kind()) {
6268 case Token::kBIT_NOT:
6269 __ notq(left);
6270 break;
6271 case Token::kNEGATE:
6272 __ negq(left);
6273 break;
6274 default:
6275 UNREACHABLE();
6276 }
6277}
6278
6279static void EmitShiftInt64ByConstant(FlowGraphCompiler* compiler,
6280 Token::Kind op_kind,
6281 Register left,
6282 const Object& right) {
6283 const int64_t shift = Integer::Cast(right).AsInt64Value();
6284 ASSERT(shift >= 0);
6285 switch (op_kind) {
6286 case Token::kSHR:
6287 __ sarq(left, compiler::Immediate(
6288 Utils::Minimum<int64_t>(shift, kBitsPerWord - 1)));
6289 break;
6290 case Token::kSHL: {
6291 ASSERT(shift < 64);
6292 __ shlq(left, compiler::Immediate(shift));
6293 break;
6294 }
6295 default:
6296 UNREACHABLE();
6297 }
6298}
6299
6300static void EmitShiftInt64ByRCX(FlowGraphCompiler* compiler,
6301 Token::Kind op_kind,
6302 Register left) {
6303 switch (op_kind) {
6304 case Token::kSHR: {
6305 __ sarq(left, RCX);
6306 break;
6307 }
6308 case Token::kSHL: {
6309 __ shlq(left, RCX);
6310 break;
6311 }
6312 default:
6313 UNREACHABLE();
6314 }
6315}
6316
6317static void EmitShiftUint32ByConstant(FlowGraphCompiler* compiler,
6318 Token::Kind op_kind,
6319 Register left,
6320 const Object& right) {
6321 const int64_t shift = Integer::Cast(right).AsInt64Value();
6322 ASSERT(shift >= 0);
6323 if (shift >= 32) {
6324 __ xorl(left, left);
6325 } else {
6326 switch (op_kind) {
6327 case Token::kSHR: {
6328 __ shrl(left, compiler::Immediate(shift));
6329 break;
6330 }
6331 case Token::kSHL: {
6332 __ shll(left, compiler::Immediate(shift));
6333 break;
6334 }
6335 default:
6336 UNREACHABLE();
6337 }
6338 }
6339}
6340
6341static void EmitShiftUint32ByRCX(FlowGraphCompiler* compiler,
6342 Token::Kind op_kind,
6343 Register left) {
6344 switch (op_kind) {
6345 case Token::kSHR: {
6346 __ shrl(left, RCX);
6347 break;
6348 }
6349 case Token::kSHL: {
6350 __ shll(left, RCX);
6351 break;
6352 }
6353 default:
6354 UNREACHABLE();
6355 }
6356}
6357
6358class ShiftInt64OpSlowPath : public ThrowErrorSlowPathCode {
6359 public:
6360 static const intptr_t kNumberOfArguments = 0;
6361
6362 ShiftInt64OpSlowPath(ShiftInt64OpInstr* instruction, intptr_t try_index)
6363 : ThrowErrorSlowPathCode(instruction,
6364 kArgumentErrorUnboxedInt64RuntimeEntry,
6365 kNumberOfArguments,
6366 try_index) {}
6367
6368 const char* name() override { return "int64 shift"; }
6369
6370 void EmitCodeAtSlowPathEntry(FlowGraphCompiler* compiler) override {
6371 const Register out = instruction()->locs()->out(0).reg();
6372 ASSERT(out == instruction()->locs()->in(0).reg());
6373
6374 compiler::Label throw_error;
6375 __ testq(RCX, RCX);
6376 __ j(LESS, &throw_error);
6377
6378 switch (instruction()->AsShiftInt64Op()->op_kind()) {
6379 case Token::kSHR:
6380 __ sarq(out, compiler::Immediate(kBitsPerInt64 - 1));
6381 break;
6382 case Token::kSHL:
6383 __ xorq(out, out);
6384 break;
6385 default:
6386 UNREACHABLE();
6387 }
6388 __ jmp(exit_label());
6389
6390 __ Bind(&throw_error);
6391
6392 // Can't pass unboxed int64 value directly to runtime call, as all
6393 // arguments are expected to be tagged (boxed).
6394 // The unboxed int64 argument is passed through a dedicated slot in Thread.
6395 // TODO(dartbug.com/33549): Clean this up when unboxed values
6396 // could be passed as arguments.
6397 __ movq(compiler::Address(THR, Thread::unboxed_int64_runtime_arg_offset()),
6398 RCX);
6399 }
6400};
6401
6402LocationSummary* ShiftInt64OpInstr::MakeLocationSummary(Zone* zone,
6403 bool opt) const {
6404 const intptr_t kNumInputs = 2;
6405 const intptr_t kNumTemps = 0;
6406 LocationSummary* summary = new (zone) LocationSummary(
6407 zone, kNumInputs, kNumTemps, LocationSummary::kCallOnSlowPath);
6408 summary->set_in(0, Location::RequiresRegister());
6409 summary->set_in(1, RangeUtils::IsPositive(shift_range())
6410 ? LocationFixedRegisterOrConstant(right(), RCX)
6411 : Location::RegisterLocation(RCX));
6412 summary->set_out(0, Location::SameAsFirstInput());
6413 return summary;
6414}
6415
6416void ShiftInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
6417 const Register left = locs()->in(0).reg();
6418 const Register out = locs()->out(0).reg();
6419 ASSERT(left == out);
6420 ASSERT(!can_overflow());
6421
6422 if (locs()->in(1).IsConstant()) {
6423 EmitShiftInt64ByConstant(compiler, op_kind(), left,
6424 locs()->in(1).constant());
6425 } else {
6426 // Code for a variable shift amount (or constant that throws).
6427 ASSERT(locs()->in(1).reg() == RCX);
6428
6429 // Jump to a slow path if shift count is > 63 or negative.
6430 ShiftInt64OpSlowPath* slow_path = NULL;
6431 if (!IsShiftCountInRange()) {
6432 slow_path =
6433 new (Z) ShiftInt64OpSlowPath(this, compiler->CurrentTryIndex());
6434 compiler->AddSlowPathCode(slow_path);
6435
6436 __ cmpq(RCX, compiler::Immediate(kShiftCountLimit));
6437 __ j(ABOVE, slow_path->entry_label());
6438 }
6439
6440 EmitShiftInt64ByRCX(compiler, op_kind(), left);
6441
6442 if (slow_path != NULL) {
6443 __ Bind(slow_path->exit_label());
6444 }
6445 }
6446}
6447
6448LocationSummary* SpeculativeShiftInt64OpInstr::MakeLocationSummary(
6449 Zone* zone,
6450 bool opt) const {
6451 const intptr_t kNumInputs = 2;
6452 const intptr_t kNumTemps = 0;
6453 LocationSummary* summary = new (zone)
6454 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
6455 summary->set_in(0, Location::RequiresRegister());
6456 summary->set_in(1, LocationFixedRegisterOrSmiConstant(right(), RCX));
6457 summary->set_out(0, Location::SameAsFirstInput());
6458 return summary;
6459}
6460
6461void SpeculativeShiftInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
6462 const Register left = locs()->in(0).reg();
6463 const Register out = locs()->out(0).reg();
6464 ASSERT(left == out);
6465 ASSERT(!can_overflow());
6466
6467 if (locs()->in(1).IsConstant()) {
6468 EmitShiftInt64ByConstant(compiler, op_kind(), left,
6469 locs()->in(1).constant());
6470 } else {
6471 ASSERT(locs()->in(1).reg() == RCX);
6472 __ SmiUntag(RCX);
6473
6474 // Deoptimize if shift count is > 63 or negative (or not a smi).
6475 if (!IsShiftCountInRange()) {
6476 ASSERT(CanDeoptimize());
6477 compiler::Label* deopt =
6478 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptBinaryInt64Op);
6479
6480 __ cmpq(RCX, compiler::Immediate(kShiftCountLimit));
6481 __ j(ABOVE, deopt);
6482 }
6483
6484 EmitShiftInt64ByRCX(compiler, op_kind(), left);
6485 }
6486}
6487
6488class ShiftUint32OpSlowPath : public ThrowErrorSlowPathCode {
6489 public:
6490 static const intptr_t kNumberOfArguments = 0;
6491
6492 ShiftUint32OpSlowPath(ShiftUint32OpInstr* instruction, intptr_t try_index)
6493 : ThrowErrorSlowPathCode(instruction,
6494 kArgumentErrorUnboxedInt64RuntimeEntry,
6495 kNumberOfArguments,
6496 try_index) {}
6497
6498 const char* name() override { return "uint32 shift"; }
6499
6500 void EmitCodeAtSlowPathEntry(FlowGraphCompiler* compiler) override {
6501 const Register out = instruction()->locs()->out(0).reg();
6502 ASSERT(out == instruction()->locs()->in(0).reg());
6503
6504 compiler::Label throw_error;
6505 __ testq(RCX, RCX);
6506 __ j(LESS, &throw_error);
6507
6508 __ xorl(out, out);
6509 __ jmp(exit_label());
6510
6511 __ Bind(&throw_error);
6512
6513 // Can't pass unboxed int64 value directly to runtime call, as all
6514 // arguments are expected to be tagged (boxed).
6515 // The unboxed int64 argument is passed through a dedicated slot in Thread.
6516 // TODO(dartbug.com/33549): Clean this up when unboxed values
6517 // could be passed as arguments.
6518 __ movq(compiler::Address(THR, Thread::unboxed_int64_runtime_arg_offset()),
6519 RCX);
6520 }
6521};
6522
6523LocationSummary* ShiftUint32OpInstr::MakeLocationSummary(Zone* zone,
6524 bool opt) const {
6525 const intptr_t kNumInputs = 2;
6526 const intptr_t kNumTemps = 0;
6527 LocationSummary* summary = new (zone) LocationSummary(
6528 zone, kNumInputs, kNumTemps, LocationSummary::kCallOnSlowPath);
6529 summary->set_in(0, Location::RequiresRegister());
6530 summary->set_in(1, RangeUtils::IsPositive(shift_range())
6531 ? LocationFixedRegisterOrConstant(right(), RCX)
6532 : Location::RegisterLocation(RCX));
6533 summary->set_out(0, Location::SameAsFirstInput());
6534 return summary;
6535}
6536
6537void ShiftUint32OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
6538 Register left = locs()->in(0).reg();
6539 Register out = locs()->out(0).reg();
6540 ASSERT(left == out);
6541
6542 if (locs()->in(1).IsConstant()) {
6543 EmitShiftUint32ByConstant(compiler, op_kind(), left,
6544 locs()->in(1).constant());
6545 } else {
6546 // Code for a variable shift amount (or constant that throws).
6547 ASSERT(locs()->in(1).reg() == RCX);
6548
6549 // Jump to a slow path if shift count is > 31 or negative.
6550 ShiftUint32OpSlowPath* slow_path = NULL;
6551 if (!IsShiftCountInRange(kUint32ShiftCountLimit)) {
6552 slow_path =
6553 new (Z) ShiftUint32OpSlowPath(this, compiler->CurrentTryIndex());
6554 compiler->AddSlowPathCode(slow_path);
6555
6556 __ cmpq(RCX, compiler::Immediate(kUint32ShiftCountLimit));
6557 __ j(ABOVE, slow_path->entry_label());
6558 }
6559
6560 EmitShiftUint32ByRCX(compiler, op_kind(), left);
6561
6562 if (slow_path != NULL) {
6563 __ Bind(slow_path->exit_label());
6564 }
6565 }
6566}
6567
6568LocationSummary* SpeculativeShiftUint32OpInstr::MakeLocationSummary(
6569 Zone* zone,
6570 bool opt) const {
6571 const intptr_t kNumInputs = 2;
6572 const intptr_t kNumTemps = 0;
6573 LocationSummary* summary = new (zone)
6574 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
6575 summary->set_in(0, Location::RequiresRegister());
6576 summary->set_in(1, LocationFixedRegisterOrSmiConstant(right(), RCX));
6577 summary->set_out(0, Location::SameAsFirstInput());
6578 return summary;
6579}
6580
6581void SpeculativeShiftUint32OpInstr::EmitNativeCode(
6582 FlowGraphCompiler* compiler) {
6583 Register left = locs()->in(0).reg();
6584 Register out = locs()->out(0).reg();
6585 ASSERT(left == out);
6586
6587 if (locs()->in(1).IsConstant()) {
6588 EmitShiftUint32ByConstant(compiler, op_kind(), left,
6589 locs()->in(1).constant());
6590 } else {
6591 ASSERT(locs()->in(1).reg() == RCX);
6592 __ SmiUntag(RCX);
6593
6594 if (!IsShiftCountInRange(kUint32ShiftCountLimit)) {
6595 if (!IsShiftCountInRange()) {
6596 // Deoptimize if shift count is negative.
6597 ASSERT(CanDeoptimize());
6598 compiler::Label* deopt =
6599 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptBinaryInt64Op);
6600
6601 __ testq(RCX, RCX);
6602 __ j(LESS, deopt);
6603 }
6604
6605 compiler::Label cont;
6606 __ cmpq(RCX, compiler::Immediate(kUint32ShiftCountLimit));
6607 __ j(LESS_EQUAL, &cont);
6608
6609 __ xorl(left, left);
6610
6611 __ Bind(&cont);
6612 }
6613
6614 EmitShiftUint32ByRCX(compiler, op_kind(), left);
6615 }
6616}
6617
6618LocationSummary* BinaryUint32OpInstr::MakeLocationSummary(Zone* zone,
6619 bool opt) const {
6620 const intptr_t kNumInputs = 2;
6621 const intptr_t kNumTemps = 0;
6622 LocationSummary* summary = new (zone)
6623 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
6624 summary->set_in(0, Location::RequiresRegister());
6625 summary->set_in(1, Location::RequiresRegister());
6626 summary->set_out(0, Location::SameAsFirstInput());
6627 return summary;
6628}
6629
6630template <typename OperandType>
6631static void EmitIntegerArithmetic(FlowGraphCompiler* compiler,
6632 Token::Kind op_kind,
6633 Register left,
6634 const OperandType& right) {
6635 switch (op_kind) {
6636 case Token::kADD:
6637 __ addl(left, right);
6638 break;
6639 case Token::kSUB:
6640 __ subl(left, right);
6641 break;
6642 case Token::kBIT_AND:
6643 __ andl(left, right);
6644 break;
6645 case Token::kBIT_OR:
6646 __ orl(left, right);
6647 break;
6648 case Token::kBIT_XOR:
6649 __ xorl(left, right);
6650 break;
6651 case Token::kMUL:
6652 __ imull(left, right);
6653 break;
6654 default:
6655 UNREACHABLE();
6656 }
6657}
6658
6659void BinaryUint32OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
6660 Register left = locs()->in(0).reg();
6661 Register right = locs()->in(1).reg();
6662 Register out = locs()->out(0).reg();
6663 ASSERT(out == left);
6664 switch (op_kind()) {
6665 case Token::kBIT_AND:
6666 case Token::kBIT_OR:
6667 case Token::kBIT_XOR:
6668 case Token::kADD:
6669 case Token::kSUB:
6670 case Token::kMUL:
6671 EmitIntegerArithmetic(compiler, op_kind(), left, right);
6672 return;
6673 default:
6674 UNREACHABLE();
6675 }
6676}
6677
6678DEFINE_BACKEND(UnaryUint32Op, (SameAsFirstInput, Register value)) {
6679 ASSERT(instr->op_kind() == Token::kBIT_NOT);
6680 __ notl(value);
6681}
6682
6683DEFINE_UNIMPLEMENTED_INSTRUCTION(BinaryInt32OpInstr)
6684
6685LocationSummary* IntConverterInstr::MakeLocationSummary(Zone* zone,
6686 bool opt) const {
6687 const intptr_t kNumInputs = 1;
6688 const intptr_t kNumTemps = 0;
6689 LocationSummary* summary = new (zone)
6690 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
6691 if (from() == kUntagged || to() == kUntagged) {
6692 ASSERT((from() == kUntagged && to() == kUnboxedIntPtr) ||
6693 (from() == kUnboxedIntPtr && to() == kUntagged));
6694 ASSERT(!CanDeoptimize());
6695 } else if (from() == kUnboxedInt64) {
6696 ASSERT(to() == kUnboxedUint32 || to() == kUnboxedInt32);
6697 } else if (to() == kUnboxedInt64) {
6698 ASSERT(from() == kUnboxedInt32 || from() == kUnboxedUint32);
6699 } else {
6700 ASSERT(to() == kUnboxedUint32 || to() == kUnboxedInt32);
6701 ASSERT(from() == kUnboxedUint32 || from() == kUnboxedInt32);
6702 }
6703
6704 summary->set_in(0, Location::RequiresRegister());
6705 summary->set_out(0, Location::SameAsFirstInput());
6706
6707 return summary;
6708}
6709
6710void IntConverterInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
6711 const bool is_nop_conversion =
6712 (from() == kUntagged && to() == kUnboxedIntPtr) ||
6713 (from() == kUnboxedIntPtr && to() == kUntagged);
6714 if (is_nop_conversion) {
6715 ASSERT(locs()->in(0).reg() == locs()->out(0).reg());
6716 return;
6717 }
6718
6719 if (from() == kUnboxedInt32 && to() == kUnboxedUint32) {
6720 const Register value = locs()->in(0).reg();
6721 const Register out = locs()->out(0).reg();
6722 // Representations are bitwise equivalent but we want to normalize
6723 // upperbits for safety reasons.
6724 // TODO(vegorov) if we ensure that we never use upperbits we could
6725 // avoid this.
6726 __ movl(out, value);
6727 } else if (from() == kUnboxedUint32 && to() == kUnboxedInt32) {
6728 // Representations are bitwise equivalent.
6729 const Register value = locs()->in(0).reg();
6730 const Register out = locs()->out(0).reg();
6731 __ movsxd(out, value);
6732 if (CanDeoptimize()) {
6733 compiler::Label* deopt =
6734 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptUnboxInteger);
6735 __ testl(out, out);
6736 __ j(NEGATIVE, deopt);
6737 }
6738 } else if (from() == kUnboxedInt64) {
6739 ASSERT(to() == kUnboxedUint32 || to() == kUnboxedInt32);
6740 const Register value = locs()->in(0).reg();
6741 const Register out = locs()->out(0).reg();
6742 if (!CanDeoptimize()) {
6743 // Copy low.
6744 __ movl(out, value);
6745 } else {
6746 compiler::Label* deopt =
6747 compiler->AddDeoptStub(deopt_id(), ICData::kDeoptUnboxInteger);
6748 // Sign extend.
6749 __ movsxd(out, value);
6750 // Compare with original value.
6751 __ cmpq(out, value);
6752 // Value cannot be held in Int32, deopt.
6753 __ j(NOT_EQUAL, deopt);
6754 }
6755 } else if (to() == kUnboxedInt64) {
6756 ASSERT((from() == kUnboxedUint32) || (from() == kUnboxedInt32));
6757 const Register value = locs()->in(0).reg();
6758 const Register out = locs()->out(0).reg();
6759 if (from() == kUnboxedUint32) {
6760 // Zero extend.
6761 __ movl(out, value);
6762 } else {
6763 // Sign extend.
6764 ASSERT(from() == kUnboxedInt32);
6765 __ movsxd(out, value);
6766 }
6767 } else {
6768 UNREACHABLE();
6769 }
6770}
6771
6772LocationSummary* StopInstr::MakeLocationSummary(Zone* zone, bool opt) const {
6773 return new (zone) LocationSummary(zone, 0, 0, LocationSummary::kNoCall);
6774}
6775
6776void StopInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
6777 __ Stop(message());
6778}
6779
6780void GraphEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
6781 BlockEntryInstr* entry = normal_entry();
6782 if (entry != nullptr) {
6783 if (!compiler->CanFallThroughTo(entry)) {
6784 FATAL("Checked function entry must have no offset");
6785 }
6786 } else {
6787 entry = osr_entry();
6788 if (!compiler->CanFallThroughTo(entry)) {
6789 __ jmp(compiler->GetJumpLabel(entry));
6790 }
6791 }
6792}
6793
6794LocationSummary* GotoInstr::MakeLocationSummary(Zone* zone, bool opt) const {
6795 return new (zone) LocationSummary(zone, 0, 0, LocationSummary::kNoCall);
6796}
6797
6798void GotoInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
6799 if (!compiler->is_optimizing()) {
6800 if (FLAG_reorder_basic_blocks) {
6801 compiler->EmitEdgeCounter(block()->preorder_number());
6802 }
6803 // Add a deoptimization descriptor for deoptimizing instructions that
6804 // may be inserted before this instruction.
6805 compiler->AddCurrentDescriptor(PcDescriptorsLayout::kDeopt, GetDeoptId(),
6806 TokenPosition::kNoSource);
6807 }
6808 if (HasParallelMove()) {
6809 compiler->parallel_move_resolver()->EmitNativeCode(parallel_move());
6810 }
6811
6812 // We can fall through if the successor is the next block in the list.
6813 // Otherwise, we need a jump.
6814 if (!compiler->CanFallThroughTo(successor())) {
6815 __ jmp(compiler->GetJumpLabel(successor()));
6816 }
6817}
6818
6819LocationSummary* IndirectGotoInstr::MakeLocationSummary(Zone* zone,
6820 bool opt) const {
6821 const intptr_t kNumInputs = 1;
6822 const intptr_t kNumTemps = 1;
6823
6824 LocationSummary* summary = new (zone)
6825 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
6826
6827 summary->set_in(0, Location::RequiresRegister());
6828 summary->set_temp(0, Location::RequiresRegister());
6829
6830 return summary;
6831}
6832
6833void IndirectGotoInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
6834 Register offset_reg = locs()->in(0).reg();
6835 Register target_address_reg = locs()->temp(0).reg();
6836
6837 {
6838 const intptr_t kRIPRelativeLeaqSize = 7;
6839 const intptr_t entry_to_rip_offset = __ CodeSize() + kRIPRelativeLeaqSize;
6840 __ leaq(target_address_reg,
6841 compiler::Address::AddressRIPRelative(-entry_to_rip_offset));
6842 ASSERT(__ CodeSize() == entry_to_rip_offset);
6843 }
6844
6845 // Load from FP+compiler::target::frame_layout.code_from_fp.
6846
6847 // Calculate the final absolute address.
6848 if (offset()->definition()->representation() == kTagged) {
6849 __ SmiUntag(offset_reg);
6850 }
6851 __ addq(target_address_reg, offset_reg);
6852
6853 // Jump to the absolute address.
6854 __ jmp(target_address_reg);
6855}
6856
6857LocationSummary* StrictCompareInstr::MakeLocationSummary(Zone* zone,
6858 bool opt) const {
6859 const intptr_t kNumInputs = 2;
6860 const intptr_t kNumTemps = 0;
6861 if (needs_number_check()) {
6862 LocationSummary* locs = new (zone)
6863 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kCall);
6864 locs->set_in(0, Location::RegisterLocation(RAX));
6865 locs->set_in(1, Location::RegisterLocation(RCX));
6866 locs->set_out(0, Location::RegisterLocation(RAX));
6867 return locs;
6868 }
6869 LocationSummary* locs = new (zone)
6870 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall);
6871 locs->set_in(0, LocationRegisterOrConstant(left()));
6872 // Only one of the inputs can be a constant. Choose register if the first one
6873 // is a constant.
6874 locs->set_in(1, locs->in(0).IsConstant()
6875 ? Location::RequiresRegister()
6876 : LocationRegisterOrConstant(right()));
6877 locs->set_out(0, Location::RequiresRegister());
6878 return locs;
6879}
6880
6881Condition StrictCompareInstr::EmitComparisonCodeRegConstant(
6882 FlowGraphCompiler* compiler,
6883 BranchLabels labels,
6884 Register reg,
6885 const Object& obj) {
6886 return compiler->EmitEqualityRegConstCompare(reg, obj, needs_number_check(),
6887 token_pos(), deopt_id());
6888}
6889
6890LocationSummary* DispatchTableCallInstr::MakeLocationSummary(Zone* zone,
6891 bool opt) const {
6892 const intptr_t kNumInputs = 1;
6893 const intptr_t kNumTemps = 0;
6894 LocationSummary* summary = new (zone)
6895 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kCall);
6896 summary->set_in(0, Location::RegisterLocation(RCX)); // ClassId
6897 return MakeCallSummary(zone, this, summary);
6898}
6899
6900LocationSummary* ClosureCallInstr::MakeLocationSummary(Zone* zone,
6901 bool opt) const {
6902 const intptr_t kNumInputs = 1;
6903 const intptr_t kNumTemps = 0;
6904 LocationSummary* summary = new (zone)
6905 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kCall);
6906 summary->set_in(0, Location::RegisterLocation(RAX)); // Function.
6907 return MakeCallSummary(zone, this, summary);
6908}
6909
6910void ClosureCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
6911 // Arguments descriptor is expected in R10.
6912 const intptr_t argument_count = ArgumentCount(); // Includes type args.
6913 const Array& arguments_descriptor =
6914 Array::ZoneHandle(Z, GetArgumentsDescriptor());
6915 __ LoadObject(R10, arguments_descriptor);
6916
6917 // Function in RAX.
6918 ASSERT(locs()->in(0).reg() == RAX);
6919 if (!FLAG_precompiled_mode || !FLAG_use_bare_instructions) {
6920 __ movq(CODE_REG, compiler::FieldAddress(
6921 RAX, compiler::target::Function::code_offset()));
6922 }
6923 __ movq(
6924 RCX,
6925 compiler::FieldAddress(
6926 RAX, compiler::target::Function::entry_point_offset(entry_kind())));
6927
6928 // RAX: Function.
6929 // R10: Arguments descriptor array.
6930 if (!FLAG_precompiled_mode) {
6931 // RBX: Smi 0 (no IC data; the lazy-compile stub expects a GC-safe value).
6932 __ xorq(RBX, RBX);
6933 }
6934 __ call(RCX);
6935 compiler->EmitCallsiteMetadata(token_pos(), deopt_id(),
6936 PcDescriptorsLayout::kOther, locs());
6937 __ Drop(argument_count);
6938}
6939
6940LocationSummary* BooleanNegateInstr::MakeLocationSummary(Zone* zone,
6941 bool opt) const {
6942 return LocationSummary::Make(zone, 1,
6943 value()->Type()->ToCid() == kBoolCid
6944 ? Location::SameAsFirstInput()
6945 : Location::RequiresRegister(),
6946 LocationSummary::kNoCall);
6947}
6948
6949void BooleanNegateInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
6950 Register input = locs()->in(0).reg();
6951 Register result = locs()->out(0).reg();
6952
6953 if (value()->Type()->ToCid() == kBoolCid) {
6954 ASSERT(input == result);
6955 __ xorq(result, compiler::Immediate(
6956 compiler::target::ObjectAlignment::kBoolValueMask));
6957 } else {
6958 ASSERT(input != result);
6959 compiler::Label done;
6960 __ LoadObject(result, Bool::True());
6961 __ CompareRegisters(result, input);
6962 __ j(NOT_EQUAL, &done, compiler::Assembler::kNearJump);
6963 __ LoadObject(result, Bool::False());
6964 __ Bind(&done);
6965 }
6966}
6967
6968LocationSummary* AllocateObjectInstr::MakeLocationSummary(Zone* zone,
6969 bool opt) const {
6970 const intptr_t kNumInputs = (type_arguments() != nullptr) ? 1 : 0;
6971 const intptr_t kNumTemps = 0;
6972 LocationSummary* locs = new (zone)
6973 LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kCall);
6974 if (type_arguments() != nullptr) {
6975 locs->set_in(0,
6976 Location::RegisterLocation(kAllocationStubTypeArgumentsReg));
6977 }
6978 locs->set_out(0, Location::RegisterLocation(RAX));
6979 return locs;
6980}
6981
6982void AllocateObjectInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
6983 if (type_arguments() != nullptr) {
6984 TypeUsageInfo* type_usage_info = compiler->thread()->type_usage_info();
6985 if (type_usage_info != nullptr) {
6986 RegisterTypeArgumentsUse(compiler->function(), type_usage_info, cls_,
6987 type_arguments()->definition());
6988 }
6989 }
6990 const Code& stub = Code::ZoneHandle(
6991 compiler->zone(), StubCode::GetAllocationStubForClass(cls()));
6992 compiler->GenerateStubCall(token_pos(), stub, PcDescriptorsLayout::kOther,
6993 locs());
6994}
6995
6996void DebugStepCheckInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
6997#ifdef PRODUCT
6998 UNREACHABLE();
6999#else
7000 ASSERT(!compiler->is_optimizing());
7001 __ CallPatchable(StubCode::DebugStepCheck());
7002 compiler->AddCurrentDescriptor(stub_kind_, deopt_id_, token_pos());
7003 compiler->RecordSafepoint(locs());
7004#endif
7005}
7006
7007} // namespace dart
7008
7009#undef __
7010
7011#endif // defined(TARGET_ARCH_X64)
7012