1// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file
2// for details. All rights reserved. Use of this source code is governed by a
3// BSD-style license that can be found in the LICENSE file.
4
5#include "vm/globals.h" // NOLINT
6#if defined(TARGET_ARCH_ARM64)
7
8#define SHOULD_NOT_INCLUDE_RUNTIME
9
10#include "vm/compiler/assembler/assembler.h"
11#include "vm/compiler/backend/locations.h"
12#include "vm/cpu.h"
13#include "vm/instructions.h"
14#include "vm/simulator.h"
15
16namespace dart {
17
18DECLARE_FLAG(bool, check_code_pointer);
19DECLARE_FLAG(bool, inline_alloc);
20DECLARE_FLAG(bool, precompiled_mode);
21DECLARE_FLAG(bool, use_slow_path);
22
23DEFINE_FLAG(bool, use_far_branches, false, "Always use far branches");
24
25namespace compiler {
26
27Assembler::Assembler(ObjectPoolBuilder* object_pool_builder,
28 bool use_far_branches)
29 : AssemblerBase(object_pool_builder),
30 use_far_branches_(use_far_branches),
31 constant_pool_allowed_(false) {
32 generate_invoke_write_barrier_wrapper_ = [&](Register reg) {
33 ldr(LR, Address(THR,
34 target::Thread::write_barrier_wrappers_thread_offset(reg)));
35 blr(LR);
36 };
37 generate_invoke_array_write_barrier_ = [&]() {
38 ldr(LR,
39 Address(THR, target::Thread::array_write_barrier_entry_point_offset()));
40 blr(LR);
41 };
42}
43
44void Assembler::Emit(int32_t value) {
45 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
46 buffer_.Emit<int32_t>(value);
47}
48
49void Assembler::Emit64(int64_t value) {
50 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
51 buffer_.Emit<int64_t>(value);
52}
53
54int32_t Assembler::BindImm19Branch(int64_t position, int64_t dest) {
55 if (use_far_branches() && !CanEncodeImm19BranchOffset(dest)) {
56 // Far branches are enabled, and we can't encode the branch offset in
57 // 19 bits.
58
59 // Grab the guarding branch instruction.
60 const int32_t guard_branch =
61 buffer_.Load<int32_t>(position + 0 * Instr::kInstrSize);
62
63 // Grab the far branch instruction.
64 const int32_t far_branch =
65 buffer_.Load<int32_t>(position + 1 * Instr::kInstrSize);
66 const Condition c = DecodeImm19BranchCondition(guard_branch);
67
68 // Grab the link to the next branch.
69 const int32_t next = DecodeImm26BranchOffset(far_branch);
70
71 // dest is the offset is from the guarding branch instruction.
72 // Correct it to be from the following instruction.
73 const int64_t offset = dest - Instr::kInstrSize;
74
75 // Encode the branch.
76 const int32_t encoded_branch = EncodeImm26BranchOffset(offset, far_branch);
77
78 // If the guard branch is conditioned on NV, replace it with a nop.
79 if (c == NV) {
80 buffer_.Store<int32_t>(position + 0 * Instr::kInstrSize,
81 Instr::kNopInstruction);
82 }
83
84 // Write the far branch into the buffer and link to the next branch.
85 buffer_.Store<int32_t>(position + 1 * Instr::kInstrSize, encoded_branch);
86 return next;
87 } else if (use_far_branches() && CanEncodeImm19BranchOffset(dest)) {
88 // We assembled a far branch, but we don't need it. Replace it with a near
89 // branch.
90
91 // Grab the guarding branch instruction.
92 const int32_t guard_branch =
93 buffer_.Load<int32_t>(position + 0 * Instr::kInstrSize);
94
95 // Grab the far branch instruction.
96 const int32_t far_branch =
97 buffer_.Load<int32_t>(position + 1 * Instr::kInstrSize);
98
99 // Grab the link to the next branch.
100 const int32_t next = DecodeImm26BranchOffset(far_branch);
101
102 // Re-target the guarding branch and flip the conditional sense.
103 int32_t encoded_guard_branch = EncodeImm19BranchOffset(dest, guard_branch);
104 const Condition c = DecodeImm19BranchCondition(encoded_guard_branch);
105 encoded_guard_branch =
106 EncodeImm19BranchCondition(InvertCondition(c), encoded_guard_branch);
107
108 // Write back the re-encoded instructions. The far branch becomes a nop.
109 buffer_.Store<int32_t>(position + 0 * Instr::kInstrSize,
110 encoded_guard_branch);
111 buffer_.Store<int32_t>(position + 1 * Instr::kInstrSize,
112 Instr::kNopInstruction);
113 return next;
114 } else {
115 const int32_t next = buffer_.Load<int32_t>(position);
116 const int32_t encoded = EncodeImm19BranchOffset(dest, next);
117 buffer_.Store<int32_t>(position, encoded);
118 return DecodeImm19BranchOffset(next);
119 }
120}
121
122int32_t Assembler::BindImm14Branch(int64_t position, int64_t dest) {
123 if (use_far_branches() && !CanEncodeImm14BranchOffset(dest)) {
124 // Far branches are enabled, and we can't encode the branch offset in
125 // 14 bits.
126
127 // Grab the guarding branch instruction.
128 const int32_t guard_branch =
129 buffer_.Load<int32_t>(position + 0 * Instr::kInstrSize);
130
131 // Grab the far branch instruction.
132 const int32_t far_branch =
133 buffer_.Load<int32_t>(position + 1 * Instr::kInstrSize);
134 const Condition c = DecodeImm14BranchCondition(guard_branch);
135
136 // Grab the link to the next branch.
137 const int32_t next = DecodeImm26BranchOffset(far_branch);
138
139 // dest is the offset is from the guarding branch instruction.
140 // Correct it to be from the following instruction.
141 const int64_t offset = dest - Instr::kInstrSize;
142
143 // Encode the branch.
144 const int32_t encoded_branch = EncodeImm26BranchOffset(offset, far_branch);
145
146 // If the guard branch is conditioned on NV, replace it with a nop.
147 if (c == NV) {
148 buffer_.Store<int32_t>(position + 0 * Instr::kInstrSize,
149 Instr::kNopInstruction);
150 }
151
152 // Write the far branch into the buffer and link to the next branch.
153 buffer_.Store<int32_t>(position + 1 * Instr::kInstrSize, encoded_branch);
154 return next;
155 } else if (use_far_branches() && CanEncodeImm14BranchOffset(dest)) {
156 // We assembled a far branch, but we don't need it. Replace it with a near
157 // branch.
158
159 // Grab the guarding branch instruction.
160 const int32_t guard_branch =
161 buffer_.Load<int32_t>(position + 0 * Instr::kInstrSize);
162
163 // Grab the far branch instruction.
164 const int32_t far_branch =
165 buffer_.Load<int32_t>(position + 1 * Instr::kInstrSize);
166
167 // Grab the link to the next branch.
168 const int32_t next = DecodeImm26BranchOffset(far_branch);
169
170 // Re-target the guarding branch and flip the conditional sense.
171 int32_t encoded_guard_branch = EncodeImm14BranchOffset(dest, guard_branch);
172 const Condition c = DecodeImm14BranchCondition(encoded_guard_branch);
173 encoded_guard_branch =
174 EncodeImm14BranchCondition(InvertCondition(c), encoded_guard_branch);
175
176 // Write back the re-encoded instructions. The far branch becomes a nop.
177 buffer_.Store<int32_t>(position + 0 * Instr::kInstrSize,
178 encoded_guard_branch);
179 buffer_.Store<int32_t>(position + 1 * Instr::kInstrSize,
180 Instr::kNopInstruction);
181 return next;
182 } else {
183 const int32_t next = buffer_.Load<int32_t>(position);
184 const int32_t encoded = EncodeImm14BranchOffset(dest, next);
185 buffer_.Store<int32_t>(position, encoded);
186 return DecodeImm14BranchOffset(next);
187 }
188}
189
190void Assembler::Bind(Label* label) {
191 ASSERT(!label->IsBound());
192 const intptr_t bound_pc = buffer_.Size();
193
194 while (label->IsLinked()) {
195 const int64_t position = label->Position();
196 const int64_t dest = bound_pc - position;
197 if (IsTestAndBranch(buffer_.Load<int32_t>(position))) {
198 label->position_ = BindImm14Branch(position, dest);
199 } else {
200 label->position_ = BindImm19Branch(position, dest);
201 }
202 }
203 label->BindTo(bound_pc);
204}
205
206static int CountLeadingZeros(uint64_t value, int width) {
207 ASSERT((width == 32) || (width == 64));
208 if (value == 0) {
209 return width;
210 }
211 int count = 0;
212 do {
213 count++;
214 } while (value >>= 1);
215 return width - count;
216}
217
218static int CountOneBits(uint64_t value, int width) {
219 // Mask out unused bits to ensure that they are not counted.
220 value &= (0xffffffffffffffffULL >> (64 - width));
221
222 value = ((value >> 1) & 0x5555555555555555) + (value & 0x5555555555555555);
223 value = ((value >> 2) & 0x3333333333333333) + (value & 0x3333333333333333);
224 value = ((value >> 4) & 0x0f0f0f0f0f0f0f0f) + (value & 0x0f0f0f0f0f0f0f0f);
225 value = ((value >> 8) & 0x00ff00ff00ff00ff) + (value & 0x00ff00ff00ff00ff);
226 value = ((value >> 16) & 0x0000ffff0000ffff) + (value & 0x0000ffff0000ffff);
227 value = ((value >> 32) & 0x00000000ffffffff) + (value & 0x00000000ffffffff);
228
229 return value;
230}
231
232// Test if a given value can be encoded in the immediate field of a logical
233// instruction.
234// If it can be encoded, the function returns true, and values pointed to by n,
235// imm_s and imm_r are updated with immediates encoded in the format required
236// by the corresponding fields in the logical instruction.
237// If it can't be encoded, the function returns false, and the operand is
238// undefined.
239bool Operand::IsImmLogical(uint64_t value, uint8_t width, Operand* imm_op) {
240 ASSERT(imm_op != NULL);
241 ASSERT((width == kWRegSizeInBits) || (width == kXRegSizeInBits));
242 ASSERT((width == kXRegSizeInBits) || (value <= 0xffffffffUL));
243 uint8_t n = 0;
244 uint8_t imm_s = 0;
245 uint8_t imm_r = 0;
246
247 // Logical immediates are encoded using parameters n, imm_s and imm_r using
248 // the following table:
249 //
250 // N imms immr size S R
251 // 1 ssssss rrrrrr 64 UInt(ssssss) UInt(rrrrrr)
252 // 0 0sssss xrrrrr 32 UInt(sssss) UInt(rrrrr)
253 // 0 10ssss xxrrrr 16 UInt(ssss) UInt(rrrr)
254 // 0 110sss xxxrrr 8 UInt(sss) UInt(rrr)
255 // 0 1110ss xxxxrr 4 UInt(ss) UInt(rr)
256 // 0 11110s xxxxxr 2 UInt(s) UInt(r)
257 // (s bits must not be all set)
258 //
259 // A pattern is constructed of size bits, where the least significant S+1
260 // bits are set. The pattern is rotated right by R, and repeated across a
261 // 32 or 64-bit value, depending on destination register width.
262 //
263 // To test if an arbitrary immediate can be encoded using this scheme, an
264 // iterative algorithm is used.
265
266 // 1. If the value has all set or all clear bits, it can't be encoded.
267 if ((value == 0) || (value == 0xffffffffffffffffULL) ||
268 ((width == kWRegSizeInBits) && (value == 0xffffffff))) {
269 return false;
270 }
271
272 int lead_zero = CountLeadingZeros(value, width);
273 int lead_one = CountLeadingZeros(~value, width);
274 int trail_zero = Utils::CountTrailingZerosWord(value);
275 int trail_one = Utils::CountTrailingZerosWord(~value);
276 int set_bits = CountOneBits(value, width);
277
278 // The fixed bits in the immediate s field.
279 // If width == 64 (X reg), start at 0xFFFFFF80.
280 // If width == 32 (W reg), start at 0xFFFFFFC0, as the iteration for 64-bit
281 // widths won't be executed.
282 int imm_s_fixed = (width == kXRegSizeInBits) ? -128 : -64;
283 int imm_s_mask = 0x3F;
284
285 for (;;) {
286 // 2. If the value is two bits wide, it can be encoded.
287 if (width == 2) {
288 n = 0;
289 imm_s = 0x3C;
290 imm_r = (value & 3) - 1;
291 *imm_op = Operand(n, imm_s, imm_r);
292 return true;
293 }
294
295 n = (width == 64) ? 1 : 0;
296 imm_s = ((imm_s_fixed | (set_bits - 1)) & imm_s_mask);
297 if ((lead_zero + set_bits) == width) {
298 imm_r = 0;
299 } else {
300 imm_r = (lead_zero > 0) ? (width - trail_zero) : lead_one;
301 }
302
303 // 3. If the sum of leading zeros, trailing zeros and set bits is equal to
304 // the bit width of the value, it can be encoded.
305 if (lead_zero + trail_zero + set_bits == width) {
306 *imm_op = Operand(n, imm_s, imm_r);
307 return true;
308 }
309
310 // 4. If the sum of leading ones, trailing ones and unset bits in the
311 // value is equal to the bit width of the value, it can be encoded.
312 if (lead_one + trail_one + (width - set_bits) == width) {
313 *imm_op = Operand(n, imm_s, imm_r);
314 return true;
315 }
316
317 // 5. If the most-significant half of the bitwise value is equal to the
318 // least-significant half, return to step 2 using the least-significant
319 // half of the value.
320 uint64_t mask = (1ULL << (width >> 1)) - 1;
321 if ((value & mask) == ((value >> (width >> 1)) & mask)) {
322 width >>= 1;
323 set_bits >>= 1;
324 imm_s_fixed >>= 1;
325 continue;
326 }
327
328 // 6. Otherwise, the value can't be encoded.
329 return false;
330 }
331}
332
333void Assembler::LoadPoolPointer(Register pp) {
334 CheckCodePointer();
335 ldr(pp, FieldAddress(CODE_REG, target::Code::object_pool_offset()));
336
337 // When in the PP register, the pool pointer is untagged. When we
338 // push it on the stack with TagAndPushPP it is tagged again. PopAndUntagPP
339 // then untags when restoring from the stack. This will make loading from the
340 // object pool only one instruction for the first 4096 entries. Otherwise,
341 // because the offset wouldn't be aligned, it would be only one instruction
342 // for the first 64 entries.
343 sub(pp, pp, Operand(kHeapObjectTag));
344 set_constant_pool_allowed(pp == PP);
345}
346
347void Assembler::LoadWordFromPoolOffset(Register dst,
348 uint32_t offset,
349 Register pp) {
350 ASSERT((pp != PP) || constant_pool_allowed());
351 ASSERT(dst != pp);
352 Operand op;
353 const uint32_t upper20 = offset & 0xfffff000;
354 if (Address::CanHoldOffset(offset)) {
355 ldr(dst, Address(pp, offset));
356 } else if (Operand::CanHold(upper20, kXRegSizeInBits, &op) ==
357 Operand::Immediate) {
358 const uint32_t lower12 = offset & 0x00000fff;
359 ASSERT(Address::CanHoldOffset(lower12));
360 add(dst, pp, op);
361 ldr(dst, Address(dst, lower12));
362 } else {
363 const uint16_t offset_low = Utils::Low16Bits(offset);
364 const uint16_t offset_high = Utils::High16Bits(offset);
365 movz(dst, Immediate(offset_low), 0);
366 if (offset_high != 0) {
367 movk(dst, Immediate(offset_high), 1);
368 }
369 ldr(dst, Address(pp, dst));
370 }
371}
372
373void Assembler::LoadWordFromPoolOffsetFixed(Register dst, uint32_t offset) {
374 ASSERT(constant_pool_allowed());
375 ASSERT(dst != PP);
376 Operand op;
377 const uint32_t upper20 = offset & 0xfffff000;
378 const uint32_t lower12 = offset & 0x00000fff;
379 const Operand::OperandType ot =
380 Operand::CanHold(upper20, kXRegSizeInBits, &op);
381 ASSERT(ot == Operand::Immediate);
382 ASSERT(Address::CanHoldOffset(lower12));
383 add(dst, PP, op);
384 ldr(dst, Address(dst, lower12));
385}
386
387void Assembler::LoadDoubleWordFromPoolOffset(Register lower,
388 Register upper,
389 uint32_t offset) {
390 // This implementation needs to be kept in sync with
391 // [InstructionPattern::DecodeLoadDoubleWordFromPool].
392 ASSERT(constant_pool_allowed());
393 ASSERT(lower != PP && upper != PP);
394 ASSERT(offset < (1 << 24));
395
396 Operand op;
397 const uint32_t upper20 = offset & 0xfffff000;
398 const uint32_t lower12 = offset & 0x00000fff;
399 if (Address::CanHoldOffset(offset, Address::PairOffset)) {
400 ldp(lower, upper, Address(PP, offset, Address::PairOffset));
401 } else if (Operand::CanHold(offset, kXRegSizeInBits, &op) ==
402 Operand::Immediate) {
403 add(TMP, PP, op);
404 ldp(lower, upper, Address(TMP, 0, Address::PairOffset));
405 } else if (Operand::CanHold(upper20, kXRegSizeInBits, &op) ==
406 Operand::Immediate &&
407 Address::CanHoldOffset(lower12, Address::PairOffset)) {
408 add(TMP, PP, op);
409 ldp(lower, upper, Address(TMP, lower12, Address::PairOffset));
410 } else {
411 const uint32_t lower12 = offset & 0xfff;
412 const uint32_t higher12 = offset & 0xfff000;
413
414 Operand op_high, op_low;
415 bool ok = Operand::CanHold(higher12, kXRegSizeInBits, &op_high) ==
416 Operand::Immediate &&
417 Operand::CanHold(lower12, kXRegSizeInBits, &op_low) ==
418 Operand::Immediate;
419 RELEASE_ASSERT(ok);
420
421 add(TMP, PP, op_high);
422 add(TMP, TMP, op_low);
423 ldp(lower, upper, Address(TMP, 0, Address::PairOffset));
424 }
425}
426
427intptr_t Assembler::FindImmediate(int64_t imm) {
428 return object_pool_builder().FindImmediate(imm);
429}
430
431bool Assembler::CanLoadFromObjectPool(const Object& object) const {
432 ASSERT(IsOriginalObject(object));
433 if (!constant_pool_allowed()) {
434 return false;
435 }
436
437 // TODO(zra, kmillikin): Also load other large immediates from the object
438 // pool
439 if (target::IsSmi(object)) {
440 // If the raw smi does not fit into a 32-bit signed int, then we'll keep
441 // the raw value in the object pool.
442 return !Utils::IsInt(32, target::ToRawSmi(object));
443 }
444 ASSERT(IsNotTemporaryScopedHandle(object));
445 ASSERT(IsInOldSpace(object));
446 return true;
447}
448
449void Assembler::LoadNativeEntry(
450 Register dst,
451 const ExternalLabel* label,
452 ObjectPoolBuilderEntry::Patchability patchable) {
453 const int32_t offset = target::ObjectPool::element_offset(
454 object_pool_builder().FindNativeFunction(label, patchable));
455 LoadWordFromPoolOffset(dst, offset);
456}
457
458void Assembler::LoadIsolate(Register dst) {
459 ldr(dst, Address(THR, target::Thread::isolate_offset()));
460}
461
462void Assembler::LoadObjectHelper(Register dst,
463 const Object& object,
464 bool is_unique) {
465 ASSERT(IsOriginalObject(object));
466 // `is_unique == true` effectively means object has to be patchable.
467 // (even if the object is null)
468 if (!is_unique) {
469 if (IsSameObject(compiler::NullObject(), object)) {
470 mov(dst, NULL_REG);
471 return;
472 }
473 if (IsSameObject(CastHandle<Object>(compiler::TrueObject()), object)) {
474 AddImmediate(dst, NULL_REG, kTrueOffsetFromNull);
475 return;
476 }
477 if (IsSameObject(CastHandle<Object>(compiler::FalseObject()), object)) {
478 AddImmediate(dst, NULL_REG, kFalseOffsetFromNull);
479 return;
480 }
481 word offset = 0;
482 if (target::CanLoadFromThread(object, &offset)) {
483 ldr(dst, Address(THR, offset));
484 return;
485 }
486 }
487 if (CanLoadFromObjectPool(object)) {
488 const int32_t offset = target::ObjectPool::element_offset(
489 is_unique ? object_pool_builder().AddObject(object)
490 : object_pool_builder().FindObject(object));
491 LoadWordFromPoolOffset(dst, offset);
492 return;
493 }
494 ASSERT(target::IsSmi(object));
495 LoadImmediate(dst, target::ToRawSmi(object));
496}
497
498void Assembler::LoadObject(Register dst, const Object& object) {
499 LoadObjectHelper(dst, object, false);
500}
501
502void Assembler::LoadUniqueObject(Register dst, const Object& object) {
503 LoadObjectHelper(dst, object, true);
504}
505
506void Assembler::CompareObject(Register reg, const Object& object) {
507 ASSERT(IsOriginalObject(object));
508 word offset = 0;
509 if (IsSameObject(compiler::NullObject(), object)) {
510 CompareRegisters(reg, NULL_REG);
511 } else if (target::CanLoadFromThread(object, &offset)) {
512 ldr(TMP, Address(THR, offset));
513 CompareRegisters(reg, TMP);
514 } else if (CanLoadFromObjectPool(object)) {
515 LoadObject(TMP, object);
516 CompareRegisters(reg, TMP);
517 } else {
518 ASSERT(target::IsSmi(object));
519 CompareImmediate(reg, target::ToRawSmi(object));
520 }
521}
522
523void Assembler::LoadImmediate(Register reg, int64_t imm) {
524 // Is it 0?
525 if (imm == 0) {
526 movz(reg, Immediate(0), 0);
527 return;
528 }
529
530 // Can we use one orri operation?
531 Operand op;
532 Operand::OperandType ot;
533 ot = Operand::CanHold(imm, kXRegSizeInBits, &op);
534 if (ot == Operand::BitfieldImm) {
535 orri(reg, ZR, Immediate(imm));
536 return;
537 }
538
539 // We may fall back on movz, movk, movn.
540 const uint32_t w0 = Utils::Low32Bits(imm);
541 const uint32_t w1 = Utils::High32Bits(imm);
542 const uint16_t h0 = Utils::Low16Bits(w0);
543 const uint16_t h1 = Utils::High16Bits(w0);
544 const uint16_t h2 = Utils::Low16Bits(w1);
545 const uint16_t h3 = Utils::High16Bits(w1);
546
547 // Special case for w1 == 0xffffffff
548 if (w1 == 0xffffffff) {
549 if (h1 == 0xffff) {
550 movn(reg, Immediate(~h0), 0);
551 } else {
552 movn(reg, Immediate(~h1), 1);
553 movk(reg, Immediate(h0), 0);
554 }
555 return;
556 }
557
558 // Special case for h3 == 0xffff
559 if (h3 == 0xffff) {
560 // We know h2 != 0xffff.
561 movn(reg, Immediate(~h2), 2);
562 if (h1 != 0xffff) {
563 movk(reg, Immediate(h1), 1);
564 }
565 if (h0 != 0xffff) {
566 movk(reg, Immediate(h0), 0);
567 }
568 return;
569 }
570
571 // Use constant pool if allowed, unless we can load imm with 2 instructions.
572 if ((w1 != 0) && constant_pool_allowed()) {
573 const int32_t offset =
574 target::ObjectPool::element_offset(FindImmediate(imm));
575 LoadWordFromPoolOffset(reg, offset);
576 return;
577 }
578
579 bool initialized = false;
580 if (h0 != 0) {
581 movz(reg, Immediate(h0), 0);
582 initialized = true;
583 }
584 if (h1 != 0) {
585 if (initialized) {
586 movk(reg, Immediate(h1), 1);
587 } else {
588 movz(reg, Immediate(h1), 1);
589 initialized = true;
590 }
591 }
592 if (h2 != 0) {
593 if (initialized) {
594 movk(reg, Immediate(h2), 2);
595 } else {
596 movz(reg, Immediate(h2), 2);
597 initialized = true;
598 }
599 }
600 if (h3 != 0) {
601 if (initialized) {
602 movk(reg, Immediate(h3), 3);
603 } else {
604 movz(reg, Immediate(h3), 3);
605 }
606 }
607}
608
609void Assembler::LoadDImmediate(VRegister vd, double immd) {
610 if (!fmovdi(vd, immd)) {
611 int64_t imm = bit_cast<int64_t, double>(immd);
612 LoadImmediate(TMP, imm);
613 fmovdr(vd, TMP);
614 }
615}
616
617void Assembler::Branch(const Code& target,
618 Register pp,
619 ObjectPoolBuilderEntry::Patchability patchable) {
620 const int32_t offset = target::ObjectPool::element_offset(
621 object_pool_builder().FindObject(ToObject(target), patchable));
622 LoadWordFromPoolOffset(CODE_REG, offset, pp);
623 ldr(TMP, FieldAddress(CODE_REG, target::Code::entry_point_offset()));
624 br(TMP);
625}
626
627void Assembler::BranchLink(const Code& target,
628 ObjectPoolBuilderEntry::Patchability patchable,
629 CodeEntryKind entry_kind) {
630 const int32_t offset = target::ObjectPool::element_offset(
631 object_pool_builder().FindObject(ToObject(target), patchable));
632 LoadWordFromPoolOffset(CODE_REG, offset);
633 ldr(TMP,
634 FieldAddress(CODE_REG, target::Code::entry_point_offset(entry_kind)));
635 blr(TMP);
636}
637
638void Assembler::BranchLinkToRuntime() {
639 ldr(LR, Address(THR, target::Thread::call_to_runtime_entry_point_offset()));
640 blr(LR);
641}
642
643void Assembler::BranchLinkWithEquivalence(const Code& target,
644 const Object& equivalence,
645 CodeEntryKind entry_kind) {
646 const int32_t offset = target::ObjectPool::element_offset(
647 object_pool_builder().FindObject(ToObject(target), equivalence));
648 LoadWordFromPoolOffset(CODE_REG, offset);
649 ldr(TMP,
650 FieldAddress(CODE_REG, target::Code::entry_point_offset(entry_kind)));
651 blr(TMP);
652}
653
654void Assembler::AddImmediate(Register dest, Register rn, int64_t imm) {
655 Operand op;
656 if (imm == 0) {
657 if (dest != rn) {
658 mov(dest, rn);
659 }
660 return;
661 }
662 if (Operand::CanHold(imm, kXRegSizeInBits, &op) == Operand::Immediate) {
663 add(dest, rn, op);
664 } else if (Operand::CanHold(-imm, kXRegSizeInBits, &op) ==
665 Operand::Immediate) {
666 sub(dest, rn, op);
667 } else {
668 // TODO(zra): Try adding top 12 bits, then bottom 12 bits.
669 ASSERT(rn != TMP2);
670 LoadImmediate(TMP2, imm);
671 add(dest, rn, Operand(TMP2));
672 }
673}
674
675void Assembler::AddImmediateSetFlags(Register dest,
676 Register rn,
677 int64_t imm,
678 OperandSize sz) {
679 ASSERT(sz == kDoubleWord || sz == kWord);
680 Operand op;
681 if (Operand::CanHold(imm, kXRegSizeInBits, &op) == Operand::Immediate) {
682 // Handles imm == kMinInt64.
683 if (sz == kDoubleWord) {
684 adds(dest, rn, op);
685 } else {
686 addsw(dest, rn, op);
687 }
688 } else if (Operand::CanHold(-imm, kXRegSizeInBits, &op) ==
689 Operand::Immediate) {
690 ASSERT(imm != kMinInt64); // Would cause erroneous overflow detection.
691 if (sz == kDoubleWord) {
692 subs(dest, rn, op);
693 } else {
694 subsw(dest, rn, op);
695 }
696 } else {
697 // TODO(zra): Try adding top 12 bits, then bottom 12 bits.
698 ASSERT(rn != TMP2);
699 LoadImmediate(TMP2, imm);
700 if (sz == kDoubleWord) {
701 adds(dest, rn, Operand(TMP2));
702 } else {
703 addsw(dest, rn, Operand(TMP2));
704 }
705 }
706}
707
708void Assembler::SubImmediateSetFlags(Register dest,
709 Register rn,
710 int64_t imm,
711 OperandSize sz) {
712 Operand op;
713 ASSERT(sz == kDoubleWord || sz == kWord);
714 if (Operand::CanHold(imm, kXRegSizeInBits, &op) == Operand::Immediate) {
715 // Handles imm == kMinInt64.
716 if (sz == kDoubleWord) {
717 subs(dest, rn, op);
718 } else {
719 subsw(dest, rn, op);
720 }
721 } else if (Operand::CanHold(-imm, kXRegSizeInBits, &op) ==
722 Operand::Immediate) {
723 ASSERT(imm != kMinInt64); // Would cause erroneous overflow detection.
724 if (sz == kDoubleWord) {
725 adds(dest, rn, op);
726 } else {
727 addsw(dest, rn, op);
728 }
729 } else {
730 // TODO(zra): Try subtracting top 12 bits, then bottom 12 bits.
731 ASSERT(rn != TMP2);
732 LoadImmediate(TMP2, imm);
733 if (sz == kDoubleWord) {
734 subs(dest, rn, Operand(TMP2));
735 } else {
736 subsw(dest, rn, Operand(TMP2));
737 }
738 }
739}
740
741void Assembler::AndImmediate(Register rd, Register rn, int64_t imm) {
742 Operand imm_op;
743 if (Operand::IsImmLogical(imm, kXRegSizeInBits, &imm_op)) {
744 andi(rd, rn, Immediate(imm));
745 } else {
746 LoadImmediate(TMP, imm);
747 and_(rd, rn, Operand(TMP));
748 }
749}
750
751void Assembler::OrImmediate(Register rd, Register rn, int64_t imm) {
752 Operand imm_op;
753 if (Operand::IsImmLogical(imm, kXRegSizeInBits, &imm_op)) {
754 orri(rd, rn, Immediate(imm));
755 } else {
756 LoadImmediate(TMP, imm);
757 orr(rd, rn, Operand(TMP));
758 }
759}
760
761void Assembler::XorImmediate(Register rd, Register rn, int64_t imm) {
762 Operand imm_op;
763 if (Operand::IsImmLogical(imm, kXRegSizeInBits, &imm_op)) {
764 eori(rd, rn, Immediate(imm));
765 } else {
766 LoadImmediate(TMP, imm);
767 eor(rd, rn, Operand(TMP));
768 }
769}
770
771void Assembler::TestImmediate(Register rn, int64_t imm) {
772 Operand imm_op;
773 if (Operand::IsImmLogical(imm, kXRegSizeInBits, &imm_op)) {
774 tsti(rn, Immediate(imm));
775 } else {
776 LoadImmediate(TMP, imm);
777 tst(rn, Operand(TMP));
778 }
779}
780
781void Assembler::CompareImmediate(Register rn, int64_t imm) {
782 Operand op;
783 if (Operand::CanHold(imm, kXRegSizeInBits, &op) == Operand::Immediate) {
784 cmp(rn, op);
785 } else if (Operand::CanHold(-static_cast<uint64_t>(imm), kXRegSizeInBits,
786 &op) == Operand::Immediate) {
787 cmn(rn, op);
788 } else {
789 ASSERT(rn != TMP2);
790 LoadImmediate(TMP2, imm);
791 cmp(rn, Operand(TMP2));
792 }
793}
794
795void Assembler::LoadFromOffset(Register dest,
796 Register base,
797 int32_t offset,
798 OperandSize sz) {
799 if (Address::CanHoldOffset(offset, Address::Offset, sz)) {
800 ldr(dest, Address(base, offset, Address::Offset, sz), sz);
801 } else {
802 ASSERT(base != TMP2);
803 AddImmediate(TMP2, base, offset);
804 ldr(dest, Address(TMP2), sz);
805 }
806}
807
808void Assembler::LoadSFromOffset(VRegister dest, Register base, int32_t offset) {
809 if (Address::CanHoldOffset(offset, Address::Offset, kSWord)) {
810 fldrs(dest, Address(base, offset, Address::Offset, kSWord));
811 } else {
812 ASSERT(base != TMP2);
813 AddImmediate(TMP2, base, offset);
814 fldrs(dest, Address(TMP2));
815 }
816}
817
818void Assembler::LoadDFromOffset(VRegister dest, Register base, int32_t offset) {
819 if (Address::CanHoldOffset(offset, Address::Offset, kDWord)) {
820 fldrd(dest, Address(base, offset, Address::Offset, kDWord));
821 } else {
822 ASSERT(base != TMP2);
823 AddImmediate(TMP2, base, offset);
824 fldrd(dest, Address(TMP2));
825 }
826}
827
828void Assembler::LoadQFromOffset(VRegister dest, Register base, int32_t offset) {
829 if (Address::CanHoldOffset(offset, Address::Offset, kQWord)) {
830 fldrq(dest, Address(base, offset, Address::Offset, kQWord));
831 } else {
832 ASSERT(base != TMP2);
833 AddImmediate(TMP2, base, offset);
834 fldrq(dest, Address(TMP2));
835 }
836}
837
838void Assembler::StoreToOffset(Register src,
839 Register base,
840 int32_t offset,
841 OperandSize sz) {
842 ASSERT(base != TMP2);
843 if (Address::CanHoldOffset(offset, Address::Offset, sz)) {
844 str(src, Address(base, offset, Address::Offset, sz), sz);
845 } else {
846 ASSERT(src != TMP2);
847 AddImmediate(TMP2, base, offset);
848 str(src, Address(TMP2), sz);
849 }
850}
851
852void Assembler::StoreSToOffset(VRegister src, Register base, int32_t offset) {
853 if (Address::CanHoldOffset(offset, Address::Offset, kSWord)) {
854 fstrs(src, Address(base, offset, Address::Offset, kSWord));
855 } else {
856 ASSERT(base != TMP2);
857 AddImmediate(TMP2, base, offset);
858 fstrs(src, Address(TMP2));
859 }
860}
861
862void Assembler::StoreDToOffset(VRegister src, Register base, int32_t offset) {
863 if (Address::CanHoldOffset(offset, Address::Offset, kDWord)) {
864 fstrd(src, Address(base, offset, Address::Offset, kDWord));
865 } else {
866 ASSERT(base != TMP2);
867 AddImmediate(TMP2, base, offset);
868 fstrd(src, Address(TMP2));
869 }
870}
871
872void Assembler::StoreQToOffset(VRegister src, Register base, int32_t offset) {
873 if (Address::CanHoldOffset(offset, Address::Offset, kQWord)) {
874 fstrq(src, Address(base, offset, Address::Offset, kQWord));
875 } else {
876 ASSERT(base != TMP2);
877 AddImmediate(TMP2, base, offset);
878 fstrq(src, Address(TMP2));
879 }
880}
881
882void Assembler::VRecps(VRegister vd, VRegister vn) {
883 ASSERT(vn != VTMP);
884 ASSERT(vd != VTMP);
885
886 // Reciprocal estimate.
887 vrecpes(vd, vn);
888 // 2 Newton-Raphson steps.
889 vrecpss(VTMP, vn, vd);
890 vmuls(vd, vd, VTMP);
891 vrecpss(VTMP, vn, vd);
892 vmuls(vd, vd, VTMP);
893}
894
895void Assembler::VRSqrts(VRegister vd, VRegister vn) {
896 ASSERT(vd != VTMP);
897 ASSERT(vn != VTMP);
898
899 // Reciprocal square root estimate.
900 vrsqrtes(vd, vn);
901 // 2 Newton-Raphson steps. xn+1 = xn * (3 - V1*xn^2) / 2.
902 // First step.
903 vmuls(VTMP, vd, vd); // VTMP <- xn^2
904 vrsqrtss(VTMP, vn, VTMP); // VTMP <- (3 - V1*VTMP) / 2.
905 vmuls(vd, vd, VTMP); // xn+1 <- xn * VTMP
906 // Second step.
907 vmuls(VTMP, vd, vd);
908 vrsqrtss(VTMP, vn, VTMP);
909 vmuls(vd, vd, VTMP);
910}
911
912// Preserves object and value registers.
913void Assembler::StoreIntoObjectFilter(Register object,
914 Register value,
915 Label* label,
916 CanBeSmi value_can_be_smi,
917 BarrierFilterMode how_to_jump) {
918 COMPILE_ASSERT((target::ObjectAlignment::kNewObjectAlignmentOffset ==
919 target::kWordSize) &&
920 (target::ObjectAlignment::kOldObjectAlignmentOffset == 0));
921
922 // Write-barrier triggers if the value is in the new space (has bit set) and
923 // the object is in the old space (has bit cleared).
924 if (value_can_be_smi == kValueIsNotSmi) {
925#if defined(DEBUG)
926 Label okay;
927 BranchIfNotSmi(value, &okay);
928 Stop("Unexpected Smi!");
929 Bind(&okay);
930#endif
931 // To check that, we compute value & ~object and skip the write barrier
932 // if the bit is not set. We can't destroy the object.
933 bic(TMP, value, Operand(object));
934 } else {
935 // For the value we are only interested in the new/old bit and the tag bit.
936 // And the new bit with the tag bit. The resulting bit will be 0 for a Smi.
937 and_(TMP, value,
938 Operand(value, LSL, target::ObjectAlignment::kNewObjectBitPosition));
939 // And the result with the negated space bit of the object.
940 bic(TMP, TMP, Operand(object));
941 }
942 if (how_to_jump == kJumpToNoUpdate) {
943 tbz(label, TMP, target::ObjectAlignment::kNewObjectBitPosition);
944 } else {
945 tbnz(label, TMP, target::ObjectAlignment::kNewObjectBitPosition);
946 }
947}
948
949void Assembler::StoreIntoObjectOffset(Register object,
950 int32_t offset,
951 Register value,
952 CanBeSmi value_can_be_smi,
953 bool lr_reserved) {
954 if (Address::CanHoldOffset(offset - kHeapObjectTag)) {
955 StoreIntoObject(object, FieldAddress(object, offset), value,
956 value_can_be_smi, lr_reserved);
957 } else {
958 AddImmediate(TMP, object, offset - kHeapObjectTag);
959 StoreIntoObject(object, Address(TMP), value, value_can_be_smi, lr_reserved);
960 }
961}
962
963void Assembler::StoreIntoObject(Register object,
964 const Address& dest,
965 Register value,
966 CanBeSmi can_be_smi,
967 bool lr_reserved) {
968 // x.slot = x. Barrier should have be removed at the IL level.
969 ASSERT(object != value);
970 ASSERT(object != LR);
971 ASSERT(value != LR);
972 ASSERT(object != TMP);
973 ASSERT(object != TMP2);
974 ASSERT(value != TMP);
975 ASSERT(value != TMP2);
976
977 str(value, dest);
978
979 // In parallel, test whether
980 // - object is old and not remembered and value is new, or
981 // - object is old and value is old and not marked and concurrent marking is
982 // in progress
983 // If so, call the WriteBarrier stub, which will either add object to the
984 // store buffer (case 1) or add value to the marking stack (case 2).
985 // Compare ObjectLayout::StorePointer.
986 Label done;
987 if (can_be_smi == kValueCanBeSmi) {
988 BranchIfSmi(value, &done);
989 }
990 ldr(TMP, FieldAddress(object, target::Object::tags_offset()), kUnsignedByte);
991 ldr(TMP2, FieldAddress(value, target::Object::tags_offset()), kUnsignedByte);
992 and_(TMP, TMP2,
993 Operand(TMP, LSR, target::ObjectLayout::kBarrierOverlapShift));
994 tst(TMP, Operand(BARRIER_MASK));
995 b(&done, ZERO);
996
997 if (!lr_reserved) Push(LR);
998 Register objectForCall = object;
999 if (value != kWriteBarrierValueReg) {
1000 // Unlikely. Only non-graph intrinsics.
1001 // TODO(rmacnak): Shuffle registers in intrinsics.
1002 if (object != kWriteBarrierValueReg) {
1003 Push(kWriteBarrierValueReg);
1004 } else {
1005 COMPILE_ASSERT(R2 != kWriteBarrierValueReg);
1006 COMPILE_ASSERT(R3 != kWriteBarrierValueReg);
1007 objectForCall = (value == R2) ? R3 : R2;
1008 PushPair(kWriteBarrierValueReg, objectForCall);
1009 mov(objectForCall, object);
1010 }
1011 mov(kWriteBarrierValueReg, value);
1012 }
1013
1014 generate_invoke_write_barrier_wrapper_(objectForCall);
1015
1016 if (value != kWriteBarrierValueReg) {
1017 if (object != kWriteBarrierValueReg) {
1018 Pop(kWriteBarrierValueReg);
1019 } else {
1020 PopPair(kWriteBarrierValueReg, objectForCall);
1021 }
1022 }
1023 if (!lr_reserved) Pop(LR);
1024 Bind(&done);
1025}
1026
1027void Assembler::StoreIntoArray(Register object,
1028 Register slot,
1029 Register value,
1030 CanBeSmi can_be_smi,
1031 bool lr_reserved) {
1032 ASSERT(object != TMP);
1033 ASSERT(object != TMP2);
1034 ASSERT(value != TMP);
1035 ASSERT(value != TMP2);
1036 ASSERT(slot != TMP);
1037 ASSERT(slot != TMP2);
1038
1039 str(value, Address(slot, 0));
1040
1041 // In parallel, test whether
1042 // - object is old and not remembered and value is new, or
1043 // - object is old and value is old and not marked and concurrent marking is
1044 // in progress
1045 // If so, call the WriteBarrier stub, which will either add object to the
1046 // store buffer (case 1) or add value to the marking stack (case 2).
1047 // Compare ObjectLayout::StorePointer.
1048 Label done;
1049 if (can_be_smi == kValueCanBeSmi) {
1050 BranchIfSmi(value, &done);
1051 }
1052 ldr(TMP, FieldAddress(object, target::Object::tags_offset()), kUnsignedByte);
1053 ldr(TMP2, FieldAddress(value, target::Object::tags_offset()), kUnsignedByte);
1054 and_(TMP, TMP2,
1055 Operand(TMP, LSR, target::ObjectLayout::kBarrierOverlapShift));
1056 tst(TMP, Operand(BARRIER_MASK));
1057 b(&done, ZERO);
1058 if (!lr_reserved) Push(LR);
1059
1060 if ((object != kWriteBarrierObjectReg) || (value != kWriteBarrierValueReg) ||
1061 (slot != kWriteBarrierSlotReg)) {
1062 // Spill and shuffle unimplemented. Currently StoreIntoArray is only used
1063 // from StoreIndexInstr, which gets these exact registers from the register
1064 // allocator.
1065 UNIMPLEMENTED();
1066 }
1067 generate_invoke_array_write_barrier_();
1068 if (!lr_reserved) Pop(LR);
1069 Bind(&done);
1070}
1071
1072void Assembler::StoreIntoObjectNoBarrier(Register object,
1073 const Address& dest,
1074 Register value) {
1075 str(value, dest);
1076#if defined(DEBUG)
1077 Label done;
1078 StoreIntoObjectFilter(object, value, &done, kValueCanBeSmi, kJumpToNoUpdate);
1079
1080 ldr(TMP, FieldAddress(object, target::Object::tags_offset()), kUnsignedByte);
1081 tsti(TMP, Immediate(1 << target::ObjectLayout::kOldAndNotRememberedBit));
1082 b(&done, ZERO);
1083
1084 Stop("Store buffer update is required");
1085 Bind(&done);
1086#endif // defined(DEBUG)
1087 // No store buffer update.
1088}
1089
1090void Assembler::StoreIntoObjectOffsetNoBarrier(Register object,
1091 int32_t offset,
1092 Register value) {
1093 if (Address::CanHoldOffset(offset - kHeapObjectTag)) {
1094 StoreIntoObjectNoBarrier(object, FieldAddress(object, offset), value);
1095 } else {
1096 AddImmediate(TMP, object, offset - kHeapObjectTag);
1097 StoreIntoObjectNoBarrier(object, Address(TMP), value);
1098 }
1099}
1100
1101void Assembler::StoreIntoObjectNoBarrier(Register object,
1102 const Address& dest,
1103 const Object& value) {
1104 ASSERT(IsOriginalObject(value));
1105 ASSERT(IsNotTemporaryScopedHandle(value));
1106 // No store buffer update.
1107 if (IsSameObject(compiler::NullObject(), value)) {
1108 str(NULL_REG, dest);
1109 } else {
1110 LoadObject(TMP2, value);
1111 str(TMP2, dest);
1112 }
1113}
1114
1115void Assembler::StoreIntoObjectOffsetNoBarrier(Register object,
1116 int32_t offset,
1117 const Object& value) {
1118 if (Address::CanHoldOffset(offset - kHeapObjectTag)) {
1119 StoreIntoObjectNoBarrier(object, FieldAddress(object, offset), value);
1120 } else {
1121 AddImmediate(TMP, object, offset - kHeapObjectTag);
1122 StoreIntoObjectNoBarrier(object, Address(TMP), value);
1123 }
1124}
1125
1126void Assembler::StoreInternalPointer(Register object,
1127 const Address& dest,
1128 Register value) {
1129 str(value, dest);
1130}
1131
1132void Assembler::ExtractClassIdFromTags(Register result, Register tags) {
1133 ASSERT(target::ObjectLayout::kClassIdTagPos == 16);
1134 ASSERT(target::ObjectLayout::kClassIdTagSize == 16);
1135 LsrImmediate(result, tags, target::ObjectLayout::kClassIdTagPos, kWord);
1136}
1137
1138void Assembler::ExtractInstanceSizeFromTags(Register result, Register tags) {
1139 ASSERT(target::ObjectLayout::kSizeTagPos == 8);
1140 ASSERT(target::ObjectLayout::kSizeTagSize == 8);
1141 ubfx(result, tags, target::ObjectLayout::kSizeTagPos,
1142 target::ObjectLayout::kSizeTagSize);
1143 LslImmediate(result, result, target::ObjectAlignment::kObjectAlignmentLog2);
1144}
1145
1146void Assembler::LoadClassId(Register result, Register object) {
1147 ASSERT(target::ObjectLayout::kClassIdTagPos == 16);
1148 ASSERT(target::ObjectLayout::kClassIdTagSize == 16);
1149 const intptr_t class_id_offset =
1150 target::Object::tags_offset() +
1151 target::ObjectLayout::kClassIdTagPos / kBitsPerByte;
1152 LoadFromOffset(result, object, class_id_offset - kHeapObjectTag,
1153 kUnsignedHalfword);
1154}
1155
1156void Assembler::LoadClassById(Register result, Register class_id) {
1157 ASSERT(result != class_id);
1158
1159 const intptr_t table_offset =
1160 target::Isolate::cached_class_table_table_offset();
1161
1162 LoadIsolate(result);
1163 LoadFromOffset(result, result, table_offset);
1164 ldr(result, Address(result, class_id, UXTX, Address::Scaled));
1165}
1166
1167void Assembler::CompareClassId(Register object,
1168 intptr_t class_id,
1169 Register scratch) {
1170 ASSERT(scratch == kNoRegister);
1171 LoadClassId(TMP, object);
1172 CompareImmediate(TMP, class_id);
1173}
1174
1175void Assembler::LoadClassIdMayBeSmi(Register result, Register object) {
1176 ASSERT(result != object);
1177 Label done;
1178 LoadImmediate(result, kSmiCid);
1179 BranchIfSmi(object, &done);
1180 LoadClassId(result, object);
1181 Bind(&done);
1182}
1183
1184void Assembler::LoadTaggedClassIdMayBeSmi(Register result, Register object) {
1185 if (result == object) {
1186 LoadClassIdMayBeSmi(TMP, object);
1187 SmiTag(result, TMP);
1188 } else {
1189 Label done;
1190 LoadImmediate(result, target::ToRawSmi(kSmiCid));
1191 BranchIfSmi(object, &done);
1192 LoadClassId(result, object);
1193 SmiTag(result);
1194 Bind(&done);
1195 }
1196}
1197
1198// Frame entry and exit.
1199void Assembler::ReserveAlignedFrameSpace(intptr_t frame_space) {
1200 // Reserve space for arguments and align frame before entering
1201 // the C++ world.
1202 if (frame_space != 0) {
1203 AddImmediate(SP, -frame_space);
1204 }
1205 if (OS::ActivationFrameAlignment() > 1) {
1206 andi(SP, SP, Immediate(~(OS::ActivationFrameAlignment() - 1)));
1207 }
1208}
1209
1210void Assembler::EmitEntryFrameVerification() {
1211#if defined(DEBUG)
1212 Label done;
1213 ASSERT(!constant_pool_allowed());
1214 LoadImmediate(TMP, target::frame_layout.exit_link_slot_from_entry_fp *
1215 target::kWordSize);
1216 add(TMP, TMP, Operand(FPREG));
1217 cmp(TMP, Operand(SPREG));
1218 b(&done, EQ);
1219
1220 Breakpoint();
1221
1222 Bind(&done);
1223#endif
1224}
1225
1226void Assembler::RestoreCodePointer() {
1227 ldr(CODE_REG,
1228 Address(FP, target::frame_layout.code_from_fp * target::kWordSize));
1229 CheckCodePointer();
1230}
1231
1232void Assembler::RestorePinnedRegisters() {
1233 ldr(BARRIER_MASK,
1234 compiler::Address(THR, target::Thread::write_barrier_mask_offset()));
1235 ldr(NULL_REG, compiler::Address(THR, target::Thread::object_null_offset()));
1236}
1237
1238void Assembler::SetupGlobalPoolAndDispatchTable() {
1239 ASSERT(FLAG_precompiled_mode && FLAG_use_bare_instructions);
1240 ldr(PP, Address(THR, target::Thread::global_object_pool_offset()));
1241 sub(PP, PP, Operand(kHeapObjectTag)); // Pool in PP is untagged!
1242 if (FLAG_use_table_dispatch) {
1243 ldr(DISPATCH_TABLE_REG,
1244 Address(THR, target::Thread::dispatch_table_array_offset()));
1245 }
1246}
1247
1248void Assembler::CheckCodePointer() {
1249#ifdef DEBUG
1250 if (!FLAG_check_code_pointer) {
1251 return;
1252 }
1253 Comment("CheckCodePointer");
1254 Label cid_ok, instructions_ok;
1255 Push(R0);
1256 CompareClassId(CODE_REG, kCodeCid);
1257 b(&cid_ok, EQ);
1258 brk(0);
1259 Bind(&cid_ok);
1260
1261 const intptr_t entry_offset =
1262 CodeSize() + target::Instructions::HeaderSize() - kHeapObjectTag;
1263 adr(R0, Immediate(-entry_offset));
1264 ldr(TMP, FieldAddress(CODE_REG, target::Code::saved_instructions_offset()));
1265 cmp(R0, Operand(TMP));
1266 b(&instructions_ok, EQ);
1267 brk(1);
1268 Bind(&instructions_ok);
1269 Pop(R0);
1270#endif
1271}
1272
1273// The ARM64 ABI requires at all times
1274// - stack limit < CSP <= stack base
1275// - CSP mod 16 = 0
1276// - we do not access stack memory below CSP
1277// Practically, this means we need to keep the C stack pointer ahead of the
1278// Dart stack pointer and 16-byte aligned for signal handlers. We set
1279// CSP to a value near the stack limit during SetupDartSP*, and use a different
1280// register within our generated code to avoid the alignment requirement.
1281// Note that Fuchsia does not have signal handlers.
1282
1283void Assembler::SetupDartSP(intptr_t reserve /* = 4096 */) {
1284 mov(SP, CSP);
1285 // The caller doesn't have a Thread available. Just kick CSP forward a bit.
1286 AddImmediate(CSP, CSP, -Utils::RoundUp(reserve, 16));
1287}
1288
1289void Assembler::SetupCSPFromThread(Register thr) {
1290 // Thread::saved_stack_limit_ is OSThread::overflow_stack_limit(), which is
1291 // OSThread::stack_limit() with some headroom. Set CSP a bit below this value
1292 // so that signal handlers won't stomp on the stack of Dart code that pushs a
1293 // bit past overflow_stack_limit before its next overflow check. (We build
1294 // frames before doing an overflow check.)
1295 ldr(TMP, Address(thr, target::Thread::saved_stack_limit_offset()));
1296 AddImmediate(CSP, TMP, -4096);
1297}
1298
1299void Assembler::RestoreCSP() {
1300 mov(CSP, SP);
1301}
1302
1303void Assembler::EnterFrame(intptr_t frame_size) {
1304 PushPair(FP, LR); // low: FP, high: LR.
1305 mov(FP, SP);
1306
1307 if (frame_size > 0) {
1308 sub(SP, SP, Operand(frame_size));
1309 }
1310}
1311
1312void Assembler::LeaveFrame() {
1313 mov(SP, FP);
1314 PopPair(FP, LR); // low: FP, high: LR.
1315}
1316
1317void Assembler::EnterDartFrame(intptr_t frame_size, Register new_pp) {
1318 ASSERT(!constant_pool_allowed());
1319 // Setup the frame.
1320 EnterFrame(0);
1321
1322 if (!(FLAG_precompiled_mode && FLAG_use_bare_instructions)) {
1323 TagAndPushPPAndPcMarker(); // Save PP and PC marker.
1324
1325 // Load the pool pointer.
1326 if (new_pp == kNoRegister) {
1327 LoadPoolPointer();
1328 } else {
1329 mov(PP, new_pp);
1330 }
1331 }
1332 set_constant_pool_allowed(true);
1333
1334 // Reserve space.
1335 if (frame_size > 0) {
1336 AddImmediate(SP, -frame_size);
1337 }
1338}
1339
1340// On entry to a function compiled for OSR, the caller's frame pointer, the
1341// stack locals, and any copied parameters are already in place. The frame
1342// pointer is already set up. The PC marker is not correct for the
1343// optimized function and there may be extra space for spill slots to
1344// allocate. We must also set up the pool pointer for the function.
1345void Assembler::EnterOsrFrame(intptr_t extra_size, Register new_pp) {
1346 ASSERT(!constant_pool_allowed());
1347 Comment("EnterOsrFrame");
1348 RestoreCodePointer();
1349 LoadPoolPointer();
1350
1351 if (extra_size > 0) {
1352 AddImmediate(SP, -extra_size);
1353 }
1354}
1355
1356void Assembler::LeaveDartFrame(RestorePP restore_pp) {
1357 if (!(FLAG_precompiled_mode && FLAG_use_bare_instructions)) {
1358 if (restore_pp == kRestoreCallerPP) {
1359 // Restore and untag PP.
1360 LoadFromOffset(
1361 PP, FP,
1362 target::frame_layout.saved_caller_pp_from_fp * target::kWordSize);
1363 sub(PP, PP, Operand(kHeapObjectTag));
1364 }
1365 }
1366 set_constant_pool_allowed(false);
1367 LeaveFrame();
1368}
1369
1370void Assembler::EnterSafepoint(Register state) {
1371 // We generate the same number of instructions whether or not the slow-path is
1372 // forced. This simplifies GenerateJitCallbackTrampolines.
1373
1374 Register addr = TMP2;
1375 ASSERT(addr != state);
1376
1377 Label slow_path, done, retry;
1378 if (FLAG_use_slow_path) {
1379 b(&slow_path);
1380 }
1381
1382 movz(addr, Immediate(target::Thread::safepoint_state_offset()), 0);
1383 add(addr, THR, Operand(addr));
1384 Bind(&retry);
1385 ldxr(state, addr);
1386 cmp(state, Operand(target::Thread::safepoint_state_unacquired()));
1387 b(&slow_path, NE);
1388
1389 movz(state, Immediate(target::Thread::safepoint_state_acquired()), 0);
1390 stxr(TMP, state, addr);
1391 cbz(&done, TMP); // 0 means stxr was successful.
1392
1393 if (!FLAG_use_slow_path) {
1394 b(&retry);
1395 }
1396
1397 Bind(&slow_path);
1398 ldr(addr, Address(THR, target::Thread::enter_safepoint_stub_offset()));
1399 ldr(addr, FieldAddress(addr, target::Code::entry_point_offset()));
1400 blr(addr);
1401
1402 Bind(&done);
1403}
1404
1405void Assembler::TransitionGeneratedToNative(Register destination,
1406 Register new_exit_frame,
1407 Register new_exit_through_ffi,
1408 bool enter_safepoint) {
1409 // Save exit frame information to enable stack walking.
1410 StoreToOffset(new_exit_frame, THR,
1411 target::Thread::top_exit_frame_info_offset());
1412
1413 StoreToOffset(new_exit_through_ffi, THR,
1414 target::Thread::exit_through_ffi_offset());
1415 Register tmp = new_exit_through_ffi;
1416
1417 // Mark that the thread is executing native code.
1418 StoreToOffset(destination, THR, target::Thread::vm_tag_offset());
1419 LoadImmediate(tmp, target::Thread::native_execution_state());
1420 StoreToOffset(tmp, THR, target::Thread::execution_state_offset());
1421
1422 if (enter_safepoint) {
1423 EnterSafepoint(tmp);
1424 }
1425}
1426
1427void Assembler::ExitSafepoint(Register state) {
1428 // We generate the same number of instructions whether or not the slow-path is
1429 // forced, for consistency with EnterSafepoint.
1430 Register addr = TMP2;
1431 ASSERT(addr != state);
1432
1433 Label slow_path, done, retry;
1434 if (FLAG_use_slow_path) {
1435 b(&slow_path);
1436 }
1437
1438 movz(addr, Immediate(target::Thread::safepoint_state_offset()), 0);
1439 add(addr, THR, Operand(addr));
1440 Bind(&retry);
1441 ldxr(state, addr);
1442 cmp(state, Operand(target::Thread::safepoint_state_acquired()));
1443 b(&slow_path, NE);
1444
1445 movz(state, Immediate(target::Thread::safepoint_state_unacquired()), 0);
1446 stxr(TMP, state, addr);
1447 cbz(&done, TMP); // 0 means stxr was successful.
1448
1449 if (!FLAG_use_slow_path) {
1450 b(&retry);
1451 }
1452
1453 Bind(&slow_path);
1454 ldr(addr, Address(THR, target::Thread::exit_safepoint_stub_offset()));
1455 ldr(addr, FieldAddress(addr, target::Code::entry_point_offset()));
1456 blr(addr);
1457
1458 Bind(&done);
1459}
1460
1461void Assembler::TransitionNativeToGenerated(Register state,
1462 bool exit_safepoint) {
1463 if (exit_safepoint) {
1464 ExitSafepoint(state);
1465 } else {
1466#if defined(DEBUG)
1467 // Ensure we've already left the safepoint.
1468 ldr(TMP, Address(THR, target::Thread::safepoint_state_offset()));
1469 Label ok;
1470 tbz(&ok, TMP, target::Thread::safepoint_state_inside_bit());
1471 Breakpoint();
1472 Bind(&ok);
1473#endif
1474 }
1475
1476 // Mark that the thread is executing Dart code.
1477 LoadImmediate(state, target::Thread::vm_tag_compiled_id());
1478 StoreToOffset(state, THR, target::Thread::vm_tag_offset());
1479 LoadImmediate(state, target::Thread::generated_execution_state());
1480 StoreToOffset(state, THR, target::Thread::execution_state_offset());
1481
1482 // Reset exit frame information in Isolate's mutator thread structure.
1483 StoreToOffset(ZR, THR, target::Thread::top_exit_frame_info_offset());
1484 LoadImmediate(state, 0);
1485 StoreToOffset(state, THR, target::Thread::exit_through_ffi_offset());
1486}
1487
1488void Assembler::EnterCallRuntimeFrame(intptr_t frame_size) {
1489 Comment("EnterCallRuntimeFrame");
1490 EnterFrame(0);
1491 if (!(FLAG_precompiled_mode && FLAG_use_bare_instructions)) {
1492 TagAndPushPPAndPcMarker(); // Save PP and PC marker.
1493 }
1494
1495 // Store fpu registers with the lowest register number at the lowest
1496 // address.
1497 for (int i = kNumberOfVRegisters - 1; i >= 0; i--) {
1498 if ((i >= kAbiFirstPreservedFpuReg) && (i <= kAbiLastPreservedFpuReg)) {
1499 // TODO(zra): When SIMD is added, we must also preserve the top
1500 // 64-bits of the callee-saved registers.
1501 continue;
1502 }
1503 // TODO(zra): Save the whole V register.
1504 VRegister reg = static_cast<VRegister>(i);
1505 PushDouble(reg);
1506 }
1507
1508 for (int i = kDartFirstVolatileCpuReg; i <= kDartLastVolatileCpuReg; i++) {
1509 const Register reg = static_cast<Register>(i);
1510 Push(reg);
1511 }
1512
1513 ReserveAlignedFrameSpace(frame_size);
1514}
1515
1516void Assembler::LeaveCallRuntimeFrame() {
1517 // SP might have been modified to reserve space for arguments
1518 // and ensure proper alignment of the stack frame.
1519 // We need to restore it before restoring registers.
1520 const intptr_t kPushedRegistersSize =
1521 kDartVolatileCpuRegCount * target::kWordSize +
1522 kDartVolatileFpuRegCount * target::kWordSize +
1523 (target::frame_layout.dart_fixed_frame_size - 2) *
1524 target::kWordSize; // From EnterStubFrame (excluding PC / FP)
1525 AddImmediate(SP, FP, -kPushedRegistersSize);
1526 for (int i = kDartLastVolatileCpuReg; i >= kDartFirstVolatileCpuReg; i--) {
1527 const Register reg = static_cast<Register>(i);
1528 Pop(reg);
1529 }
1530
1531 for (int i = 0; i < kNumberOfVRegisters; i++) {
1532 if ((i >= kAbiFirstPreservedFpuReg) && (i <= kAbiLastPreservedFpuReg)) {
1533 // TODO(zra): When SIMD is added, we must also restore the top
1534 // 64-bits of the callee-saved registers.
1535 continue;
1536 }
1537 // TODO(zra): Restore the whole V register.
1538 VRegister reg = static_cast<VRegister>(i);
1539 PopDouble(reg);
1540 }
1541
1542 LeaveStubFrame();
1543}
1544
1545void Assembler::CallRuntime(const RuntimeEntry& entry,
1546 intptr_t argument_count) {
1547 entry.Call(this, argument_count);
1548}
1549
1550void Assembler::EnterStubFrame() {
1551 EnterDartFrame(0);
1552}
1553
1554void Assembler::LeaveStubFrame() {
1555 LeaveDartFrame();
1556}
1557
1558void Assembler::EnterCFrame(intptr_t frame_space) {
1559 EnterFrame(0);
1560 ReserveAlignedFrameSpace(frame_space);
1561}
1562
1563void Assembler::LeaveCFrame() {
1564 LeaveFrame();
1565}
1566
1567// R0 receiver, R5 ICData entries array
1568// Preserve R4 (ARGS_DESC_REG), not required today, but maybe later.
1569void Assembler::MonomorphicCheckedEntryJIT() {
1570 has_monomorphic_entry_ = true;
1571 const bool saved_use_far_branches = use_far_branches();
1572 set_use_far_branches(false);
1573 const intptr_t start = CodeSize();
1574
1575 Label immediate, miss;
1576 Bind(&miss);
1577 ldr(IP0, Address(THR, target::Thread::switchable_call_miss_entry_offset()));
1578 br(IP0);
1579
1580 Comment("MonomorphicCheckedEntry");
1581 ASSERT_EQUAL(CodeSize() - start,
1582 target::Instructions::kMonomorphicEntryOffsetJIT);
1583
1584 const intptr_t cid_offset = target::Array::element_offset(0);
1585 const intptr_t count_offset = target::Array::element_offset(1);
1586
1587 // Sadly this cannot use ldp because ldp requires aligned offsets.
1588 ldr(R1, FieldAddress(R5, cid_offset));
1589 ldr(R2, FieldAddress(R5, count_offset));
1590 LoadClassIdMayBeSmi(IP0, R0);
1591 add(R2, R2, Operand(target::ToRawSmi(1)));
1592 cmp(R1, Operand(IP0, LSL, 1));
1593 b(&miss, NE);
1594 str(R2, FieldAddress(R5, count_offset));
1595 LoadImmediate(R4, 0); // GC-safe for OptimizeInvokedFunction
1596
1597 // Fall through to unchecked entry.
1598 ASSERT_EQUAL(CodeSize() - start,
1599 target::Instructions::kPolymorphicEntryOffsetJIT);
1600
1601 set_use_far_branches(saved_use_far_branches);
1602}
1603
1604// R0 receiver, R5 guarded cid as Smi.
1605// Preserve R4 (ARGS_DESC_REG), not required today, but maybe later.
1606void Assembler::MonomorphicCheckedEntryAOT() {
1607 has_monomorphic_entry_ = true;
1608 bool saved_use_far_branches = use_far_branches();
1609 set_use_far_branches(false);
1610
1611 const intptr_t start = CodeSize();
1612
1613 Label immediate, miss;
1614 Bind(&miss);
1615 ldr(IP0, Address(THR, target::Thread::switchable_call_miss_entry_offset()));
1616 br(IP0);
1617
1618 Comment("MonomorphicCheckedEntry");
1619 ASSERT_EQUAL(CodeSize() - start,
1620 target::Instructions::kMonomorphicEntryOffsetAOT);
1621 LoadClassId(IP0, R0);
1622 cmp(R5, Operand(IP0, LSL, 1));
1623 b(&miss, NE);
1624
1625 // Fall through to unchecked entry.
1626 ASSERT_EQUAL(CodeSize() - start,
1627 target::Instructions::kPolymorphicEntryOffsetAOT);
1628
1629 set_use_far_branches(saved_use_far_branches);
1630}
1631
1632void Assembler::BranchOnMonomorphicCheckedEntryJIT(Label* label) {
1633 has_monomorphic_entry_ = true;
1634 while (CodeSize() < target::Instructions::kMonomorphicEntryOffsetJIT) {
1635 brk(0);
1636 }
1637 b(label);
1638 while (CodeSize() < target::Instructions::kPolymorphicEntryOffsetJIT) {
1639 brk(0);
1640 }
1641}
1642
1643#ifndef PRODUCT
1644void Assembler::MaybeTraceAllocation(intptr_t cid,
1645 Register temp_reg,
1646 Label* trace) {
1647 ASSERT(cid > 0);
1648
1649 const intptr_t shared_table_offset =
1650 target::Isolate::shared_class_table_offset();
1651 const intptr_t table_offset =
1652 target::SharedClassTable::class_heap_stats_table_offset();
1653 const intptr_t class_offset = target::ClassTable::ClassOffsetFor(cid);
1654
1655 LoadIsolate(temp_reg);
1656 ldr(temp_reg, Address(temp_reg, shared_table_offset));
1657 ldr(temp_reg, Address(temp_reg, table_offset));
1658 AddImmediate(temp_reg, class_offset);
1659 ldr(temp_reg, Address(temp_reg, 0), kUnsignedByte);
1660 cbnz(trace, temp_reg);
1661}
1662#endif // !PRODUCT
1663
1664void Assembler::TryAllocate(const Class& cls,
1665 Label* failure,
1666 Register instance_reg,
1667 Register top_reg,
1668 bool tag_result) {
1669 ASSERT(failure != NULL);
1670 const intptr_t instance_size = target::Class::GetInstanceSize(cls);
1671 if (FLAG_inline_alloc &&
1672 target::Heap::IsAllocatableInNewSpace(instance_size)) {
1673 // If this allocation is traced, program will jump to failure path
1674 // (i.e. the allocation stub) which will allocate the object and trace the
1675 // allocation call site.
1676 const classid_t cid = target::Class::GetId(cls);
1677 NOT_IN_PRODUCT(MaybeTraceAllocation(cid, /*temp_reg=*/top_reg, failure));
1678
1679 const Register kEndReg = TMP;
1680
1681 // instance_reg: potential next object start.
1682 RELEASE_ASSERT((target::Thread::top_offset() + target::kWordSize) ==
1683 target::Thread::end_offset());
1684 ldp(instance_reg, kEndReg,
1685 Address(THR, target::Thread::top_offset(), Address::PairOffset));
1686
1687 // TODO(koda): Protect against unsigned overflow here.
1688 AddImmediate(top_reg, instance_reg, instance_size);
1689 cmp(kEndReg, Operand(top_reg));
1690 b(failure, LS); // Unsigned lower or equal.
1691
1692 // Successfully allocated the object, now update top to point to
1693 // next object start and store the class in the class field of object.
1694 str(top_reg, Address(THR, target::Thread::top_offset()));
1695
1696 const uint32_t tags =
1697 target::MakeTagWordForNewSpaceObject(cid, instance_size);
1698 // Extends the 32 bit tags with zeros, which is the uninitialized
1699 // hash code.
1700 LoadImmediate(TMP, tags);
1701 StoreToOffset(TMP, instance_reg, target::Object::tags_offset());
1702
1703 if (tag_result) {
1704 AddImmediate(instance_reg, kHeapObjectTag);
1705 }
1706 } else {
1707 b(failure);
1708 }
1709}
1710
1711void Assembler::TryAllocateArray(intptr_t cid,
1712 intptr_t instance_size,
1713 Label* failure,
1714 Register instance,
1715 Register end_address,
1716 Register temp1,
1717 Register temp2) {
1718 if (FLAG_inline_alloc &&
1719 target::Heap::IsAllocatableInNewSpace(instance_size)) {
1720 // If this allocation is traced, program will jump to failure path
1721 // (i.e. the allocation stub) which will allocate the object and trace the
1722 // allocation call site.
1723 NOT_IN_PRODUCT(MaybeTraceAllocation(cid, temp1, failure));
1724 // Potential new object start.
1725 ldr(instance, Address(THR, target::Thread::top_offset()));
1726 AddImmediateSetFlags(end_address, instance, instance_size);
1727 b(failure, CS); // Fail on unsigned overflow.
1728
1729 // Check if the allocation fits into the remaining space.
1730 // instance: potential new object start.
1731 // end_address: potential next object start.
1732 ldr(temp2, Address(THR, target::Thread::end_offset()));
1733 cmp(end_address, Operand(temp2));
1734 b(failure, CS);
1735
1736 // Successfully allocated the object(s), now update top to point to
1737 // next object start and initialize the object.
1738 str(end_address, Address(THR, target::Thread::top_offset()));
1739 add(instance, instance, Operand(kHeapObjectTag));
1740 NOT_IN_PRODUCT(LoadImmediate(temp2, instance_size));
1741
1742 // Initialize the tags.
1743 // instance: new object start as a tagged pointer.
1744 const uint32_t tags =
1745 target::MakeTagWordForNewSpaceObject(cid, instance_size);
1746 // Extends the 32 bit tags with zeros, which is the uninitialized
1747 // hash code.
1748 LoadImmediate(temp2, tags);
1749 str(temp2, FieldAddress(instance, target::Object::tags_offset()));
1750 } else {
1751 b(failure);
1752 }
1753}
1754
1755void Assembler::GenerateUnRelocatedPcRelativeCall(intptr_t offset_into_target) {
1756 // Emit "bl <offset>".
1757 EmitUnconditionalBranchOp(BL, 0);
1758
1759 PcRelativeCallPattern pattern(buffer_.contents() + buffer_.Size() -
1760 PcRelativeCallPattern::kLengthInBytes);
1761 pattern.set_distance(offset_into_target);
1762}
1763
1764void Assembler::GenerateUnRelocatedPcRelativeTailCall(
1765 intptr_t offset_into_target) {
1766 // Emit "b <offset>".
1767 EmitUnconditionalBranchOp(B, 0);
1768 PcRelativeTailCallPattern pattern(buffer_.contents() + buffer_.Size() -
1769 PcRelativeTailCallPattern::kLengthInBytes);
1770 pattern.set_distance(offset_into_target);
1771}
1772
1773Address Assembler::ElementAddressForIntIndex(bool is_external,
1774 intptr_t cid,
1775 intptr_t index_scale,
1776 Register array,
1777 intptr_t index) const {
1778 const int64_t offset = index * index_scale + HeapDataOffset(is_external, cid);
1779 ASSERT(Utils::IsInt(32, offset));
1780 const OperandSize size = Address::OperandSizeFor(cid);
1781 ASSERT(Address::CanHoldOffset(offset, Address::Offset, size));
1782 return Address(array, static_cast<int32_t>(offset), Address::Offset, size);
1783}
1784
1785void Assembler::ComputeElementAddressForIntIndex(Register address,
1786 bool is_external,
1787 intptr_t cid,
1788 intptr_t index_scale,
1789 Register array,
1790 intptr_t index) {
1791 const int64_t offset = index * index_scale + HeapDataOffset(is_external, cid);
1792 AddImmediate(address, array, offset);
1793}
1794
1795Address Assembler::ElementAddressForRegIndex(bool is_external,
1796 intptr_t cid,
1797 intptr_t index_scale,
1798 bool index_unboxed,
1799 Register array,
1800 Register index,
1801 Register temp) {
1802 return ElementAddressForRegIndexWithSize(
1803 is_external, cid, Address::OperandSizeFor(cid), index_scale,
1804 index_unboxed, array, index, temp);
1805}
1806
1807Address Assembler::ElementAddressForRegIndexWithSize(bool is_external,
1808 intptr_t cid,
1809 OperandSize size,
1810 intptr_t index_scale,
1811 bool index_unboxed,
1812 Register array,
1813 Register index,
1814 Register temp) {
1815 // If unboxed, index is expected smi-tagged, (i.e, LSL 1) for all arrays.
1816 const intptr_t boxing_shift = index_unboxed ? 0 : -kSmiTagShift;
1817 const intptr_t shift = Utils::ShiftForPowerOfTwo(index_scale) + boxing_shift;
1818 const int32_t offset = HeapDataOffset(is_external, cid);
1819 ASSERT(array != temp);
1820 ASSERT(index != temp);
1821 if ((offset == 0) && (shift == 0)) {
1822 return Address(array, index, UXTX, Address::Unscaled);
1823 } else if (shift < 0) {
1824 ASSERT(shift == -1);
1825 add(temp, array, Operand(index, ASR, 1));
1826 } else {
1827 add(temp, array, Operand(index, LSL, shift));
1828 }
1829 ASSERT(Address::CanHoldOffset(offset, Address::Offset, size));
1830 return Address(temp, offset, Address::Offset, size);
1831}
1832
1833void Assembler::ComputeElementAddressForRegIndex(Register address,
1834 bool is_external,
1835 intptr_t cid,
1836 intptr_t index_scale,
1837 bool index_unboxed,
1838 Register array,
1839 Register index) {
1840 // If unboxed, index is expected smi-tagged, (i.e, LSL 1) for all arrays.
1841 const intptr_t boxing_shift = index_unboxed ? 0 : -kSmiTagShift;
1842 const intptr_t shift = Utils::ShiftForPowerOfTwo(index_scale) + boxing_shift;
1843 const int32_t offset = HeapDataOffset(is_external, cid);
1844 if (shift == 0) {
1845 add(address, array, Operand(index));
1846 } else if (shift < 0) {
1847 ASSERT(shift == -1);
1848 add(address, array, Operand(index, ASR, 1));
1849 } else {
1850 add(address, array, Operand(index, LSL, shift));
1851 }
1852 if (offset != 0) {
1853 AddImmediate(address, offset);
1854 }
1855}
1856
1857void Assembler::LoadFieldAddressForRegOffset(Register address,
1858 Register instance,
1859 Register offset_in_words_as_smi) {
1860 add(address, instance,
1861 Operand(offset_in_words_as_smi, LSL,
1862 target::kWordSizeLog2 - kSmiTagShift));
1863 AddImmediate(address, -kHeapObjectTag);
1864}
1865
1866void Assembler::PushRegisters(const RegisterSet& regs) {
1867 const intptr_t fpu_regs_count = regs.FpuRegisterCount();
1868 if (fpu_regs_count > 0) {
1869 // Store fpu registers with the lowest register number at the lowest
1870 // address.
1871 for (intptr_t i = kNumberOfVRegisters - 1; i >= 0; --i) {
1872 VRegister fpu_reg = static_cast<VRegister>(i);
1873 if (regs.ContainsFpuRegister(fpu_reg)) {
1874 PushQuad(fpu_reg);
1875 }
1876 }
1877 }
1878
1879 // The order in which the registers are pushed must match the order
1880 // in which the registers are encoded in the safe point's stack map.
1881 Register prev = kNoRegister;
1882 for (intptr_t i = kNumberOfCpuRegisters - 1; i >= 0; --i) {
1883 Register reg = static_cast<Register>(i);
1884 if (regs.ContainsRegister(reg)) {
1885 if (prev != kNoRegister) {
1886 PushPair(/*low=*/reg, /*high=*/prev);
1887 prev = kNoRegister;
1888 } else {
1889 prev = reg;
1890 }
1891 }
1892 }
1893 if (prev != kNoRegister) {
1894 Push(prev);
1895 }
1896}
1897
1898void Assembler::PopRegisters(const RegisterSet& regs) {
1899 bool pop_single = (regs.CpuRegisterCount() & 1) == 1;
1900 Register prev = kNoRegister;
1901 for (intptr_t i = 0; i < kNumberOfCpuRegisters; ++i) {
1902 Register reg = static_cast<Register>(i);
1903 if (regs.ContainsRegister(reg)) {
1904 if (pop_single) {
1905 // Emit the leftover pop at the beginning instead of the end to
1906 // mirror PushRegisters.
1907 Pop(reg);
1908 pop_single = false;
1909 } else if (prev != kNoRegister) {
1910 PopPair(/*low=*/prev, /*high=*/reg);
1911 prev = kNoRegister;
1912 } else {
1913 prev = reg;
1914 }
1915 }
1916 }
1917 ASSERT(prev == kNoRegister);
1918
1919 const intptr_t fpu_regs_count = regs.FpuRegisterCount();
1920 if (fpu_regs_count > 0) {
1921 // Fpu registers have the lowest register number at the lowest address.
1922 for (intptr_t i = 0; i < kNumberOfVRegisters; ++i) {
1923 VRegister fpu_reg = static_cast<VRegister>(i);
1924 if (regs.ContainsFpuRegister(fpu_reg)) {
1925 PopQuad(fpu_reg);
1926 }
1927 }
1928 }
1929}
1930
1931void Assembler::PushNativeCalleeSavedRegisters() {
1932 // Save the callee-saved registers.
1933 for (int i = kAbiFirstPreservedCpuReg; i <= kAbiLastPreservedCpuReg; i++) {
1934 const Register r = static_cast<Register>(i);
1935 // We use str instead of the Push macro because we will be pushing the PP
1936 // register when it is not holding a pool-pointer since we are coming from
1937 // C++ code.
1938 str(r, Address(SP, -1 * target::kWordSize, Address::PreIndex));
1939 }
1940
1941 // Save the bottom 64-bits of callee-saved V registers.
1942 for (int i = kAbiFirstPreservedFpuReg; i <= kAbiLastPreservedFpuReg; i++) {
1943 const VRegister r = static_cast<VRegister>(i);
1944 PushDouble(r);
1945 }
1946}
1947
1948void Assembler::PopNativeCalleeSavedRegisters() {
1949 // Restore the bottom 64-bits of callee-saved V registers.
1950 for (int i = kAbiLastPreservedFpuReg; i >= kAbiFirstPreservedFpuReg; i--) {
1951 const VRegister r = static_cast<VRegister>(i);
1952 PopDouble(r);
1953 }
1954
1955 // Restore C++ ABI callee-saved registers.
1956 for (int i = kAbiLastPreservedCpuReg; i >= kAbiFirstPreservedCpuReg; i--) {
1957 Register r = static_cast<Register>(i);
1958 // We use ldr instead of the Pop macro because we will be popping the PP
1959 // register when it is not holding a pool-pointer since we are returning to
1960 // C++ code. We also skip the dart stack pointer SP, since we are still
1961 // using it as the stack pointer.
1962 ldr(r, Address(SP, 1 * target::kWordSize, Address::PostIndex));
1963 }
1964}
1965
1966bool Assembler::CanGenerateXCbzTbz(Register rn, Condition cond) {
1967 if (rn == CSP) {
1968 return false;
1969 }
1970 switch (cond) {
1971 case EQ: // equal
1972 case NE: // not equal
1973 case MI: // minus/negative
1974 case LT: // signed less than
1975 case PL: // plus/positive or zero
1976 case GE: // signed greater than or equal
1977 return true;
1978 default:
1979 return false;
1980 }
1981}
1982
1983void Assembler::GenerateXCbzTbz(Register rn, Condition cond, Label* label) {
1984 constexpr int32_t bit_no = 63;
1985 constexpr OperandSize sz = kDoubleWord;
1986 ASSERT(rn != CSP);
1987 switch (cond) {
1988 case EQ: // equal
1989 cbz(label, rn, sz);
1990 return;
1991 case NE: // not equal
1992 cbnz(label, rn, sz);
1993 return;
1994 case MI: // minus/negative
1995 case LT: // signed less than
1996 tbnz(label, rn, bit_no);
1997 return;
1998 case PL: // plus/positive or zero
1999 case GE: // signed greater than or equal
2000 tbz(label, rn, bit_no);
2001 return;
2002 default:
2003 // Only conditions above allow single instruction emission.
2004 UNREACHABLE();
2005 }
2006}
2007
2008} // namespace compiler
2009
2010} // namespace dart
2011
2012#endif // defined(TARGET_ARCH_ARM64)
2013