1// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
2// for details. All rights reserved. Use of this source code is governed by a
3// BSD-style license that can be found in the LICENSE file.
4
5#include "vm/globals.h" // Needed here to get TARGET_ARCH_X64.
6#if defined(TARGET_ARCH_X64)
7
8#include "vm/code_patcher.h"
9#include "vm/cpu.h"
10#include "vm/dart_entry.h"
11#include "vm/instructions.h"
12#include "vm/object.h"
13#include "vm/object_store.h"
14#include "vm/raw_object.h"
15#include "vm/reverse_pc_lookup_cache.h"
16
17namespace dart {
18
19class UnoptimizedCall : public ValueObject {
20 public:
21 UnoptimizedCall(uword return_address, const Code& code)
22 : object_pool_(ObjectPool::Handle(code.GetObjectPool())),
23 code_index_(-1),
24 argument_index_(-1) {
25 uword pc = return_address;
26
27 // callq [CODE_REG + entry_point_offset]
28 static int16_t call_pattern[] = {
29 0x41, 0xff, 0x54, 0x24, -1,
30 };
31 if (MatchesPattern(pc, call_pattern, ARRAY_SIZE(call_pattern))) {
32 pc -= ARRAY_SIZE(call_pattern);
33 } else {
34 FATAL1("Failed to decode at %" Px, pc);
35 }
36
37 // movq CODE_REG, [PP + offset]
38 static int16_t load_code_disp8[] = {
39 0x4d, 0x8b, 0x67, -1, //
40 };
41 static int16_t load_code_disp32[] = {
42 0x4d, 0x8b, 0xa7, -1, -1, -1, -1,
43 };
44 if (MatchesPattern(pc, load_code_disp8, ARRAY_SIZE(load_code_disp8))) {
45 pc -= ARRAY_SIZE(load_code_disp8);
46 code_index_ = IndexFromPPLoadDisp8(pc + 3);
47 } else if (MatchesPattern(pc, load_code_disp32,
48 ARRAY_SIZE(load_code_disp32))) {
49 pc -= ARRAY_SIZE(load_code_disp32);
50 code_index_ = IndexFromPPLoadDisp32(pc + 3);
51 } else {
52 FATAL1("Failed to decode at %" Px, pc);
53 }
54 ASSERT(Object::Handle(object_pool_.ObjectAt(code_index_)).IsCode());
55
56 // movq RBX, [PP + offset]
57 static int16_t load_argument_disp8[] = {
58 0x49, 0x8b, 0x5f, -1, //
59 };
60 static int16_t load_argument_disp32[] = {
61 0x49, 0x8b, 0x9f, -1, -1, -1, -1,
62 };
63 if (MatchesPattern(pc, load_argument_disp8,
64 ARRAY_SIZE(load_argument_disp8))) {
65 pc -= ARRAY_SIZE(load_argument_disp8);
66 argument_index_ = IndexFromPPLoadDisp8(pc + 3);
67 } else if (MatchesPattern(pc, load_argument_disp32,
68 ARRAY_SIZE(load_argument_disp32))) {
69 pc -= ARRAY_SIZE(load_argument_disp32);
70 argument_index_ = IndexFromPPLoadDisp32(pc + 3);
71 } else {
72 FATAL1("Failed to decode at %" Px, pc);
73 }
74 }
75
76 intptr_t argument_index() const { return argument_index_; }
77
78 CodePtr target() const {
79 Code& code = Code::Handle();
80 code ^= object_pool_.ObjectAt(code_index_);
81 return code.raw();
82 }
83
84 void set_target(const Code& target) const {
85 object_pool_.SetObjectAt(code_index_, target);
86 // No need to flush the instruction cache, since the code is not modified.
87 }
88
89 protected:
90 const ObjectPool& object_pool_;
91 intptr_t code_index_;
92 intptr_t argument_index_;
93
94 private:
95 uword start_;
96 DISALLOW_IMPLICIT_CONSTRUCTORS(UnoptimizedCall);
97};
98
99class NativeCall : public UnoptimizedCall {
100 public:
101 NativeCall(uword return_address, const Code& code)
102 : UnoptimizedCall(return_address, code) {}
103
104 NativeFunction native_function() const {
105 return reinterpret_cast<NativeFunction>(
106 object_pool_.RawValueAt(argument_index()));
107 }
108
109 void set_native_function(NativeFunction func) const {
110 object_pool_.SetRawValueAt(argument_index(), reinterpret_cast<uword>(func));
111 }
112
113 private:
114 DISALLOW_IMPLICIT_CONSTRUCTORS(NativeCall);
115};
116
117class InstanceCall : public UnoptimizedCall {
118 public:
119 InstanceCall(uword return_address, const Code& code)
120 : UnoptimizedCall(return_address, code) {
121#if defined(DEBUG)
122 Object& test_data = Object::Handle(data());
123 ASSERT(test_data.IsArray() || test_data.IsICData() ||
124 test_data.IsMegamorphicCache());
125 if (test_data.IsICData()) {
126 ASSERT(ICData::Cast(test_data).NumArgsTested() > 0);
127 }
128#endif // DEBUG
129 }
130
131 ObjectPtr data() const { return object_pool_.ObjectAt(argument_index()); }
132 void set_data(const Object& data) const {
133 ASSERT(data.IsArray() || data.IsICData() || data.IsMegamorphicCache());
134 object_pool_.SetObjectAt(argument_index(), data);
135 }
136
137 private:
138 DISALLOW_IMPLICIT_CONSTRUCTORS(InstanceCall);
139};
140
141class UnoptimizedStaticCall : public UnoptimizedCall {
142 public:
143 UnoptimizedStaticCall(uword return_address, const Code& caller_code)
144 : UnoptimizedCall(return_address, caller_code) {
145#if defined(DEBUG)
146 ICData& test_ic_data = ICData::Handle();
147 test_ic_data ^= ic_data();
148 ASSERT(test_ic_data.NumArgsTested() >= 0);
149#endif // DEBUG
150 }
151
152 ObjectPtr ic_data() const { return object_pool_.ObjectAt(argument_index()); }
153
154 private:
155 DISALLOW_IMPLICIT_CONSTRUCTORS(UnoptimizedStaticCall);
156};
157
158// The expected pattern of a call where the target is loaded from
159// the object pool.
160class PoolPointerCall : public ValueObject {
161 public:
162 explicit PoolPointerCall(uword return_address, const Code& caller_code)
163 : object_pool_(ObjectPool::Handle(caller_code.GetObjectPool())),
164 code_index_(-1) {
165 uword pc = return_address;
166
167 // callq [CODE_REG + entry_point_offset]
168 static int16_t call_pattern[] = {
169 0x41, 0xff, 0x54, 0x24, -1,
170 };
171 if (MatchesPattern(pc, call_pattern, ARRAY_SIZE(call_pattern))) {
172 pc -= ARRAY_SIZE(call_pattern);
173 } else {
174 FATAL1("Failed to decode at %" Px, pc);
175 }
176
177 // movq CODE_REG, [PP + offset]
178 static int16_t load_code_disp8[] = {
179 0x4d, 0x8b, 0x67, -1, //
180 };
181 static int16_t load_code_disp32[] = {
182 0x4d, 0x8b, 0xa7, -1, -1, -1, -1,
183 };
184 if (MatchesPattern(pc, load_code_disp8, ARRAY_SIZE(load_code_disp8))) {
185 pc -= ARRAY_SIZE(load_code_disp8);
186 code_index_ = IndexFromPPLoadDisp8(pc + 3);
187 } else if (MatchesPattern(pc, load_code_disp32,
188 ARRAY_SIZE(load_code_disp32))) {
189 pc -= ARRAY_SIZE(load_code_disp32);
190 code_index_ = IndexFromPPLoadDisp32(pc + 3);
191 } else {
192 FATAL1("Failed to decode at %" Px, pc);
193 }
194 ASSERT(Object::Handle(object_pool_.ObjectAt(code_index_)).IsCode());
195 }
196
197 CodePtr Target() const {
198 Code& code = Code::Handle();
199 code ^= object_pool_.ObjectAt(code_index_);
200 return code.raw();
201 }
202
203 void SetTarget(const Code& target) const {
204 object_pool_.SetObjectAt(code_index_, target);
205 // No need to flush the instruction cache, since the code is not modified.
206 }
207
208 protected:
209 const ObjectPool& object_pool_;
210 intptr_t code_index_;
211
212 private:
213 DISALLOW_IMPLICIT_CONSTRUCTORS(PoolPointerCall);
214};
215
216// Instance call that can switch between a direct monomorphic call, an IC call,
217// and a megamorphic call.
218// load guarded cid load ICData load MegamorphicCache
219// load monomorphic target <-> load ICLookup stub -> load MMLookup stub
220// call target.entry call stub.entry call stub.entry
221class SwitchableCallBase : public ValueObject {
222 public:
223 explicit SwitchableCallBase(const Code& code)
224 : object_pool_(ObjectPool::Handle(code.GetObjectPool())),
225 target_index_(-1),
226 data_index_(-1) {}
227
228 intptr_t data_index() const { return data_index_; }
229 intptr_t target_index() const { return target_index_; }
230
231 ObjectPtr data() const { return object_pool_.ObjectAt(data_index()); }
232
233 void SetData(const Object& data) const {
234 ASSERT(!Object::Handle(object_pool_.ObjectAt(data_index())).IsCode());
235 object_pool_.SetObjectAt(data_index(), data);
236 // No need to flush the instruction cache, since the code is not modified.
237 }
238
239 protected:
240 ObjectPool& object_pool_;
241 intptr_t target_index_;
242 intptr_t data_index_;
243
244 private:
245 DISALLOW_IMPLICIT_CONSTRUCTORS(SwitchableCallBase);
246};
247
248// See [SwitchableCallBase] for a switchable calls in general.
249//
250// The target slot is always a [Code] object: Either the code of the
251// monomorphic function or a stub code.
252class SwitchableCall : public SwitchableCallBase {
253 public:
254 SwitchableCall(uword return_address, const Code& code)
255 : SwitchableCallBase(code) {
256 uword pc = return_address;
257
258 // callq RCX
259 static int16_t call_pattern[] = {
260 0xff, 0xd1, //
261 };
262 if (MatchesPattern(pc, call_pattern, ARRAY_SIZE(call_pattern))) {
263 pc -= ARRAY_SIZE(call_pattern);
264 } else {
265 FATAL1("Failed to decode at %" Px, pc);
266 }
267
268 // movq RBX, [PP + offset]
269 static int16_t load_data_disp8[] = {
270 0x49, 0x8b, 0x5f, -1, //
271 };
272 static int16_t load_data_disp32[] = {
273 0x49, 0x8b, 0x9f, -1, -1, -1, -1,
274 };
275 if (MatchesPattern(pc, load_data_disp8, ARRAY_SIZE(load_data_disp8))) {
276 pc -= ARRAY_SIZE(load_data_disp8);
277 data_index_ = IndexFromPPLoadDisp8(pc + 3);
278 } else if (MatchesPattern(pc, load_data_disp32,
279 ARRAY_SIZE(load_data_disp32))) {
280 pc -= ARRAY_SIZE(load_data_disp32);
281 data_index_ = IndexFromPPLoadDisp32(pc + 3);
282 } else {
283 FATAL1("Failed to decode at %" Px, pc);
284 }
285 ASSERT(!Object::Handle(object_pool_.ObjectAt(data_index_)).IsCode());
286
287 // movq rcx, [CODE_REG + entrypoint_offset]
288 static int16_t load_entry_pattern[] = {
289 0x49, 0x8b, 0x4c, 0x24, -1,
290 };
291 if (MatchesPattern(pc, load_entry_pattern,
292 ARRAY_SIZE(load_entry_pattern))) {
293 pc -= ARRAY_SIZE(load_entry_pattern);
294 } else {
295 FATAL1("Failed to decode at %" Px, pc);
296 }
297
298 // movq CODE_REG, [PP + offset]
299 static int16_t load_code_disp8[] = {
300 0x4d, 0x8b, 0x67, -1, //
301 };
302 static int16_t load_code_disp32[] = {
303 0x4d, 0x8b, 0xa7, -1, -1, -1, -1,
304 };
305 if (MatchesPattern(pc, load_code_disp8, ARRAY_SIZE(load_code_disp8))) {
306 pc -= ARRAY_SIZE(load_code_disp8);
307 target_index_ = IndexFromPPLoadDisp8(pc + 3);
308 } else if (MatchesPattern(pc, load_code_disp32,
309 ARRAY_SIZE(load_code_disp32))) {
310 pc -= ARRAY_SIZE(load_code_disp32);
311 target_index_ = IndexFromPPLoadDisp32(pc + 3);
312 } else {
313 FATAL1("Failed to decode at %" Px, pc);
314 }
315 ASSERT(Object::Handle(object_pool_.ObjectAt(target_index_)).IsCode());
316 }
317
318 void SetTarget(const Code& target) const {
319 ASSERT(Object::Handle(object_pool_.ObjectAt(target_index())).IsCode());
320 object_pool_.SetObjectAt(target_index(), target);
321 // No need to flush the instruction cache, since the code is not modified.
322 }
323
324 CodePtr target() const {
325 return static_cast<CodePtr>(object_pool_.ObjectAt(target_index()));
326 }
327};
328
329// See [SwitchableCallBase] for a switchable calls in general.
330//
331// The target slot is always a direct entrypoint address: Either the entry point
332// of the monomorphic function or a stub entry point.
333class BareSwitchableCall : public SwitchableCallBase {
334 public:
335 BareSwitchableCall(uword return_address, const Code& code)
336 : SwitchableCallBase(code) {
337 object_pool_ = ObjectPool::RawCast(
338 Isolate::Current()->object_store()->global_object_pool());
339
340 uword pc = return_address;
341
342 // callq RCX
343 static int16_t call_pattern[] = {
344 0xff, 0xd1, //
345 };
346 if (MatchesPattern(pc, call_pattern, ARRAY_SIZE(call_pattern))) {
347 pc -= ARRAY_SIZE(call_pattern);
348 } else {
349 FATAL1("Failed to decode at %" Px, pc);
350 }
351
352 // movq RBX, [PP + offset]
353 static int16_t load_data_disp8[] = {
354 0x49, 0x8b, 0x5f, -1, //
355 };
356 static int16_t load_data_disp32[] = {
357 0x49, 0x8b, 0x9f, -1, -1, -1, -1,
358 };
359 if (MatchesPattern(pc, load_data_disp8, ARRAY_SIZE(load_data_disp8))) {
360 pc -= ARRAY_SIZE(load_data_disp8);
361 data_index_ = IndexFromPPLoadDisp8(pc + 3);
362 } else if (MatchesPattern(pc, load_data_disp32,
363 ARRAY_SIZE(load_data_disp32))) {
364 pc -= ARRAY_SIZE(load_data_disp32);
365 data_index_ = IndexFromPPLoadDisp32(pc + 3);
366 } else {
367 FATAL1("Failed to decode at %" Px, pc);
368 }
369 ASSERT(!Object::Handle(object_pool_.ObjectAt(data_index_)).IsCode());
370
371 // movq RCX, [PP + offset]
372 static int16_t load_code_disp8[] = {
373 0x49, 0x8b, 0x4f, -1, //
374 };
375 static int16_t load_code_disp32[] = {
376 0x49, 0x8b, 0x8f, -1, -1, -1, -1,
377 };
378 if (MatchesPattern(pc, load_code_disp8, ARRAY_SIZE(load_code_disp8))) {
379 pc -= ARRAY_SIZE(load_code_disp8);
380 target_index_ = IndexFromPPLoadDisp8(pc + 3);
381 } else if (MatchesPattern(pc, load_code_disp32,
382 ARRAY_SIZE(load_code_disp32))) {
383 pc -= ARRAY_SIZE(load_code_disp32);
384 target_index_ = IndexFromPPLoadDisp32(pc + 3);
385 } else {
386 FATAL1("Failed to decode at %" Px, pc);
387 }
388 ASSERT(object_pool_.TypeAt(target_index_) ==
389 ObjectPool::EntryType::kImmediate);
390 }
391
392 void SetTarget(const Code& target) const {
393 ASSERT(object_pool_.TypeAt(target_index()) ==
394 ObjectPool::EntryType::kImmediate);
395 object_pool_.SetRawValueAt(target_index(), target.MonomorphicEntryPoint());
396 }
397
398 CodePtr target() const {
399 const uword pc = object_pool_.RawValueAt(target_index());
400 CodePtr result = ReversePc::Lookup(IsolateGroup::Current(), pc);
401 if (result != Code::null()) {
402 return result;
403 }
404 result = ReversePc::Lookup(Dart::vm_isolate()->group(), pc);
405 if (result != Code::null()) {
406 return result;
407 }
408 UNREACHABLE();
409 }
410};
411
412CodePtr CodePatcher::GetStaticCallTargetAt(uword return_address,
413 const Code& code) {
414 ASSERT(code.ContainsInstructionAt(return_address));
415 PoolPointerCall call(return_address, code);
416 return call.Target();
417}
418
419void CodePatcher::PatchStaticCallAt(uword return_address,
420 const Code& code,
421 const Code& new_target) {
422 PatchPoolPointerCallAt(return_address, code, new_target);
423}
424
425void CodePatcher::PatchPoolPointerCallAt(uword return_address,
426 const Code& code,
427 const Code& new_target) {
428 ASSERT(code.ContainsInstructionAt(return_address));
429 PoolPointerCall call(return_address, code);
430 call.SetTarget(new_target);
431}
432
433CodePtr CodePatcher::GetInstanceCallAt(uword return_address,
434 const Code& caller_code,
435 Object* data) {
436 ASSERT(caller_code.ContainsInstructionAt(return_address));
437 InstanceCall call(return_address, caller_code);
438 if (data != NULL) {
439 *data = call.data();
440 }
441 return call.target();
442}
443
444void CodePatcher::PatchInstanceCallAt(uword return_address,
445 const Code& caller_code,
446 const Object& data,
447 const Code& target) {
448 auto thread = Thread::Current();
449 thread->isolate_group()->RunWithStoppedMutators([&]() {
450 PatchInstanceCallAtWithMutatorsStopped(thread, return_address, caller_code,
451 data, target);
452 });
453}
454
455void CodePatcher::PatchInstanceCallAtWithMutatorsStopped(
456 Thread* thread,
457 uword return_address,
458 const Code& caller_code,
459 const Object& data,
460 const Code& target) {
461 ASSERT(caller_code.ContainsInstructionAt(return_address));
462 InstanceCall call(return_address, caller_code);
463 call.set_data(data);
464 call.set_target(target);
465}
466
467void CodePatcher::InsertDeoptimizationCallAt(uword start) {
468 UNREACHABLE();
469}
470
471FunctionPtr CodePatcher::GetUnoptimizedStaticCallAt(uword return_address,
472 const Code& caller_code,
473 ICData* ic_data_result) {
474 ASSERT(caller_code.ContainsInstructionAt(return_address));
475 UnoptimizedStaticCall static_call(return_address, caller_code);
476 ICData& ic_data = ICData::Handle();
477 ic_data ^= static_call.ic_data();
478 if (ic_data_result != NULL) {
479 *ic_data_result = ic_data.raw();
480 }
481 return ic_data.GetTargetAt(0);
482}
483
484void CodePatcher::PatchSwitchableCallAt(uword return_address,
485 const Code& caller_code,
486 const Object& data,
487 const Code& target) {
488 auto thread = Thread::Current();
489 // Ensure all threads are suspended as we update data and target pair.
490 thread->isolate_group()->RunWithStoppedMutators([&]() {
491 PatchSwitchableCallAtWithMutatorsStopped(thread, return_address,
492 caller_code, data, target);
493 });
494}
495
496void CodePatcher::PatchSwitchableCallAtWithMutatorsStopped(
497 Thread* thread,
498 uword return_address,
499 const Code& caller_code,
500 const Object& data,
501 const Code& target) {
502 ASSERT(caller_code.ContainsInstructionAt(return_address));
503 if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
504 BareSwitchableCall call(return_address, caller_code);
505 call.SetData(data);
506 call.SetTarget(target);
507 } else {
508 SwitchableCall call(return_address, caller_code);
509 call.SetData(data);
510 call.SetTarget(target);
511 }
512}
513
514CodePtr CodePatcher::GetSwitchableCallTargetAt(uword return_address,
515 const Code& caller_code) {
516 ASSERT(caller_code.ContainsInstructionAt(return_address));
517 if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
518 BareSwitchableCall call(return_address, caller_code);
519 return call.target();
520 } else {
521 SwitchableCall call(return_address, caller_code);
522 return call.target();
523 }
524}
525
526ObjectPtr CodePatcher::GetSwitchableCallDataAt(uword return_address,
527 const Code& caller_code) {
528 ASSERT(caller_code.ContainsInstructionAt(return_address));
529 if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
530 BareSwitchableCall call(return_address, caller_code);
531 return call.data();
532 } else {
533 SwitchableCall call(return_address, caller_code);
534 return call.data();
535 }
536}
537
538void CodePatcher::PatchNativeCallAt(uword return_address,
539 const Code& caller_code,
540 NativeFunction target,
541 const Code& trampoline) {
542 Thread::Current()->isolate_group()->RunWithStoppedMutators([&]() {
543 ASSERT(caller_code.ContainsInstructionAt(return_address));
544 NativeCall call(return_address, caller_code);
545 call.set_target(trampoline);
546 call.set_native_function(target);
547 });
548}
549
550CodePtr CodePatcher::GetNativeCallAt(uword return_address,
551 const Code& caller_code,
552 NativeFunction* target) {
553 ASSERT(caller_code.ContainsInstructionAt(return_address));
554 NativeCall call(return_address, caller_code);
555 *target = call.native_function();
556 return call.target();
557}
558
559} // namespace dart
560
561#endif // defined TARGET_ARCH_X64
562