1/*
2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25#include "precompiled.hpp"
26#include "gc/shared/barrierSet.hpp"
27#include "gc/shared/barrierSetAssembler.hpp"
28#include "gc/shared/barrierSetNMethod.hpp"
29#include "gc/shared/collectedHeap.hpp"
30#include "interpreter/interp_masm.hpp"
31#include "memory/universe.hpp"
32#include "runtime/jniHandles.hpp"
33#include "runtime/sharedRuntime.hpp"
34#include "runtime/thread.hpp"
35
36#define __ masm->
37
38void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
39 Register dst, Address src, Register tmp1, Register tmp_thread) {
40 bool in_heap = (decorators & IN_HEAP) != 0;
41 bool in_native = (decorators & IN_NATIVE) != 0;
42 bool is_not_null = (decorators & IS_NOT_NULL) != 0;
43 bool atomic = (decorators & MO_RELAXED) != 0;
44
45 switch (type) {
46 case T_OBJECT:
47 case T_ARRAY: {
48 if (in_heap) {
49#ifdef _LP64
50 if (UseCompressedOops) {
51 __ movl(dst, src);
52 if (is_not_null) {
53 __ decode_heap_oop_not_null(dst);
54 } else {
55 __ decode_heap_oop(dst);
56 }
57 } else
58#endif
59 {
60 __ movptr(dst, src);
61 }
62 } else {
63 assert(in_native, "why else?");
64 __ movptr(dst, src);
65 }
66 break;
67 }
68 case T_BOOLEAN: __ load_unsigned_byte(dst, src); break;
69 case T_BYTE: __ load_signed_byte(dst, src); break;
70 case T_CHAR: __ load_unsigned_short(dst, src); break;
71 case T_SHORT: __ load_signed_short(dst, src); break;
72 case T_INT: __ movl (dst, src); break;
73 case T_ADDRESS: __ movptr(dst, src); break;
74 case T_FLOAT:
75 assert(dst == noreg, "only to ftos");
76 __ load_float(src);
77 break;
78 case T_DOUBLE:
79 assert(dst == noreg, "only to dtos");
80 __ load_double(src);
81 break;
82 case T_LONG:
83 assert(dst == noreg, "only to ltos");
84#ifdef _LP64
85 __ movq(rax, src);
86#else
87 if (atomic) {
88 __ fild_d(src); // Must load atomically
89 __ subptr(rsp,2*wordSize); // Make space for store
90 __ fistp_d(Address(rsp,0));
91 __ pop(rax);
92 __ pop(rdx);
93 } else {
94 __ movl(rax, src);
95 __ movl(rdx, src.plus_disp(wordSize));
96 }
97#endif
98 break;
99 default: Unimplemented();
100 }
101}
102
103void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
104 Address dst, Register val, Register tmp1, Register tmp2) {
105 bool in_heap = (decorators & IN_HEAP) != 0;
106 bool in_native = (decorators & IN_NATIVE) != 0;
107 bool is_not_null = (decorators & IS_NOT_NULL) != 0;
108 bool atomic = (decorators & MO_RELAXED) != 0;
109
110 switch (type) {
111 case T_OBJECT:
112 case T_ARRAY: {
113 if (in_heap) {
114 if (val == noreg) {
115 assert(!is_not_null, "inconsistent access");
116#ifdef _LP64
117 if (UseCompressedOops) {
118 __ movl(dst, (int32_t)NULL_WORD);
119 } else {
120 __ movslq(dst, (int32_t)NULL_WORD);
121 }
122#else
123 __ movl(dst, (int32_t)NULL_WORD);
124#endif
125 } else {
126#ifdef _LP64
127 if (UseCompressedOops) {
128 assert(!dst.uses(val), "not enough registers");
129 if (is_not_null) {
130 __ encode_heap_oop_not_null(val);
131 } else {
132 __ encode_heap_oop(val);
133 }
134 __ movl(dst, val);
135 } else
136#endif
137 {
138 __ movptr(dst, val);
139 }
140 }
141 } else {
142 assert(in_native, "why else?");
143 assert(val != noreg, "not supported");
144 __ movptr(dst, val);
145 }
146 break;
147 }
148 case T_BOOLEAN:
149 __ andl(val, 0x1); // boolean is true if LSB is 1
150 __ movb(dst, val);
151 break;
152 case T_BYTE:
153 __ movb(dst, val);
154 break;
155 case T_SHORT:
156 __ movw(dst, val);
157 break;
158 case T_CHAR:
159 __ movw(dst, val);
160 break;
161 case T_INT:
162 __ movl(dst, val);
163 break;
164 case T_LONG:
165 assert(val == noreg, "only tos");
166#ifdef _LP64
167 __ movq(dst, rax);
168#else
169 if (atomic) {
170 __ push(rdx);
171 __ push(rax); // Must update atomically with FIST
172 __ fild_d(Address(rsp,0)); // So load into FPU register
173 __ fistp_d(dst); // and put into memory atomically
174 __ addptr(rsp, 2*wordSize);
175 } else {
176 __ movptr(dst, rax);
177 __ movptr(dst.plus_disp(wordSize), rdx);
178 }
179#endif
180 break;
181 case T_FLOAT:
182 assert(val == noreg, "only tos");
183 __ store_float(dst);
184 break;
185 case T_DOUBLE:
186 assert(val == noreg, "only tos");
187 __ store_double(dst);
188 break;
189 case T_ADDRESS:
190 __ movptr(dst, val);
191 break;
192 default: Unimplemented();
193 }
194}
195
196#ifndef _LP64
197void BarrierSetAssembler::obj_equals(MacroAssembler* masm,
198 Address obj1, jobject obj2) {
199 __ cmpoop_raw(obj1, obj2);
200}
201
202void BarrierSetAssembler::obj_equals(MacroAssembler* masm,
203 Register obj1, jobject obj2) {
204 __ cmpoop_raw(obj1, obj2);
205}
206#endif
207void BarrierSetAssembler::obj_equals(MacroAssembler* masm,
208 Register obj1, Address obj2) {
209 __ cmpptr(obj1, obj2);
210}
211
212void BarrierSetAssembler::obj_equals(MacroAssembler* masm,
213 Register obj1, Register obj2) {
214 __ cmpptr(obj1, obj2);
215}
216
217void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
218 Register obj, Register tmp, Label& slowpath) {
219 __ clear_jweak_tag(obj);
220 __ movptr(obj, Address(obj, 0));
221}
222
223void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm,
224 Register thread, Register obj,
225 Register var_size_in_bytes,
226 int con_size_in_bytes,
227 Register t1,
228 Register t2,
229 Label& slow_case) {
230 assert_different_registers(obj, t1, t2);
231 assert_different_registers(obj, var_size_in_bytes, t1);
232 Register end = t2;
233 if (!thread->is_valid()) {
234#ifdef _LP64
235 thread = r15_thread;
236#else
237 assert(t1->is_valid(), "need temp reg");
238 thread = t1;
239 __ get_thread(thread);
240#endif
241 }
242
243 __ verify_tlab();
244
245 __ movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
246 if (var_size_in_bytes == noreg) {
247 __ lea(end, Address(obj, con_size_in_bytes));
248 } else {
249 __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
250 }
251 __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
252 __ jcc(Assembler::above, slow_case);
253
254 // update the tlab top pointer
255 __ movptr(Address(thread, JavaThread::tlab_top_offset()), end);
256
257 // recover var_size_in_bytes if necessary
258 if (var_size_in_bytes == end) {
259 __ subptr(var_size_in_bytes, obj);
260 }
261 __ verify_tlab();
262}
263
264// Defines obj, preserves var_size_in_bytes
265void BarrierSetAssembler::eden_allocate(MacroAssembler* masm,
266 Register thread, Register obj,
267 Register var_size_in_bytes,
268 int con_size_in_bytes,
269 Register t1,
270 Label& slow_case) {
271 assert(obj == rax, "obj must be in rax, for cmpxchg");
272 assert_different_registers(obj, var_size_in_bytes, t1);
273 if (!Universe::heap()->supports_inline_contig_alloc()) {
274 __ jmp(slow_case);
275 } else {
276 Register end = t1;
277 Label retry;
278 __ bind(retry);
279 ExternalAddress heap_top((address) Universe::heap()->top_addr());
280 __ movptr(obj, heap_top);
281 if (var_size_in_bytes == noreg) {
282 __ lea(end, Address(obj, con_size_in_bytes));
283 } else {
284 __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
285 }
286 // if end < obj then we wrapped around => object too long => slow case
287 __ cmpptr(end, obj);
288 __ jcc(Assembler::below, slow_case);
289 __ cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr()));
290 __ jcc(Assembler::above, slow_case);
291 // Compare obj with the top addr, and if still equal, store the new top addr in
292 // end at the address of the top addr pointer. Sets ZF if was equal, and clears
293 // it otherwise. Use lock prefix for atomicity on MPs.
294 __ locked_cmpxchgptr(end, heap_top);
295 __ jcc(Assembler::notEqual, retry);
296 incr_allocated_bytes(masm, thread, var_size_in_bytes, con_size_in_bytes, thread->is_valid() ? noreg : t1);
297 }
298}
299
300void BarrierSetAssembler::incr_allocated_bytes(MacroAssembler* masm, Register thread,
301 Register var_size_in_bytes,
302 int con_size_in_bytes,
303 Register t1) {
304 if (!thread->is_valid()) {
305#ifdef _LP64
306 thread = r15_thread;
307#else
308 assert(t1->is_valid(), "need temp reg");
309 thread = t1;
310 __ get_thread(thread);
311#endif
312 }
313
314#ifdef _LP64
315 if (var_size_in_bytes->is_valid()) {
316 __ addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
317 } else {
318 __ addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
319 }
320#else
321 if (var_size_in_bytes->is_valid()) {
322 __ addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
323 } else {
324 __ addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
325 }
326 __ adcl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())+4), 0);
327#endif
328}
329
330void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) {
331 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
332 if (bs_nm == NULL) {
333 return;
334 }
335#ifndef _LP64
336 ShouldNotReachHere();
337#else
338 Label continuation;
339 Register thread = LP64_ONLY(r15_thread);
340 Address disarmed_addr(thread, in_bytes(bs_nm->thread_disarmed_offset()));
341 __ align(8);
342 __ cmpl(disarmed_addr, 0);
343 __ jcc(Assembler::equal, continuation);
344 __ call(RuntimeAddress(StubRoutines::x86::method_entry_barrier()));
345 __ bind(continuation);
346#endif
347}
348
349void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
350 BarrierSetNMethod* bs = BarrierSet::barrier_set()->barrier_set_nmethod();
351 if (bs == NULL) {
352 return;
353 }
354
355 Label bad_call;
356 __ cmpptr(rbx, 0); // rbx contains the incoming method for c2i adapters.
357 __ jcc(Assembler::equal, bad_call);
358
359 // Pointer chase to the method holder to find out if the method is concurrently unloading.
360 Label method_live;
361 __ load_method_holder_cld(rscratch1, rbx);
362
363 // Is it a strong CLD?
364 __ movl(rscratch2, Address(rscratch1, ClassLoaderData::keep_alive_offset()));
365 __ cmpptr(rscratch2, 0);
366 __ jcc(Assembler::greater, method_live);
367
368 // Is it a weak but alive CLD?
369 __ movptr(rscratch1, Address(rscratch1, ClassLoaderData::holder_offset()));
370 __ resolve_weak_handle(rscratch1, rscratch2);
371 __ cmpptr(rscratch1, 0);
372 __ jcc(Assembler::notEqual, method_live);
373
374 __ bind(bad_call);
375 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
376 __ bind(method_live);
377}
378