1/*
2 * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24#ifndef SHARE_GC_Z_ZBARRIER_INLINE_HPP
25#define SHARE_GC_Z_ZBARRIER_INLINE_HPP
26
27#include "gc/z/zAddress.inline.hpp"
28#include "gc/z/zBarrier.hpp"
29#include "gc/z/zOop.inline.hpp"
30#include "gc/z/zResurrection.inline.hpp"
31#include "runtime/atomic.hpp"
32
33template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path>
34inline oop ZBarrier::barrier(volatile oop* p, oop o) {
35 uintptr_t addr = ZOop::to_address(o);
36
37retry:
38 // Fast path
39 if (fast_path(addr)) {
40 return ZOop::from_address(addr);
41 }
42
43 // Slow path
44 const uintptr_t good_addr = slow_path(addr);
45
46 // Self heal, but only if the address was actually updated by the slow path,
47 // which might not be the case, e.g. when marking through an already good oop.
48 if (p != NULL && good_addr != addr) {
49 const uintptr_t prev_addr = Atomic::cmpxchg(good_addr, (volatile uintptr_t*)p, addr);
50 if (prev_addr != addr) {
51 // Some other thread overwrote the oop. If this oop was updated by a
52 // weak barrier the new oop might not be good, in which case we need
53 // to re-apply this barrier.
54 addr = prev_addr;
55 goto retry;
56 }
57 }
58
59 return ZOop::from_address(good_addr);
60}
61
62template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path>
63inline oop ZBarrier::weak_barrier(volatile oop* p, oop o) {
64 const uintptr_t addr = ZOop::to_address(o);
65
66 // Fast path
67 if (fast_path(addr)) {
68 // Return the good address instead of the weak good address
69 // to ensure that the currently active heap view is used.
70 return ZOop::from_address(ZAddress::good_or_null(addr));
71 }
72
73 // Slow path
74 uintptr_t good_addr = slow_path(addr);
75
76 // Self heal unless the address returned from the slow path is null,
77 // in which case resurrection was blocked and we must let the reference
78 // processor clear the oop. Mutators are not allowed to clear oops in
79 // these cases, since that would be similar to calling Reference.clear(),
80 // which would make the reference non-discoverable or silently dropped
81 // by the reference processor.
82 if (p != NULL && good_addr != 0) {
83 // The slow path returns a good/marked address, but we never mark oops
84 // in a weak load barrier so we always self heal with the remapped address.
85 const uintptr_t weak_good_addr = ZAddress::remapped(good_addr);
86 const uintptr_t prev_addr = Atomic::cmpxchg(weak_good_addr, (volatile uintptr_t*)p, addr);
87 if (prev_addr != addr) {
88 // Some other thread overwrote the oop. The new
89 // oop is guaranteed to be weak good or null.
90 assert(ZAddress::is_weak_good_or_null(prev_addr), "Bad weak overwrite");
91
92 // Return the good address instead of the weak good address
93 // to ensure that the currently active heap view is used.
94 good_addr = ZAddress::good_or_null(prev_addr);
95 }
96 }
97
98 return ZOop::from_address(good_addr);
99}
100
101template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path>
102inline void ZBarrier::root_barrier(oop* p, oop o) {
103 const uintptr_t addr = ZOop::to_address(o);
104
105 // Fast path
106 if (fast_path(addr)) {
107 return;
108 }
109
110 // Slow path
111 const uintptr_t good_addr = slow_path(addr);
112
113 // Non-atomic healing helps speed up root scanning. This is safe to do
114 // since we are always healing roots in a safepoint, or under a lock,
115 // which ensures we are never racing with mutators modifying roots while
116 // we are healing them. It's also safe in case multiple GC threads try
117 // to heal the same root if it is aligned, since they would always heal
118 // the root in the same way and it does not matter in which order it
119 // happens. For misaligned oops, there needs to be mutual exclusion.
120 *p = ZOop::from_address(good_addr);
121}
122
123inline bool ZBarrier::is_null_fast_path(uintptr_t addr) {
124 return ZAddress::is_null(addr);
125}
126
127inline bool ZBarrier::is_good_or_null_fast_path(uintptr_t addr) {
128 return ZAddress::is_good_or_null(addr);
129}
130
131inline bool ZBarrier::is_weak_good_or_null_fast_path(uintptr_t addr) {
132 return ZAddress::is_weak_good_or_null(addr);
133}
134
135inline bool ZBarrier::is_resurrection_blocked(volatile oop* p, oop* o) {
136 const bool is_blocked = ZResurrection::is_blocked();
137
138 // Reload oop after checking the resurrection blocked state. This is
139 // done to prevent a race where we first load an oop, which is logically
140 // null but not yet cleared, then this oop is cleared by the reference
141 // processor and resurrection is unblocked. At this point the mutator
142 // would see the unblocked state and pass this invalid oop through the
143 // normal barrier path, which would incorrectly try to mark this oop.
144 if (p != NULL) {
145 // First assign to reloaded_o to avoid compiler warning about
146 // implicit dereference of volatile oop.
147 const oop reloaded_o = *p;
148 *o = reloaded_o;
149 }
150
151 return is_blocked;
152}
153
154//
155// Load barrier
156//
157inline oop ZBarrier::load_barrier_on_oop(oop o) {
158 return load_barrier_on_oop_field_preloaded((oop*)NULL, o);
159}
160
161inline oop ZBarrier::load_barrier_on_oop_field(volatile oop* p) {
162 const oop o = *p;
163 return load_barrier_on_oop_field_preloaded(p, o);
164}
165
166inline oop ZBarrier::load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) {
167 return barrier<is_good_or_null_fast_path, load_barrier_on_oop_slow_path>(p, o);
168}
169
170inline void ZBarrier::load_barrier_on_oop_array(volatile oop* p, size_t length) {
171 for (volatile const oop* const end = p + length; p < end; p++) {
172 load_barrier_on_oop_field(p);
173 }
174}
175
176inline oop ZBarrier::load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o) {
177 if (is_resurrection_blocked(p, &o)) {
178 return weak_barrier<is_good_or_null_fast_path, weak_load_barrier_on_weak_oop_slow_path>(p, o);
179 }
180
181 return load_barrier_on_oop_field_preloaded(p, o);
182}
183
184inline oop ZBarrier::load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) {
185 if (is_resurrection_blocked(p, &o)) {
186 return weak_barrier<is_good_or_null_fast_path, weak_load_barrier_on_phantom_oop_slow_path>(p, o);
187 }
188
189 return load_barrier_on_oop_field_preloaded(p, o);
190}
191
192inline void ZBarrier::load_barrier_on_root_oop_field(oop* p) {
193 const oop o = *p;
194 root_barrier<is_good_or_null_fast_path, load_barrier_on_oop_slow_path>(p, o);
195}
196
197//
198// Weak load barrier
199//
200inline oop ZBarrier::weak_load_barrier_on_oop_field(volatile oop* p) {
201 assert(!ZResurrection::is_blocked(), "Should not be called during resurrection blocked phase");
202 const oop o = *p;
203 return weak_load_barrier_on_oop_field_preloaded(p, o);
204}
205
206inline oop ZBarrier::weak_load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) {
207 return weak_barrier<is_weak_good_or_null_fast_path, weak_load_barrier_on_oop_slow_path>(p, o);
208}
209
210inline oop ZBarrier::weak_load_barrier_on_weak_oop(oop o) {
211 return weak_load_barrier_on_weak_oop_field_preloaded((oop*)NULL, o);
212}
213
214inline oop ZBarrier::weak_load_barrier_on_weak_oop_field(volatile oop* p) {
215 const oop o = *p;
216 return weak_load_barrier_on_weak_oop_field_preloaded(p, o);
217}
218
219inline oop ZBarrier::weak_load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o) {
220 if (is_resurrection_blocked(p, &o)) {
221 return weak_barrier<is_good_or_null_fast_path, weak_load_barrier_on_weak_oop_slow_path>(p, o);
222 }
223
224 return weak_load_barrier_on_oop_field_preloaded(p, o);
225}
226
227inline oop ZBarrier::weak_load_barrier_on_phantom_oop(oop o) {
228 return weak_load_barrier_on_phantom_oop_field_preloaded((oop*)NULL, o);
229}
230
231inline oop ZBarrier::weak_load_barrier_on_phantom_oop_field(volatile oop* p) {
232 const oop o = *p;
233 return weak_load_barrier_on_phantom_oop_field_preloaded(p, o);
234}
235
236inline oop ZBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) {
237 if (is_resurrection_blocked(p, &o)) {
238 return weak_barrier<is_good_or_null_fast_path, weak_load_barrier_on_phantom_oop_slow_path>(p, o);
239 }
240
241 return weak_load_barrier_on_oop_field_preloaded(p, o);
242}
243
244//
245// Is alive barrier
246//
247inline bool ZBarrier::is_alive_barrier_on_weak_oop(oop o) {
248 // Check if oop is logically non-null. This operation
249 // is only valid when resurrection is blocked.
250 assert(ZResurrection::is_blocked(), "Invalid phase");
251 return weak_load_barrier_on_weak_oop(o) != NULL;
252}
253
254inline bool ZBarrier::is_alive_barrier_on_phantom_oop(oop o) {
255 // Check if oop is logically non-null. This operation
256 // is only valid when resurrection is blocked.
257 assert(ZResurrection::is_blocked(), "Invalid phase");
258 return weak_load_barrier_on_phantom_oop(o) != NULL;
259}
260
261//
262// Keep alive barrier
263//
264inline void ZBarrier::keep_alive_barrier_on_weak_oop_field(volatile oop* p) {
265 // This operation is only valid when resurrection is blocked.
266 assert(ZResurrection::is_blocked(), "Invalid phase");
267 const oop o = *p;
268 barrier<is_good_or_null_fast_path, keep_alive_barrier_on_weak_oop_slow_path>(p, o);
269}
270
271inline void ZBarrier::keep_alive_barrier_on_phantom_oop_field(volatile oop* p) {
272 // This operation is only valid when resurrection is blocked.
273 assert(ZResurrection::is_blocked(), "Invalid phase");
274 const oop o = *p;
275 barrier<is_good_or_null_fast_path, keep_alive_barrier_on_phantom_oop_slow_path>(p, o);
276}
277
278inline void ZBarrier::keep_alive_barrier_on_phantom_root_oop_field(oop* p) {
279 // This operation is only valid when resurrection is blocked.
280 assert(ZResurrection::is_blocked(), "Invalid phase");
281 const oop o = *p;
282 root_barrier<is_good_or_null_fast_path, keep_alive_barrier_on_phantom_oop_slow_path>(p, o);
283}
284
285//
286// Mark barrier
287//
288inline void ZBarrier::mark_barrier_on_oop_field(volatile oop* p, bool finalizable) {
289 // The fast path only checks for null since the GC worker
290 // threads doing marking wants to mark through good oops.
291 const oop o = *p;
292
293 if (finalizable) {
294 barrier<is_null_fast_path, mark_barrier_on_finalizable_oop_slow_path>(p, o);
295 } else {
296 barrier<is_null_fast_path, mark_barrier_on_oop_slow_path>(p, o);
297 }
298}
299
300inline void ZBarrier::mark_barrier_on_oop_array(volatile oop* p, size_t length, bool finalizable) {
301 for (volatile const oop* const end = p + length; p < end; p++) {
302 mark_barrier_on_oop_field(p, finalizable);
303 }
304}
305
306inline void ZBarrier::mark_barrier_on_root_oop_field(oop* p) {
307 const oop o = *p;
308 root_barrier<is_good_or_null_fast_path, mark_barrier_on_root_oop_slow_path>(p, o);
309}
310
311//
312// Relocate barrier
313//
314inline void ZBarrier::relocate_barrier_on_root_oop_field(oop* p) {
315 const oop o = *p;
316 root_barrier<is_good_or_null_fast_path, relocate_barrier_on_root_oop_slow_path>(p, o);
317}
318
319#endif // SHARE_GC_Z_ZBARRIER_INLINE_HPP
320