1/*
2 * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24#include "precompiled.hpp"
25#include "gc/z/zAddress.inline.hpp"
26#include "gc/z/zBarrier.inline.hpp"
27#include "gc/z/zForwarding.inline.hpp"
28#include "gc/z/zHeap.hpp"
29#include "gc/z/zOopClosures.inline.hpp"
30#include "gc/z/zPage.hpp"
31#include "gc/z/zRelocate.hpp"
32#include "gc/z/zRelocationSet.inline.hpp"
33#include "gc/z/zRootsIterator.hpp"
34#include "gc/z/zStat.hpp"
35#include "gc/z/zTask.hpp"
36#include "gc/z/zThreadLocalAllocBuffer.hpp"
37#include "gc/z/zWorkers.hpp"
38#include "logging/log.hpp"
39
40static const ZStatCounter ZCounterRelocationContention("Contention", "Relocation Contention", ZStatUnitOpsPerSecond);
41
42ZRelocate::ZRelocate(ZWorkers* workers) :
43 _workers(workers) {}
44
45class ZRelocateRootsIteratorClosure : public ZRootsIteratorClosure {
46public:
47 virtual void do_thread(Thread* thread) {
48 // Update thread local address bad mask
49 ZThreadLocalData::set_address_bad_mask(thread, ZAddressBadMask);
50
51 // Remap TLAB
52 ZThreadLocalAllocBuffer::remap(thread);
53 }
54
55 virtual void do_oop(oop* p) {
56 ZBarrier::relocate_barrier_on_root_oop_field(p);
57 }
58
59 virtual void do_oop(narrowOop* p) {
60 ShouldNotReachHere();
61 }
62};
63
64class ZRelocateRootsTask : public ZTask {
65private:
66 ZRootsIterator _roots;
67 ZRelocateRootsIteratorClosure _cl;
68
69public:
70 ZRelocateRootsTask() :
71 ZTask("ZRelocateRootsTask"),
72 _roots() {}
73
74 virtual void work() {
75 // During relocation we need to visit the JVMTI
76 // export weak roots to rehash the JVMTI tag map
77 _roots.oops_do(&_cl, true /* visit_jvmti_weak_export */);
78 }
79};
80
81void ZRelocate::start() {
82 ZRelocateRootsTask task;
83 _workers->run_parallel(&task);
84}
85
86uintptr_t ZRelocate::relocate_object_inner(ZForwarding* forwarding, uintptr_t from_index, uintptr_t from_offset) const {
87 ZForwardingCursor cursor;
88
89 // Lookup forwarding entry
90 const ZForwardingEntry entry = forwarding->find(from_index, &cursor);
91 if (entry.populated() && entry.from_index() == from_index) {
92 // Already relocated, return new address
93 return entry.to_offset();
94 }
95
96 assert(ZHeap::heap()->is_object_live(ZAddress::good(from_offset)), "Should be live");
97
98 if (forwarding->is_pinned()) {
99 // In-place forward
100 return forwarding->insert(from_index, from_offset, &cursor);
101 }
102
103 // Allocate object
104 const uintptr_t from_good = ZAddress::good(from_offset);
105 const size_t size = ZUtils::object_size(from_good);
106 const uintptr_t to_good = ZHeap::heap()->alloc_object_for_relocation(size);
107 if (to_good == 0) {
108 // Failed, in-place forward
109 return forwarding->insert(from_index, from_offset, &cursor);
110 }
111
112 // Copy object
113 ZUtils::object_copy(from_good, to_good, size);
114
115 // Insert forwarding entry
116 const uintptr_t to_offset = ZAddress::offset(to_good);
117 const uintptr_t to_offset_final = forwarding->insert(from_index, to_offset, &cursor);
118 if (to_offset_final == to_offset) {
119 // Relocation succeeded
120 return to_offset;
121 }
122
123 // Relocation contention
124 ZStatInc(ZCounterRelocationContention);
125 log_trace(gc)("Relocation contention, thread: " PTR_FORMAT " (%s), forwarding: " PTR_FORMAT
126 ", entry: " UINT32_FORMAT ", oop: " PTR_FORMAT ", size: " SIZE_FORMAT,
127 ZThread::id(), ZThread::name(), p2i(forwarding), cursor, from_good, size);
128
129 // Try undo allocation
130 ZHeap::heap()->undo_alloc_object_for_relocation(to_good, size);
131
132 return to_offset_final;
133}
134
135uintptr_t ZRelocate::relocate_object(ZForwarding* forwarding, uintptr_t from_addr) const {
136 const uintptr_t from_offset = ZAddress::offset(from_addr);
137 const uintptr_t from_index = (from_offset - forwarding->start()) >> forwarding->object_alignment_shift();
138 const uintptr_t to_offset = relocate_object_inner(forwarding, from_index, from_offset);
139
140 if (from_offset == to_offset) {
141 // In-place forwarding, pin page
142 forwarding->set_pinned();
143 }
144
145 return ZAddress::good(to_offset);
146}
147
148uintptr_t ZRelocate::forward_object(ZForwarding* forwarding, uintptr_t from_addr) const {
149 const uintptr_t from_offset = ZAddress::offset(from_addr);
150 const uintptr_t from_index = (from_offset - forwarding->start()) >> forwarding->object_alignment_shift();
151 const ZForwardingEntry entry = forwarding->find(from_index);
152
153 assert(entry.populated(), "Should be forwarded");
154 assert(entry.from_index() == from_index, "Should be forwarded");
155
156 return ZAddress::good(entry.to_offset());
157}
158
159class ZRelocateObjectClosure : public ObjectClosure {
160private:
161 ZRelocate* const _relocate;
162 ZForwarding* const _forwarding;
163
164public:
165 ZRelocateObjectClosure(ZRelocate* relocate, ZForwarding* forwarding) :
166 _relocate(relocate),
167 _forwarding(forwarding) {}
168
169 virtual void do_object(oop o) {
170 _relocate->relocate_object(_forwarding, ZOop::to_address(o));
171 }
172};
173
174bool ZRelocate::work(ZRelocationSetParallelIterator* iter) {
175 bool success = true;
176
177 // Relocate pages in the relocation set
178 for (ZForwarding* forwarding; iter->next(&forwarding);) {
179 // Relocate objects in page
180 ZRelocateObjectClosure cl(this, forwarding);
181 forwarding->page()->object_iterate(&cl);
182
183 if (ZVerifyForwarding) {
184 forwarding->verify();
185 }
186
187 if (forwarding->is_pinned()) {
188 // Relocation failed, page is now pinned
189 success = false;
190 } else {
191 // Relocation succeeded, release page
192 forwarding->release_page();
193 }
194 }
195
196 return success;
197}
198
199class ZRelocateTask : public ZTask {
200private:
201 ZRelocate* const _relocate;
202 ZRelocationSetParallelIterator _iter;
203 bool _failed;
204
205public:
206 ZRelocateTask(ZRelocate* relocate, ZRelocationSet* relocation_set) :
207 ZTask("ZRelocateTask"),
208 _relocate(relocate),
209 _iter(relocation_set),
210 _failed(false) {}
211
212 virtual void work() {
213 if (!_relocate->work(&_iter)) {
214 _failed = true;
215 }
216 }
217
218 bool failed() const {
219 return _failed;
220 }
221};
222
223bool ZRelocate::relocate(ZRelocationSet* relocation_set) {
224 ZRelocateTask task(this, relocation_set);
225 _workers->run_concurrent(&task);
226 return !task.failed();
227}
228