1/*
2 * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25#ifndef SHARE_GC_CMS_CONCURRENTMARKSWEEPGENERATION_INLINE_HPP
26#define SHARE_GC_CMS_CONCURRENTMARKSWEEPGENERATION_INLINE_HPP
27
28#include "gc/cms/cmsHeap.hpp"
29#include "gc/cms/cmsLockVerifier.hpp"
30#include "gc/cms/compactibleFreeListSpace.inline.hpp"
31#include "gc/cms/concurrentMarkSweepGeneration.hpp"
32#include "gc/cms/concurrentMarkSweepThread.hpp"
33#include "gc/cms/parNewGeneration.hpp"
34#include "gc/shared/gcUtil.hpp"
35#include "utilities/align.hpp"
36#include "utilities/bitMap.inline.hpp"
37
38inline void CMSBitMap::clear_all() {
39 assert_locked();
40 // CMS bitmaps are usually cover large memory regions
41 _bm.clear_large();
42 return;
43}
44
45inline size_t CMSBitMap::heapWordToOffset(HeapWord* addr) const {
46 return (pointer_delta(addr, _bmStartWord)) >> _shifter;
47}
48
49inline HeapWord* CMSBitMap::offsetToHeapWord(size_t offset) const {
50 return _bmStartWord + (offset << _shifter);
51}
52
53inline size_t CMSBitMap::heapWordDiffToOffsetDiff(size_t diff) const {
54 assert((diff & ((1 << _shifter) - 1)) == 0, "argument check");
55 return diff >> _shifter;
56}
57
58inline void CMSBitMap::mark(HeapWord* addr) {
59 assert_locked();
60 assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize),
61 "outside underlying space?");
62 _bm.set_bit(heapWordToOffset(addr));
63}
64
65inline bool CMSBitMap::par_mark(HeapWord* addr) {
66 assert_locked();
67 assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize),
68 "outside underlying space?");
69 return _bm.par_at_put(heapWordToOffset(addr), true);
70}
71
72inline void CMSBitMap::par_clear(HeapWord* addr) {
73 assert_locked();
74 assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize),
75 "outside underlying space?");
76 _bm.par_at_put(heapWordToOffset(addr), false);
77}
78
79inline void CMSBitMap::mark_range(MemRegion mr) {
80 NOT_PRODUCT(region_invariant(mr));
81 // Range size is usually just 1 bit.
82 _bm.set_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()),
83 BitMap::small_range);
84}
85
86inline void CMSBitMap::clear_range(MemRegion mr) {
87 NOT_PRODUCT(region_invariant(mr));
88 // Range size is usually just 1 bit.
89 _bm.clear_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()),
90 BitMap::small_range);
91}
92
93inline void CMSBitMap::par_mark_range(MemRegion mr) {
94 NOT_PRODUCT(region_invariant(mr));
95 // Range size is usually just 1 bit.
96 _bm.par_set_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()),
97 BitMap::small_range);
98}
99
100inline void CMSBitMap::par_clear_range(MemRegion mr) {
101 NOT_PRODUCT(region_invariant(mr));
102 // Range size is usually just 1 bit.
103 _bm.par_clear_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()),
104 BitMap::small_range);
105}
106
107inline void CMSBitMap::mark_large_range(MemRegion mr) {
108 NOT_PRODUCT(region_invariant(mr));
109 // Range size must be greater than 32 bytes.
110 _bm.set_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()),
111 BitMap::large_range);
112}
113
114inline void CMSBitMap::clear_large_range(MemRegion mr) {
115 NOT_PRODUCT(region_invariant(mr));
116 // Range size must be greater than 32 bytes.
117 _bm.clear_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()),
118 BitMap::large_range);
119}
120
121inline void CMSBitMap::par_mark_large_range(MemRegion mr) {
122 NOT_PRODUCT(region_invariant(mr));
123 // Range size must be greater than 32 bytes.
124 _bm.par_set_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()),
125 BitMap::large_range);
126}
127
128inline void CMSBitMap::par_clear_large_range(MemRegion mr) {
129 NOT_PRODUCT(region_invariant(mr));
130 // Range size must be greater than 32 bytes.
131 _bm.par_clear_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()),
132 BitMap::large_range);
133}
134
135// Starting at "addr" (inclusive) return a memory region
136// corresponding to the first maximally contiguous marked ("1") region.
137inline MemRegion CMSBitMap::getAndClearMarkedRegion(HeapWord* addr) {
138 return getAndClearMarkedRegion(addr, endWord());
139}
140
141// Starting at "start_addr" (inclusive) return a memory region
142// corresponding to the first maximal contiguous marked ("1") region
143// strictly less than end_addr.
144inline MemRegion CMSBitMap::getAndClearMarkedRegion(HeapWord* start_addr,
145 HeapWord* end_addr) {
146 HeapWord *start, *end;
147 assert_locked();
148 start = getNextMarkedWordAddress (start_addr, end_addr);
149 end = getNextUnmarkedWordAddress(start, end_addr);
150 assert(start <= end, "Consistency check");
151 MemRegion mr(start, end);
152 if (!mr.is_empty()) {
153 clear_range(mr);
154 }
155 return mr;
156}
157
158inline bool CMSBitMap::isMarked(HeapWord* addr) const {
159 assert_locked();
160 assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize),
161 "outside underlying space?");
162 return _bm.at(heapWordToOffset(addr));
163}
164
165// The same as isMarked() but without a lock check.
166inline bool CMSBitMap::par_isMarked(HeapWord* addr) const {
167 assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize),
168 "outside underlying space?");
169 return _bm.at(heapWordToOffset(addr));
170}
171
172
173inline bool CMSBitMap::isUnmarked(HeapWord* addr) const {
174 assert_locked();
175 assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize),
176 "outside underlying space?");
177 return !_bm.at(heapWordToOffset(addr));
178}
179
180// Return the HeapWord address corresponding to next "1" bit
181// (inclusive).
182inline HeapWord* CMSBitMap::getNextMarkedWordAddress(HeapWord* addr) const {
183 return getNextMarkedWordAddress(addr, endWord());
184}
185
186// Return the least HeapWord address corresponding to next "1" bit
187// starting at start_addr (inclusive) but strictly less than end_addr.
188inline HeapWord* CMSBitMap::getNextMarkedWordAddress(
189 HeapWord* start_addr, HeapWord* end_addr) const {
190 assert_locked();
191 size_t nextOffset = _bm.get_next_one_offset(
192 heapWordToOffset(start_addr),
193 heapWordToOffset(end_addr));
194 HeapWord* nextAddr = offsetToHeapWord(nextOffset);
195 assert(nextAddr >= start_addr &&
196 nextAddr <= end_addr, "get_next_one postcondition");
197 assert((nextAddr == end_addr) ||
198 isMarked(nextAddr), "get_next_one postcondition");
199 return nextAddr;
200}
201
202
203// Return the HeapWord address corresponding to the next "0" bit
204// (inclusive).
205inline HeapWord* CMSBitMap::getNextUnmarkedWordAddress(HeapWord* addr) const {
206 return getNextUnmarkedWordAddress(addr, endWord());
207}
208
209// Return the HeapWord address corresponding to the next "0" bit
210// (inclusive).
211inline HeapWord* CMSBitMap::getNextUnmarkedWordAddress(
212 HeapWord* start_addr, HeapWord* end_addr) const {
213 assert_locked();
214 size_t nextOffset = _bm.get_next_zero_offset(
215 heapWordToOffset(start_addr),
216 heapWordToOffset(end_addr));
217 HeapWord* nextAddr = offsetToHeapWord(nextOffset);
218 assert(nextAddr >= start_addr &&
219 nextAddr <= end_addr, "get_next_zero postcondition");
220 assert((nextAddr == end_addr) ||
221 isUnmarked(nextAddr), "get_next_zero postcondition");
222 return nextAddr;
223}
224
225inline bool CMSBitMap::isAllClear() const {
226 assert_locked();
227 return getNextMarkedWordAddress(startWord()) >= endWord();
228}
229
230inline void CMSBitMap::iterate(BitMapClosure* cl, HeapWord* left,
231 HeapWord* right) {
232 assert_locked();
233 left = MAX2(_bmStartWord, left);
234 right = MIN2(_bmStartWord + _bmWordSize, right);
235 if (right > left) {
236 _bm.iterate(cl, heapWordToOffset(left), heapWordToOffset(right));
237 }
238}
239
240inline void CMSCollector::save_sweep_limits() {
241 _cmsGen->save_sweep_limit();
242}
243
244inline bool CMSCollector::is_dead_obj(oop obj) const {
245 HeapWord* addr = (HeapWord*)obj;
246 assert((_cmsGen->cmsSpace()->is_in_reserved(addr)
247 && _cmsGen->cmsSpace()->block_is_obj(addr)),
248 "must be object");
249 return should_unload_classes() &&
250 _collectorState == Sweeping &&
251 !_markBitMap.isMarked(addr);
252}
253
254inline bool CMSCollector::should_abort_preclean() const {
255 // We are in the midst of an "abortable preclean" and either
256 // scavenge is done or foreground GC wants to take over collection
257 return _collectorState == AbortablePreclean &&
258 (_abort_preclean || _foregroundGCIsActive ||
259 CMSHeap::heap()->incremental_collection_will_fail(true /* consult_young */));
260}
261
262inline size_t CMSCollector::get_eden_used() const {
263 return _young_gen->eden()->used();
264}
265
266inline size_t CMSCollector::get_eden_capacity() const {
267 return _young_gen->eden()->capacity();
268}
269
270inline bool CMSStats::valid() const {
271 return _valid_bits == _ALL_VALID;
272}
273
274inline void CMSStats::record_gc0_begin() {
275 if (_gc0_begin_time.is_updated()) {
276 float last_gc0_period = _gc0_begin_time.seconds();
277 _gc0_period = AdaptiveWeightedAverage::exp_avg(_gc0_period,
278 last_gc0_period, _gc0_alpha);
279 _gc0_alpha = _saved_alpha;
280 _valid_bits |= _GC0_VALID;
281 }
282 _cms_used_at_gc0_begin = _cms_gen->cmsSpace()->used();
283
284 _gc0_begin_time.update();
285}
286
287inline void CMSStats::record_gc0_end(size_t cms_gen_bytes_used) {
288 float last_gc0_duration = _gc0_begin_time.seconds();
289 _gc0_duration = AdaptiveWeightedAverage::exp_avg(_gc0_duration,
290 last_gc0_duration, _gc0_alpha);
291
292 // Amount promoted.
293 _cms_used_at_gc0_end = cms_gen_bytes_used;
294
295 size_t promoted_bytes = 0;
296 if (_cms_used_at_gc0_end >= _cms_used_at_gc0_begin) {
297 promoted_bytes = _cms_used_at_gc0_end - _cms_used_at_gc0_begin;
298 }
299
300 // If the young gen collection was skipped, then the
301 // number of promoted bytes will be 0 and adding it to the
302 // average will incorrectly lessen the average. It is, however,
303 // also possible that no promotion was needed.
304 //
305 // _gc0_promoted used to be calculated as
306 // _gc0_promoted = AdaptiveWeightedAverage::exp_avg(_gc0_promoted,
307 // promoted_bytes, _gc0_alpha);
308 _cms_gen->gc_stats()->avg_promoted()->sample(promoted_bytes);
309 _gc0_promoted = (size_t) _cms_gen->gc_stats()->avg_promoted()->average();
310
311 // Amount directly allocated.
312 size_t allocated_bytes = _cms_gen->direct_allocated_words() * HeapWordSize;
313 _cms_gen->reset_direct_allocated_words();
314 _cms_allocated = AdaptiveWeightedAverage::exp_avg(_cms_allocated,
315 allocated_bytes, _gc0_alpha);
316}
317
318inline void CMSStats::record_cms_begin() {
319 _cms_timer.stop();
320
321 // This is just an approximate value, but is good enough.
322 _cms_used_at_cms_begin = _cms_used_at_gc0_end;
323
324 _cms_period = AdaptiveWeightedAverage::exp_avg((float)_cms_period,
325 (float) _cms_timer.seconds(), _cms_alpha);
326 _cms_begin_time.update();
327
328 _cms_timer.reset();
329 _cms_timer.start();
330}
331
332inline void CMSStats::record_cms_end() {
333 _cms_timer.stop();
334
335 float cur_duration = _cms_timer.seconds();
336 _cms_duration = AdaptiveWeightedAverage::exp_avg(_cms_duration,
337 cur_duration, _cms_alpha);
338
339 _cms_end_time.update();
340 _cms_alpha = _saved_alpha;
341 _allow_duty_cycle_reduction = true;
342 _valid_bits |= _CMS_VALID;
343
344 _cms_timer.start();
345}
346
347inline double CMSStats::cms_time_since_begin() const {
348 return _cms_begin_time.seconds();
349}
350
351inline double CMSStats::cms_time_since_end() const {
352 return _cms_end_time.seconds();
353}
354
355inline double CMSStats::promotion_rate() const {
356 assert(valid(), "statistics not valid yet");
357 return gc0_promoted() / gc0_period();
358}
359
360inline double CMSStats::cms_allocation_rate() const {
361 assert(valid(), "statistics not valid yet");
362 return cms_allocated() / gc0_period();
363}
364
365inline double CMSStats::cms_consumption_rate() const {
366 assert(valid(), "statistics not valid yet");
367 return (gc0_promoted() + cms_allocated()) / gc0_period();
368}
369
370inline void ConcurrentMarkSweepGeneration::save_sweep_limit() {
371 cmsSpace()->save_sweep_limit();
372}
373
374inline MemRegion ConcurrentMarkSweepGeneration::used_region_at_save_marks() const {
375 return _cmsSpace->used_region_at_save_marks();
376}
377
378template <typename OopClosureType>
379void ConcurrentMarkSweepGeneration::oop_since_save_marks_iterate(OopClosureType* cl) {
380 cl->set_generation(this);
381 cmsSpace()->oop_since_save_marks_iterate(cl);
382 cl->reset_generation();
383 save_marks();
384}
385
386inline void MarkFromRootsClosure::do_yield_check() {
387 if (ConcurrentMarkSweepThread::should_yield() &&
388 !_collector->foregroundGCIsActive() &&
389 _yield) {
390 do_yield_work();
391 }
392}
393
394inline void ParMarkFromRootsClosure::do_yield_check() {
395 if (ConcurrentMarkSweepThread::should_yield() &&
396 !_collector->foregroundGCIsActive()) {
397 do_yield_work();
398 }
399}
400
401inline void PushOrMarkClosure::do_yield_check() {
402 _parent->do_yield_check();
403}
404
405inline void ParPushOrMarkClosure::do_yield_check() {
406 _parent->do_yield_check();
407}
408
409// Return value of "true" indicates that the on-going preclean
410// should be aborted.
411inline bool ScanMarkedObjectsAgainCarefullyClosure::do_yield_check() {
412 if (ConcurrentMarkSweepThread::should_yield() &&
413 !_collector->foregroundGCIsActive() &&
414 _yield) {
415 // Sample young gen size before and after yield
416 _collector->sample_eden();
417 do_yield_work();
418 _collector->sample_eden();
419 return _collector->should_abort_preclean();
420 }
421 return false;
422}
423
424inline void SurvivorSpacePrecleanClosure::do_yield_check() {
425 if (ConcurrentMarkSweepThread::should_yield() &&
426 !_collector->foregroundGCIsActive() &&
427 _yield) {
428 // Sample young gen size before and after yield
429 _collector->sample_eden();
430 do_yield_work();
431 _collector->sample_eden();
432 }
433}
434
435inline void SweepClosure::do_yield_check(HeapWord* addr) {
436 if (ConcurrentMarkSweepThread::should_yield() &&
437 !_collector->foregroundGCIsActive() &&
438 _yield) {
439 do_yield_work(addr);
440 }
441}
442
443inline void MarkRefsIntoAndScanClosure::do_yield_check() {
444 // The conditions are ordered for the remarking phase
445 // when _yield is false.
446 if (_yield &&
447 !_collector->foregroundGCIsActive() &&
448 ConcurrentMarkSweepThread::should_yield()) {
449 do_yield_work();
450 }
451}
452
453
454inline void ModUnionClosure::do_MemRegion(MemRegion mr) {
455 // Align the end of mr so it's at a card boundary.
456 // This is superfluous except at the end of the space;
457 // we should do better than this XXX
458 MemRegion mr2(mr.start(), align_up(mr.end(),
459 CardTable::card_size /* bytes */));
460 _t->mark_range(mr2);
461}
462
463inline void ModUnionClosurePar::do_MemRegion(MemRegion mr) {
464 // Align the end of mr so it's at a card boundary.
465 // This is superfluous except at the end of the space;
466 // we should do better than this XXX
467 MemRegion mr2(mr.start(), align_up(mr.end(),
468 CardTable::card_size /* bytes */));
469 _t->par_mark_range(mr2);
470}
471
472#endif // SHARE_GC_CMS_CONCURRENTMARKSWEEPGENERATION_INLINE_HPP
473