g1CardLiveData.cpp revision 11123:6784b64bacf0
156893Sfenner/* 256893Sfenner * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. 356893Sfenner * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4127668Sbms * 556893Sfenner * This code is free software; you can redistribute it and/or modify it 656893Sfenner * under the terms of the GNU General Public License version 2 only, as 756893Sfenner * published by the Free Software Foundation. 856893Sfenner * 956893Sfenner * This code is distributed in the hope that it will be useful, but WITHOUT 1056893Sfenner * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1156893Sfenner * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 1256893Sfenner * version 2 for more details (a copy is included in the LICENSE file that 1356893Sfenner * accompanied this code). 1456893Sfenner * 1556893Sfenner * You should have received a copy of the GNU General Public License version 16127668Sbms * 2 along with this work; if not, write to the Free Software Foundation, 1756893Sfenner * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 1856893Sfenner * 1956893Sfenner * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 2056893Sfenner * or visit www.oracle.com if you need additional information or have any 2156893Sfenner * questions. 2256893Sfenner * 2356893Sfenner */ 2456893Sfenner 2556893Sfenner#include "precompiled.hpp" 2656893Sfenner#include "gc/g1/g1CollectedHeap.inline.hpp" 2756893Sfenner#include "gc/g1/g1ConcurrentMark.inline.hpp" 2856893Sfenner#include "gc/g1/g1CardLiveData.inline.hpp" 2956893Sfenner#include "gc/g1/suspendibleThreadSet.hpp" 3056893Sfenner#include "gc/shared/workgroup.hpp" 3156893Sfenner#include "logging/log.hpp" 32127668Sbms#include "memory/universe.hpp" 33146773Ssam#include "runtime/atomic.inline.hpp" 3456893Sfenner#include "runtime/globals.hpp" 3556893Sfenner#include "runtime/os.hpp" 3656893Sfenner#include "utilities/bitMap.inline.hpp" 3756893Sfenner#include "utilities/debug.hpp" 3856893Sfenner 3956893SfennerG1CardLiveData::G1CardLiveData() : 40127668Sbms _max_capacity(0), 41127668Sbms _cards_per_region(0), 4256893Sfenner _gc_timestamp_at_create(0), 4356893Sfenner _live_regions(NULL), 4456893Sfenner _live_regions_size_in_bits(0), 4556893Sfenner _live_cards(NULL), 4656893Sfenner _live_cards_size_in_bits(0) { 4756893Sfenner} 4856893Sfenner 4956893SfennerG1CardLiveData::~G1CardLiveData() { 5056893Sfenner free_large_bitmap(_live_cards, _live_cards_size_in_bits); 5156893Sfenner free_large_bitmap(_live_regions, _live_regions_size_in_bits); 5256893Sfenner} 5375115Sfenner 5475115SfennerG1CardLiveData::bm_word_t* G1CardLiveData::allocate_large_bitmap(size_t size_in_bits) { 5575115Sfenner size_t size_in_words = BitMap::calc_size_in_words(size_in_bits); 5675115Sfenner 5775115Sfenner bm_word_t* map = MmapArrayAllocator<bm_word_t, mtGC>::allocate(size_in_words); 5856893Sfenner 5956893Sfenner return map; 6056893Sfenner} 6156893Sfenner 62127668Sbmsvoid G1CardLiveData::free_large_bitmap(bm_word_t* bitmap, size_t size_in_bits) { 63127668Sbms MmapArrayAllocator<bm_word_t, mtGC>::free(bitmap, BitMap::calc_size_in_words(size_in_bits)); 64127668Sbms} 65127668Sbms 66127668Sbmsvoid G1CardLiveData::initialize(size_t max_capacity, uint num_max_regions) { 67127668Sbms assert(max_capacity % num_max_regions == 0, 68127668Sbms "Given capacity must be evenly divisible by region size."); 69127668Sbms size_t region_size = max_capacity / num_max_regions; 70127668Sbms assert(region_size % (G1SATBCardTableModRefBS::card_size * BitsPerWord) == 0, 71127668Sbms "Region size must be evenly divisible by area covered by a single word."); 72127668Sbms _max_capacity = max_capacity; 73127668Sbms _cards_per_region = region_size / G1SATBCardTableModRefBS::card_size; 74127668Sbms 75127668Sbms _live_regions_size_in_bits = live_region_bitmap_size_in_bits(); 76127668Sbms _live_regions = allocate_large_bitmap(_live_regions_size_in_bits); 77127668Sbms _live_cards_size_in_bits = live_card_bitmap_size_in_bits(); 78127668Sbms _live_cards = allocate_large_bitmap(_live_cards_size_in_bits); 79127668Sbms} 80127668Sbms 81127668Sbmsvoid G1CardLiveData::pretouch() { 82127668Sbms live_cards_bm().pretouch(); 83127668Sbms live_regions_bm().pretouch(); 84127668Sbms} 85127668Sbms 86127668Sbmssize_t G1CardLiveData::live_region_bitmap_size_in_bits() const { 87127668Sbms return _max_capacity / (_cards_per_region << G1SATBCardTableModRefBS::card_shift); 88127668Sbms} 89127668Sbms 90127668Sbmssize_t G1CardLiveData::live_card_bitmap_size_in_bits() const { 91127668Sbms return _max_capacity >> G1SATBCardTableModRefBS::card_shift; 9275115Sfenner} 93127668Sbms 9456893Sfenner// Helper class that provides functionality to generate the Live Data Count 9556893Sfenner// information. 9656893Sfennerclass G1CardLiveDataHelper VALUE_OBJ_CLASS_SPEC { 9756893Sfennerprivate: 9856893Sfenner BitMapView _region_bm; 9956893Sfenner BitMapView _card_bm; 10056893Sfenner 10156893Sfenner // The card number of the bottom of the G1 heap. 10256893Sfenner // Used in biasing indices into accounting card bitmaps. 10356893Sfenner BitMap::idx_t _heap_card_bias; 104127668Sbms 10556893Sfenner // Utility routine to set an exclusive range of bits on the given 10656893Sfenner // bitmap, optimized for very small ranges. 10756893Sfenner // There must be at least one bit to set. 10856893Sfenner void set_card_bitmap_range(BitMap::idx_t start_idx, 109127668Sbms BitMap::idx_t end_idx) { 11056893Sfenner 11156893Sfenner // Set the exclusive bit range [start_idx, end_idx). 11256893Sfenner assert((end_idx - start_idx) > 0, "at least one bit"); 11356893Sfenner 11456893Sfenner // For small ranges use a simple loop; otherwise use set_range. 115127668Sbms // The range is made up of the cards that are spanned by an object/mem 116127668Sbms // region so 8 cards will allow up to object sizes up to 4K to be handled 11756893Sfenner // using the loop. 11856893Sfenner if ((end_idx - start_idx) <= 8) { 11956893Sfenner for (BitMap::idx_t i = start_idx; i < end_idx; i += 1) { 12056893Sfenner _card_bm.set_bit(i); 12156893Sfenner } 12256893Sfenner } else { 12375115Sfenner _card_bm.set_range(start_idx, end_idx); 12475115Sfenner } 12556893Sfenner } 12675115Sfenner 12756893Sfenner // We cache the last mark set. This avoids setting the same bit multiple times. 12856893Sfenner // This is particularly interesting for dense bitmaps, as this avoids doing 12956893Sfenner // lots of work most of the time. 13056893Sfenner BitMap::idx_t _last_marked_bit_idx; 13156893Sfenner 13256893Sfenner void clear_card_bitmap_range(HeapWord* start, HeapWord* end) { 13356893Sfenner BitMap::idx_t start_idx = card_live_bitmap_index_for(start); 134127668Sbms BitMap::idx_t end_idx = card_live_bitmap_index_for((HeapWord*)align_ptr_up(end, CardTableModRefBS::card_size)); 13556893Sfenner 13656893Sfenner _card_bm.clear_range(start_idx, end_idx); 13756893Sfenner } 13856893Sfenner 13956893Sfenner // Mark the card liveness bitmap for the object spanning from start to end. 14056893Sfenner void mark_card_bitmap_range(HeapWord* start, HeapWord* end) { 14156893Sfenner BitMap::idx_t start_idx = card_live_bitmap_index_for(start); 14256893Sfenner BitMap::idx_t end_idx = card_live_bitmap_index_for((HeapWord*)align_ptr_up(end, CardTableModRefBS::card_size)); 14356893Sfenner 14456893Sfenner assert((end_idx - start_idx) > 0, "Trying to mark zero sized range."); 14556893Sfenner 14656893Sfenner if (start_idx == _last_marked_bit_idx) { 14756893Sfenner start_idx++; 14856893Sfenner } 14956893Sfenner if (start_idx == end_idx) { 15056893Sfenner return; 15156893Sfenner } 15256893Sfenner 15356893Sfenner // Set the bits in the card bitmap for the cards spanned by this object. 15456893Sfenner set_card_bitmap_range(start_idx, end_idx); 15556893Sfenner _last_marked_bit_idx = end_idx - 1; 15656893Sfenner } 15756893Sfenner 15856893Sfenner void reset_mark_cache() { 15956893Sfenner _last_marked_bit_idx = (BitMap::idx_t)-1; 16056893Sfenner } 16156893Sfenner 16256893Sfennerpublic: 16356893Sfenner // Returns the index in the per-card liveness count bitmap 16456893Sfenner // for the given address 16556893Sfenner inline BitMap::idx_t card_live_bitmap_index_for(HeapWord* addr) { 16656893Sfenner // Below, the term "card num" means the result of shifting an address 16756893Sfenner // by the card shift -- address 0 corresponds to card number 0. One 16856893Sfenner // must subtract the card num of the bottom of the heap to obtain a 16956893Sfenner // card table index. 17056893Sfenner BitMap::idx_t card_num = uintptr_t(addr) >> CardTableModRefBS::card_shift; 17156893Sfenner return card_num - _heap_card_bias; 17256893Sfenner } 17356893Sfenner 17456893Sfenner // Takes a region that's not empty (i.e., it has at least one 17556893Sfenner // live object in it and sets its corresponding bit on the region 17656893Sfenner // bitmap to 1. 17756893Sfenner void set_bit_for_region(HeapRegion* hr) { 17856893Sfenner _region_bm.par_set_bit(hr->hrm_index()); 17956893Sfenner } 18056893Sfenner 18156893Sfenner void reset_live_data(HeapRegion* hr) { 18256893Sfenner clear_card_bitmap_range(hr->next_top_at_mark_start(), hr->end()); 18356893Sfenner } 18456893Sfenner 18556893Sfenner // Mark the range of bits covered by allocations done since the last marking 18656893Sfenner // in the given heap region, i.e. from NTAMS to top of the given region. 18756893Sfenner // Returns if there has been some allocation in this region since the last marking. 18856893Sfenner bool mark_allocated_since_marking(HeapRegion* hr) { 18956893Sfenner reset_mark_cache(); 19056893Sfenner 19156893Sfenner HeapWord* ntams = hr->next_top_at_mark_start(); 19256893Sfenner HeapWord* top = hr->top(); 19356893Sfenner 19456893Sfenner assert(hr->bottom() <= ntams && ntams <= hr->end(), "Preconditions."); 19575115Sfenner 19656893Sfenner // Mark the allocated-since-marking portion... 19756893Sfenner if (ntams < top) { 19856893Sfenner mark_card_bitmap_range(ntams, top); 19956893Sfenner return true; 20056893Sfenner } else { 20156893Sfenner return false; 20256893Sfenner } 20356893Sfenner } 20456893Sfenner 20556893Sfenner // Mark the range of bits covered by live objects on the mark bitmap between 20656893Sfenner // bottom and NTAMS of the given region. 20756893Sfenner // Returns the number of live bytes marked within that area for the given 20856893Sfenner // heap region. 20956893Sfenner size_t mark_marked_during_marking(G1CMBitMap* mark_bitmap, HeapRegion* hr) { 21056893Sfenner reset_mark_cache(); 21156893Sfenner 21256893Sfenner size_t marked_bytes = 0; 21356893Sfenner 21456893Sfenner HeapWord* ntams = hr->next_top_at_mark_start(); 21556893Sfenner HeapWord* start = hr->bottom(); 21656893Sfenner 21756893Sfenner if (ntams <= start) { 21856893Sfenner // Skip empty regions. 21956893Sfenner return 0; 22056893Sfenner } 22156893Sfenner if (hr->is_humongous()) { 22256893Sfenner HeapRegion* start_region = hr->humongous_start_region(); 22356893Sfenner if (mark_bitmap->isMarked(start_region->bottom())) { 22456893Sfenner mark_card_bitmap_range(start, hr->top()); 22556893Sfenner return pointer_delta(hr->top(), start, 1); 22656893Sfenner } else { 22756893Sfenner // Humongous start object was actually dead. 22856893Sfenner return 0; 22956893Sfenner } 23056893Sfenner } 23156893Sfenner 23256893Sfenner assert(start <= hr->end() && start <= ntams && ntams <= hr->end(), 23356893Sfenner "Preconditions not met - " 23456893Sfenner "start: " PTR_FORMAT ", ntams: " PTR_FORMAT ", end: " PTR_FORMAT, 23556893Sfenner p2i(start), p2i(ntams), p2i(hr->end())); 23656893Sfenner 23756893Sfenner // Find the first marked object at or after "start". 23856893Sfenner start = mark_bitmap->getNextMarkedWordAddress(start, ntams); 23956893Sfenner while (start < ntams) { 24056893Sfenner oop obj = oop(start); 24156893Sfenner size_t obj_size = obj->size(); 24256893Sfenner HeapWord* obj_end = start + obj_size; 24356893Sfenner 24456893Sfenner assert(obj_end <= hr->end(), "Humongous objects must have been handled elsewhere."); 24556893Sfenner 24656893Sfenner mark_card_bitmap_range(start, obj_end); 24756893Sfenner 24856893Sfenner // Add the size of this object to the number of marked bytes. 24956893Sfenner marked_bytes += obj_size * HeapWordSize; 25056893Sfenner 25156893Sfenner // Find the next marked object after this one. 25256893Sfenner start = mark_bitmap->getNextMarkedWordAddress(obj_end, ntams); 25356893Sfenner } 25456893Sfenner 25556893Sfenner return marked_bytes; 25656893Sfenner } 25756893Sfenner 25856893Sfenner G1CardLiveDataHelper(G1CardLiveData* live_data, HeapWord* base_address) : 25956893Sfenner _region_bm(live_data->live_regions_bm()), 26056893Sfenner _card_bm(live_data->live_cards_bm()) { 26175115Sfenner // Calculate the card number for the bottom of the heap. Used 26256893Sfenner // in biasing indexes into the accounting card bitmaps. 26356893Sfenner _heap_card_bias = 26456893Sfenner uintptr_t(base_address) >> CardTableModRefBS::card_shift; 26556893Sfenner } 26656893Sfenner}; 26756893Sfenner 26856893Sfennerclass G1CreateCardLiveDataTask: public AbstractGangTask { 26956893Sfenner // Aggregate the counting data that was constructed concurrently 27056893Sfenner // with marking. 27156893Sfenner class G1CreateLiveDataClosure : public HeapRegionClosure { 27256893Sfenner G1CardLiveDataHelper _helper; 27356893Sfenner 27456893Sfenner G1CMBitMap* _mark_bitmap; 27556893Sfenner 27656893Sfenner G1ConcurrentMark* _cm; 27756893Sfenner public: 27856893Sfenner G1CreateLiveDataClosure(G1CollectedHeap* g1h, 27956893Sfenner G1ConcurrentMark* cm, 28056893Sfenner G1CMBitMap* mark_bitmap, 28156893Sfenner G1CardLiveData* live_data) : 28256893Sfenner HeapRegionClosure(), 28356893Sfenner _helper(live_data, g1h->reserved_region().start()), 28456893Sfenner _mark_bitmap(mark_bitmap), 28556893Sfenner _cm(cm) { } 28656893Sfenner 28756893Sfenner bool doHeapRegion(HeapRegion* hr) { 28856893Sfenner size_t marked_bytes = _helper.mark_marked_during_marking(_mark_bitmap, hr); 28956893Sfenner if (marked_bytes > 0) { 29056893Sfenner hr->add_to_marked_bytes(marked_bytes); 29156893Sfenner } 29256893Sfenner 29356893Sfenner return (_cm->do_yield_check() && _cm->has_aborted()); 29456893Sfenner } 29556893Sfenner }; 29656893Sfenner 29756893Sfenner G1ConcurrentMark* _cm; 29856893Sfenner G1CardLiveData* _live_data; 29956893Sfenner HeapRegionClaimer _hr_claimer; 30056893Sfenner 30156893Sfennerpublic: 30256893Sfenner G1CreateCardLiveDataTask(G1CMBitMap* bitmap, 30356893Sfenner G1CardLiveData* live_data, 30456893Sfenner uint n_workers) : 30556893Sfenner AbstractGangTask("G1 Create Live Data"), 30656893Sfenner _live_data(live_data), 30756893Sfenner _hr_claimer(n_workers) { 30856893Sfenner } 30956893Sfenner 31056893Sfenner void work(uint worker_id) { 31156893Sfenner SuspendibleThreadSetJoiner sts_join; 31256893Sfenner 31356893Sfenner G1CollectedHeap* g1h = G1CollectedHeap::heap(); 31456893Sfenner G1ConcurrentMark* cm = g1h->concurrent_mark(); 31556893Sfenner G1CreateLiveDataClosure cl(g1h, cm, cm->nextMarkBitMap(), _live_data); 31656893Sfenner g1h->heap_region_par_iterate(&cl, worker_id, &_hr_claimer); 31756893Sfenner } 31856893Sfenner}; 31956893Sfenner 32056893Sfennervoid G1CardLiveData::create(WorkGang* workers, G1CMBitMap* mark_bitmap) { 32156893Sfenner _gc_timestamp_at_create = G1CollectedHeap::heap()->get_gc_time_stamp(); 32256893Sfenner 32356893Sfenner uint n_workers = workers->active_workers(); 324127668Sbms 32556893Sfenner G1CreateCardLiveDataTask cl(mark_bitmap, 32656893Sfenner this, 32756893Sfenner n_workers); 328127668Sbms workers->run_task(&cl); 32956893Sfenner} 330124488Sfenner 331124488Sfennerclass G1FinalizeCardLiveDataTask: public AbstractGangTask { 33256893Sfenner // Finalizes the liveness counting data. 33356893Sfenner // Sets the bits corresponding to the interval [NTAMS, top] 33456893Sfenner // (which contains the implicitly live objects) in the 335127668Sbms // card liveness bitmap. Also sets the bit for each region 336124488Sfenner // containing live data, in the region liveness bitmap. 337127668Sbms class G1FinalizeCardLiveDataClosure: public HeapRegionClosure { 33856893Sfenner private: 33956893Sfenner G1CardLiveDataHelper _helper; 34056893Sfenner 341127668Sbms uint _gc_timestamp_at_create; 342127668Sbms 343127668Sbms bool has_been_reclaimed(HeapRegion* hr) const { 34456893Sfenner return hr->get_gc_time_stamp() > _gc_timestamp_at_create; 34556893Sfenner } 346127668Sbms public: 347127668Sbms G1FinalizeCardLiveDataClosure(G1CollectedHeap* g1h, 348127668Sbms G1CMBitMap* bitmap, 34956893Sfenner G1CardLiveData* live_data) : 35075115Sfenner HeapRegionClosure(), 35156893Sfenner _helper(live_data, g1h->reserved_region().start()), 35256893Sfenner _gc_timestamp_at_create(live_data->gc_timestamp_at_create()) { } 35356893Sfenner 35475115Sfenner bool doHeapRegion(HeapRegion* hr) { 35556893Sfenner if (has_been_reclaimed(hr)) { 35656893Sfenner _helper.reset_live_data(hr); 35756893Sfenner } 358127668Sbms bool allocated_since_marking = _helper.mark_allocated_since_marking(hr); 35956893Sfenner if (allocated_since_marking || hr->next_marked_bytes() > 0) { 36056893Sfenner _helper.set_bit_for_region(hr); 36156893Sfenner } 36256893Sfenner return false; 36356893Sfenner } 36456893Sfenner }; 365127668Sbms 36656893Sfenner G1CMBitMap* _bitmap; 36756893Sfenner 36856893Sfenner G1CardLiveData* _live_data; 36956893Sfenner 37056893Sfenner HeapRegionClaimer _hr_claimer; 37156893Sfenner 372127668Sbmspublic: 37356893Sfenner G1FinalizeCardLiveDataTask(G1CMBitMap* bitmap, G1CardLiveData* live_data, uint n_workers) : 37456893Sfenner AbstractGangTask("G1 Finalize Card Live Data"), 37556893Sfenner _bitmap(bitmap), 37656893Sfenner _live_data(live_data), 37756893Sfenner _hr_claimer(n_workers) { 378127668Sbms } 379127668Sbms 38056893Sfenner void work(uint worker_id) { 38156893Sfenner G1FinalizeCardLiveDataClosure cl(G1CollectedHeap::heap(), _bitmap, _live_data); 38256893Sfenner 38356893Sfenner G1CollectedHeap::heap()->heap_region_par_iterate(&cl, worker_id, &_hr_claimer); 38456893Sfenner } 385127668Sbms}; 386127668Sbms 38756893Sfennervoid G1CardLiveData::finalize(WorkGang* workers, G1CMBitMap* mark_bitmap) { 38875115Sfenner // Finalize the live data. 38956893Sfenner G1FinalizeCardLiveDataTask cl(mark_bitmap, 39056893Sfenner this, 39156893Sfenner workers->active_workers()); 39275115Sfenner workers->run_task(&cl); 39356893Sfenner} 39456893Sfenner 39556893Sfennerclass G1ClearCardLiveDataTask : public AbstractGangTask { 396127668Sbms BitMapView _bitmap; 39756893Sfenner size_t _num_chunks; 39856893Sfenner size_t _cur_chunk; 39956893Sfennerpublic: 40056893Sfenner G1ClearCardLiveDataTask(const BitMapView& bitmap, size_t num_tasks) : 40156893Sfenner AbstractGangTask("G1 Clear Card Live Data"), 40256893Sfenner _bitmap(bitmap), 403127668Sbms _num_chunks(num_tasks), 40456893Sfenner _cur_chunk(0) { 40556893Sfenner } 40656893Sfenner 40756893Sfenner static size_t chunk_size() { return M; } 40856893Sfenner 40956893Sfenner virtual void work(uint worker_id) { 410127668Sbms while (true) { 411127668Sbms size_t to_process = Atomic::add(1, &_cur_chunk) - 1; 41256893Sfenner if (to_process >= _num_chunks) { 41356893Sfenner break; 41456893Sfenner } 41556893Sfenner 41656893Sfenner BitMap::idx_t start = M * BitsPerByte * to_process; 417127668Sbms BitMap::idx_t end = MIN2(start + M * BitsPerByte, _bitmap.size()); 418146773Ssam _bitmap.clear_range(start, end); 419146773Ssam } 420146773Ssam } 421146773Ssam}; 42256893Sfenner 423127668Sbmsvoid G1CardLiveData::clear(WorkGang* workers) { 424127668Sbms guarantee(Universe::is_fully_initialized(), "Should not call this during initialization."); 425127668Sbms 42675115Sfenner size_t const num_chunks = align_size_up(live_cards_bm().size_in_bytes(), G1ClearCardLiveDataTask::chunk_size()) / G1ClearCardLiveDataTask::chunk_size(); 427127668Sbms uint const num_workers = (uint)MIN2(num_chunks, (size_t)workers->active_workers()); 42856893Sfenner 42956893Sfenner G1ClearCardLiveDataTask cl(live_cards_bm(), num_chunks); 43056893Sfenner 43156893Sfenner log_debug(gc, ergo)("Running %s using %u workers for " SIZE_FORMAT " work units.", cl.name(), num_workers, num_chunks); 43256893Sfenner workers->run_task(&cl, num_workers); 433127668Sbms 43475115Sfenner // The region live bitmap is always very small, even for huge heaps. Clear 43575115Sfenner // directly. 43675115Sfenner live_regions_bm().clear(); 43756893Sfenner} 43856893Sfenner 43975115Sfennerclass G1VerifyCardLiveDataTask: public AbstractGangTask { 44056893Sfenner // Heap region closure used for verifying the live count data 44156893Sfenner // that was created concurrently and finalized during 44256893Sfenner // the remark pause. This closure is applied to the heap 44356893Sfenner // regions during the STW cleanup pause. 44475115Sfenner class G1VerifyCardLiveDataClosure: public HeapRegionClosure { 44556893Sfenner private: 44656893Sfenner G1CollectedHeap* _g1h; 44775115Sfenner G1CMBitMap* _mark_bitmap; 44856893Sfenner G1CardLiveDataHelper _helper; 44956893Sfenner 45056893Sfenner G1CardLiveData* _act_live_data; 45175115Sfenner 45256893Sfenner G1CardLiveData* _exp_live_data; 45356893Sfenner 45456893Sfenner int _failures; 45575115Sfenner 45656893Sfenner // Completely recreates the live data count for the given heap region and 45756893Sfenner // returns the number of bytes marked. 45875115Sfenner size_t create_live_data_count(HeapRegion* hr) { 45975115Sfenner size_t bytes_marked = _helper.mark_marked_during_marking(_mark_bitmap, hr); 460127668Sbms bool allocated_since_marking = _helper.mark_allocated_since_marking(hr); 46175115Sfenner if (allocated_since_marking || bytes_marked > 0) { 46275115Sfenner _helper.set_bit_for_region(hr); 46375115Sfenner } 46475115Sfenner return bytes_marked; 46556893Sfenner } 46675115Sfenner public: 467127668Sbms G1VerifyCardLiveDataClosure(G1CollectedHeap* g1h, 46856893Sfenner G1CMBitMap* mark_bitmap, 469127668Sbms G1CardLiveData* act_live_data, 470127668Sbms G1CardLiveData* exp_live_data) : 47156893Sfenner _g1h(g1h), 47256893Sfenner _mark_bitmap(mark_bitmap), 473127668Sbms _helper(exp_live_data, g1h->reserved_region().start()), 474127668Sbms _act_live_data(act_live_data), 475127668Sbms _exp_live_data(exp_live_data), 47656893Sfenner _failures(0) { } 47756893Sfenner 478127668Sbms int failures() const { return _failures; } 479146773Ssam 480146773Ssam bool doHeapRegion(HeapRegion* hr) { 481146773Ssam int failures = 0; 48256893Sfenner 483127668Sbms // Walk the marking bitmap for this region and set the corresponding bits 484127668Sbms // in the expected region and card bitmaps. 485127668Sbms size_t exp_marked_bytes = create_live_data_count(hr); 48656893Sfenner size_t act_marked_bytes = hr->next_marked_bytes(); 48756893Sfenner // Verify the marked bytes for this region. 48856893Sfenner 48956893Sfenner if (exp_marked_bytes != act_marked_bytes) { 490127668Sbms log_error(gc)("Expected marked bytes " SIZE_FORMAT " != actual marked bytes " SIZE_FORMAT " in region %u", exp_marked_bytes, act_marked_bytes, hr->hrm_index()); 49175115Sfenner failures += 1; 49256893Sfenner } else if (exp_marked_bytes > HeapRegion::GrainBytes) { 49375115Sfenner log_error(gc)("Expected marked bytes " SIZE_FORMAT " larger than possible " SIZE_FORMAT " in region %u", exp_marked_bytes, HeapRegion::GrainBytes, hr->hrm_index()); 49475115Sfenner failures += 1; 49556893Sfenner } 496127668Sbms 497127668Sbms // Verify the bit, for this region, in the actual and expected 49856893Sfenner // (which was just calculated) region bit maps. 49956893Sfenner uint index = hr->hrm_index(); 50075115Sfenner 501127668Sbms bool expected = _exp_live_data->is_region_live(index); 50256893Sfenner bool actual = _act_live_data->is_region_live(index); 50356893Sfenner if (expected != actual) { 504127668Sbms log_error(gc)("Expected liveness %d not equal actual %d in region %u", expected, actual, hr->hrm_index()); 50556893Sfenner failures += 1; 50656893Sfenner } 507127668Sbms 508127668Sbms // Verify that the card bit maps for the cards spanned by the current 509127668Sbms // region match. 51056893Sfenner BitMap::idx_t start_idx = _helper.card_live_bitmap_index_for(hr->bottom()); 51156893Sfenner BitMap::idx_t end_idx = _helper.card_live_bitmap_index_for(hr->top()); 512127668Sbms 51356893Sfenner for (BitMap::idx_t i = start_idx; i < end_idx; i+=1) { 51456893Sfenner expected = _exp_live_data->is_card_live_at(i); 51556893Sfenner actual = _act_live_data->is_card_live_at(i); 516127668Sbms 51775115Sfenner if (expected != actual) { 51875115Sfenner log_error(gc)("Expected card liveness %d not equal actual card liveness %d at card " SIZE_FORMAT " in region %u", expected, actual, i, hr->hrm_index()); 51956893Sfenner failures += 1; 52056893Sfenner } 521127668Sbms } 52256893Sfenner 52375115Sfenner _failures += failures; 52456893Sfenner 52556893Sfenner // We could stop iteration over the heap when we 526127668Sbms // find the first violating region by returning true. 52756893Sfenner return false; 52856893Sfenner } 52956893Sfenner }; 530127668Sbmsprotected: 531127668Sbms G1CollectedHeap* _g1h; 53256893Sfenner G1CMBitMap* _mark_bitmap; 533127668Sbms 53456893Sfenner G1CardLiveData* _act_live_data; 53556893Sfenner 53656893Sfenner G1CardLiveData _exp_live_data; 53756893Sfenner 538127668Sbms int _failures; 539127668Sbms 540127668Sbms HeapRegionClaimer _hr_claimer; 541127668Sbms 54256893Sfennerpublic: 54356893Sfenner G1VerifyCardLiveDataTask(G1CMBitMap* bitmap, 544127668Sbms G1CardLiveData* act_live_data, 545127668Sbms uint n_workers) 54675115Sfenner : AbstractGangTask("G1 Verify Card Live Data"), 54775115Sfenner _g1h(G1CollectedHeap::heap()), 54875115Sfenner _mark_bitmap(bitmap), 54975115Sfenner _act_live_data(act_live_data), 55056893Sfenner _exp_live_data(), 55156893Sfenner _failures(0), 55256893Sfenner _hr_claimer(n_workers) { 55356893Sfenner assert(VerifyDuringGC, "don't call this otherwise"); 55456893Sfenner _exp_live_data.initialize(_g1h->max_capacity(), _g1h->max_regions()); 555127668Sbms } 556127668Sbms 557127668Sbms void work(uint worker_id) { 558127668Sbms G1VerifyCardLiveDataClosure cl(_g1h, 559127668Sbms _mark_bitmap, 56056893Sfenner _act_live_data, 561127668Sbms &_exp_live_data); 562127668Sbms _g1h->heap_region_par_iterate(&cl, worker_id, &_hr_claimer); 563127668Sbms 564127668Sbms Atomic::add(cl.failures(), &_failures); 565127668Sbms } 56656893Sfenner 56756893Sfenner int failures() const { return _failures; } 568127668Sbms}; 569127668Sbms 570127668Sbmsvoid G1CardLiveData::verify(WorkGang* workers, G1CMBitMap* actual_bitmap) { 571127668Sbms ResourceMark rm; 57256893Sfenner 573127668Sbms G1VerifyCardLiveDataTask cl(actual_bitmap, 574127668Sbms this, 575127668Sbms workers->active_workers()); 576127668Sbms workers->run_task(&cl); 577127668Sbms 57856893Sfenner guarantee(cl.failures() == 0, "Unexpected accounting failures"); 579127668Sbms} 58056893Sfenner 58156893Sfenner#ifndef PRODUCT 58256893Sfennervoid G1CardLiveData::verify_is_clear() { 58356893Sfenner assert(live_cards_bm().count_one_bits() == 0, "Live cards bitmap must be clear."); 584127668Sbms assert(live_regions_bm().count_one_bits() == 0, "Live regions bitmap must be clear."); 58575115Sfenner} 58656893Sfenner#endif 58756893Sfenner