src/hotspot/share/gc/g1/g1ConcurrentRebuildAndScrub.cpp - openjdk/jdk25u - Git at Google

 /*
  * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 only, as
  * published by the Free Software Foundation.
  *
  * This code is distributed in the hope that it will be useful, but WITHOUT
  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  * version 2 for more details (a copy is included in the LICENSE file that
  * accompanied this code).
  *
  * You should have received a copy of the GNU General Public License version
  * 2 along with this work; if not, write to the Free Software Foundation,
  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  *
  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  * or visit www.oracle.com if you need additional information or have any
  * questions.
  *
  */


 #include "gc/g1/g1ConcurrentMark.inline.hpp"
 #include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
 #include "gc/g1/g1ConcurrentRebuildAndScrub.hpp"
 #include "gc/g1/g1HeapRegion.inline.hpp"
 #include "gc/g1/g1HeapRegionManager.inline.hpp"
 #include "gc/shared/gc_globals.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
 #include "gc/shared/workerThread.hpp"
 #include "logging/log.hpp"
 #include "memory/memRegion.hpp"
 #include "oops/oopsHierarchy.hpp"
 #include "utilities/debug.hpp"
 #include "utilities/globalDefinitions.hpp"

 // Worker task that scans the objects in the old generation to rebuild the remembered
 // set and at the same time scrubs dead objects by replacing them with filler objects
 // to make them completely parseable.
 //
 // The remark pause recorded two pointers within the regions:
 //
 // parsable_bottom (pb): this is the TAMS of the recent marking for that region. Objects
 //                       below that may or may not be dead (as per mark bitmap).
 //                       This task needs to remove the dead objects, replacing them
 //                       with filler objects so that they can be walked through later.
 //
 // top_at_rebuild_start (tars): at rebuild phase start we record the current top: up to
 //                              this address (live) objects need to be scanned for references
 //                              that might need to be added to the remembered sets.
 //
 // Note that bottom <= parsable_bottom <= tars; if there is no tars (i.e. null),
 // obviously there can not be a parsable_bottom.
 //
 // We need to scrub and scan objects to rebuild remembered sets until parsable_bottom;
 // we need to scan objects to rebuild remembered sets until tars.
 // Regions might have been reclaimed while scrubbing them after having yielded for
 // a pause.
 class G1RebuildRSAndScrubTask : public WorkerTask {
   G1ConcurrentMark* _cm;
   G1HeapRegionClaimer _hr_claimer;

   const bool _should_rebuild_remset;

   class G1RebuildRSAndScrubRegionClosure : public G1HeapRegionClosure {
     G1ConcurrentMark* _cm;
     const G1CMBitMap* _bitmap;

     G1RebuildRemSetClosure _rebuild_closure;

     const bool _should_rebuild_remset;

     size_t _processed_words;

     const size_t ProcessingYieldLimitInWords = G1RebuildRemSetChunkSize / HeapWordSize;

     void reset_processed_words() {
       _processed_words = 0;
     }

     void add_processed_words(size_t processed) {
       _processed_words += processed;
     }

     // Yield if enough has been processed. Return whether we should stop
     // processing this region because either the concurrent marking cycle has been
     // aborted or the region has been reclaimed.
     bool yield_if_necessary(G1HeapRegion* hr) {
       if (_processed_words >= ProcessingYieldLimitInWords) {
         reset_processed_words();
         // If a yield occurs (potential young-gc pause), must recheck for
         // potential regions reclamation.
         if (_cm->do_yield_check() && !should_rebuild_or_scrub(hr)) {
           return true;
         }
       }
       return _cm->has_aborted() || !should_rebuild_or_scrub(hr);
     }

     // Returns whether the top at rebuild start value for the given region indicates
     // that there is some rebuild or scrubbing work.
     //
     // Based on the results of G1RemSetTrackingPolicy::needs_scan_for_rebuild(),
     // the value may be changed to null during rebuilding if the region has either:
     //  - been allocated after rebuild start, or
     //  - been reclaimed by a collection.
     bool should_rebuild_or_scrub(G1HeapRegion* hr) const {
       return _cm->top_at_rebuild_start(hr) != nullptr;
     }

     // Helper used by both humongous objects and when chunking an object larger than the
     // G1RebuildRemSetChunkSize. The heap region is needed check whether the region has
     // been reclaimed during yielding.
     void scan_large_object(G1HeapRegion* hr, const oop obj, MemRegion scan_range) {
       HeapWord* start = scan_range.start();
       HeapWord* limit = scan_range.end();
       do {
         MemRegion mr(start, MIN2(start + ProcessingYieldLimitInWords, limit));
         obj->oop_iterate(&_rebuild_closure, mr);

         // Update processed words and yield, for humongous objects we will yield
         // after each chunk.
         add_processed_words(mr.word_size());

         if (yield_if_necessary(hr)) {
           return;
         }

         // Step to next chunk of the large object.
         start = mr.end();
       } while (start < limit);
     }

     // Scan for references into regions that need remembered set update for the given
     // live object. Returns the offset to the next object.
     size_t scan_object(G1HeapRegion* hr, HeapWord* current) {
       oop obj = cast_to_oop(current);
       size_t obj_size = obj->size();

       if (!_should_rebuild_remset) {
         // Not rebuilding, just step to next object.
         add_processed_words(obj_size);
       } else if (obj_size > ProcessingYieldLimitInWords) {
         // Large object, needs to be chunked to avoid stalling safepoints.
         MemRegion mr(current, obj_size);
         scan_large_object(hr, obj, mr);
         // No need to add to _processed_words, this is all handled by the above call;
         // we also ignore the marking abort result of scan_large_object - we will check
         // again right afterwards.
       } else {
         // Object smaller than yield limit, process it fully.
         obj->oop_iterate(&_rebuild_closure);
         // Update how much we have processed. Yield check in main loop
         // will handle this case.
         add_processed_words(obj_size);
       }

       return obj_size;
     }

     // Scan or scrub depending on if addr is marked.
     HeapWord* scan_or_scrub(G1HeapRegion* hr, HeapWord* addr, HeapWord* limit) {
       if (_bitmap->is_marked(addr)) {
         //  Live object, need to scan to rebuild remembered sets for this object.
         return addr + scan_object(hr, addr);
       } else {
         // Found dead object (which klass has potentially been unloaded). Scrub to next marked object.
         HeapWord* scrub_end = _bitmap->get_next_marked_addr(addr, limit);
         hr->fill_range_with_dead_objects(addr, scrub_end);
         // Return the next object to handle.
         return scrub_end;
       }
     }

     // Scan and scrub the given region to tars.
     void scan_and_scrub_region(G1HeapRegion* hr, HeapWord* const pb) {
       assert(should_rebuild_or_scrub(hr), "must be");

       log_trace(gc, marking)("Scrub and rebuild region: " HR_FORMAT " pb: " PTR_FORMAT " TARS: " PTR_FORMAT " TAMS: " PTR_FORMAT,
                              HR_FORMAT_PARAMS(hr), p2i(pb), p2i(_cm->top_at_rebuild_start(hr)), p2i(_cm->top_at_mark_start(hr)));

       {
         // Step 1: Scan the given region from bottom to parsable_bottom.
         HeapWord* start = hr->bottom();
         HeapWord* limit = pb;
         while (start < limit) {
           start = scan_or_scrub(hr, start, limit);

           if (yield_if_necessary(hr)) {
             return;
           }
         }
       }

       // Scrubbing completed for this region - notify that we are done with it, resetting
       // pb to bottom.
       hr->note_end_of_scrubbing();

       {
         // Step 2: Rebuild from TAMS (= parsable_bottom) to TARS.
         HeapWord* start = pb;
         HeapWord* limit = _cm->top_at_rebuild_start(hr);
         while (start < limit) {
           start += scan_object(hr, start);

           if (yield_if_necessary(hr)) {
             return;
           }
         }
       }
     }

     // Scan a humongous region for remembered set updates. Scans in chunks to avoid
     // stalling safepoints.
     void scan_humongous_region(G1HeapRegion* hr, HeapWord* const pb) {
       assert(should_rebuild_or_scrub(hr), "must be");

       if (!_should_rebuild_remset) {
         // When not rebuilding there is nothing to do for humongous objects.
         return;
       }

       // At this point we should only have live humongous objects, that
       // means it must either be:
       // - marked
       // - or seen as fully parsable, i.e. allocated after the marking started
       oop humongous = cast_to_oop(hr->humongous_start_region()->bottom());
       assert(_bitmap->is_marked(humongous) || pb == hr->bottom(),
              "Humongous object not live");

       log_trace(gc, marking)("Rebuild for humongous region: " HR_FORMAT " pb: " PTR_FORMAT " TARS: " PTR_FORMAT,
                               HR_FORMAT_PARAMS(hr), p2i(pb), p2i(_cm->top_at_rebuild_start(hr)));

       // Scan the humongous object in chunks from bottom to top to rebuild remembered sets.
       HeapWord* humongous_end = hr->humongous_start_region()->bottom() + humongous->size();
       MemRegion mr(hr->bottom(), MIN2(hr->top(), humongous_end));

       scan_large_object(hr, humongous, mr);
     }

   public:
     G1RebuildRSAndScrubRegionClosure(G1ConcurrentMark* cm, bool should_rebuild_remset, uint worker_id) :
       _cm(cm),
       _bitmap(_cm->mark_bitmap()),
       _rebuild_closure(G1CollectedHeap::heap(), worker_id),
       _should_rebuild_remset(should_rebuild_remset),
       _processed_words(0) { }

     bool do_heap_region(G1HeapRegion* hr) {
       // Avoid stalling safepoints and stop iteration if mark cycle has been aborted.
       _cm->do_yield_check();
       if (_cm->has_aborted()) {
         return true;
       }

       HeapWord* const pb = hr->parsable_bottom_acquire();

       if (!should_rebuild_or_scrub(hr)) {
         // Region has been allocated during this phase, no need to either scrub or
         // scan to rebuild remembered sets.
         log_trace(gc, marking)("Scrub and rebuild region skipped for " HR_FORMAT " pb: " PTR_FORMAT,
                                HR_FORMAT_PARAMS(hr), p2i(pb));
         assert(hr->bottom() == pb, "Region must be fully parsable");
         return false;
       }

       if (hr->needs_scrubbing()) {
         // This is a region with potentially unparsable (dead) objects.
         scan_and_scrub_region(hr, pb);
       } else {
         assert(hr->is_humongous(), "must be, but %u is %s", hr->hrm_index(), hr->get_short_type_str());
         // No need to scrub humongous, but we should scan it to rebuild remsets.
         scan_humongous_region(hr, pb);
       }

       return _cm->has_aborted();
     }
   };

 public:
   G1RebuildRSAndScrubTask(G1ConcurrentMark* cm, bool should_rebuild_remset, uint num_workers) :
     WorkerTask("Scrub dead objects"),
     _cm(cm),
     _hr_claimer(num_workers),
     _should_rebuild_remset(should_rebuild_remset) { }

   void work(uint worker_id) {
     SuspendibleThreadSetJoiner sts_join;

     G1CollectedHeap* g1h = G1CollectedHeap::heap();
     G1RebuildRSAndScrubRegionClosure cl(_cm, _should_rebuild_remset, worker_id);
     g1h->heap_region_par_iterate_from_worker_offset(&cl, &_hr_claimer, worker_id);
   }
 };

 void G1ConcurrentRebuildAndScrub::rebuild_and_scrub(G1ConcurrentMark* cm, bool should_rebuild_remset, WorkerThreads* workers) {
   uint num_workers = workers->active_workers();

   G1RebuildRSAndScrubTask task(cm, should_rebuild_remset, num_workers);
   workers->run_task(&task, num_workers);
 }
	/*
	* Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
	*
	* This code is free software; you can redistribute it and/or modify it
	* under the terms of the GNU General Public License version 2 only, as
	* published by the Free Software Foundation.
	*
	* This code is distributed in the hope that it will be useful, but WITHOUT
	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	* version 2 for more details (a copy is included in the LICENSE file that
	* accompanied this code).
	*
	* You should have received a copy of the GNU General Public License version
	* 2 along with this work; if not, write to the Free Software Foundation,
	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
	*
	* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
	* or visit www.oracle.com if you need additional information or have any
	* questions.
	*
	*/


	#include "gc/g1/g1ConcurrentMark.inline.hpp"
	#include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
	#include "gc/g1/g1ConcurrentRebuildAndScrub.hpp"
	#include "gc/g1/g1HeapRegion.inline.hpp"
	#include "gc/g1/g1HeapRegionManager.inline.hpp"
	#include "gc/shared/gc_globals.hpp"
	#include "gc/shared/suspendibleThreadSet.hpp"
	#include "gc/shared/workerThread.hpp"
	#include "logging/log.hpp"
	#include "memory/memRegion.hpp"
	#include "oops/oopsHierarchy.hpp"
	#include "utilities/debug.hpp"
	#include "utilities/globalDefinitions.hpp"

	// Worker task that scans the objects in the old generation to rebuild the remembered
	// set and at the same time scrubs dead objects by replacing them with filler objects
	// to make them completely parseable.
	//
	// The remark pause recorded two pointers within the regions:
	//
	// parsable_bottom (pb): this is the TAMS of the recent marking for that region. Objects
	// below that may or may not be dead (as per mark bitmap).
	// This task needs to remove the dead objects, replacing them
	// with filler objects so that they can be walked through later.
	//
	// top_at_rebuild_start (tars): at rebuild phase start we record the current top: up to
	// this address (live) objects need to be scanned for references
	// that might need to be added to the remembered sets.
	//
	// Note that bottom <= parsable_bottom <= tars; if there is no tars (i.e. null),
	// obviously there can not be a parsable_bottom.
	//
	// We need to scrub and scan objects to rebuild remembered sets until parsable_bottom;
	// we need to scan objects to rebuild remembered sets until tars.
	// Regions might have been reclaimed while scrubbing them after having yielded for
	// a pause.
	class G1RebuildRSAndScrubTask : public WorkerTask {
	G1ConcurrentMark* _cm;
	G1HeapRegionClaimer _hr_claimer;

	const bool _should_rebuild_remset;

	class G1RebuildRSAndScrubRegionClosure : public G1HeapRegionClosure {
	G1ConcurrentMark* _cm;
	const G1CMBitMap* _bitmap;

	G1RebuildRemSetClosure _rebuild_closure;

	const bool _should_rebuild_remset;

	size_t _processed_words;

	const size_t ProcessingYieldLimitInWords = G1RebuildRemSetChunkSize / HeapWordSize;

	void reset_processed_words() {
	_processed_words = 0;
	}

	void add_processed_words(size_t processed) {
	_processed_words += processed;
	}

	// Yield if enough has been processed. Return whether we should stop
	// processing this region because either the concurrent marking cycle has been
	// aborted or the region has been reclaimed.
	bool yield_if_necessary(G1HeapRegion* hr) {
	if (_processed_words >= ProcessingYieldLimitInWords) {
	reset_processed_words();
	// If a yield occurs (potential young-gc pause), must recheck for
	// potential regions reclamation.
	if (_cm->do_yield_check() && !should_rebuild_or_scrub(hr)) {
	return true;
	}
	}
	return _cm->has_aborted() \|\| !should_rebuild_or_scrub(hr);
	}

	// Returns whether the top at rebuild start value for the given region indicates
	// that there is some rebuild or scrubbing work.
	//
	// Based on the results of G1RemSetTrackingPolicy::needs_scan_for_rebuild(),
	// the value may be changed to null during rebuilding if the region has either:
	// - been allocated after rebuild start, or
	// - been reclaimed by a collection.
	bool should_rebuild_or_scrub(G1HeapRegion* hr) const {
	return _cm->top_at_rebuild_start(hr) != nullptr;
	}

	// Helper used by both humongous objects and when chunking an object larger than the
	// G1RebuildRemSetChunkSize. The heap region is needed check whether the region has
	// been reclaimed during yielding.
	void scan_large_object(G1HeapRegion* hr, const oop obj, MemRegion scan_range) {
	HeapWord* start = scan_range.start();
	HeapWord* limit = scan_range.end();
	do {
	MemRegion mr(start, MIN2(start + ProcessingYieldLimitInWords, limit));
	obj->oop_iterate(&_rebuild_closure, mr);

	// Update processed words and yield, for humongous objects we will yield
	// after each chunk.
	add_processed_words(mr.word_size());

	if (yield_if_necessary(hr)) {
	return;
	}

	// Step to next chunk of the large object.
	start = mr.end();
	} while (start < limit);
	}

	// Scan for references into regions that need remembered set update for the given
	// live object. Returns the offset to the next object.
	size_t scan_object(G1HeapRegion* hr, HeapWord* current) {
	oop obj = cast_to_oop(current);
	size_t obj_size = obj->size();

	if (!_should_rebuild_remset) {
	// Not rebuilding, just step to next object.
	add_processed_words(obj_size);
	} else if (obj_size > ProcessingYieldLimitInWords) {
	// Large object, needs to be chunked to avoid stalling safepoints.
	MemRegion mr(current, obj_size);
	scan_large_object(hr, obj, mr);
	// No need to add to _processed_words, this is all handled by the above call;
	// we also ignore the marking abort result of scan_large_object - we will check
	// again right afterwards.
	} else {
	// Object smaller than yield limit, process it fully.
	obj->oop_iterate(&_rebuild_closure);
	// Update how much we have processed. Yield check in main loop
	// will handle this case.
	add_processed_words(obj_size);
	}

	return obj_size;
	}

	// Scan or scrub depending on if addr is marked.
	HeapWord* scan_or_scrub(G1HeapRegion* hr, HeapWord* addr, HeapWord* limit) {
	if (_bitmap->is_marked(addr)) {
	// Live object, need to scan to rebuild remembered sets for this object.
	return addr + scan_object(hr, addr);
	} else {
	// Found dead object (which klass has potentially been unloaded). Scrub to next marked object.
	HeapWord* scrub_end = _bitmap->get_next_marked_addr(addr, limit);
	hr->fill_range_with_dead_objects(addr, scrub_end);
	// Return the next object to handle.
	return scrub_end;
	}
	}

	// Scan and scrub the given region to tars.
	void scan_and_scrub_region(G1HeapRegion* hr, HeapWord* const pb) {
	assert(should_rebuild_or_scrub(hr), "must be");

	log_trace(gc, marking)("Scrub and rebuild region: " HR_FORMAT " pb: " PTR_FORMAT " TARS: " PTR_FORMAT " TAMS: " PTR_FORMAT,
	HR_FORMAT_PARAMS(hr), p2i(pb), p2i(_cm->top_at_rebuild_start(hr)), p2i(_cm->top_at_mark_start(hr)));

	{
	// Step 1: Scan the given region from bottom to parsable_bottom.
	HeapWord* start = hr->bottom();
	HeapWord* limit = pb;
	while (start < limit) {
	start = scan_or_scrub(hr, start, limit);

	if (yield_if_necessary(hr)) {
	return;
	}
	}
	}

	// Scrubbing completed for this region - notify that we are done with it, resetting
	// pb to bottom.
	hr->note_end_of_scrubbing();

	{
	// Step 2: Rebuild from TAMS (= parsable_bottom) to TARS.
	HeapWord* start = pb;
	HeapWord* limit = _cm->top_at_rebuild_start(hr);
	while (start < limit) {
	start += scan_object(hr, start);

	if (yield_if_necessary(hr)) {
	return;
	}
	}
	}
	}

	// Scan a humongous region for remembered set updates. Scans in chunks to avoid
	// stalling safepoints.
	void scan_humongous_region(G1HeapRegion* hr, HeapWord* const pb) {
	assert(should_rebuild_or_scrub(hr), "must be");

	if (!_should_rebuild_remset) {
	// When not rebuilding there is nothing to do for humongous objects.
	return;
	}

	// At this point we should only have live humongous objects, that
	// means it must either be:
	// - marked
	// - or seen as fully parsable, i.e. allocated after the marking started
	oop humongous = cast_to_oop(hr->humongous_start_region()->bottom());
	assert(_bitmap->is_marked(humongous) \|\| pb == hr->bottom(),
	"Humongous object not live");

	log_trace(gc, marking)("Rebuild for humongous region: " HR_FORMAT " pb: " PTR_FORMAT " TARS: " PTR_FORMAT,
	HR_FORMAT_PARAMS(hr), p2i(pb), p2i(_cm->top_at_rebuild_start(hr)));

	// Scan the humongous object in chunks from bottom to top to rebuild remembered sets.
	HeapWord* humongous_end = hr->humongous_start_region()->bottom() + humongous->size();
	MemRegion mr(hr->bottom(), MIN2(hr->top(), humongous_end));

	scan_large_object(hr, humongous, mr);
	}

	public:
	G1RebuildRSAndScrubRegionClosure(G1ConcurrentMark* cm, bool should_rebuild_remset, uint worker_id) :
	_cm(cm),
	_bitmap(_cm->mark_bitmap()),
	_rebuild_closure(G1CollectedHeap::heap(), worker_id),
	_should_rebuild_remset(should_rebuild_remset),
	_processed_words(0) { }

	bool do_heap_region(G1HeapRegion* hr) {
	// Avoid stalling safepoints and stop iteration if mark cycle has been aborted.
	_cm->do_yield_check();
	if (_cm->has_aborted()) {
	return true;
	}

	HeapWord* const pb = hr->parsable_bottom_acquire();

	if (!should_rebuild_or_scrub(hr)) {
	// Region has been allocated during this phase, no need to either scrub or
	// scan to rebuild remembered sets.
	log_trace(gc, marking)("Scrub and rebuild region skipped for " HR_FORMAT " pb: " PTR_FORMAT,
	HR_FORMAT_PARAMS(hr), p2i(pb));
	assert(hr->bottom() == pb, "Region must be fully parsable");
	return false;
	}

	if (hr->needs_scrubbing()) {
	// This is a region with potentially unparsable (dead) objects.
	scan_and_scrub_region(hr, pb);
	} else {
	assert(hr->is_humongous(), "must be, but %u is %s", hr->hrm_index(), hr->get_short_type_str());
	// No need to scrub humongous, but we should scan it to rebuild remsets.
	scan_humongous_region(hr, pb);
	}

	return _cm->has_aborted();
	}
	};

	public:
	G1RebuildRSAndScrubTask(G1ConcurrentMark* cm, bool should_rebuild_remset, uint num_workers) :
	WorkerTask("Scrub dead objects"),
	_cm(cm),
	_hr_claimer(num_workers),
	_should_rebuild_remset(should_rebuild_remset) { }

	void work(uint worker_id) {
	SuspendibleThreadSetJoiner sts_join;

	G1CollectedHeap* g1h = G1CollectedHeap::heap();
	G1RebuildRSAndScrubRegionClosure cl(_cm, _should_rebuild_remset, worker_id);
	g1h->heap_region_par_iterate_from_worker_offset(&cl, &_hr_claimer, worker_id);
	}
	};

	void G1ConcurrentRebuildAndScrub::rebuild_and_scrub(G1ConcurrentMark* cm, bool should_rebuild_remset, WorkerThreads* workers) {
	uint num_workers = workers->active_workers();

	G1RebuildRSAndScrubTask task(cm, should_rebuild_remset, num_workers);
	workers->run_task(&task, num_workers);
	}