src/hotspot/share/gc/serial/cardTableRS.cpp - openjdk/jdk25u - Git at Google

 /*
  * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 only, as
  * published by the Free Software Foundation.
  *
  * This code is distributed in the hope that it will be useful, but WITHOUT
  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  * version 2 for more details (a copy is included in the LICENSE file that
  * accompanied this code).
  *
  * You should have received a copy of the GNU General Public License version
  * 2 along with this work; if not, write to the Free Software Foundation,
  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  *
  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  * or visit www.oracle.com if you need additional information or have any
  * questions.
  *
  */

 #include "classfile/classLoaderDataGraph.hpp"
 #include "gc/serial/cardTableRS.hpp"
 #include "gc/serial/generation.hpp"
 #include "gc/serial/serialHeap.inline.hpp"
 #include "gc/shared/space.hpp"
 #include "memory/iterator.inline.hpp"
 #include "utilities/align.hpp"

 void CardTableRS::scan_old_to_young_refs(TenuredGeneration* tg, HeapWord* saved_top) {
   const MemRegion ur    = tg->used_region();
   const MemRegion urasm = MemRegion(tg->space()->bottom(), saved_top);

   assert(ur.contains(urasm),
          "[" PTR_FORMAT ", " PTR_FORMAT ") is not contained in "
          "[" PTR_FORMAT ", " PTR_FORMAT ")",
          p2i(urasm.start()), p2i(urasm.end()), p2i(ur.start()), p2i(ur.end()));

   if (!urasm.is_empty()) {
     OldGenScanClosure cl(SerialHeap::heap()->young_gen());
     non_clean_card_iterate(tg, urasm, &cl);
   }
 }

 void CardTableRS::maintain_old_to_young_invariant(TenuredGeneration* old_gen,
                                                   bool is_young_gen_empty) {
   if (is_young_gen_empty) {
     clear_MemRegion(old_gen->prev_used_region());
   } else {
     MemRegion used_mr = old_gen->used_region();
     MemRegion prev_used_mr = old_gen->prev_used_region();
     if (used_mr.end() < prev_used_mr.end()) {
       // Shrunk; need to clear the previously-used but now-unused parts.
       clear_MemRegion(MemRegion(used_mr.end(), prev_used_mr.end()));
     }
     // No idea which card contains old-to-young pointer, so dirtying cards for
     // the entire used part of old-gen conservatively.
     dirty_MemRegion(used_mr);
   }
 }

 class SerialCheckForUnmarkedOops : public BasicOopIterateClosure {
   DefNewGeneration* _young_gen;
   CardTableRS* _card_table;
   HeapWord*    _unmarked_addr;

   template <class T> void do_oop_work(T* p) {
     oop obj = RawAccess<>::oop_load(p);
     if (_young_gen->is_in_reserved(obj) &&
         !_card_table->is_dirty_for_addr(p)) {
       // Don't overwrite the first missing card mark
       if (_unmarked_addr == nullptr) {
         _unmarked_addr = (HeapWord*)p;
       }
     }
   }

  public:
   SerialCheckForUnmarkedOops(DefNewGeneration* young_gen, CardTableRS* card_table) :
     _young_gen(young_gen),
     _card_table(card_table),
     _unmarked_addr(nullptr) {}

   void do_oop(oop* p)       override { do_oop_work(p); }
   void do_oop(narrowOop* p) override { do_oop_work(p); }

   bool has_unmarked_oop() {
     return _unmarked_addr != nullptr;
   }
 };

 void CardTableRS::verify() {
   class CheckForUnmarkedObjects : public ObjectClosure {
     DefNewGeneration* _young_gen;
     CardTableRS* _card_table;

    public:
     CheckForUnmarkedObjects() {
       SerialHeap* heap = SerialHeap::heap();
       _young_gen = heap->young_gen();
       _card_table = heap->rem_set();
     }

     void do_object(oop obj) override {
       SerialCheckForUnmarkedOops object_check(_young_gen, _card_table);
       obj->oop_iterate(&object_check);
       // If this obj is imprecisely-marked, the card for obj-start must be dirty.
       if (object_check.has_unmarked_oop()) {
         guarantee(_card_table->is_dirty_for_addr(obj), "Found unmarked old-to-young pointer");
       }
     }
   } check;

   SerialHeap::heap()->old_gen()->object_iterate(&check);
 }

 CardTableRS::CardTableRS(MemRegion whole_heap) :
   CardTable(whole_heap) { }

 // Implemented word-iteration to skip long consecutive clean cards.
 CardTable::CardValue* CardTableRS::find_first_dirty_card(CardValue* const start_card,
                                                          CardValue* const end_card) {
   using Word = uintptr_t;

   CardValue* current_card = start_card;

   while (!is_aligned(current_card, sizeof(Word))) {
     if (current_card >= end_card) {
       return end_card;
     }
     if (is_dirty(current_card)) {
       return current_card;
     }
     ++current_card;
   }

   // Word comparison
   while (current_card + sizeof(Word) <= end_card) {
     Word* current_word = reinterpret_cast<Word*>(current_card);
     if (*current_word != (Word)clean_card_row_val()) {
       // Found a dirty card in this word; fall back to per-CardValue comparison.
       break;
     }
     current_card += sizeof(Word);
   }

   // Per-CardValue comparison.
   for (/* empty */; current_card < end_card; ++current_card) {
     if (is_dirty(current_card)) {
       return current_card;
     }
   }

   return end_card;
 }

 // Because non-objArray objs can be imprecisely marked (only the obj-start card
 // is dirty instead of the part containing old-to-young pointers), if the
 // obj-start of a non-objArray is dirty, all cards that the obj resides on,
 // except the final one, are unconditionally considered as dirty. This is
 // because that obj will be iterated (scanned for old-to-young pointers) as a
 // whole.
 template<typename Func>
 CardTable::CardValue* CardTableRS::find_first_clean_card(CardValue* const start_card,
                                                          CardValue* const end_card,
                                                          Func& object_start) {
   for (CardValue* current_card = start_card; current_card < end_card; /* empty */) {
     if (is_dirty(current_card)) {
       current_card++;
       continue;
     }

     // A potential candidate.
     HeapWord* addr = addr_for(current_card);
     HeapWord* obj_start_addr = object_start(addr);

     if (obj_start_addr == addr) {
       return current_card;
     }

     // Final obj in dirty-chunk crosses card-boundary.
     oop obj = cast_to_oop(obj_start_addr);
     if (obj->is_objArray()) {
       // ObjArrays are always precisely-marked so we are not allowed to jump to
       // the end of the current object.
       return current_card;
     }

     // Final card occupied by obj.
     CardValue* obj_final_card = byte_for(obj_start_addr + obj->size() - 1);
     if (is_clean(obj_final_card)) {
       return obj_final_card;
     }

     // Continue the search after this known-dirty card...
     current_card = obj_final_card + 1;
   }

   return end_card;
 }

 void CardTableRS::clear_cards(CardValue* start, CardValue* end) {
   size_t num_cards = pointer_delta(end, start, sizeof(CardValue));
   memset(start, clean_card_val(), num_cards);
 }

 static void prefetch_write(void *p) {
   if (PrefetchScanIntervalInBytes >= 0) {
     Prefetch::write(p, PrefetchScanIntervalInBytes);
   }
 }

 static void scan_obj_with_limit(oop obj,
                                 OldGenScanClosure* cl,
                                 HeapWord* start,
                                 HeapWord* end) {
   if (!obj->is_typeArray()) {
     prefetch_write(start);
     obj->oop_iterate(cl, MemRegion(start, end));
   }
 }

 void CardTableRS::non_clean_card_iterate(TenuredGeneration* tg,
                                          MemRegion mr,
                                          OldGenScanClosure* cl) {
   struct {
     HeapWord* start_addr;
     HeapWord* end_addr;
   } cached_obj { nullptr, mr.start() };

   auto object_start = [&] (const HeapWord* const addr) {
     if (addr < cached_obj.end_addr) {
       assert(cached_obj.start_addr != nullptr, "inv");
       return cached_obj.start_addr;
     }
     HeapWord* result = tg->block_start(addr);

     cached_obj.start_addr = result;
     cached_obj.end_addr = result + cast_to_oop(result)->size();

     return result;
   };

   CardValue* const start_card = byte_for(mr.start());
   CardValue* const end_card = byte_for(mr.last()) + 1;

   // if mr.end() is not card-aligned, that final card should not be cleared
   // because it can be annotated dirty due to old-to-young pointers in
   // newly-promoted objs on that card.
   CardValue* const clear_limit_card = is_card_aligned(mr.end()) ? end_card - 1
                                                                 : end_card - 2;

   for (CardValue* current_card = start_card; current_card < end_card; /* empty */) {
     CardValue* const dirty_l = find_first_dirty_card(current_card, end_card);
     if (dirty_l == end_card) {
       // No dirty cards to iterate.
       return;
     }

     HeapWord* const addr_l = addr_for(dirty_l);
     HeapWord* obj_addr = object_start(addr_l);

     CardValue* const dirty_r = find_first_clean_card(dirty_l + 1,
                                                      end_card,
                                                      object_start);
     assert(dirty_l < dirty_r, "inv");
     HeapWord* const addr_r = dirty_r == end_card ? mr.end()
                                                  : addr_for(dirty_r);

     clear_cards(MIN2(dirty_l, clear_limit_card),
                 MIN2(dirty_r, clear_limit_card));

     while (true) {
       assert(obj_addr < addr_r, "inv");

       oop obj = cast_to_oop(obj_addr);
       const bool is_obj_array = obj->is_objArray();
       HeapWord* const obj_end_addr = obj_addr + obj->size();

       if (is_obj_array) {
         // ObjArrays are always precise-marked.
         scan_obj_with_limit(obj, cl, addr_l, addr_r);
       } else {
         scan_obj_with_limit(obj, cl, addr_l, obj_end_addr);
       }

       if (obj_end_addr >= addr_r) {
         current_card = dirty_r + 1;
         break;
       }

       // Move to next obj inside this dirty chunk.
       obj_addr = obj_end_addr;
     }
   }
 }

 bool CardTableRS::is_in_young(const void* p) const {
   return SerialHeap::heap()->is_in_young(p);
 }
	/*
	* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
	*
	* This code is free software; you can redistribute it and/or modify it
	* under the terms of the GNU General Public License version 2 only, as
	* published by the Free Software Foundation.
	*
	* This code is distributed in the hope that it will be useful, but WITHOUT
	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	* version 2 for more details (a copy is included in the LICENSE file that
	* accompanied this code).
	*
	* You should have received a copy of the GNU General Public License version
	* 2 along with this work; if not, write to the Free Software Foundation,
	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
	*
	* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
	* or visit www.oracle.com if you need additional information or have any
	* questions.
	*
	*/

	#include "classfile/classLoaderDataGraph.hpp"
	#include "gc/serial/cardTableRS.hpp"
	#include "gc/serial/generation.hpp"
	#include "gc/serial/serialHeap.inline.hpp"
	#include "gc/shared/space.hpp"
	#include "memory/iterator.inline.hpp"
	#include "utilities/align.hpp"

	void CardTableRS::scan_old_to_young_refs(TenuredGeneration* tg, HeapWord* saved_top) {
	const MemRegion ur = tg->used_region();
	const MemRegion urasm = MemRegion(tg->space()->bottom(), saved_top);

	assert(ur.contains(urasm),
	"[" PTR_FORMAT ", " PTR_FORMAT ") is not contained in "
	"[" PTR_FORMAT ", " PTR_FORMAT ")",
	p2i(urasm.start()), p2i(urasm.end()), p2i(ur.start()), p2i(ur.end()));

	if (!urasm.is_empty()) {
	OldGenScanClosure cl(SerialHeap::heap()->young_gen());
	non_clean_card_iterate(tg, urasm, &cl);
	}
	}

	void CardTableRS::maintain_old_to_young_invariant(TenuredGeneration* old_gen,
	bool is_young_gen_empty) {
	if (is_young_gen_empty) {
	clear_MemRegion(old_gen->prev_used_region());
	} else {
	MemRegion used_mr = old_gen->used_region();
	MemRegion prev_used_mr = old_gen->prev_used_region();
	if (used_mr.end() < prev_used_mr.end()) {
	// Shrunk; need to clear the previously-used but now-unused parts.
	clear_MemRegion(MemRegion(used_mr.end(), prev_used_mr.end()));
	}
	// No idea which card contains old-to-young pointer, so dirtying cards for
	// the entire used part of old-gen conservatively.
	dirty_MemRegion(used_mr);
	}
	}

	class SerialCheckForUnmarkedOops : public BasicOopIterateClosure {
	DefNewGeneration* _young_gen;
	CardTableRS* _card_table;
	HeapWord* _unmarked_addr;

	template <class T> void do_oop_work(T* p) {
	oop obj = RawAccess<>::oop_load(p);
	if (_young_gen->is_in_reserved(obj) &&
	!_card_table->is_dirty_for_addr(p)) {
	// Don't overwrite the first missing card mark
	if (_unmarked_addr == nullptr) {
	_unmarked_addr = (HeapWord*)p;
	}
	}
	}

	public:
	SerialCheckForUnmarkedOops(DefNewGeneration* young_gen, CardTableRS* card_table) :
	_young_gen(young_gen),
	_card_table(card_table),
	_unmarked_addr(nullptr) {}

	void do_oop(oop* p) override { do_oop_work(p); }
	void do_oop(narrowOop* p) override { do_oop_work(p); }

	bool has_unmarked_oop() {
	return _unmarked_addr != nullptr;
	}
	};

	void CardTableRS::verify() {
	class CheckForUnmarkedObjects : public ObjectClosure {
	DefNewGeneration* _young_gen;
	CardTableRS* _card_table;

	public:
	CheckForUnmarkedObjects() {
	SerialHeap* heap = SerialHeap::heap();
	_young_gen = heap->young_gen();
	_card_table = heap->rem_set();
	}

	void do_object(oop obj) override {
	SerialCheckForUnmarkedOops object_check(_young_gen, _card_table);
	obj->oop_iterate(&object_check);
	// If this obj is imprecisely-marked, the card for obj-start must be dirty.
	if (object_check.has_unmarked_oop()) {
	guarantee(_card_table->is_dirty_for_addr(obj), "Found unmarked old-to-young pointer");
	}
	}
	} check;

	SerialHeap::heap()->old_gen()->object_iterate(&check);
	}

	CardTableRS::CardTableRS(MemRegion whole_heap) :
	CardTable(whole_heap) { }

	// Implemented word-iteration to skip long consecutive clean cards.
	CardTable::CardValue* CardTableRS::find_first_dirty_card(CardValue* const start_card,
	CardValue* const end_card) {
	using Word = uintptr_t;

	CardValue* current_card = start_card;

	while (!is_aligned(current_card, sizeof(Word))) {
	if (current_card >= end_card) {
	return end_card;
	}
	if (is_dirty(current_card)) {
	return current_card;
	}
	++current_card;
	}

	// Word comparison
	while (current_card + sizeof(Word) <= end_card) {
	Word* current_word = reinterpret_cast<Word*>(current_card);
	if (*current_word != (Word)clean_card_row_val()) {
	// Found a dirty card in this word; fall back to per-CardValue comparison.
	break;
	}
	current_card += sizeof(Word);
	}

	// Per-CardValue comparison.
	for (/* empty */; current_card < end_card; ++current_card) {
	if (is_dirty(current_card)) {
	return current_card;
	}
	}

	return end_card;
	}

	// Because non-objArray objs can be imprecisely marked (only the obj-start card
	// is dirty instead of the part containing old-to-young pointers), if the
	// obj-start of a non-objArray is dirty, all cards that the obj resides on,
	// except the final one, are unconditionally considered as dirty. This is
	// because that obj will be iterated (scanned for old-to-young pointers) as a
	// whole.
	template<typename Func>
	CardTable::CardValue* CardTableRS::find_first_clean_card(CardValue* const start_card,
	CardValue* const end_card,
	Func& object_start) {
	for (CardValue* current_card = start_card; current_card < end_card; /* empty */) {
	if (is_dirty(current_card)) {
	current_card++;
	continue;
	}

	// A potential candidate.
	HeapWord* addr = addr_for(current_card);
	HeapWord* obj_start_addr = object_start(addr);

	if (obj_start_addr == addr) {
	return current_card;
	}

	// Final obj in dirty-chunk crosses card-boundary.
	oop obj = cast_to_oop(obj_start_addr);
	if (obj->is_objArray()) {
	// ObjArrays are always precisely-marked so we are not allowed to jump to
	// the end of the current object.
	return current_card;
	}

	// Final card occupied by obj.
	CardValue* obj_final_card = byte_for(obj_start_addr + obj->size() - 1);
	if (is_clean(obj_final_card)) {
	return obj_final_card;
	}

	// Continue the search after this known-dirty card...
	current_card = obj_final_card + 1;
	}

	return end_card;
	}

	void CardTableRS::clear_cards(CardValue* start, CardValue* end) {
	size_t num_cards = pointer_delta(end, start, sizeof(CardValue));
	memset(start, clean_card_val(), num_cards);
	}

	static void prefetch_write(void *p) {
	if (PrefetchScanIntervalInBytes >= 0) {
	Prefetch::write(p, PrefetchScanIntervalInBytes);
	}
	}

	static void scan_obj_with_limit(oop obj,
	OldGenScanClosure* cl,
	HeapWord* start,
	HeapWord* end) {
	if (!obj->is_typeArray()) {
	prefetch_write(start);
	obj->oop_iterate(cl, MemRegion(start, end));
	}
	}

	void CardTableRS::non_clean_card_iterate(TenuredGeneration* tg,
	MemRegion mr,
	OldGenScanClosure* cl) {
	struct {
	HeapWord* start_addr;
	HeapWord* end_addr;
	} cached_obj { nullptr, mr.start() };

	auto object_start = [&] (const HeapWord* const addr) {
	if (addr < cached_obj.end_addr) {
	assert(cached_obj.start_addr != nullptr, "inv");
	return cached_obj.start_addr;
	}
	HeapWord* result = tg->block_start(addr);

	cached_obj.start_addr = result;
	cached_obj.end_addr = result + cast_to_oop(result)->size();

	return result;
	};

	CardValue* const start_card = byte_for(mr.start());
	CardValue* const end_card = byte_for(mr.last()) + 1;

	// if mr.end() is not card-aligned, that final card should not be cleared
	// because it can be annotated dirty due to old-to-young pointers in
	// newly-promoted objs on that card.
	CardValue* const clear_limit_card = is_card_aligned(mr.end()) ? end_card - 1
	: end_card - 2;

	for (CardValue* current_card = start_card; current_card < end_card; /* empty */) {
	CardValue* const dirty_l = find_first_dirty_card(current_card, end_card);
	if (dirty_l == end_card) {
	// No dirty cards to iterate.
	return;
	}

	HeapWord* const addr_l = addr_for(dirty_l);
	HeapWord* obj_addr = object_start(addr_l);

	CardValue* const dirty_r = find_first_clean_card(dirty_l + 1,
	end_card,
	object_start);
	assert(dirty_l < dirty_r, "inv");
	HeapWord* const addr_r = dirty_r == end_card ? mr.end()
	: addr_for(dirty_r);

	clear_cards(MIN2(dirty_l, clear_limit_card),
	MIN2(dirty_r, clear_limit_card));

	while (true) {
	assert(obj_addr < addr_r, "inv");

	oop obj = cast_to_oop(obj_addr);
	const bool is_obj_array = obj->is_objArray();
	HeapWord* const obj_end_addr = obj_addr + obj->size();

	if (is_obj_array) {
	// ObjArrays are always precise-marked.
	scan_obj_with_limit(obj, cl, addr_l, addr_r);
	} else {
	scan_obj_with_limit(obj, cl, addr_l, obj_end_addr);
	}

	if (obj_end_addr >= addr_r) {
	current_card = dirty_r + 1;
	break;
	}

	// Move to next obj inside this dirty chunk.
	obj_addr = obj_end_addr;
	}
	}
	}

	bool CardTableRS::is_in_young(const void* p) const {
	return SerialHeap::heap()->is_in_young(p);
	}