src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp

/*
 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
 * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 *
 */

#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp"
#include "gc/shenandoah/shenandoahAsserts.hpp"
#include "gc/shenandoah/shenandoahCardTable.hpp"
#include "gc/shenandoah/shenandoahClosures.inline.hpp"
#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
#include "gc/shenandoah/shenandoahFreeSet.hpp"
#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
#include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
#include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
#include "gc/shenandoah/shenandoahOldGeneration.hpp"
#include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
#include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
#include "gc/shenandoah/shenandoahUtils.hpp"
#include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
#include "gc/shenandoah/shenandoahYoungGeneration.hpp"
#include "runtime/threads.hpp"
#include "utilities/events.hpp"

class ShenandoahFlushAllSATB : public ThreadClosure {
private:
  SATBMarkQueueSet& _satb_qset;

public:
  explicit ShenandoahFlushAllSATB(SATBMarkQueueSet& satb_qset) :
    _satb_qset(satb_qset) {}

  void do_thread(Thread* thread) override {
    // Transfer any partial buffer to the qset for completed buffer processing.
    _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
  }
};

class ShenandoahProcessOldSATB : public SATBBufferClosure {
private:
  ShenandoahObjToScanQueue*       _queue;
  ShenandoahHeap*                 _heap;
  ShenandoahMarkingContext* const _mark_context;
  size_t                          _trashed_oops;

public:
  explicit ShenandoahProcessOldSATB(ShenandoahObjToScanQueue* q) :
    _queue(q),
    _heap(ShenandoahHeap::heap()),
    _mark_context(_heap->marking_context()),
    _trashed_oops(0) {}

  void do_buffer(void** buffer, size_t size) override {
    assert(size == 0 || !_heap->has_forwarded_objects() || _heap->is_concurrent_old_mark_in_progress(), "Forwarded objects are not expected here");
    for (size_t i = 0; i < size; ++i) {
      oop *p = (oop *) &buffer[i];
      ShenandoahHeapRegion* region = _heap->heap_region_containing(*p);
      if (region->is_old() && region->is_active()) {
          ShenandoahMark::mark_through_ref<oop, OLD>(p, _queue, nullptr, _mark_context, false);
      } else {
        _trashed_oops++;
      }
    }
  }

  size_t trashed_oops() const {
    return _trashed_oops;
  }
};

class ShenandoahPurgeSATBTask : public WorkerTask {
private:
  ShenandoahObjToScanQueueSet* _mark_queues;
  // Keep track of the number of oops that are not transferred to mark queues.
  // This is volatile because workers update it, but the vm thread reads it.
  volatile size_t             _trashed_oops;

public:
  explicit ShenandoahPurgeSATBTask(ShenandoahObjToScanQueueSet* queues) :
    WorkerTask("Purge SATB"),
    _mark_queues(queues),
    _trashed_oops(0) {
    Threads::change_thread_claim_token();
  }

  ~ShenandoahPurgeSATBTask() {
    if (_trashed_oops > 0) {
      log_debug(gc)("Purged %zu oops from old generation SATB buffers", _trashed_oops);
    }
  }

  void work(uint worker_id) override {
    ShenandoahParallelWorkerSession worker_session(worker_id);
    ShenandoahSATBMarkQueueSet &satb_queues = ShenandoahBarrierSet::satb_mark_queue_set();
    ShenandoahFlushAllSATB flusher(satb_queues);
    Threads::possibly_parallel_threads_do(true /* is_par */, &flusher);

    ShenandoahObjToScanQueue* mark_queue = _mark_queues->queue(worker_id);
    ShenandoahProcessOldSATB processor(mark_queue);
    while (satb_queues.apply_closure_to_completed_buffer(&processor)) {}

    Atomic::add(&_trashed_oops, processor.trashed_oops());
  }
};

class ShenandoahTransferOldSATBTask : public WorkerTask {
  ShenandoahSATBMarkQueueSet&  _satb_queues;
  ShenandoahObjToScanQueueSet* _mark_queues;
  // Keep track of the number of oops that are not transferred to mark queues.
  // This is volatile because workers update it, but the control thread reads it.
  volatile size_t              _trashed_oops;

public:
  explicit ShenandoahTransferOldSATBTask(ShenandoahSATBMarkQueueSet& satb_queues, ShenandoahObjToScanQueueSet* mark_queues) :
    WorkerTask("Transfer SATB"),
    _satb_queues(satb_queues),
    _mark_queues(mark_queues),
    _trashed_oops(0) {}

  ~ShenandoahTransferOldSATBTask() {
    if (_trashed_oops > 0) {
      log_debug(gc)("Purged %zu oops from old generation SATB buffers", _trashed_oops);
    }
  }

  void work(uint worker_id) override {
    ShenandoahObjToScanQueue* mark_queue = _mark_queues->queue(worker_id);
    ShenandoahProcessOldSATB processor(mark_queue);
    while (_satb_queues.apply_closure_to_completed_buffer(&processor)) {}

    Atomic::add(&_trashed_oops, processor.trashed_oops());
  }
};

class ShenandoahConcurrentCoalesceAndFillTask : public WorkerTask {
private:
  uint                    _nworkers;
  ShenandoahHeapRegion**  _coalesce_and_fill_region_array;
  uint                    _coalesce_and_fill_region_count;
  volatile bool           _is_preempted;

public:
  ShenandoahConcurrentCoalesceAndFillTask(uint nworkers,
                                          ShenandoahHeapRegion** coalesce_and_fill_region_array,
                                          uint region_count) :
    WorkerTask("Shenandoah Concurrent Coalesce and Fill"),
    _nworkers(nworkers),
    _coalesce_and_fill_region_array(coalesce_and_fill_region_array),
    _coalesce_and_fill_region_count(region_count),
    _is_preempted(false) {
  }

  void work(uint worker_id) override {
    ShenandoahWorkerTimingsTracker timer(ShenandoahPhaseTimings::conc_coalesce_and_fill, ShenandoahPhaseTimings::ScanClusters, worker_id);
    for (uint region_idx = worker_id; region_idx < _coalesce_and_fill_region_count; region_idx += _nworkers) {
      ShenandoahHeapRegion* r = _coalesce_and_fill_region_array[region_idx];
      if (r->is_humongous()) {
        // There is only one object in this region and it is not garbage,
        // so no need to coalesce or fill.
        continue;
      }

      if (!r->oop_coalesce_and_fill(true)) {
        // Coalesce and fill has been preempted
        Atomic::store(&_is_preempted, true);
        return;
      }
    }
  }

  // Value returned from is_completed() is only valid after all worker thread have terminated.
  bool is_completed() {
    return !Atomic::load(&_is_preempted);
  }
};

ShenandoahOldGeneration::ShenandoahOldGeneration(uint max_queues, size_t max_capacity, size_t soft_max_capacity)
  : ShenandoahGeneration(OLD, max_queues, max_capacity, soft_max_capacity),
    _coalesce_and_fill_region_array(NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, ShenandoahHeap::heap()->num_regions(), mtGC)),
    _old_heuristics(nullptr),
    _region_balance(0),
    _promoted_reserve(0),
    _promoted_expended(0),
    _promotion_potential(0),
    _pad_for_promote_in_place(0),
    _promotable_humongous_regions(0),
    _promotable_regular_regions(0),
    _is_parsable(true),
    _card_scan(nullptr),
    _state(WAITING_FOR_BOOTSTRAP),
    _growth_before_compaction(INITIAL_GROWTH_BEFORE_COMPACTION),
    _min_growth_before_compaction ((ShenandoahMinOldGenGrowthPercent * FRACTIONAL_DENOMINATOR) / 100)
{
  _live_bytes_after_last_mark = ShenandoahHeap::heap()->capacity() * INITIAL_LIVE_FRACTION / FRACTIONAL_DENOMINATOR;
  // Always clear references for old generation
  ref_processor()->set_soft_reference_policy(true);

  if (ShenandoahCardBarrier) {
    ShenandoahCardTable* card_table = ShenandoahBarrierSet::barrier_set()->card_table();
    size_t card_count = card_table->cards_required(ShenandoahHeap::heap()->reserved_region().word_size());
    auto rs = new ShenandoahDirectCardMarkRememberedSet(card_table, card_count);
    _card_scan = new ShenandoahScanRemembered(rs);
  }
}

void ShenandoahOldGeneration::set_promoted_reserve(size_t new_val) {
  shenandoah_assert_heaplocked_or_safepoint();
  _promoted_reserve = new_val;
}

size_t ShenandoahOldGeneration::get_promoted_reserve() const {
  return _promoted_reserve;
}

void ShenandoahOldGeneration::augment_promoted_reserve(size_t increment) {
  shenandoah_assert_heaplocked_or_safepoint();
  _promoted_reserve += increment;
}

void ShenandoahOldGeneration::reset_promoted_expended() {
  shenandoah_assert_heaplocked_or_safepoint();
  Atomic::store(&_promoted_expended, (size_t) 0);
}

size_t ShenandoahOldGeneration::expend_promoted(size_t increment) {
  shenandoah_assert_heaplocked_or_safepoint();
  assert(get_promoted_expended() + increment <= get_promoted_reserve(), "Do not expend more promotion than budgeted");
  return Atomic::add(&_promoted_expended, increment);
}

size_t ShenandoahOldGeneration::unexpend_promoted(size_t decrement) {
  return Atomic::sub(&_promoted_expended, decrement);
}

size_t ShenandoahOldGeneration::get_promoted_expended() const {
  return Atomic::load(&_promoted_expended);
}

bool ShenandoahOldGeneration::can_allocate(const ShenandoahAllocRequest &req) const {
  assert(req.type() != ShenandoahAllocRequest::_alloc_gclab, "GCLAB pertains only to young-gen memory");

  const size_t requested_bytes = req.size() * HeapWordSize;
  // The promotion reserve may also be used for evacuations. If we can promote this object,
  // then we can also evacuate it.
  if (can_promote(requested_bytes)) {
    // The promotion reserve should be able to accommodate this request. The request
    // might still fail if alignment with the card table increases the size. The request
    // may also fail if the heap is badly fragmented and the free set cannot find room for it.
    return true;
  }

  if (req.type() == ShenandoahAllocRequest::_alloc_plab) {
    // The promotion reserve cannot accommodate this plab request. Check if we still have room for
    // evacuations. Note that we cannot really know how much of the plab will be used for evacuations,
    // so here we only check that some evacuation reserve still exists.
    return get_evacuation_reserve() > 0;
  }

  // This is a shared allocation request. We've already checked that it can't be promoted, so if
  // it is a promotion, we return false. Otherwise, it is a shared evacuation request, and we allow
  // the allocation to proceed.
  return !req.is_promotion();
}

void
ShenandoahOldGeneration::configure_plab_for_current_thread(const ShenandoahAllocRequest &req) {
  // Note: Even when a mutator is performing a promotion outside a LAB, we use a 'shared_gc' request.
  if (req.is_gc_alloc()) {
    const size_t actual_size = req.actual_size() * HeapWordSize;
    if (req.type() ==  ShenandoahAllocRequest::_alloc_plab) {
      // We've created a new plab. Now we configure it whether it will be used for promotions
      // and evacuations - or just evacuations.
      Thread* thread = Thread::current();
      ShenandoahThreadLocalData::reset_plab_promoted(thread);

      // The actual size of the allocation may be larger than the requested bytes (due to alignment on card boundaries).
      // If this puts us over our promotion budget, we need to disable future PLAB promotions for this thread.
      if (can_promote(actual_size)) {
        // Assume the entirety of this PLAB will be used for promotion.  This prevents promotion from overreach.
        // When we retire this plab, we'll unexpend what we don't really use.
        expend_promoted(actual_size);
        ShenandoahThreadLocalData::enable_plab_promotions(thread);
        ShenandoahThreadLocalData::set_plab_actual_size(thread, actual_size);
      } else {
        // Disable promotions in this thread because entirety of this PLAB must be available to hold old-gen evacuations.
        ShenandoahThreadLocalData::disable_plab_promotions(thread);
        ShenandoahThreadLocalData::set_plab_actual_size(thread, 0);
      }
    } else if (req.is_promotion()) {
      // Shared promotion.
      expend_promoted(actual_size);
    }
  }
}

size_t ShenandoahOldGeneration::get_live_bytes_after_last_mark() const {
  return _live_bytes_after_last_mark;
}

void ShenandoahOldGeneration::set_live_bytes_after_last_mark(size_t bytes) {
  if (bytes == 0) {
    // Restart search for best old-gen size to the initial state
    _live_bytes_after_last_mark = ShenandoahHeap::heap()->capacity() * INITIAL_LIVE_FRACTION / FRACTIONAL_DENOMINATOR;
    _growth_before_compaction = INITIAL_GROWTH_BEFORE_COMPACTION;
  } else {
    _live_bytes_after_last_mark = bytes;
    _growth_before_compaction /= 2;
    if (_growth_before_compaction < _min_growth_before_compaction) {
      _growth_before_compaction = _min_growth_before_compaction;
    }
  }
}

void ShenandoahOldGeneration::handle_failed_transfer() {
  _old_heuristics->trigger_cannot_expand();
}

size_t ShenandoahOldGeneration::usage_trigger_threshold() const {
  size_t result = _live_bytes_after_last_mark + (_live_bytes_after_last_mark * _growth_before_compaction) / FRACTIONAL_DENOMINATOR;
  return result;
}

bool ShenandoahOldGeneration::contains(ShenandoahAffiliation affiliation) const {
  return affiliation == OLD_GENERATION;
}
bool ShenandoahOldGeneration::contains(ShenandoahHeapRegion* region) const {
  return region->is_old();
}

void ShenandoahOldGeneration::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) {
  ShenandoahIncludeRegionClosure<OLD_GENERATION> old_regions_cl(cl);
  ShenandoahHeap::heap()->parallel_heap_region_iterate(&old_regions_cl);
}

void ShenandoahOldGeneration::heap_region_iterate(ShenandoahHeapRegionClosure* cl) {
  ShenandoahIncludeRegionClosure<OLD_GENERATION> old_regions_cl(cl);
  ShenandoahHeap::heap()->heap_region_iterate(&old_regions_cl);
}

void ShenandoahOldGeneration::set_concurrent_mark_in_progress(bool in_progress) {
  ShenandoahHeap::heap()->set_concurrent_old_mark_in_progress(in_progress);
}

bool ShenandoahOldGeneration::is_concurrent_mark_in_progress() {
  return ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress();
}

void ShenandoahOldGeneration::cancel_marking() {
  if (is_concurrent_mark_in_progress()) {
    log_debug(gc)("Abandon SATB buffers");
    ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking();
  }

  ShenandoahGeneration::cancel_marking();
}

void ShenandoahOldGeneration::cancel_gc() {
  shenandoah_assert_safepoint();
  if (is_idle()) {
#ifdef ASSERT
    validate_waiting_for_bootstrap();
#endif
  } else {
    log_info(gc)("Terminating old gc cycle.");
    // Stop marking
    cancel_marking();
    // Stop tracking old regions
    abandon_collection_candidates();
    // Remove old generation access to young generation mark queues
    ShenandoahHeap::heap()->young_generation()->set_old_gen_task_queues(nullptr);
    // Transition to IDLE now.
    transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP);
  }
}

void ShenandoahOldGeneration::prepare_gc() {
  // Now that we have made the old generation parsable, it is safe to reset the mark bitmap.
  assert(state() != FILLING, "Cannot reset old without making it parsable");

  ShenandoahGeneration::prepare_gc();
}

bool ShenandoahOldGeneration::entry_coalesce_and_fill() {
  ShenandoahHeap* const heap = ShenandoahHeap::heap();

  static const char* msg = "Coalescing and filling (Old)";
  ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_coalesce_and_fill);

  TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
  EventMark em("%s", msg);
  ShenandoahWorkerScope scope(heap->workers(),
                              ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
                              msg);

  return coalesce_and_fill();
}

// Make the old generation regions parsable, so they can be safely
// scanned when looking for objects in memory indicated by dirty cards.
bool ShenandoahOldGeneration::coalesce_and_fill() {
  transition_to(FILLING);

  // This code will see the same set of regions to fill on each resumption as it did
  // on the initial run. That's okay because each region keeps track of its own coalesce
  // and fill state. Regions that were filled on a prior attempt will not try to fill again.
  uint coalesce_and_fill_regions_count = _old_heuristics->get_coalesce_and_fill_candidates(_coalesce_and_fill_region_array);
  assert(coalesce_and_fill_regions_count <= ShenandoahHeap::heap()->num_regions(), "Sanity");
  if (coalesce_and_fill_regions_count == 0) {
    // No regions need to be filled.
    abandon_collection_candidates();
    return true;
  }

  ShenandoahHeap* const heap = ShenandoahHeap::heap();
  WorkerThreads* workers = heap->workers();
  uint nworkers = workers->active_workers();
  ShenandoahConcurrentCoalesceAndFillTask task(nworkers, _coalesce_and_fill_region_array, coalesce_and_fill_regions_count);

  log_debug(gc)("Starting (or resuming) coalesce-and-fill of " UINT32_FORMAT " old heap regions", coalesce_and_fill_regions_count);
  workers->run_task(&task);
  if (task.is_completed()) {
    // We no longer need to track regions that need to be coalesced and filled.
    abandon_collection_candidates();
    return true;
  } else {
    // Coalesce-and-fill has been preempted. We'll finish that effort in the future.  Do not invoke
    // ShenandoahGeneration::prepare_gc() until coalesce-and-fill is done because it resets the mark bitmap
    // and invokes set_mark_incomplete().  Coalesce-and-fill depends on the mark bitmap.
    log_debug(gc)("Suspending coalesce-and-fill of old heap regions");
    return false;
  }
}

void ShenandoahOldGeneration::concurrent_transfer_pointers_from_satb() const {
  const ShenandoahHeap* heap = ShenandoahHeap::heap();
  assert(heap->is_concurrent_old_mark_in_progress(), "Only necessary during old marking.");
  log_debug(gc)("Transfer SATB buffers");

  // Step 1. All threads need to 'complete' partially filled, thread local SATB buffers. This
  // is accomplished in ShenandoahConcurrentGC::complete_abbreviated_cycle using a Handshake
  // operation.
  // Step 2. Use worker threads to transfer oops from old, active regions in the completed
  // SATB buffers to old generation mark queues.
  ShenandoahSATBMarkQueueSet& satb_queues = ShenandoahBarrierSet::satb_mark_queue_set();
  ShenandoahTransferOldSATBTask transfer_task(satb_queues, task_queues());
  heap->workers()->run_task(&transfer_task);
}

void ShenandoahOldGeneration::transfer_pointers_from_satb() const {
  const ShenandoahHeap* heap = ShenandoahHeap::heap();
  assert(heap->is_concurrent_old_mark_in_progress(), "Only necessary during old marking.");
  log_debug(gc)("Transfer SATB buffers");
  ShenandoahPurgeSATBTask purge_satb_task(task_queues());
  heap->workers()->run_task(&purge_satb_task);
}

bool ShenandoahOldGeneration::contains(oop obj) const {
  return ShenandoahHeap::heap()->is_in_old(obj);
}

void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent) {
  ShenandoahHeap* heap = ShenandoahHeap::heap();
  assert(!heap->is_full_gc_in_progress(), "Only for concurrent and degenerated GC");

  {
    ShenandoahGCPhase phase(concurrent ?
        ShenandoahPhaseTimings::final_update_region_states :
        ShenandoahPhaseTimings::degen_gc_final_update_region_states);
    ShenandoahFinalMarkUpdateRegionStateClosure cl(complete_marking_context());

    parallel_heap_region_iterate(&cl);
    heap->assert_pinned_region_status();
  }

  {
    // This doesn't actually choose a collection set, but prepares a list of
    // regions as 'candidates' for inclusion in a mixed collection.
    ShenandoahGCPhase phase(concurrent ?
        ShenandoahPhaseTimings::choose_cset :
        ShenandoahPhaseTimings::degen_gc_choose_cset);
    ShenandoahHeapLocker locker(heap->lock());
    _old_heuristics->prepare_for_old_collections();
  }

  {
    // Though we did not choose a collection set above, we still may have
    // freed up immediate garbage regions so proceed with rebuilding the free set.
    ShenandoahGCPhase phase(concurrent ?
        ShenandoahPhaseTimings::final_rebuild_freeset :
        ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset);
    ShenandoahHeapLocker locker(heap->lock());
    size_t cset_young_regions, cset_old_regions;
    size_t first_old, last_old, num_old;
    heap->free_set()->prepare_to_rebuild(cset_young_regions, cset_old_regions, first_old, last_old, num_old);
    // This is just old-gen completion.  No future budgeting required here.  The only reason to rebuild the freeset here
    // is in case there was any immediate old garbage identified.
    heap->free_set()->finish_rebuild(cset_young_regions, cset_old_regions, num_old);
  }
}

const char* ShenandoahOldGeneration::state_name(State state) {
  switch (state) {
    case WAITING_FOR_BOOTSTRAP:   return "Waiting for Bootstrap";
    case FILLING:                 return "Coalescing";
    case BOOTSTRAPPING:           return "Bootstrapping";
    case MARKING:                 return "Marking";
    case EVACUATING:              return "Evacuating";
    case EVACUATING_AFTER_GLOBAL: return "Evacuating (G)";
    default:
      ShouldNotReachHere();
      return "Unknown";
  }
}

void ShenandoahOldGeneration::transition_to(State new_state) {
  if (_state != new_state) {
    log_debug(gc, thread)("Old generation transition from %s to %s", state_name(_state), state_name(new_state));
    EventMark event("Old was %s, now is %s", state_name(_state), state_name(new_state));
    validate_transition(new_state);
    _state = new_state;
  }
}

#ifdef ASSERT
// This diagram depicts the expected state transitions for marking the old generation
// and preparing for old collections. When a young generation cycle executes, the
// remembered set scan must visit objects in old regions. Visiting an object which
// has become dead on previous old cycles will result in crashes. To avoid visiting
// such objects, the remembered set scan will use the old generation mark bitmap when
// possible. It is _not_ possible to use the old generation bitmap when old marking
// is active (bitmap is not complete). For this reason, the old regions are made
// parsable _before_ the old generation bitmap is reset. The diagram does not depict
// cancellation of old collections by global or full collections.
//
// When a global collection supersedes an old collection, the global mark still
// "completes" the old mark bitmap. Subsequent remembered set scans may use the
// old generation mark bitmap, but any uncollected old regions must still be made parsable
// before the next old generation cycle begins. For this reason, a global collection may
// create mixed collection candidates and coalesce and fill candidates and will put
// the old generation in the respective states (EVACUATING or FILLING). After a Full GC,
// the mark bitmaps are all reset, all regions are parsable and the mark context will
// not be "complete". After a Full GC, remembered set scans will _not_ use the mark bitmap
// and we expect the old generation to be waiting for bootstrap.
//
//                              +-----------------+
//               +------------> |     FILLING     | <---+
//               |   +--------> |                 |     |
//               |   |          +-----------------+     |
//               |   |            |                     |
//               |   |            | Filling Complete    | <-> A global collection may
//               |   |            v                     |     move the old generation
//               |   |          +-----------------+     |     directly from waiting for
//           +-- |-- |--------> |     WAITING     |     |     bootstrap to filling or
//           |   |   |    +---- |  FOR BOOTSTRAP  | ----+     evacuating. It may also
//           |   |   |    |     +-----------------+           move from filling to waiting
//           |   |   |    |       |                           for bootstrap.
//           |   |   |    |       | Reset Bitmap
//           |   |   |    |       v
//           |   |   |    |     +-----------------+     +----------------------+
//           |   |   |    |     |    BOOTSTRAP    | <-> |       YOUNG GC       |
//           |   |   |    |     |                 |     | (RSet Parses Region) |
//           |   |   |    |     +-----------------+     +----------------------+
//           |   |   |    |       |
//           |   |   |    |       | Old Marking
//           |   |   |    |       v
//           |   |   |    |     +-----------------+     +----------------------+
//           |   |   |    |     |     MARKING     | <-> |       YOUNG GC       |
//           |   |   +--------- |                 |     | (RSet Parses Region) |
//           |   |        |     +-----------------+     +----------------------+
//           |   |        |       |
//           |   |        |       | Has Evacuation Candidates
//           |   |        |       v
//           |   |        |     +-----------------+     +--------------------+
//           |   |        +---> |    EVACUATING   | <-> |      YOUNG GC      |
//           |   +------------- |                 |     | (RSet Uses Bitmap) |
//           |                  +-----------------+     +--------------------+
//           |                    |
//           |                    | Global Cycle Coalesces and Fills Old Regions
//           |                    v
//           |                  +-----------------+     +--------------------+
//           +----------------- |    EVACUATING   | <-> |      YOUNG GC      |
//                              |   AFTER GLOBAL  |     | (RSet Uses Bitmap) |
//                              +-----------------+     +--------------------+
//
//
void ShenandoahOldGeneration::validate_transition(State new_state) {
  ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
  switch (new_state) {
    case FILLING:
      assert(_state != BOOTSTRAPPING, "Cannot begin making old regions parsable after bootstrapping");
      assert(is_mark_complete(), "Cannot begin filling without first completing marking, state is '%s'", state_name(_state));
      assert(_old_heuristics->has_coalesce_and_fill_candidates(), "Cannot begin filling without something to fill.");
      break;
    case WAITING_FOR_BOOTSTRAP:
      // GC cancellation can send us back here from any state.
      validate_waiting_for_bootstrap();
      break;
    case BOOTSTRAPPING:
      assert(_state == WAITING_FOR_BOOTSTRAP, "Cannot reset bitmap without making old regions parsable, state is '%s'", state_name(_state));
      assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Cannot bootstrap with mixed collection candidates");
      assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot still be making old regions parsable.");
      break;
    case MARKING:
      assert(_state == BOOTSTRAPPING, "Must have finished bootstrapping before marking, state is '%s'", state_name(_state));
      assert(heap->young_generation()->old_gen_task_queues() != nullptr, "Young generation needs old mark queues.");
      assert(heap->is_concurrent_old_mark_in_progress(), "Should be marking old now.");
      break;
    case EVACUATING_AFTER_GLOBAL:
      assert(_state == EVACUATING, "Must have been evacuating, state is '%s'", state_name(_state));
      break;
    case EVACUATING:
      assert(_state == WAITING_FOR_BOOTSTRAP || _state == MARKING, "Cannot have old collection candidates without first marking, state is '%s'", state_name(_state));
      assert(_old_heuristics->unprocessed_old_collection_candidates() > 0, "Must have collection candidates here.");
      break;
    default:
      fatal("Unknown new state");
  }
}

bool ShenandoahOldGeneration::validate_waiting_for_bootstrap() {
  ShenandoahHeap* heap = ShenandoahHeap::heap();
  assert(!heap->is_concurrent_old_mark_in_progress(), "Cannot become ready for bootstrap during old mark.");
  assert(heap->young_generation()->old_gen_task_queues() == nullptr, "Cannot become ready for bootstrap when still setup for bootstrapping.");
  assert(!is_concurrent_mark_in_progress(), "Cannot be marking in IDLE");
  assert(!heap->young_generation()->is_bootstrap_cycle(), "Cannot have old mark queues if IDLE");
  assert(!_old_heuristics->has_coalesce_and_fill_candidates(), "Cannot have coalesce and fill candidates in IDLE");
  assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Cannot have mixed collection candidates in IDLE");
  return true;
}
#endif

ShenandoahHeuristics* ShenandoahOldGeneration::initialize_heuristics(ShenandoahMode* gc_mode) {
  _old_heuristics = new ShenandoahOldHeuristics(this, ShenandoahGenerationalHeap::heap());
  _old_heuristics->set_guaranteed_gc_interval(ShenandoahGuaranteedOldGCInterval);
  _heuristics = _old_heuristics;
  return _heuristics;
}

void ShenandoahOldGeneration::record_success_concurrent(bool abbreviated) {
  heuristics()->record_success_concurrent();
  ShenandoahHeap::heap()->shenandoah_policy()->record_success_old();
}

void ShenandoahOldGeneration::handle_failed_evacuation() {
  if (_failed_evacuation.try_set()) {
    log_debug(gc)("Old gen evac failure.");
  }
}

void ShenandoahOldGeneration::handle_failed_promotion(Thread* thread, size_t size) {
  // We squelch excessive reports to reduce noise in logs.
  const size_t MaxReportsPerEpoch = 4;
  static size_t last_report_epoch = 0;
  static size_t epoch_report_count = 0;
  auto heap = ShenandoahGenerationalHeap::heap();

  size_t promotion_reserve;
  size_t promotion_expended;

  const size_t gc_id = heap->control_thread()->get_gc_id();

  if ((gc_id != last_report_epoch) || (epoch_report_count++ < MaxReportsPerEpoch)) {
    {
      // Promotion failures should be very rare.  Invest in providing useful diagnostic info.
      ShenandoahHeapLocker locker(heap->lock());
      promotion_reserve = get_promoted_reserve();
      promotion_expended = get_promoted_expended();
    }
    PLAB* const plab = ShenandoahThreadLocalData::plab(thread);
    const size_t words_remaining = (plab == nullptr)? 0: plab->words_remaining();
    const char* promote_enabled = ShenandoahThreadLocalData::allow_plab_promotions(thread)? "enabled": "disabled";

    log_info(gc, ergo)("Promotion failed, size %zu, has plab? %s, PLAB remaining: %zu"
                       ", plab promotions %s, promotion reserve: %zu, promotion expended: %zu"
                       ", old capacity: %zu, old_used: %zu, old unaffiliated regions: %zu",
                       size * HeapWordSize, plab == nullptr? "no": "yes",
                       words_remaining * HeapWordSize, promote_enabled, promotion_reserve, promotion_expended,
                       max_capacity(), used(), free_unaffiliated_regions());

    if ((gc_id == last_report_epoch) && (epoch_report_count >= MaxReportsPerEpoch)) {
      log_debug(gc, ergo)("Squelching additional promotion failure reports for current epoch");
    } else if (gc_id != last_report_epoch) {
      last_report_epoch = gc_id;
      epoch_report_count = 1;
    }
  }
}

void ShenandoahOldGeneration::handle_evacuation(HeapWord* obj, size_t words, bool promotion) {
  // Only register the copy of the object that won the evacuation race.
  _card_scan->register_object_without_lock(obj);

  // Mark the entire range of the evacuated object as dirty.  At next remembered set scan,
  // we will clear dirty bits that do not hold interesting pointers.  It's more efficient to
  // do this in batch, in a background GC thread than to try to carefully dirty only cards
  // that hold interesting pointers right now.
  _card_scan->mark_range_as_dirty(obj, words);

  if (promotion) {
    // This evacuation was a promotion, track this as allocation against old gen
    increase_allocated(words * HeapWordSize);
  }
}

bool ShenandoahOldGeneration::has_unprocessed_collection_candidates() {
  return _old_heuristics->unprocessed_old_collection_candidates() > 0;
}

size_t ShenandoahOldGeneration::unprocessed_collection_candidates_live_memory() {
  return _old_heuristics->unprocessed_old_collection_candidates_live_memory();
}

void ShenandoahOldGeneration::abandon_collection_candidates() {
  _old_heuristics->abandon_collection_candidates();
}

void ShenandoahOldGeneration::prepare_for_mixed_collections_after_global_gc() {
  assert(is_mark_complete(), "Expected old generation mark to be complete after global cycle.");
  _old_heuristics->prepare_for_old_collections();
  log_info(gc, ergo)("After choosing global collection set, mixed candidates: " UINT32_FORMAT ", coalescing candidates: %zu",
               _old_heuristics->unprocessed_old_collection_candidates(),
               _old_heuristics->coalesce_and_fill_candidates_count());
}

void ShenandoahOldGeneration::parallel_heap_region_iterate_free(ShenandoahHeapRegionClosure* cl) {
  // Iterate over old and free regions (exclude young).
  ShenandoahExcludeRegionClosure<YOUNG_GENERATION> exclude_cl(cl);
  ShenandoahGeneration::parallel_heap_region_iterate_free(&exclude_cl);
}

void ShenandoahOldGeneration::set_parsable(bool parsable) {
  _is_parsable = parsable;
  if (_is_parsable) {
    // The current state would have been chosen during final mark of the global
    // collection, _before_ any decisions about class unloading have been made.
    //
    // After unloading classes, we have made the old generation regions parsable.
    // We can skip filling or transition to a state that knows everything has
    // already been filled.
    switch (state()) {
      case ShenandoahOldGeneration::EVACUATING:
        transition_to(ShenandoahOldGeneration::EVACUATING_AFTER_GLOBAL);
        break;
      case ShenandoahOldGeneration::FILLING:
        assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Expected no mixed collection candidates");
        assert(_old_heuristics->coalesce_and_fill_candidates_count() > 0, "Expected coalesce and fill candidates");
        // When the heuristic put the old generation in this state, it didn't know
        // that we would unload classes and make everything parsable. But, we know
        // that now so we can override this state.
        abandon_collection_candidates();
        transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP);
        break;
      default:
        // We can get here during a full GC. The full GC will cancel anything
        // happening in the old generation and return it to the waiting for bootstrap
        // state. The full GC will then record that the old regions are parsable
        // after rebuilding the remembered set.
        assert(is_idle(), "Unexpected state %s at end of global GC", state_name());
        break;
    }
  }
}

void ShenandoahOldGeneration::complete_mixed_evacuations() {
  assert(is_doing_mixed_evacuations(), "Mixed evacuations should be in progress");
  if (!_old_heuristics->has_coalesce_and_fill_candidates()) {
    // No candidate regions to coalesce and fill
    transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP);
    return;
  }

  if (state() == ShenandoahOldGeneration::EVACUATING) {
    transition_to(ShenandoahOldGeneration::FILLING);
    return;
  }

  // Here, we have no more candidates for mixed collections. The candidates for coalescing
  // and filling have already been processed during the global cycle, so there is nothing
  // more to do.
  assert(state() == ShenandoahOldGeneration::EVACUATING_AFTER_GLOBAL, "Should be evacuating after a global cycle");
  abandon_collection_candidates();
  transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP);
}

void ShenandoahOldGeneration::abandon_mixed_evacuations() {
  switch(state()) {
    case ShenandoahOldGeneration::EVACUATING:
      transition_to(ShenandoahOldGeneration::FILLING);
      break;
    case ShenandoahOldGeneration::EVACUATING_AFTER_GLOBAL:
      abandon_collection_candidates();
      transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP);
      break;
    default:
      log_warning(gc)("Abandon mixed evacuations in unexpected state: %s", state_name(state()));
      ShouldNotReachHere();
      break;
  }
}

void ShenandoahOldGeneration::clear_cards_for(ShenandoahHeapRegion* region) {
  _card_scan->mark_range_as_empty(region->bottom(), pointer_delta(region->end(), region->bottom()));
}

void ShenandoahOldGeneration::mark_card_as_dirty(void* location) {
  _card_scan->mark_card_as_dirty((HeapWord*)location);
}