src/hotspot/share/gc/shared/referenceProcessor.hpp - openjdk/jdk25u - Git at Google

 /*
  * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 only, as
  * published by the Free Software Foundation.
  *
  * This code is distributed in the hope that it will be useful, but WITHOUT
  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  * version 2 for more details (a copy is included in the LICENSE file that
  * accompanied this code).
  *
  * You should have received a copy of the GNU General Public License version
  * 2 along with this work; if not, write to the Free Software Foundation,
  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  *
  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  * or visit www.oracle.com if you need additional information or have any
  * questions.
  *
  */

 #ifndef SHARE_GC_SHARED_REFERENCEPROCESSOR_HPP
 #define SHARE_GC_SHARED_REFERENCEPROCESSOR_HPP

 #include "gc/shared/referenceDiscoverer.hpp"
 #include "gc/shared/referencePolicy.hpp"
 #include "gc/shared/referenceProcessorStats.hpp"
 #include "gc/shared/workerThread.hpp"
 #include "memory/referenceType.hpp"
 #include "oops/instanceRefKlass.hpp"

 class GCTimer;
 class ReferencePolicy;
 class ReferenceProcessorPhaseTimes;
 class RefProcTask;
 class RefProcProxyTask;

 // Provides a callback to the garbage collector to set the given value to the
 // discovered field of the j.l.ref.Reference instance. This is called during STW
 // reference processing when iterating over the discovered lists for all
 // discovered references.
 // Typically garbage collectors may just call the barrier, but for some garbage
 // collectors the barrier environment (e.g. card table) may not be set up correctly
 // at the point of invocation.
 class EnqueueDiscoveredFieldClosure {
 public:
   // For the given j.l.ref.Reference discovered field address, set the discovered
   // field to value and apply any barriers to it.
   virtual void enqueue(HeapWord* discovered_field_addr, oop value) = 0;
 };

 // EnqueueDiscoveredFieldClosure that executes the default barrier on the discovered
 // field of the j.l.ref.Reference with the given value.
 class BarrierEnqueueDiscoveredFieldClosure : public EnqueueDiscoveredFieldClosure {
 public:
   void enqueue(HeapWord* discovered_field_addr, oop value) override;
 };

 // List of discovered references.
 class DiscoveredList {
 public:
   DiscoveredList() : _oop_head(nullptr), _compressed_head(narrowOop::null), _len(0) { }
   inline oop head() const;
   HeapWord* adr_head() {
     return UseCompressedOops ? (HeapWord*)&_compressed_head :
                                (HeapWord*)&_oop_head;
   }
   inline void add_as_head(oop o);
   inline void set_head(oop o);
   inline bool is_empty() const;
   size_t length()               { return _len; }
   void   set_length(size_t len) { _len = len;  }
   void   inc_length(size_t inc) { _len += inc; assert(_len > 0, "Error"); }
   void   dec_length(size_t dec) { _len -= dec; }

   inline void clear();
 private:
   // Set value depending on UseCompressedOops. This could be a template class
   // but then we have to fix all the instantiations and declarations that use this class.
   oop       _oop_head;
   narrowOop _compressed_head;
   size_t _len;
 };

 // Iterator for the list of discovered references.
 class DiscoveredListIterator {
   DiscoveredList&    _refs_list;
   HeapWord*          _prev_discovered_addr;
   oop                _prev_discovered;
   oop                _current_discovered;
   HeapWord*          _current_discovered_addr;
   oop                _next_discovered;

   oop                _referent;

   OopClosure*        _keep_alive;
   BoolObjectClosure* _is_alive;
   EnqueueDiscoveredFieldClosure* _enqueue;

   DEBUG_ONLY(
   oop                _first_seen; // cyclic linked list check
   )

   size_t             _processed;
   size_t             _removed;

 public:
   inline DiscoveredListIterator(DiscoveredList&    refs_list,
                                 OopClosure*        keep_alive,
                                 BoolObjectClosure* is_alive,
                                 EnqueueDiscoveredFieldClosure* enqueue);

   // End Of List.
   inline bool has_next() const { return _current_discovered != nullptr; }

   // Get oop to the Reference object.
   inline oop obj() const { return _current_discovered; }

   // Get oop to the referent object.
   inline oop referent() const { return _referent; }

   // Returns true if referent is alive.
   inline bool is_referent_alive() const {
     return _is_alive->do_object_b(_referent);
   }

   // Loads data for the current reference.
   // The "allow_null_referent" argument tells us to allow for the possibility
   // of a null referent in the discovered Reference object. This typically
   // happens in the case of concurrent collectors that may have done the
   // discovery concurrently, or interleaved, with mutator execution.
   void load_ptrs(DEBUG_ONLY(bool allow_null_referent));

   // Move to the next discovered reference.
   inline void next() {
     _prev_discovered_addr = _current_discovered_addr;
     _prev_discovered = _current_discovered;
     move_to_next();
   }

   // Remove the current reference from the list
   void remove();

   // Apply the keep_alive function to the referent address.
   void make_referent_alive();

   // Do enqueuing work, i.e. notifying the GC about the changed discovered pointers.
   void enqueue();

   // Move enqueued references to the reference pending list.
   void complete_enqueue();

   // null out referent pointer.
   void clear_referent();

   // Statistics
   inline size_t processed() const { return _processed; }
   inline size_t removed() const { return _removed; }

   inline void move_to_next() {
     if (_current_discovered == _next_discovered) {
       // End of the list.
       _current_discovered = nullptr;
     } else {
       _current_discovered = _next_discovered;
     }
     assert(_current_discovered != _first_seen, "cyclic ref_list found");
     _processed++;
   }
 };

 // The ReferenceProcessor class encapsulates the per-"collector" processing
 // of java.lang.Reference objects for GC. The interface is useful for supporting
 // a generational abstraction, in particular when there are multiple
 // generations that are being independently collected -- possibly
 // concurrently and/or incrementally.
 // ReferenceProcessor class abstracts away from a generational setting
 // by using a closure that determines whether a given reference or referent are
 // subject to this ReferenceProcessor's discovery, thus allowing its use in a
 // straightforward manner in a general, non-generational, non-contiguous generation
 // (or heap) setting.
 class ReferenceProcessor : public ReferenceDiscoverer {
   friend class RefProcTask;
   friend class RefProcKeepAliveFinalPhaseTask;
 public:
   // Names of sub-phases of reference processing. Indicates the type of the reference
   // processed and the associated phase number at the end.
   enum RefProcSubPhases {
     ProcessSoftRefSubPhase,
     ProcessWeakRefSubPhase,
     ProcessFinalRefSubPhase,
     KeepAliveFinalRefsSubPhase,
     ProcessPhantomRefsSubPhase,
     RefSubPhaseMax
   };

   // Main phases of reference processing.
   enum RefProcPhases {
     SoftWeakFinalRefsPhase,
     KeepAliveFinalRefsPhase,
     PhantomRefsPhase,
     RefPhaseMax
   };

 private:
   size_t total_count(DiscoveredList lists[]) const;
   void verify_total_count_zero(DiscoveredList lists[], const char* type) NOT_DEBUG_RETURN;

   // The SoftReference master timestamp clock
   static jlong _soft_ref_timestamp_clock;

   BoolObjectClosure* _is_subject_to_discovery; // determines whether a given oop is subject
                                                // to this ReferenceProcessor's discovery
                                                // (and further processing).

   bool        _discovering_refs;        // true when discovery enabled
   bool        _discovery_is_concurrent; // if discovery is concurrent to the mutator
   bool        _discovery_is_mt;         // true if reference discovery is MT.

   uint        _next_id;                 // round-robin mod _num_queues counter in
                                         // support of work distribution

   // For collectors that do not keep GC liveness information
   // in the object header, this field holds a closure that
   // helps the reference processor determine the reachability
   // of an oop.
   BoolObjectClosure* _is_alive_non_header;

   // Soft ref clearing policies
   // . the default policy
   static ReferencePolicy*   _default_soft_ref_policy;
   // . the "clear all" policy
   static ReferencePolicy*   _always_clear_soft_ref_policy;
   // . the current policy below is either one of the above
   ReferencePolicy*          _current_soft_ref_policy;

   // The discovered ref lists themselves

   // The active MT'ness degree of the queues below
   uint            _num_queues;
   // The maximum MT'ness degree of the queues below
   uint            _max_num_queues;

   // Master array of discovered oops
   DiscoveredList* _discovered_refs;

   // Arrays of lists of oops, one per thread (pointers into master array above)
   DiscoveredList* _discoveredSoftRefs;
   DiscoveredList* _discoveredWeakRefs;
   DiscoveredList* _discoveredFinalRefs;
   DiscoveredList* _discoveredPhantomRefs;

   void run_task(RefProcTask& task, RefProcProxyTask& proxy_task, bool marks_oops_alive);

   // Drop Soft/Weak/Final references with a null or live referent, and clear
   // and enqueue non-Final references.
   void process_soft_weak_final_refs(RefProcProxyTask& proxy_task,
                                     ReferenceProcessorPhaseTimes& phase_times);

   // Keep alive followers of Final references, and enqueue.
   void process_final_keep_alive(RefProcProxyTask& proxy_task,
                                 ReferenceProcessorPhaseTimes& phase_times);

   // Drop and keep alive live Phantom references, or clear and enqueue if dead.
   void process_phantom_refs(RefProcProxyTask& proxy_task,
                             ReferenceProcessorPhaseTimes& phase_times);

   // Work methods used by the process_* methods. All methods return the number of
   // removed elements.

   // Traverse the list and remove any Refs whose referents are alive,
   // or null if discovery is concurrent. Enqueue and clear the reference for
   // others if do_enqueue_and_clear is set.
   size_t process_discovered_list_work(DiscoveredList&    refs_list,
                                       BoolObjectClosure* is_alive,
                                       OopClosure*        keep_alive,
                                       EnqueueDiscoveredFieldClosure* enqueue,
                                       bool               do_enqueue_and_clear);

   // Keep alive followers of referents for FinalReferences. Must only be called for
   // those.
   size_t process_final_keep_alive_work(DiscoveredList& refs_list,
                                        OopClosure* keep_alive,
                                        EnqueueDiscoveredFieldClosure* enqueue);


   void setup_policy(bool always_clear) {
     _current_soft_ref_policy = always_clear ?
                                _always_clear_soft_ref_policy : _default_soft_ref_policy;
     _current_soft_ref_policy->setup();   // snapshot the policy threshold
   }
 public:
   static int number_of_subclasses_of_ref() { return (REF_PHANTOM - REF_NONE); }

   uint num_queues() const                  { return _num_queues; }
   uint max_num_queues() const              { return _max_num_queues; }
   void set_active_mt_degree(uint v);

   void start_discovery(bool always_clear) {
     enable_discovery();
     setup_policy(always_clear);
   }

   // "Preclean" all the discovered reference lists by removing references whose
   // referents are null or strongly reachable (`is_alive` returns true).
   // Note: when a referent is strongly reachable, we assume it's already marked
   // through, so this method doesn't perform (and doesn't need to) any marking
   // work at all. Currently, this assumption holds because G1 uses SATB and the
   // marking status of an object is *not* updated when `Reference.get()` is
   // called.
   // `yield` is a closure that may be used to incrementalize or abort the
   // precleaning process. The caller is responsible for taking care of
   // potential interference with concurrent operations on these lists (or
   // predicates involved) by other threads.
   void preclean_discovered_references(BoolObjectClosure* is_alive,
                                       EnqueueDiscoveredFieldClosure* enqueue,
                                       YieldClosure*      yield,
                                       GCTimer*           gc_timer);

 private:
   // Returns the name of the discovered reference list
   // occupying the i / _num_queues slot.
   const char* list_name(uint i);

   // "Preclean" the given discovered reference list by removing references with
   // the attributes mentioned in preclean_discovered_references().
   // Supports both normal and fine grain yielding.
   // Returns whether the operation should be aborted.
   bool preclean_discovered_reflist(DiscoveredList&    refs_list,
                                    BoolObjectClosure* is_alive,
                                    EnqueueDiscoveredFieldClosure* enqueue,
                                    YieldClosure*      yield);

   // round-robin mod _num_queues (not: _not_ mod _max_num_queues)
   uint next_id() {
     uint id = _next_id;
     assert(!_discovery_is_mt, "Round robin should only be used in serial discovery");
     if (++_next_id == _num_queues) {
       _next_id = 0;
     }
     assert(_next_id < _num_queues, "_next_id %u _num_queues %u _max_num_queues %u", _next_id, _num_queues, _max_num_queues);
     return id;
   }
   DiscoveredList* get_discovered_list(ReferenceType rt);
   inline bool set_discovered_link(HeapWord* discovered_addr, oop next_discovered);
   inline void add_to_discovered_list(DiscoveredList& refs_list, oop obj,
                                      HeapWord* discovered_addr);
   inline bool set_discovered_link_st(HeapWord* discovered_addr,
                                      oop next_discovered);
   inline bool set_discovered_link_mt(HeapWord* discovered_addr,
                                      oop next_discovered);

   void clear_discovered_references(DiscoveredList& refs_list);

   void log_reflist(const char* prefix, DiscoveredList list[], uint num_active_queues);
   void log_reflist_counts(DiscoveredList ref_lists[], uint num_active_queues) PRODUCT_RETURN;

   // Balances reference queues.
   void balance_queues(DiscoveredList refs_lists[]);
   bool need_balance_queues(DiscoveredList refs_lists[]);

   // If there is need to balance the given queue, do it.
   void maybe_balance_queues(DiscoveredList refs_lists[]);

   // Update (advance) the soft ref master clock field.
   void update_soft_ref_master_clock();

   bool is_subject_to_discovery(oop const obj) const;

 public:
   // Default parameters give you a vanilla reference processor.
   ReferenceProcessor(BoolObjectClosure* is_subject_to_discovery,
                      uint mt_processing_degree = 1,
                      uint mt_discovery_degree  = 1,
                      bool concurrent_discovery = false,
                      BoolObjectClosure* is_alive_non_header = nullptr);

   static void init_statics();

   // get and set "is_alive_non_header" field
   BoolObjectClosure* is_alive_non_header() {
     return _is_alive_non_header;
   }
   void set_is_alive_non_header(BoolObjectClosure* is_alive_non_header) {
     _is_alive_non_header = is_alive_non_header;
   }

   BoolObjectClosure* is_subject_to_discovery_closure() const { return _is_subject_to_discovery; }
   void set_is_subject_to_discovery_closure(BoolObjectClosure* cl) { _is_subject_to_discovery = cl; }

   // start and stop weak ref discovery
   void enable_discovery();
   void disable_discovery()  { _discovering_refs = false; }
   bool discovery_enabled()  { return _discovering_refs;  }

   // whether discovery is concurrent to the mutator, or done in an stw pause.
   bool discovery_is_concurrent() const { return _discovery_is_concurrent; }
   bool discovery_is_stw() const { return !discovery_is_concurrent(); }

   // whether discovery is done by multiple threads same-old-timeously
   bool discovery_is_mt() const { return _discovery_is_mt; }
   void set_mt_discovery(bool mt) { _discovery_is_mt = mt; }

   // Whether we are in a phase when _processing_ is MT.
   bool processing_is_mt() const;

   // iterate over oops
   void weak_oops_do(OopClosure* f);       // weak roots

   // Discover a Reference object, using appropriate discovery criteria
   virtual bool discover_reference(oop obj, ReferenceType rt);

   // Process references found during GC (called by the garbage collector)
   ReferenceProcessorStats
   process_discovered_references(RefProcProxyTask& proxy_task,
                                 ReferenceProcessorPhaseTimes& phase_times);

   // If a discovery is in process that is being superseded, abandon it: all
   // the discovered lists will be empty, and all the objects on them will
   // have null discovered fields.  Must be called only at a safepoint.
   void abandon_partial_discovery();

   size_t total_reference_count(ReferenceType rt) const;

   // debugging
   void verify_no_references_recorded() PRODUCT_RETURN;
   void verify_referent(oop obj)        PRODUCT_RETURN;
 };

 // A subject-to-discovery closure that uses a single memory span to determine the area that
 // is subject to discovery. Useful for collectors which have contiguous generations.
 class SpanSubjectToDiscoveryClosure : public BoolObjectClosure {
   MemRegion _span;

 public:
   SpanSubjectToDiscoveryClosure() : BoolObjectClosure(), _span() { }

   void set_span(MemRegion mr) {
     _span = mr;
   }

   virtual bool do_object_b(oop obj) {
     return _span.contains(obj);
   }
 };

 // A utility class to temporarily mutate the subject discovery closure of the
 // given ReferenceProcessor in the scope that contains it.
 class ReferenceProcessorSubjectToDiscoveryMutator : StackObj {
   ReferenceProcessor* _rp;
   BoolObjectClosure* _saved_cl;

 public:
   ReferenceProcessorSubjectToDiscoveryMutator(ReferenceProcessor* rp, BoolObjectClosure* cl):
     _rp(rp) {
     _saved_cl = _rp->is_subject_to_discovery_closure();
     _rp->set_is_subject_to_discovery_closure(cl);
   }

   ~ReferenceProcessorSubjectToDiscoveryMutator() {
     _rp->set_is_subject_to_discovery_closure(_saved_cl);
   }
 };

 // A utility class to temporarily change the MT'ness of
 // reference discovery for the given ReferenceProcessor
 // in the scope that contains it.
 class ReferenceProcessorMTDiscoveryMutator: StackObj {
  private:
   ReferenceProcessor* _rp;
   bool                _saved_mt;

  public:
   ReferenceProcessorMTDiscoveryMutator(ReferenceProcessor* rp,
                                        bool mt):
     _rp(rp) {
     _saved_mt = _rp->discovery_is_mt();
     _rp->set_mt_discovery(mt);
   }

   ~ReferenceProcessorMTDiscoveryMutator() {
     _rp->set_mt_discovery(_saved_mt);
   }
 };

 // A utility class to temporarily change the disposition
 // of the "is_alive_non_header" closure field of the
 // given ReferenceProcessor in the scope that contains it.
 class ReferenceProcessorIsAliveMutator: StackObj {
  private:
   ReferenceProcessor* _rp;
   BoolObjectClosure*  _saved_cl;

  public:
   ReferenceProcessorIsAliveMutator(ReferenceProcessor* rp,
                                    BoolObjectClosure*  cl):
     _rp(rp) {
     _saved_cl = _rp->is_alive_non_header();
     _rp->set_is_alive_non_header(cl);
   }

   ~ReferenceProcessorIsAliveMutator() {
     _rp->set_is_alive_non_header(_saved_cl);
   }
 };

 enum class RefProcThreadModel { Multi, Single };

 /*
  * This is the (base) task that handles reference processing that does not depend on
  * the chosen GC (Serial, Parallel or G1). This RefProcTask will be called from a subclass
  * of RefProcProxyTask. The RefProcProxyTask will give the behaviour of the selected GC by
  * calling rp_work with the gc-specific closures.
  */
 class RefProcTask : StackObj {
 protected:
   ReferenceProcessor& _ref_processor;
   ReferenceProcessorPhaseTimes* _phase_times;

   // Used for tracking how much time a worker spends in a (sub)phase.
   uint tracker_id(uint worker_id) const {
     return _ref_processor.processing_is_mt() ? worker_id : 0;
   }

   void process_discovered_list(uint worker_id,
                                ReferenceType ref_type,
                                BoolObjectClosure* is_alive,
                                OopClosure* keep_alive,
                                EnqueueDiscoveredFieldClosure* enqueue);
 public:
   RefProcTask(ReferenceProcessor& ref_processor,
               ReferenceProcessorPhaseTimes* phase_times)
     : _ref_processor(ref_processor),
       _phase_times(phase_times) {}

   virtual void rp_work(uint worker_id,
                        BoolObjectClosure* is_alive,
                        OopClosure* keep_alive,
                        EnqueueDiscoveredFieldClosure* enqueue,
                        VoidClosure* complete_gc) = 0;
 };

 /*
  * This is the (base) task that handles reference processing that do depend on
  * the chosen GC (Serial, Parallel or G1). This RefProcProxyTask will call a subclass
  * of RefProcTask that will handle reference processing in a generic way for Serial,
  * Parallel and G1. This proxy will add the relevant closures, task terminators etc.
  */
 class RefProcProxyTask : public WorkerTask {
 protected:
   const uint _max_workers;
   RefProcTask* _rp_task;
   RefProcThreadModel _tm;
   uint _queue_count;
   bool _marks_oops_alive;

 public:
   RefProcProxyTask(const char* name, uint max_workers) : WorkerTask(name), _max_workers(max_workers), _rp_task(nullptr),_tm(RefProcThreadModel::Single), _queue_count(0), _marks_oops_alive(false) {}

   void prepare_run_task(RefProcTask& rp_task, uint queue_count, RefProcThreadModel tm, bool marks_oops_alive) {
     _rp_task = &rp_task;
     _tm = tm;
     _queue_count = queue_count;
     _marks_oops_alive = marks_oops_alive;
     prepare_run_task_hook();
   }

   virtual void prepare_run_task_hook() {}
 };

 // Temporarily change the number of workers based on given reference count.
 // This ergonomically decided worker count will be used to activate worker threads.
 class RefProcMTDegreeAdjuster : public StackObj {
   typedef ReferenceProcessor::RefProcPhases RefProcPhases;

   ReferenceProcessor* _rp;
   uint                _saved_num_queues;

   // Calculate based on total of references.
   uint ergo_proc_thread_count(size_t ref_count,
                               uint max_threads,
                               RefProcPhases phase) const;

   bool use_max_threads(RefProcPhases phase) const;

 public:
   RefProcMTDegreeAdjuster(ReferenceProcessor* rp,
                           RefProcPhases phase,
                           size_t ref_count);
   ~RefProcMTDegreeAdjuster();
 };

 #endif // SHARE_GC_SHARED_REFERENCEPROCESSOR_HPP
	/*
	* Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
	*
	* This code is free software; you can redistribute it and/or modify it
	* under the terms of the GNU General Public License version 2 only, as
	* published by the Free Software Foundation.
	*
	* This code is distributed in the hope that it will be useful, but WITHOUT
	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	* version 2 for more details (a copy is included in the LICENSE file that
	* accompanied this code).
	*
	* You should have received a copy of the GNU General Public License version
	* 2 along with this work; if not, write to the Free Software Foundation,
	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
	*
	* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
	* or visit www.oracle.com if you need additional information or have any
	* questions.
	*
	*/

	#ifndef SHARE_GC_SHARED_REFERENCEPROCESSOR_HPP
	#define SHARE_GC_SHARED_REFERENCEPROCESSOR_HPP

	#include "gc/shared/referenceDiscoverer.hpp"
	#include "gc/shared/referencePolicy.hpp"
	#include "gc/shared/referenceProcessorStats.hpp"
	#include "gc/shared/workerThread.hpp"
	#include "memory/referenceType.hpp"
	#include "oops/instanceRefKlass.hpp"

	class GCTimer;
	class ReferencePolicy;
	class ReferenceProcessorPhaseTimes;
	class RefProcTask;
	class RefProcProxyTask;

	// Provides a callback to the garbage collector to set the given value to the
	// discovered field of the j.l.ref.Reference instance. This is called during STW
	// reference processing when iterating over the discovered lists for all
	// discovered references.
	// Typically garbage collectors may just call the barrier, but for some garbage
	// collectors the barrier environment (e.g. card table) may not be set up correctly
	// at the point of invocation.
	class EnqueueDiscoveredFieldClosure {
	public:
	// For the given j.l.ref.Reference discovered field address, set the discovered
	// field to value and apply any barriers to it.
	virtual void enqueue(HeapWord* discovered_field_addr, oop value) = 0;
	};

	// EnqueueDiscoveredFieldClosure that executes the default barrier on the discovered
	// field of the j.l.ref.Reference with the given value.
	class BarrierEnqueueDiscoveredFieldClosure : public EnqueueDiscoveredFieldClosure {
	public:
	void enqueue(HeapWord* discovered_field_addr, oop value) override;
	};

	// List of discovered references.
	class DiscoveredList {
	public:
	DiscoveredList() : _oop_head(nullptr), _compressed_head(narrowOop::null), _len(0) { }
	inline oop head() const;
	HeapWord* adr_head() {
	return UseCompressedOops ? (HeapWord*)&_compressed_head :
	(HeapWord*)&_oop_head;
	}
	inline void add_as_head(oop o);
	inline void set_head(oop o);
	inline bool is_empty() const;
	size_t length() { return _len; }
	void set_length(size_t len) { _len = len; }
	void inc_length(size_t inc) { _len += inc; assert(_len > 0, "Error"); }
	void dec_length(size_t dec) { _len -= dec; }

	inline void clear();
	private:
	// Set value depending on UseCompressedOops. This could be a template class
	// but then we have to fix all the instantiations and declarations that use this class.
	oop _oop_head;
	narrowOop _compressed_head;
	size_t _len;
	};

	// Iterator for the list of discovered references.
	class DiscoveredListIterator {
	DiscoveredList& _refs_list;
	HeapWord* _prev_discovered_addr;
	oop _prev_discovered;
	oop _current_discovered;
	HeapWord* _current_discovered_addr;
	oop _next_discovered;

	oop _referent;

	OopClosure* _keep_alive;
	BoolObjectClosure* _is_alive;
	EnqueueDiscoveredFieldClosure* _enqueue;

	DEBUG_ONLY(
	oop _first_seen; // cyclic linked list check
	)

	size_t _processed;
	size_t _removed;

	public:
	inline DiscoveredListIterator(DiscoveredList& refs_list,
	OopClosure* keep_alive,
	BoolObjectClosure* is_alive,
	EnqueueDiscoveredFieldClosure* enqueue);

	// End Of List.
	inline bool has_next() const { return _current_discovered != nullptr; }

	// Get oop to the Reference object.
	inline oop obj() const { return _current_discovered; }

	// Get oop to the referent object.
	inline oop referent() const { return _referent; }

	// Returns true if referent is alive.
	inline bool is_referent_alive() const {
	return _is_alive->do_object_b(_referent);
	}

	// Loads data for the current reference.
	// The "allow_null_referent" argument tells us to allow for the possibility
	// of a null referent in the discovered Reference object. This typically
	// happens in the case of concurrent collectors that may have done the
	// discovery concurrently, or interleaved, with mutator execution.
	void load_ptrs(DEBUG_ONLY(bool allow_null_referent));

	// Move to the next discovered reference.
	inline void next() {
	_prev_discovered_addr = _current_discovered_addr;
	_prev_discovered = _current_discovered;
	move_to_next();
	}

	// Remove the current reference from the list
	void remove();

	// Apply the keep_alive function to the referent address.
	void make_referent_alive();

	// Do enqueuing work, i.e. notifying the GC about the changed discovered pointers.
	void enqueue();

	// Move enqueued references to the reference pending list.
	void complete_enqueue();

	// null out referent pointer.
	void clear_referent();

	// Statistics
	inline size_t processed() const { return _processed; }
	inline size_t removed() const { return _removed; }

	inline void move_to_next() {
	if (_current_discovered == _next_discovered) {
	// End of the list.
	_current_discovered = nullptr;
	} else {
	_current_discovered = _next_discovered;
	}
	assert(_current_discovered != _first_seen, "cyclic ref_list found");
	_processed++;
	}
	};

	// The ReferenceProcessor class encapsulates the per-"collector" processing
	// of java.lang.Reference objects for GC. The interface is useful for supporting
	// a generational abstraction, in particular when there are multiple
	// generations that are being independently collected -- possibly
	// concurrently and/or incrementally.
	// ReferenceProcessor class abstracts away from a generational setting
	// by using a closure that determines whether a given reference or referent are
	// subject to this ReferenceProcessor's discovery, thus allowing its use in a
	// straightforward manner in a general, non-generational, non-contiguous generation
	// (or heap) setting.
	class ReferenceProcessor : public ReferenceDiscoverer {
	friend class RefProcTask;
	friend class RefProcKeepAliveFinalPhaseTask;
	public:
	// Names of sub-phases of reference processing. Indicates the type of the reference
	// processed and the associated phase number at the end.
	enum RefProcSubPhases {
	ProcessSoftRefSubPhase,
	ProcessWeakRefSubPhase,
	ProcessFinalRefSubPhase,
	KeepAliveFinalRefsSubPhase,
	ProcessPhantomRefsSubPhase,
	RefSubPhaseMax
	};

	// Main phases of reference processing.
	enum RefProcPhases {
	SoftWeakFinalRefsPhase,
	KeepAliveFinalRefsPhase,
	PhantomRefsPhase,
	RefPhaseMax
	};

	private:
	size_t total_count(DiscoveredList lists[]) const;
	void verify_total_count_zero(DiscoveredList lists[], const char* type) NOT_DEBUG_RETURN;

	// The SoftReference master timestamp clock
	static jlong _soft_ref_timestamp_clock;

	BoolObjectClosure* _is_subject_to_discovery; // determines whether a given oop is subject
	// to this ReferenceProcessor's discovery
	// (and further processing).

	bool _discovering_refs; // true when discovery enabled
	bool _discovery_is_concurrent; // if discovery is concurrent to the mutator
	bool _discovery_is_mt; // true if reference discovery is MT.

	uint _next_id; // round-robin mod _num_queues counter in
	// support of work distribution

	// For collectors that do not keep GC liveness information
	// in the object header, this field holds a closure that
	// helps the reference processor determine the reachability
	// of an oop.
	BoolObjectClosure* _is_alive_non_header;

	// Soft ref clearing policies
	// . the default policy
	static ReferencePolicy* _default_soft_ref_policy;
	// . the "clear all" policy
	static ReferencePolicy* _always_clear_soft_ref_policy;
	// . the current policy below is either one of the above
	ReferencePolicy* _current_soft_ref_policy;

	// The discovered ref lists themselves

	// The active MT'ness degree of the queues below
	uint _num_queues;
	// The maximum MT'ness degree of the queues below
	uint _max_num_queues;

	// Master array of discovered oops
	DiscoveredList* _discovered_refs;

	// Arrays of lists of oops, one per thread (pointers into master array above)
	DiscoveredList* _discoveredSoftRefs;
	DiscoveredList* _discoveredWeakRefs;
	DiscoveredList* _discoveredFinalRefs;
	DiscoveredList* _discoveredPhantomRefs;

	void run_task(RefProcTask& task, RefProcProxyTask& proxy_task, bool marks_oops_alive);

	// Drop Soft/Weak/Final references with a null or live referent, and clear
	// and enqueue non-Final references.
	void process_soft_weak_final_refs(RefProcProxyTask& proxy_task,
	ReferenceProcessorPhaseTimes& phase_times);

	// Keep alive followers of Final references, and enqueue.
	void process_final_keep_alive(RefProcProxyTask& proxy_task,
	ReferenceProcessorPhaseTimes& phase_times);

	// Drop and keep alive live Phantom references, or clear and enqueue if dead.
	void process_phantom_refs(RefProcProxyTask& proxy_task,
	ReferenceProcessorPhaseTimes& phase_times);

	// Work methods used by the process_* methods. All methods return the number of
	// removed elements.

	// Traverse the list and remove any Refs whose referents are alive,
	// or null if discovery is concurrent. Enqueue and clear the reference for
	// others if do_enqueue_and_clear is set.
	size_t process_discovered_list_work(DiscoveredList& refs_list,
	BoolObjectClosure* is_alive,
	OopClosure* keep_alive,
	EnqueueDiscoveredFieldClosure* enqueue,
	bool do_enqueue_and_clear);

	// Keep alive followers of referents for FinalReferences. Must only be called for
	// those.
	size_t process_final_keep_alive_work(DiscoveredList& refs_list,
	OopClosure* keep_alive,
	EnqueueDiscoveredFieldClosure* enqueue);


	void setup_policy(bool always_clear) {
	_current_soft_ref_policy = always_clear ?
	_always_clear_soft_ref_policy : _default_soft_ref_policy;
	_current_soft_ref_policy->setup(); // snapshot the policy threshold
	}
	public:
	static int number_of_subclasses_of_ref() { return (REF_PHANTOM - REF_NONE); }

	uint num_queues() const { return _num_queues; }
	uint max_num_queues() const { return _max_num_queues; }
	void set_active_mt_degree(uint v);

	void start_discovery(bool always_clear) {
	enable_discovery();
	setup_policy(always_clear);
	}

	// "Preclean" all the discovered reference lists by removing references whose
	// referents are null or strongly reachable (`is_alive` returns true).
	// Note: when a referent is strongly reachable, we assume it's already marked
	// through, so this method doesn't perform (and doesn't need to) any marking
	// work at all. Currently, this assumption holds because G1 uses SATB and the
	// marking status of an object is not updated when `Reference.get()` is
	// called.
	// `yield` is a closure that may be used to incrementalize or abort the
	// precleaning process. The caller is responsible for taking care of
	// potential interference with concurrent operations on these lists (or
	// predicates involved) by other threads.
	void preclean_discovered_references(BoolObjectClosure* is_alive,
	EnqueueDiscoveredFieldClosure* enqueue,
	YieldClosure* yield,
	GCTimer* gc_timer);

	private:
	// Returns the name of the discovered reference list
	// occupying the i / _num_queues slot.
	const char* list_name(uint i);

	// "Preclean" the given discovered reference list by removing references with
	// the attributes mentioned in preclean_discovered_references().
	// Supports both normal and fine grain yielding.
	// Returns whether the operation should be aborted.
	bool preclean_discovered_reflist(DiscoveredList& refs_list,
	BoolObjectClosure* is_alive,
	EnqueueDiscoveredFieldClosure* enqueue,
	YieldClosure* yield);

	// round-robin mod _num_queues (not: _not_ mod _max_num_queues)
	uint next_id() {
	uint id = _next_id;
	assert(!_discovery_is_mt, "Round robin should only be used in serial discovery");
	if (++_next_id == _num_queues) {
	_next_id = 0;
	}
	assert(_next_id < _num_queues, "_next_id %u _num_queues %u _max_num_queues %u", _next_id, _num_queues, _max_num_queues);
	return id;
	}
	DiscoveredList* get_discovered_list(ReferenceType rt);
	inline bool set_discovered_link(HeapWord* discovered_addr, oop next_discovered);
	inline void add_to_discovered_list(DiscoveredList& refs_list, oop obj,
	HeapWord* discovered_addr);
	inline bool set_discovered_link_st(HeapWord* discovered_addr,
	oop next_discovered);
	inline bool set_discovered_link_mt(HeapWord* discovered_addr,
	oop next_discovered);

	void clear_discovered_references(DiscoveredList& refs_list);

	void log_reflist(const char* prefix, DiscoveredList list[], uint num_active_queues);
	void log_reflist_counts(DiscoveredList ref_lists[], uint num_active_queues) PRODUCT_RETURN;

	// Balances reference queues.
	void balance_queues(DiscoveredList refs_lists[]);
	bool need_balance_queues(DiscoveredList refs_lists[]);

	// If there is need to balance the given queue, do it.
	void maybe_balance_queues(DiscoveredList refs_lists[]);

	// Update (advance) the soft ref master clock field.
	void update_soft_ref_master_clock();

	bool is_subject_to_discovery(oop const obj) const;

	public:
	// Default parameters give you a vanilla reference processor.
	ReferenceProcessor(BoolObjectClosure* is_subject_to_discovery,
	uint mt_processing_degree = 1,
	uint mt_discovery_degree = 1,
	bool concurrent_discovery = false,
	BoolObjectClosure* is_alive_non_header = nullptr);

	static void init_statics();

	// get and set "is_alive_non_header" field
	BoolObjectClosure* is_alive_non_header() {
	return _is_alive_non_header;
	}
	void set_is_alive_non_header(BoolObjectClosure* is_alive_non_header) {
	_is_alive_non_header = is_alive_non_header;
	}

	BoolObjectClosure* is_subject_to_discovery_closure() const { return _is_subject_to_discovery; }
	void set_is_subject_to_discovery_closure(BoolObjectClosure* cl) { _is_subject_to_discovery = cl; }

	// start and stop weak ref discovery
	void enable_discovery();
	void disable_discovery() { _discovering_refs = false; }
	bool discovery_enabled() { return _discovering_refs; }

	// whether discovery is concurrent to the mutator, or done in an stw pause.
	bool discovery_is_concurrent() const { return _discovery_is_concurrent; }
	bool discovery_is_stw() const { return !discovery_is_concurrent(); }

	// whether discovery is done by multiple threads same-old-timeously
	bool discovery_is_mt() const { return _discovery_is_mt; }
	void set_mt_discovery(bool mt) { _discovery_is_mt = mt; }

	// Whether we are in a phase when _processing_ is MT.
	bool processing_is_mt() const;

	// iterate over oops
	void weak_oops_do(OopClosure* f); // weak roots

	// Discover a Reference object, using appropriate discovery criteria
	virtual bool discover_reference(oop obj, ReferenceType rt);

	// Process references found during GC (called by the garbage collector)
	ReferenceProcessorStats
	process_discovered_references(RefProcProxyTask& proxy_task,
	ReferenceProcessorPhaseTimes& phase_times);

	// If a discovery is in process that is being superseded, abandon it: all
	// the discovered lists will be empty, and all the objects on them will
	// have null discovered fields. Must be called only at a safepoint.
	void abandon_partial_discovery();

	size_t total_reference_count(ReferenceType rt) const;

	// debugging
	void verify_no_references_recorded() PRODUCT_RETURN;
	void verify_referent(oop obj) PRODUCT_RETURN;
	};

	// A subject-to-discovery closure that uses a single memory span to determine the area that
	// is subject to discovery. Useful for collectors which have contiguous generations.
	class SpanSubjectToDiscoveryClosure : public BoolObjectClosure {
	MemRegion _span;

	public:
	SpanSubjectToDiscoveryClosure() : BoolObjectClosure(), _span() { }

	void set_span(MemRegion mr) {
	_span = mr;
	}

	virtual bool do_object_b(oop obj) {
	return _span.contains(obj);
	}
	};

	// A utility class to temporarily mutate the subject discovery closure of the
	// given ReferenceProcessor in the scope that contains it.
	class ReferenceProcessorSubjectToDiscoveryMutator : StackObj {
	ReferenceProcessor* _rp;
	BoolObjectClosure* _saved_cl;

	public:
	ReferenceProcessorSubjectToDiscoveryMutator(ReferenceProcessor* rp, BoolObjectClosure* cl):
	_rp(rp) {
	_saved_cl = _rp->is_subject_to_discovery_closure();
	_rp->set_is_subject_to_discovery_closure(cl);
	}

	~ReferenceProcessorSubjectToDiscoveryMutator() {
	_rp->set_is_subject_to_discovery_closure(_saved_cl);
	}
	};

	// A utility class to temporarily change the MT'ness of
	// reference discovery for the given ReferenceProcessor
	// in the scope that contains it.
	class ReferenceProcessorMTDiscoveryMutator: StackObj {
	private:
	ReferenceProcessor* _rp;
	bool _saved_mt;

	public:
	ReferenceProcessorMTDiscoveryMutator(ReferenceProcessor* rp,
	bool mt):
	_rp(rp) {
	_saved_mt = _rp->discovery_is_mt();
	_rp->set_mt_discovery(mt);
	}

	~ReferenceProcessorMTDiscoveryMutator() {
	_rp->set_mt_discovery(_saved_mt);
	}
	};

	// A utility class to temporarily change the disposition
	// of the "is_alive_non_header" closure field of the
	// given ReferenceProcessor in the scope that contains it.
	class ReferenceProcessorIsAliveMutator: StackObj {
	private:
	ReferenceProcessor* _rp;
	BoolObjectClosure* _saved_cl;

	public:
	ReferenceProcessorIsAliveMutator(ReferenceProcessor* rp,
	BoolObjectClosure* cl):
	_rp(rp) {
	_saved_cl = _rp->is_alive_non_header();
	_rp->set_is_alive_non_header(cl);
	}

	~ReferenceProcessorIsAliveMutator() {
	_rp->set_is_alive_non_header(_saved_cl);
	}
	};

	enum class RefProcThreadModel { Multi, Single };

	/*
	* This is the (base) task that handles reference processing that does not depend on
	* the chosen GC (Serial, Parallel or G1). This RefProcTask will be called from a subclass
	* of RefProcProxyTask. The RefProcProxyTask will give the behaviour of the selected GC by
	* calling rp_work with the gc-specific closures.
	*/
	class RefProcTask : StackObj {
	protected:
	ReferenceProcessor& _ref_processor;
	ReferenceProcessorPhaseTimes* _phase_times;

	// Used for tracking how much time a worker spends in a (sub)phase.
	uint tracker_id(uint worker_id) const {
	return _ref_processor.processing_is_mt() ? worker_id : 0;
	}

	void process_discovered_list(uint worker_id,
	ReferenceType ref_type,
	BoolObjectClosure* is_alive,
	OopClosure* keep_alive,
	EnqueueDiscoveredFieldClosure* enqueue);
	public:
	RefProcTask(ReferenceProcessor& ref_processor,
	ReferenceProcessorPhaseTimes* phase_times)
	: _ref_processor(ref_processor),
	_phase_times(phase_times) {}

	virtual void rp_work(uint worker_id,
	BoolObjectClosure* is_alive,
	OopClosure* keep_alive,
	EnqueueDiscoveredFieldClosure* enqueue,
	VoidClosure* complete_gc) = 0;
	};

	/*
	* This is the (base) task that handles reference processing that do depend on
	* the chosen GC (Serial, Parallel or G1). This RefProcProxyTask will call a subclass
	* of RefProcTask that will handle reference processing in a generic way for Serial,
	* Parallel and G1. This proxy will add the relevant closures, task terminators etc.
	*/
	class RefProcProxyTask : public WorkerTask {
	protected:
	const uint _max_workers;
	RefProcTask* _rp_task;
	RefProcThreadModel _tm;
	uint _queue_count;
	bool _marks_oops_alive;

	public:
	RefProcProxyTask(const char* name, uint max_workers) : WorkerTask(name), _max_workers(max_workers), _rp_task(nullptr),_tm(RefProcThreadModel::Single), _queue_count(0), _marks_oops_alive(false) {}

	void prepare_run_task(RefProcTask& rp_task, uint queue_count, RefProcThreadModel tm, bool marks_oops_alive) {
	_rp_task = &rp_task;
	_tm = tm;
	_queue_count = queue_count;
	_marks_oops_alive = marks_oops_alive;
	prepare_run_task_hook();
	}

	virtual void prepare_run_task_hook() {}
	};

	// Temporarily change the number of workers based on given reference count.
	// This ergonomically decided worker count will be used to activate worker threads.
	class RefProcMTDegreeAdjuster : public StackObj {
	typedef ReferenceProcessor::RefProcPhases RefProcPhases;

	ReferenceProcessor* _rp;
	uint _saved_num_queues;

	// Calculate based on total of references.
	uint ergo_proc_thread_count(size_t ref_count,
	uint max_threads,
	RefProcPhases phase) const;

	bool use_max_threads(RefProcPhases phase) const;

	public:
	RefProcMTDegreeAdjuster(ReferenceProcessor* rp,
	RefProcPhases phase,
	size_t ref_count);
	~RefProcMTDegreeAdjuster();
	};

	#endif // SHARE_GC_SHARED_REFERENCEPROCESSOR_HPP