| /* |
| * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. |
| * Copyright (c) 2021, 2025 SAP SE. All rights reserved. |
| * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| * |
| * This code is free software; you can redistribute it and/or modify it |
| * under the terms of the GNU General Public License version 2 only, as |
| * published by the Free Software Foundation. |
| * |
| * This code is distributed in the hope that it will be useful, but WITHOUT |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| * version 2 for more details (a copy is included in the LICENSE file that |
| * accompanied this code). |
| * |
| * You should have received a copy of the GNU General Public License version |
| * 2 along with this work; if not, write to the Free Software Foundation, |
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| * |
| * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| * or visit www.oracle.com if you need additional information or have any |
| * questions. |
| */ |
| |
| #include "asm/macroAssembler.inline.hpp" |
| #include "asm/register.hpp" |
| #include "code/codeBlob.hpp" |
| #include "code/vmreg.inline.hpp" |
| #include "gc/z/zAddress.hpp" |
| #include "gc/z/zBarrier.inline.hpp" |
| #include "gc/z/zBarrierSet.hpp" |
| #include "gc/z/zBarrierSetAssembler.hpp" |
| #include "gc/z/zBarrierSetRuntime.hpp" |
| #include "gc/z/zThreadLocalData.hpp" |
| #include "memory/resourceArea.hpp" |
| #include "register_ppc.hpp" |
| #include "runtime/jniHandles.hpp" |
| #include "runtime/sharedRuntime.hpp" |
| #include "utilities/globalDefinitions.hpp" |
| #include "utilities/macros.hpp" |
| #ifdef COMPILER1 |
| #include "c1/c1_LIRAssembler.hpp" |
| #include "c1/c1_MacroAssembler.hpp" |
| #include "gc/z/c1/zBarrierSetC1.hpp" |
| #endif // COMPILER1 |
| #ifdef COMPILER2 |
| #include "gc/z/c2/zBarrierSetC2.hpp" |
| #include "opto/output.hpp" |
| #endif // COMPILER2 |
| |
| #undef __ |
| #define __ masm-> |
| |
| // Helper for saving and restoring registers across a runtime call that does |
| // not have any live vector registers. |
| class ZRuntimeCallSpill { |
| MacroAssembler* _masm; |
| Register _result; |
| bool _needs_frame, _preserve_gp_registers, _preserve_fp_registers; |
| int _nbytes_save; |
| |
| void save() { |
| MacroAssembler* masm = _masm; |
| |
| if (_needs_frame) { |
| if (_preserve_gp_registers) { |
| bool preserve_R3 = _result != R3_ARG1; |
| _nbytes_save = (MacroAssembler::num_volatile_gp_regs |
| + (_preserve_fp_registers ? MacroAssembler::num_volatile_fp_regs : 0) |
| - (preserve_R3 ? 0 : 1) |
| ) * BytesPerWord; |
| __ save_volatile_gprs(R1_SP, -_nbytes_save, _preserve_fp_registers, preserve_R3); |
| } |
| |
| __ save_LR(R0); |
| __ push_frame_reg_args(_nbytes_save, R0); |
| } |
| } |
| |
| void restore() { |
| MacroAssembler* masm = _masm; |
| |
| Register result = R3_RET; |
| if (_needs_frame) { |
| __ pop_frame(); |
| __ restore_LR(R0); |
| |
| if (_preserve_gp_registers) { |
| bool restore_R3 = _result != R3_ARG1; |
| if (restore_R3 && _result != noreg) { |
| __ mr(R0, R3_RET); |
| result = R0; |
| } |
| __ restore_volatile_gprs(R1_SP, -_nbytes_save, _preserve_fp_registers, restore_R3); |
| } |
| } |
| if (_result != noreg) { |
| __ mr_if_needed(_result, result); |
| } |
| } |
| |
| public: |
| ZRuntimeCallSpill(MacroAssembler* masm, Register result, MacroAssembler::PreservationLevel preservation_level) |
| : _masm(masm), |
| _result(result), |
| _needs_frame(preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR), |
| _preserve_gp_registers(preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS), |
| _preserve_fp_registers(preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS), |
| _nbytes_save(0) { |
| save(); |
| } |
| ~ZRuntimeCallSpill() { |
| restore(); |
| } |
| }; |
| |
| |
| void ZBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, |
| Register base, RegisterOrConstant ind_or_offs, Register dst, |
| Register tmp1, Register tmp2, |
| MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null) { |
| __ block_comment("load_at (zgc) {"); |
| |
| // Check whether a special gc barrier is required for this particular load |
| // (e.g. whether it's a reference load or not) |
| if (!ZBarrierSet::barrier_needed(decorators, type)) { |
| BarrierSetAssembler::load_at(masm, decorators, type, base, ind_or_offs, dst, |
| tmp1, tmp2, preservation_level, L_handle_null); |
| return; |
| } |
| |
| if (ind_or_offs.is_register()) { |
| assert_different_registers(base, ind_or_offs.as_register(), tmp1, tmp2, R0, noreg); |
| assert_different_registers(dst, ind_or_offs.as_register(), tmp1, tmp2, R0, noreg); |
| } else { |
| assert_different_registers(base, tmp1, tmp2, R0, noreg); |
| assert_different_registers(dst, tmp1, tmp2, R0, noreg); |
| } |
| |
| /* ==== Load the pointer using the standard implementation for the actual heap access |
| and the decompression of compressed pointers ==== */ |
| // Result of 'load_at' (standard implementation) will be written back to 'dst'. |
| // As 'base' is required for the C-call, it must be reserved in case of a register clash. |
| Register saved_base = base; |
| if (base == dst) { |
| __ mr(tmp2, base); |
| saved_base = tmp2; |
| } |
| |
| __ ld(dst, ind_or_offs, base); |
| |
| /* ==== Check whether pointer is dirty ==== */ |
| Label done, uncolor; |
| |
| const bool on_non_strong = |
| (decorators & ON_WEAK_OOP_REF) != 0 || |
| (decorators & ON_PHANTOM_OOP_REF) != 0; |
| |
| // Load bad mask into scratch register. |
| if (on_non_strong) { |
| __ ld(tmp1, in_bytes(ZThreadLocalData::mark_bad_mask_offset()), R16_thread); |
| } else { |
| __ ld(tmp1, in_bytes(ZThreadLocalData::load_bad_mask_offset()), R16_thread); |
| } |
| |
| // The color bits of the to-be-tested pointer do not have to be equivalent to the 'bad_mask' testing bits. |
| // A pointer is classified as dirty if any of the color bits that also match the bad mask is set. |
| // Conversely, it follows that the logical AND of the bad mask and the pointer must be zero |
| // if the pointer is not dirty. |
| // Only dirty pointers must be processed by this barrier, so we can skip it in case the latter condition holds true. |
| __ and_(tmp1, tmp1, dst); |
| __ beq(CR0, uncolor); |
| |
| /* ==== Invoke barrier ==== */ |
| { |
| ZRuntimeCallSpill rcs(masm, dst, preservation_level); |
| |
| // Setup arguments |
| if (saved_base != R3_ARG1 && ind_or_offs.register_or_noreg() != R3_ARG1) { |
| __ mr_if_needed(R3_ARG1, dst); |
| __ add(R4_ARG2, ind_or_offs, saved_base); |
| } else if (dst != R4_ARG2) { |
| __ add(R4_ARG2, ind_or_offs, saved_base); |
| __ mr(R3_ARG1, dst); |
| } else { |
| __ add(R0, ind_or_offs, saved_base); |
| __ mr(R3_ARG1, dst); |
| __ mr(R4_ARG2, R0); |
| } |
| |
| __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators)); |
| } |
| |
| // Slow-path has already uncolored |
| if (L_handle_null != nullptr) { |
| __ cmpdi(CR0, dst, 0); |
| __ beq(CR0, *L_handle_null); |
| } |
| __ b(done); |
| |
| __ bind(uncolor); |
| if (L_handle_null == nullptr) { |
| __ srdi(dst, dst, ZPointerLoadShift); |
| } else { |
| __ srdi_(dst, dst, ZPointerLoadShift); |
| __ beq(CR0, *L_handle_null); |
| } |
| |
| __ bind(done); |
| __ block_comment("} load_at (zgc)"); |
| } |
| |
| static void load_least_significant_16_oop_bits(MacroAssembler* masm, Register dst, RegisterOrConstant ind_or_offs, Register base) { |
| assert_different_registers(dst, base); |
| #ifndef VM_LITTLE_ENDIAN |
| const int BE_offset = 6; |
| if (ind_or_offs.is_register()) { |
| __ addi(dst, ind_or_offs.as_register(), BE_offset); |
| __ lhzx(dst, base, dst); |
| } else { |
| __ lhz(dst, ind_or_offs.as_constant() + BE_offset, base); |
| } |
| #else |
| __ lhz(dst, ind_or_offs, base); |
| #endif |
| } |
| |
| static void emit_store_fast_path_check(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, bool is_atomic, Label& medium_path) { |
| if (is_atomic) { |
| assert(ZPointerLoadShift + LogMinObjAlignmentInBytes >= 16, "or replace following code"); |
| load_least_significant_16_oop_bits(masm, R0, ind_or_offs, base); |
| // Atomic operations must ensure that the contents of memory are store-good before |
| // an atomic operation can execute. |
| // A not relocatable object could have spurious raw null pointers in its fields after |
| // getting promoted to the old generation. |
| __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBits); |
| __ cmplwi(CR0, R0, barrier_Relocation::unpatched); |
| } else { |
| __ ld(R0, ind_or_offs, base); |
| // Stores on relocatable objects never need to deal with raw null pointers in fields. |
| // Raw null pointers may only exist in the young generation, as they get pruned when |
| // the object is relocated to old. And no pre-write barrier needs to perform any action |
| // in the young generation. |
| __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreBadMask); |
| __ andi_(R0, R0, barrier_Relocation::unpatched); |
| } |
| __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CR0, Assembler::equal), medium_path); |
| } |
| |
| void ZBarrierSetAssembler::store_barrier_fast(MacroAssembler* masm, |
| Register ref_base, |
| RegisterOrConstant ind_or_offset, |
| Register rnew_zaddress, |
| Register rnew_zpointer, |
| bool in_nmethod, |
| bool is_atomic, |
| Label& medium_path, |
| Label& medium_path_continuation) const { |
| assert_different_registers(ref_base, rnew_zpointer); |
| assert_different_registers(ind_or_offset.register_or_noreg(), rnew_zpointer); |
| assert_different_registers(rnew_zaddress, rnew_zpointer); |
| |
| if (in_nmethod) { |
| emit_store_fast_path_check(masm, ref_base, ind_or_offset, is_atomic, medium_path); |
| __ bind(medium_path_continuation); |
| __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBits); |
| __ li(rnew_zpointer, barrier_Relocation::unpatched); // Load color bits. |
| if (rnew_zaddress == noreg) { // noreg encodes null. |
| if (ZPointerLoadShift >= 16) { |
| __ rldicl(rnew_zpointer, rnew_zpointer, 0, 64 - ZPointerLoadShift); // Clear sign extension from li. |
| } |
| } |
| } else { |
| __ ld(R0, ind_or_offset, ref_base); |
| __ ld(rnew_zpointer, in_bytes(ZThreadLocalData::store_bad_mask_offset()), R16_thread); |
| __ and_(R0, R0, rnew_zpointer); |
| __ bne(CR0, medium_path); |
| __ bind(medium_path_continuation); |
| __ ld(rnew_zpointer, in_bytes(ZThreadLocalData::store_good_mask_offset()), R16_thread); |
| } |
| if (rnew_zaddress != noreg) { // noreg encodes null. |
| __ rldimi(rnew_zpointer, rnew_zaddress, ZPointerLoadShift, 0); // Insert shifted pointer. |
| } |
| } |
| |
| static void store_barrier_buffer_add(MacroAssembler* masm, |
| Register ref_base, |
| RegisterOrConstant ind_or_offs, |
| Register tmp1, |
| Label& slow_path) { |
| __ ld(tmp1, in_bytes(ZThreadLocalData::store_barrier_buffer_offset()), R16_thread); |
| |
| // Combined pointer bump and check if the buffer is disabled or full |
| __ ld(R0, in_bytes(ZStoreBarrierBuffer::current_offset()), tmp1); |
| __ addic_(R0, R0, -(int)sizeof(ZStoreBarrierEntry)); |
| __ blt(CR0, slow_path); |
| __ std(R0, in_bytes(ZStoreBarrierBuffer::current_offset()), tmp1); |
| |
| // Entry is at ZStoreBarrierBuffer (tmp1) + buffer_offset + scaled index (R0) |
| __ add(tmp1, tmp1, R0); |
| |
| // Compute and log the store address |
| Register store_addr = ref_base; |
| if (!ind_or_offs.is_constant() || ind_or_offs.as_constant() != 0) { |
| __ add(R0, ind_or_offs, ref_base); |
| store_addr = R0; |
| } |
| __ std(store_addr, in_bytes(ZStoreBarrierBuffer::buffer_offset()) + in_bytes(ZStoreBarrierEntry::p_offset()), tmp1); |
| |
| // Load and log the prev value |
| __ ld(R0, ind_or_offs, ref_base); |
| __ std(R0, in_bytes(ZStoreBarrierBuffer::buffer_offset()) + in_bytes(ZStoreBarrierEntry::prev_offset()), tmp1); |
| } |
| |
| void ZBarrierSetAssembler::store_barrier_medium(MacroAssembler* masm, |
| Register ref_base, |
| RegisterOrConstant ind_or_offs, |
| Register tmp, |
| bool is_atomic, |
| Label& medium_path_continuation, |
| Label& slow_path) const { |
| assert_different_registers(ref_base, tmp, R0); |
| |
| // The reason to end up in the medium path is that the pre-value was not 'good'. |
| |
| if (is_atomic) { |
| // Atomic accesses can get to the medium fast path because the value was a |
| // raw null value. If it was not null, then there is no doubt we need to take a slow path. |
| __ ld(tmp, ind_or_offs, ref_base); |
| __ cmpdi(CR0, tmp, 0); |
| __ bne(CR0, slow_path); |
| |
| // If we get this far, we know there is a young raw null value in the field. |
| // Try to self-heal null values for atomic accesses |
| bool need_restore = false; |
| if (!ind_or_offs.is_constant() || ind_or_offs.as_constant() != 0) { |
| __ add(ref_base, ref_base, ind_or_offs); |
| need_restore = true; |
| } |
| __ ld(R0, in_bytes(ZThreadLocalData::store_good_mask_offset()), R16_thread); |
| __ cmpxchgd(CR0, tmp, (intptr_t)0, R0, ref_base, |
| MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), |
| noreg, need_restore ? nullptr : &slow_path); |
| if (need_restore) { |
| __ sub(ref_base, ref_base, ind_or_offs); |
| __ bne(CR0, slow_path); |
| } |
| } else { |
| // A non-atomic relocatable object won't get to the medium fast path due to a |
| // raw null in the young generation. We only get here because the field is bad. |
| // In this path we don't need any self healing, so we can avoid a runtime call |
| // most of the time by buffering the store barrier to be applied lazily. |
| store_barrier_buffer_add(masm, |
| ref_base, |
| ind_or_offs, |
| tmp, |
| slow_path); |
| } |
| __ b(medium_path_continuation); |
| } |
| |
| // The Z store barrier only verifies the pointers it is operating on and is thus a sole debugging measure. |
| void ZBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, |
| Register base, RegisterOrConstant ind_or_offs, Register val, |
| Register tmp1, Register tmp2, Register tmp3, |
| MacroAssembler::PreservationLevel preservation_level) { |
| __ block_comment("store_at (zgc) {"); |
| |
| bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; |
| |
| if (is_reference_type(type)) { |
| assert_different_registers(base, val, tmp1, tmp2, tmp3); |
| |
| if (dest_uninitialized) { |
| // tmp1 = (val << ZPointerLoadShift) | store_good_mask |
| __ ld(tmp1, in_bytes(ZThreadLocalData::store_good_mask_offset()), R16_thread); |
| if (val != noreg) { // noreg encodes null. |
| __ rldimi(tmp1, val, ZPointerLoadShift, 0); |
| } |
| } else { |
| Label done; |
| Label medium; |
| Label medium_continuation; // bound in store_barrier_fast |
| Label slow; |
| |
| store_barrier_fast(masm, base, ind_or_offs, val, tmp1, false, false, medium, medium_continuation); |
| __ b(done); |
| __ bind(medium); |
| store_barrier_medium(masm, base, ind_or_offs, tmp1, false, medium_continuation, slow); |
| __ bind(slow); |
| { |
| ZRuntimeCallSpill rcs(masm, noreg, preservation_level); |
| __ add(R3_ARG1, ind_or_offs, base); |
| __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), R3_ARG1); |
| } |
| __ b(medium_continuation); |
| |
| __ bind(done); |
| } |
| BarrierSetAssembler::store_at(masm, decorators, type, base, ind_or_offs, tmp1, tmp2, tmp3, noreg, preservation_level); |
| } else { |
| BarrierSetAssembler::store_at(masm, decorators, type, base, ind_or_offs, val, tmp1, tmp2, tmp3, preservation_level); |
| } |
| |
| __ block_comment("} store_at (zgc)"); |
| } |
| |
| /* arraycopy */ |
| const Register _load_bad_mask = R6, _store_bad_mask = R7, _store_good_mask = R8; |
| |
| void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type, |
| Register src, Register dst, Register count, |
| Register preserve1, Register preserve2) { |
| bool is_checkcast_copy = (decorators & ARRAYCOPY_CHECKCAST) != 0, |
| dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; |
| |
| if (!ZBarrierSet::barrier_needed(decorators, type) || is_checkcast_copy) { |
| // Barrier not needed |
| return; |
| } |
| |
| __ block_comment("arraycopy_prologue (zgc) {"); |
| |
| load_copy_masks(masm, _load_bad_mask, _store_bad_mask, _store_good_mask, dest_uninitialized); |
| |
| __ block_comment("} arraycopy_prologue (zgc)"); |
| } |
| |
| void ZBarrierSetAssembler::load_copy_masks(MacroAssembler* masm, |
| Register load_bad_mask, |
| Register store_bad_mask, |
| Register store_good_mask, |
| bool dest_uninitialized) const { |
| __ ld(load_bad_mask, in_bytes(ZThreadLocalData::load_bad_mask_offset()), R16_thread); |
| __ ld(store_good_mask, in_bytes(ZThreadLocalData::store_good_mask_offset()), R16_thread); |
| if (dest_uninitialized) { |
| DEBUG_ONLY( __ li(store_bad_mask, -1); ) |
| } else { |
| __ ld(store_bad_mask, in_bytes(ZThreadLocalData::store_bad_mask_offset()), R16_thread); |
| } |
| } |
| void ZBarrierSetAssembler::copy_load_at_fast(MacroAssembler* masm, |
| Register zpointer, |
| Register addr, |
| Register load_bad_mask, |
| Label& slow_path, |
| Label& continuation) const { |
| __ ldx(zpointer, addr); |
| __ and_(R0, zpointer, load_bad_mask); |
| __ bne(CR0, slow_path); |
| __ bind(continuation); |
| } |
| void ZBarrierSetAssembler::copy_load_at_slow(MacroAssembler* masm, |
| Register zpointer, |
| Register addr, |
| Register tmp, |
| Label& slow_path, |
| Label& continuation) const { |
| __ align(32); |
| __ bind(slow_path); |
| __ mfctr(tmp); // preserve loop counter |
| { |
| ZRuntimeCallSpill rcs(masm, R0, MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS); |
| assert(zpointer != R4_ARG2, "or change argument setup"); |
| __ mr_if_needed(R4_ARG2, addr); |
| __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(), zpointer, R4_ARG2); |
| } |
| __ sldi(zpointer, R0, ZPointerLoadShift); // Slow-path has uncolored; revert |
| __ mtctr(tmp); // restore loop counter |
| __ b(continuation); |
| } |
| void ZBarrierSetAssembler::copy_store_at_fast(MacroAssembler* masm, |
| Register zpointer, |
| Register addr, |
| Register store_bad_mask, |
| Register store_good_mask, |
| Label& medium_path, |
| Label& continuation, |
| bool dest_uninitialized) const { |
| if (!dest_uninitialized) { |
| __ ldx(R0, addr); |
| __ and_(R0, R0, store_bad_mask); |
| __ bne(CR0, medium_path); |
| __ bind(continuation); |
| } |
| __ rldimi(zpointer, store_good_mask, 0, 64 - ZPointerLoadShift); // Replace color bits. |
| __ stdx(zpointer, addr); |
| } |
| void ZBarrierSetAssembler::copy_store_at_slow(MacroAssembler* masm, |
| Register addr, |
| Register tmp, |
| Label& medium_path, |
| Label& continuation, |
| bool dest_uninitialized) const { |
| if (!dest_uninitialized) { |
| Label slow_path; |
| __ align(32); |
| __ bind(medium_path); |
| store_barrier_medium(masm, addr, (intptr_t)0, tmp, false, continuation, slow_path); |
| __ bind(slow_path); |
| __ mfctr(tmp); // preserve loop counter |
| { |
| ZRuntimeCallSpill rcs(masm, noreg, MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS); |
| __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), addr); |
| } |
| __ mtctr(tmp); // restore loop counter |
| __ b(continuation); |
| } |
| } |
| |
| // Arguments for generated stub: |
| // from: R3_ARG1 |
| // to: R4_ARG2 |
| // count: R5_ARG3 (int >= 0) |
| void ZBarrierSetAssembler::generate_disjoint_oop_copy(MacroAssembler* masm, bool dest_uninitialized) { |
| const Register zpointer = R2, tmp = R9; |
| Label done, loop, load_bad, load_good, store_bad, store_good; |
| __ cmpdi(CR0, R5_ARG3, 0); |
| __ beq(CR0, done); |
| __ mtctr(R5_ARG3); |
| |
| __ align(32); |
| __ bind(loop); |
| copy_load_at_fast(masm, zpointer, R3_ARG1, _load_bad_mask, load_bad, load_good); |
| copy_store_at_fast(masm, zpointer, R4_ARG2, _store_bad_mask, _store_good_mask, store_bad, store_good, dest_uninitialized); |
| __ addi(R3_ARG1, R3_ARG1, 8); |
| __ addi(R4_ARG2, R4_ARG2, 8); |
| __ bdnz(loop); |
| |
| __ bind(done); |
| __ li(R3_RET, 0); |
| __ blr(); |
| |
| copy_load_at_slow(masm, zpointer, R3_ARG1, tmp, load_bad, load_good); |
| copy_store_at_slow(masm, R4_ARG2, tmp, store_bad, store_good, dest_uninitialized); |
| } |
| |
| void ZBarrierSetAssembler::generate_conjoint_oop_copy(MacroAssembler* masm, bool dest_uninitialized) { |
| const Register zpointer = R2, tmp = R9; |
| Label done, loop, load_bad, load_good, store_bad, store_good; |
| __ sldi_(R0, R5_ARG3, 3); |
| __ beq(CR0, done); |
| __ mtctr(R5_ARG3); |
| // Point behind last elements and copy backwards. |
| __ add(R3_ARG1, R3_ARG1, R0); |
| __ add(R4_ARG2, R4_ARG2, R0); |
| |
| __ align(32); |
| __ bind(loop); |
| __ addi(R3_ARG1, R3_ARG1, -8); |
| __ addi(R4_ARG2, R4_ARG2, -8); |
| copy_load_at_fast(masm, zpointer, R3_ARG1, _load_bad_mask, load_bad, load_good); |
| copy_store_at_fast(masm, zpointer, R4_ARG2, _store_bad_mask, _store_good_mask, store_bad, store_good, dest_uninitialized); |
| __ bdnz(loop); |
| |
| __ bind(done); |
| __ li(R3_RET, 0); |
| __ blr(); |
| |
| copy_load_at_slow(masm, zpointer, R3_ARG1, tmp, load_bad, load_good); |
| copy_store_at_slow(masm, R4_ARG2, tmp, store_bad, store_good, dest_uninitialized); |
| } |
| |
| |
| // Verify a colored pointer. |
| void ZBarrierSetAssembler::check_oop(MacroAssembler *masm, Register obj, const char* msg) { |
| if (!VerifyOops) { |
| return; |
| } |
| Label done, skip_uncolor; |
| // Skip (colored) null. |
| __ srdi_(R0, obj, ZPointerLoadShift); |
| __ beq(CR0, done); |
| |
| // Check if ZAddressHeapBase << ZPointerLoadShift is set. If so, we need to uncolor. |
| __ rldicl_(R0, obj, 64 - ZAddressHeapBaseShift - ZPointerLoadShift, 63); |
| __ mr(R0, obj); |
| __ beq(CR0, skip_uncolor); |
| __ srdi(R0, obj, ZPointerLoadShift); |
| __ bind(skip_uncolor); |
| |
| __ verify_oop(R0, msg); |
| __ bind(done); |
| } |
| |
| |
| void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env, |
| Register obj, Register tmp, Label& slowpath) { |
| __ block_comment("try_resolve_jobject_in_native (zgc) {"); |
| |
| Label done, tagged, weak_tagged, check_color; |
| Address load_bad_mask = load_bad_mask_from_jni_env(jni_env), |
| mark_bad_mask = mark_bad_mask_from_jni_env(jni_env); |
| |
| // Test for tag |
| __ andi_(tmp, obj, JNIHandles::tag_mask); |
| __ bne(CR0, tagged); |
| |
| // Resolve local handle |
| __ ld(dst, 0, obj); |
| __ b(done); |
| |
| __ bind(tagged); |
| |
| // Test for weak tag |
| __ andi_(tmp, obj, JNIHandles::TypeTag::weak_global); |
| __ clrrdi(dst, obj, JNIHandles::tag_size); // Untag. |
| __ bne(CR0, weak_tagged); |
| |
| // Resolve global handle |
| __ ld(dst, 0, dst); |
| __ ld(tmp, load_bad_mask); |
| __ b(check_color); |
| |
| __ bind(weak_tagged); |
| |
| // Resolve weak handle |
| __ ld(dst, 0, dst); |
| __ ld(tmp, mark_bad_mask); |
| |
| __ bind(check_color); |
| __ and_(tmp, tmp, dst); |
| __ bne(CR0, slowpath); |
| |
| // Uncolor |
| __ srdi(dst, dst, ZPointerLoadShift); |
| |
| __ bind(done); |
| |
| __ block_comment("} try_resolve_jobject_in_native (zgc)"); |
| } |
| |
| #undef __ |
| |
| #ifdef COMPILER1 |
| #define __ ce->masm()-> |
| |
| static void z_uncolor(LIR_Assembler* ce, LIR_Opr ref) { |
| Register r = ref->as_register(); |
| __ srdi(r, r, ZPointerLoadShift); |
| } |
| |
| static void check_color(LIR_Assembler* ce, LIR_Opr ref, bool on_non_strong) { |
| int relocFormat = on_non_strong ? ZBarrierRelocationFormatMarkBadMask |
| : ZBarrierRelocationFormatLoadBadMask; |
| __ relocate(barrier_Relocation::spec(), relocFormat); |
| __ andi_(R0, ref->as_register(), barrier_Relocation::unpatched); |
| } |
| |
| static void z_color(LIR_Assembler* ce, LIR_Opr ref) { |
| __ sldi(ref->as_register(), ref->as_register(), ZPointerLoadShift); |
| __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBits); |
| __ ori(ref->as_register(), ref->as_register(), barrier_Relocation::unpatched); |
| } |
| |
| void ZBarrierSetAssembler::generate_c1_uncolor(LIR_Assembler* ce, LIR_Opr ref) const { |
| z_uncolor(ce, ref); |
| } |
| |
| void ZBarrierSetAssembler::generate_c1_color(LIR_Assembler* ce, LIR_Opr ref) const { |
| z_color(ce, ref); |
| } |
| |
| void ZBarrierSetAssembler::generate_c1_load_barrier(LIR_Assembler* ce, |
| LIR_Opr ref, |
| ZLoadBarrierStubC1* stub, |
| bool on_non_strong) const { |
| check_color(ce, ref, on_non_strong); |
| __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CR0, Assembler::equal), *stub->entry()); |
| z_uncolor(ce, ref); |
| __ bind(*stub->continuation()); |
| } |
| |
| // Code emitted by code stub "ZLoadBarrierStubC1" which in turn is emitted by ZBarrierSetC1::load_barrier. |
| // Invokes the runtime stub which is defined just below. |
| void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce, |
| ZLoadBarrierStubC1* stub) const { |
| __ block_comment("c1_load_barrier_stub (zgc) {"); |
| |
| __ bind(*stub->entry()); |
| |
| /* ==== Determine relevant data registers and ensure register sanity ==== */ |
| Register ref = stub->ref()->as_register(); |
| Register ref_addr = noreg; |
| |
| // Determine reference address |
| if (stub->tmp()->is_valid()) { |
| // 'tmp' register is given, so address might have an index or a displacement. |
| ce->leal(stub->ref_addr(), stub->tmp()); |
| ref_addr = stub->tmp()->as_pointer_register(); |
| } else { |
| // 'tmp' register is not given, so address must have neither an index nor a displacement. |
| // The address' base register is thus usable as-is. |
| assert(stub->ref_addr()->as_address_ptr()->disp() == 0, "illegal displacement"); |
| assert(!stub->ref_addr()->as_address_ptr()->index()->is_valid(), "illegal index"); |
| |
| ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register(); |
| } |
| |
| assert_different_registers(ref, ref_addr, R0, noreg); |
| |
| /* ==== Invoke stub ==== */ |
| // Pass arguments via stack. The stack pointer will be bumped by the stub. |
| __ std(ref, -1 * BytesPerWord, R1_SP); |
| __ std(ref_addr, -2 * BytesPerWord, R1_SP); |
| |
| __ load_const_optimized(R0, stub->runtime_stub(), /* temp */ ref); |
| __ call_stub(R0); |
| |
| // The runtime stub passes the result via the R0 register, overriding the previously-loaded stub address. |
| __ mr(ref, R0); |
| __ b(*stub->continuation()); |
| |
| __ block_comment("} c1_load_barrier_stub (zgc)"); |
| } |
| |
| void ZBarrierSetAssembler::generate_c1_store_barrier(LIR_Assembler* ce, |
| LIR_Address* addr, |
| LIR_Opr new_zaddress, |
| LIR_Opr new_zpointer, |
| ZStoreBarrierStubC1* stub) const { |
| Register rnew_zaddress = new_zaddress->as_register(); |
| Register rnew_zpointer = new_zpointer->as_register(); |
| |
| Register rbase = addr->base()->as_pointer_register(); |
| RegisterOrConstant ind_or_offs = (addr->index()->is_illegal()) |
| ? (RegisterOrConstant)addr->disp() |
| : (RegisterOrConstant)addr->index()->as_pointer_register(); |
| |
| store_barrier_fast(ce->masm(), |
| rbase, |
| ind_or_offs, |
| rnew_zaddress, |
| rnew_zpointer, |
| true, |
| stub->is_atomic(), |
| *stub->entry(), |
| *stub->continuation()); |
| } |
| |
| void ZBarrierSetAssembler::generate_c1_store_barrier_stub(LIR_Assembler* ce, |
| ZStoreBarrierStubC1* stub) const { |
| // Stub entry |
| __ bind(*stub->entry()); |
| |
| Label slow; |
| |
| LIR_Address* addr = stub->ref_addr()->as_address_ptr(); |
| assert(addr->index()->is_illegal() || addr->disp() == 0, "can't have both"); |
| Register rbase = addr->base()->as_pointer_register(); |
| RegisterOrConstant ind_or_offs = (addr->index()->is_illegal()) |
| ? (RegisterOrConstant)addr->disp() |
| : (RegisterOrConstant)addr->index()->as_pointer_register(); |
| Register new_zpointer = stub->new_zpointer()->as_register(); |
| |
| store_barrier_medium(ce->masm(), |
| rbase, |
| ind_or_offs, |
| new_zpointer, // temp |
| stub->is_atomic(), |
| *stub->continuation(), |
| slow); |
| |
| __ bind(slow); |
| |
| __ load_const_optimized(/*stub address*/ new_zpointer, stub->runtime_stub(), R0); |
| __ add(R0, ind_or_offs, rbase); // pass store address in R0 |
| __ mtctr(new_zpointer); |
| __ bctrl(); |
| |
| // Stub exit |
| __ b(*stub->continuation()); |
| } |
| |
| #undef __ |
| #define __ sasm-> |
| |
| // Code emitted by runtime code stub which in turn is emitted by ZBarrierSetC1::generate_c1_runtime_stubs. |
| void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm, |
| DecoratorSet decorators) const { |
| __ block_comment("c1_load_barrier_runtime_stub (zgc) {"); |
| |
| const int stack_parameters = 2; |
| const int nbytes_save = (MacroAssembler::num_volatile_regs + stack_parameters) * BytesPerWord; |
| |
| __ save_volatile_gprs(R1_SP, -nbytes_save); |
| __ save_LR(R0); |
| |
| // Load arguments back again from the stack. |
| __ ld(R3_ARG1, -1 * BytesPerWord, R1_SP); // ref |
| __ ld(R4_ARG2, -2 * BytesPerWord, R1_SP); // ref_addr |
| |
| __ push_frame_reg_args(nbytes_save, R0); |
| |
| __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators)); |
| |
| __ verify_oop(R3_RET, "Bad pointer after barrier invocation"); |
| __ mr(R0, R3_RET); |
| |
| __ pop_frame(); |
| __ restore_LR(R3_RET); |
| __ restore_volatile_gprs(R1_SP, -nbytes_save); |
| |
| __ blr(); |
| |
| __ block_comment("} c1_load_barrier_runtime_stub (zgc)"); |
| } |
| |
| void ZBarrierSetAssembler::generate_c1_store_barrier_runtime_stub(StubAssembler* sasm, |
| bool self_healing) const { |
| __ block_comment("c1_store_barrier_runtime_stub (zgc) {"); |
| |
| const int nbytes_save = MacroAssembler::num_volatile_regs * BytesPerWord; |
| __ save_volatile_gprs(R1_SP, -nbytes_save); |
| __ mr(R3_ARG1, R0); // store address |
| |
| __ save_LR(R0); |
| __ push_frame_reg_args(nbytes_save, R0); |
| |
| if (self_healing) { |
| __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr()); |
| } else { |
| __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr()); |
| } |
| |
| __ pop_frame(); |
| __ restore_LR(R3_RET); |
| __ restore_volatile_gprs(R1_SP, -nbytes_save); |
| |
| __ blr(); |
| |
| __ block_comment("} c1_store_barrier_runtime_stub (zgc)"); |
| } |
| |
| #undef __ |
| #endif // COMPILER1 |
| |
| #ifdef COMPILER2 |
| |
| #undef __ |
| #define __ _masm-> |
| |
| class ZSetupArguments { |
| MacroAssembler* const _masm; |
| const Register _ref; |
| const Address _ref_addr; |
| |
| public: |
| ZSetupArguments(MacroAssembler* masm, ZLoadBarrierStubC2* stub) |
| : _masm(masm), |
| _ref(stub->ref()), |
| _ref_addr(stub->ref_addr()) { |
| |
| // Desired register/argument configuration: |
| // _ref: R3_ARG1 |
| // _ref_addr: R4_ARG2 |
| |
| // '_ref_addr' can be unspecified. In that case, the barrier will not heal the reference. |
| if (_ref_addr.base() == noreg) { |
| assert_different_registers(_ref, R0, noreg); |
| |
| __ mr_if_needed(R3_ARG1, _ref); |
| __ li(R4_ARG2, 0); |
| } else { |
| assert_different_registers(_ref, _ref_addr.base(), R0, noreg); |
| assert(!_ref_addr.index()->is_valid(), "reference addresses must not contain an index component"); |
| |
| if (_ref != R4_ARG2) { |
| // Calculate address first as the address' base register might clash with R4_ARG2 |
| __ addi(R4_ARG2, _ref_addr.base(), _ref_addr.disp()); |
| __ mr_if_needed(R3_ARG1, _ref); |
| } else if (_ref_addr.base() != R3_ARG1) { |
| __ mr(R3_ARG1, _ref); |
| __ addi(R4_ARG2, _ref_addr.base(), _ref_addr.disp()); // Clobbering _ref |
| } else { |
| // Arguments are provided in inverse order (i.e. _ref == R4_ARG2, _ref_addr == R3_ARG1) |
| __ mr(R0, _ref); |
| __ addi(R4_ARG2, _ref_addr.base(), _ref_addr.disp()); |
| __ mr(R3_ARG1, R0); |
| } |
| } |
| } |
| }; |
| |
| #undef __ |
| #define __ masm-> |
| |
| void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const { |
| Assembler::InlineSkippedInstructionsCounter skipped_counter(masm); |
| __ block_comment("generate_c2_load_barrier_stub (zgc) {"); |
| |
| __ bind(*stub->entry()); |
| |
| Register ref = stub->ref(); |
| Address ref_addr = stub->ref_addr(); |
| |
| assert_different_registers(ref, ref_addr.base()); |
| |
| { |
| SaveLiveRegisters save_live_registers(masm, stub); |
| ZSetupArguments setup_arguments(masm, stub); |
| |
| __ call_VM_leaf(stub->slow_path()); |
| __ mr_if_needed(ref, R3_RET); |
| } |
| |
| __ b(*stub->continuation()); |
| |
| __ block_comment("} generate_c2_load_barrier_stub (zgc)"); |
| } |
| |
| void ZBarrierSetAssembler::generate_c2_store_barrier_stub(MacroAssembler* masm, ZStoreBarrierStubC2* stub) const { |
| Assembler::InlineSkippedInstructionsCounter skipped_counter(masm); |
| __ block_comment("ZStoreBarrierStubC2"); |
| |
| // Stub entry |
| __ bind(*stub->entry()); |
| |
| Label slow; |
| |
| Address addr = stub->ref_addr(); |
| Register rbase = addr.base(); |
| RegisterOrConstant ind_or_offs = (addr.index() == noreg) |
| ? (RegisterOrConstant)addr.disp() |
| : (RegisterOrConstant)addr.index(); |
| |
| if (!stub->is_native()) { |
| store_barrier_medium(masm, |
| rbase, |
| ind_or_offs, |
| stub->new_zpointer(), |
| stub->is_atomic(), |
| *stub->continuation(), |
| slow); |
| } |
| |
| __ bind(slow); |
| { |
| SaveLiveRegisters save_live_registers(masm, stub); |
| __ add(R3_ARG1, ind_or_offs, rbase); |
| if (stub->is_native()) { |
| __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_native_oop_field_without_healing_addr(), R3_ARG1); |
| } else if (stub->is_atomic()) { |
| __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr(), R3_ARG1); |
| } else if (stub->is_nokeepalive()) { |
| __ call_VM_leaf(ZBarrierSetRuntime::no_keepalive_store_barrier_on_oop_field_without_healing_addr(), R3_ARG1); |
| } else { |
| __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), R3_ARG1); |
| } |
| } |
| |
| // Stub exit |
| __ b(*stub->continuation()); |
| } |
| |
| #undef __ |
| #endif // COMPILER2 |
| |
| static uint16_t patch_barrier_relocation_value(int format) { |
| switch (format) { |
| case ZBarrierRelocationFormatLoadBadMask: |
| return (uint16_t)ZPointerLoadBadMask; |
| case ZBarrierRelocationFormatMarkBadMask: |
| return (uint16_t)ZPointerMarkBadMask; |
| case ZBarrierRelocationFormatStoreGoodBits: |
| return (uint16_t)ZPointerStoreGoodMask; |
| case ZBarrierRelocationFormatStoreBadMask: |
| return (uint16_t)ZPointerStoreBadMask; |
| default: |
| ShouldNotReachHere(); |
| return 0; |
| } |
| } |
| |
| void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) { |
| #ifdef ASSERT |
| int inst = *(int*)addr; |
| if (format == ZBarrierRelocationFormatStoreGoodBits) { |
| assert(Assembler::is_li(inst) || Assembler::is_ori(inst) || Assembler::is_cmpli(inst), |
| "unexpected instruction 0x%04x", inst); |
| // Note: li uses sign extend, but these bits will get cleared by rldimi. |
| } else { |
| assert(Assembler::is_andi(inst), "unexpected instruction 0x%04x", inst); |
| } |
| #endif |
| // Patch the signed/unsigned 16 bit immediate field of the instruction. |
| *(uint16_t*)(addr BIG_ENDIAN_ONLY(+2)) = patch_barrier_relocation_value(format); |
| ICache::ppc64_flush_icache_bytes(addr, BytesPerInstWord); |
| } |
