| /* | |
| * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved. | |
| * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |
| * | |
| * This code is free software; you can redistribute it and/or modify it | |
| * under the terms of the GNU General Public License version 2 only, as | |
| * published by the Free Software Foundation. | |
| * | |
| * This code is distributed in the hope that it will be useful, but WITHOUT | |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| * version 2 for more details (a copy is included in the LICENSE file that | |
| * accompanied this code). | |
| * | |
| * You should have received a copy of the GNU General Public License version | |
| * 2 along with this work; if not, write to the Free Software Foundation, | |
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| * | |
| * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA | |
| * or visit www.oracle.com if you need additional information or have any | |
| * questions. | |
| * | |
| */ | |
| #include "classfile/classLoaderData.hpp" | |
| #include "gc/shared/barrierSet.hpp" | |
| #include "gc/shared/barrierSetAssembler.hpp" | |
| #include "gc/shared/barrierSetNMethod.hpp" | |
| #include "gc/shared/collectedHeap.hpp" | |
| #include "interpreter/interp_masm.hpp" | |
| #include "memory/universe.hpp" | |
| #include "runtime/javaThread.hpp" | |
| #include "runtime/jniHandles.hpp" | |
| #include "runtime/sharedRuntime.hpp" | |
| #include "runtime/stubRoutines.hpp" | |
| #ifdef COMPILER2 | |
| #include "code/vmreg.inline.hpp" | |
| #include "gc/shared/c2/barrierSetC2.hpp" | |
| #endif // COMPILER2 | |
| #define __ masm-> | |
| void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, | |
| Register dst, Address src, Register tmp1, Register tmp2) { | |
| // LR is live. It must be saved around calls. | |
| bool in_heap = (decorators & IN_HEAP) != 0; | |
| bool in_native = (decorators & IN_NATIVE) != 0; | |
| bool is_not_null = (decorators & IS_NOT_NULL) != 0; | |
| switch (type) { | |
| case T_OBJECT: | |
| case T_ARRAY: { | |
| if (in_heap) { | |
| if (UseCompressedOops) { | |
| __ ldrw(dst, src); | |
| if (is_not_null) { | |
| __ decode_heap_oop_not_null(dst); | |
| } else { | |
| __ decode_heap_oop(dst); | |
| } | |
| } else { | |
| __ ldr(dst, src); | |
| } | |
| } else { | |
| assert(in_native, "why else?"); | |
| __ ldr(dst, src); | |
| } | |
| break; | |
| } | |
| case T_BOOLEAN: __ load_unsigned_byte (dst, src); break; | |
| case T_BYTE: __ load_signed_byte (dst, src); break; | |
| case T_CHAR: __ load_unsigned_short(dst, src); break; | |
| case T_SHORT: __ load_signed_short (dst, src); break; | |
| case T_INT: __ ldrw (dst, src); break; | |
| case T_LONG: __ ldr (dst, src); break; | |
| case T_ADDRESS: __ ldr (dst, src); break; | |
| case T_FLOAT: __ ldrs (v0, src); break; | |
| case T_DOUBLE: __ ldrd (v0, src); break; | |
| default: Unimplemented(); | |
| } | |
| } | |
| void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, | |
| Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) { | |
| bool in_heap = (decorators & IN_HEAP) != 0; | |
| bool in_native = (decorators & IN_NATIVE) != 0; | |
| switch (type) { | |
| case T_OBJECT: | |
| case T_ARRAY: { | |
| val = val == noreg ? zr : val; | |
| if (in_heap) { | |
| if (UseCompressedOops) { | |
| assert(!dst.uses(val), "not enough registers"); | |
| if (val != zr) { | |
| __ encode_heap_oop(val); | |
| } | |
| __ strw(val, dst); | |
| } else { | |
| __ str(val, dst); | |
| } | |
| } else { | |
| assert(in_native, "why else?"); | |
| __ str(val, dst); | |
| } | |
| break; | |
| } | |
| case T_BOOLEAN: | |
| __ andw(val, val, 0x1); // boolean is true if LSB is 1 | |
| __ strb(val, dst); | |
| break; | |
| case T_BYTE: __ strb(val, dst); break; | |
| case T_CHAR: __ strh(val, dst); break; | |
| case T_SHORT: __ strh(val, dst); break; | |
| case T_INT: __ strw(val, dst); break; | |
| case T_LONG: __ str (val, dst); break; | |
| case T_ADDRESS: __ str (val, dst); break; | |
| case T_FLOAT: __ strs(v0, dst); break; | |
| case T_DOUBLE: __ strd(v0, dst); break; | |
| default: Unimplemented(); | |
| } | |
| } | |
| void BarrierSetAssembler::copy_load_at(MacroAssembler* masm, | |
| DecoratorSet decorators, | |
| BasicType type, | |
| size_t bytes, | |
| Register dst1, | |
| Register dst2, | |
| Address src, | |
| Register tmp) { | |
| if (bytes == 1) { | |
| assert(dst2 == noreg, "invariant"); | |
| __ ldrb(dst1, src); | |
| } else if (bytes == 2) { | |
| assert(dst2 == noreg, "invariant"); | |
| __ ldrh(dst1, src); | |
| } else if (bytes == 4) { | |
| assert(dst2 == noreg, "invariant"); | |
| __ ldrw(dst1, src); | |
| } else if (bytes == 8) { | |
| assert(dst2 == noreg, "invariant"); | |
| __ ldr(dst1, src); | |
| } else if (bytes == 16) { | |
| assert(dst2 != noreg, "invariant"); | |
| assert(dst2 != dst1, "invariant"); | |
| __ ldp(dst1, dst2, src); | |
| } else { | |
| // Not the right size | |
| ShouldNotReachHere(); | |
| } | |
| if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) { | |
| __ decode_heap_oop(dst1); | |
| } | |
| } | |
| void BarrierSetAssembler::copy_store_at(MacroAssembler* masm, | |
| DecoratorSet decorators, | |
| BasicType type, | |
| size_t bytes, | |
| Address dst, | |
| Register src1, | |
| Register src2, | |
| Register tmp1, | |
| Register tmp2, | |
| Register tmp3) { | |
| if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) { | |
| __ encode_heap_oop(src1); | |
| } | |
| if (bytes == 1) { | |
| assert(src2 == noreg, "invariant"); | |
| __ strb(src1, dst); | |
| } else if (bytes == 2) { | |
| assert(src2 == noreg, "invariant"); | |
| __ strh(src1, dst); | |
| } else if (bytes == 4) { | |
| assert(src2 == noreg, "invariant"); | |
| __ strw(src1, dst); | |
| } else if (bytes == 8) { | |
| assert(src2 == noreg, "invariant"); | |
| __ str(src1, dst); | |
| } else if (bytes == 16) { | |
| assert(src2 != noreg, "invariant"); | |
| assert(src2 != src1, "invariant"); | |
| __ stp(src1, src2, dst); | |
| } else { | |
| // Not the right size | |
| ShouldNotReachHere(); | |
| } | |
| } | |
| void BarrierSetAssembler::copy_load_at(MacroAssembler* masm, | |
| DecoratorSet decorators, | |
| BasicType type, | |
| size_t bytes, | |
| FloatRegister dst1, | |
| FloatRegister dst2, | |
| Address src, | |
| Register tmp1, | |
| Register tmp2, | |
| FloatRegister vec_tmp) { | |
| if (bytes == 32) { | |
| __ ldpq(dst1, dst2, src); | |
| } else { | |
| ShouldNotReachHere(); | |
| } | |
| } | |
| void BarrierSetAssembler::copy_store_at(MacroAssembler* masm, | |
| DecoratorSet decorators, | |
| BasicType type, | |
| size_t bytes, | |
| Address dst, | |
| FloatRegister src1, | |
| FloatRegister src2, | |
| Register tmp1, | |
| Register tmp2, | |
| Register tmp3, | |
| FloatRegister vec_tmp1, | |
| FloatRegister vec_tmp2, | |
| FloatRegister vec_tmp3) { | |
| if (bytes == 32) { | |
| __ stpq(src1, src2, dst); | |
| } else { | |
| ShouldNotReachHere(); | |
| } | |
| } | |
| void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env, | |
| Register obj, Register tmp, Label& slowpath) { | |
| // If mask changes we need to ensure that the inverse is still encodable as an immediate | |
| STATIC_ASSERT(JNIHandles::tag_mask == 0b11); | |
| __ andr(obj, obj, ~JNIHandles::tag_mask); | |
| __ ldr(obj, Address(obj, 0)); // *obj | |
| } | |
| // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes. | |
| void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj, | |
| Register var_size_in_bytes, | |
| int con_size_in_bytes, | |
| Register t1, | |
| Register t2, | |
| Label& slow_case) { | |
| assert_different_registers(obj, t2); | |
| assert_different_registers(obj, var_size_in_bytes); | |
| Register end = t2; | |
| // verify_tlab(); | |
| __ ldr(obj, Address(rthread, JavaThread::tlab_top_offset())); | |
| if (var_size_in_bytes == noreg) { | |
| __ lea(end, Address(obj, con_size_in_bytes)); | |
| } else { | |
| __ lea(end, Address(obj, var_size_in_bytes)); | |
| } | |
| __ ldr(rscratch1, Address(rthread, JavaThread::tlab_end_offset())); | |
| __ cmp(end, rscratch1); | |
| __ br(Assembler::HI, slow_case); | |
| // update the tlab top pointer | |
| __ str(end, Address(rthread, JavaThread::tlab_top_offset())); | |
| // recover var_size_in_bytes if necessary | |
| if (var_size_in_bytes == end) { | |
| __ sub(var_size_in_bytes, var_size_in_bytes, obj); | |
| } | |
| // verify_tlab(); | |
| } | |
| static volatile uint32_t _patching_epoch = 0; | |
| address BarrierSetAssembler::patching_epoch_addr() { | |
| return (address)&_patching_epoch; | |
| } | |
| void BarrierSetAssembler::increment_patching_epoch() { | |
| Atomic::inc(&_patching_epoch); | |
| } | |
| void BarrierSetAssembler::clear_patching_epoch() { | |
| _patching_epoch = 0; | |
| } | |
| void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label* slow_path, Label* continuation, Label* guard) { | |
| BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); | |
| Label local_guard; | |
| Label skip_barrier; | |
| NMethodPatchingType patching_type = nmethod_patching_type(); | |
| if (slow_path == nullptr) { | |
| guard = &local_guard; | |
| } | |
| // If the slow path is out of line in a stub, we flip the condition | |
| Assembler::Condition condition = slow_path == nullptr ? Assembler::EQ : Assembler::NE; | |
| Label& barrier_target = slow_path == nullptr ? skip_barrier : *slow_path; | |
| __ ldrw(rscratch1, *guard); | |
| if (patching_type == NMethodPatchingType::stw_instruction_and_data_patch) { | |
| // With STW patching, no data or instructions are updated concurrently, | |
| // which means there isn't really any need for any fencing for neither | |
| // data nor instruction modifications happening concurrently. The | |
| // instruction patching is handled with isb fences on the way back | |
| // from the safepoint to Java. So here we can do a plain conditional | |
| // branch with no fencing. | |
| Address thread_disarmed_addr(rthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset())); | |
| __ ldrw(rscratch2, thread_disarmed_addr); | |
| __ cmp(rscratch1, rscratch2); | |
| } else if (patching_type == NMethodPatchingType::conc_instruction_and_data_patch) { | |
| // If we patch code we need both a code patching and a loadload | |
| // fence. It's not super cheap, so we use a global epoch mechanism | |
| // to hide them in a slow path. | |
| // The high level idea of the global epoch mechanism is to detect | |
| // when any thread has performed the required fencing, after the | |
| // last nmethod was disarmed. This implies that the required | |
| // fencing has been performed for all preceding nmethod disarms | |
| // as well. Therefore, we do not need any further fencing. | |
| __ lea(rscratch2, ExternalAddress((address)&_patching_epoch)); | |
| // Embed an artificial data dependency to order the guard load | |
| // before the epoch load. | |
| __ orr(rscratch2, rscratch2, rscratch1, Assembler::LSR, 32); | |
| // Read the global epoch value. | |
| __ ldrw(rscratch2, rscratch2); | |
| // Combine the guard value (low order) with the epoch value (high order). | |
| __ orr(rscratch1, rscratch1, rscratch2, Assembler::LSL, 32); | |
| // Compare the global values with the thread-local values. | |
| Address thread_disarmed_and_epoch_addr(rthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset())); | |
| __ ldr(rscratch2, thread_disarmed_and_epoch_addr); | |
| __ cmp(rscratch1, rscratch2); | |
| } else { | |
| assert(patching_type == NMethodPatchingType::conc_data_patch, "must be"); | |
| // Subsequent loads of oops must occur after load of guard value. | |
| // BarrierSetNMethod::disarm sets guard with release semantics. | |
| __ membar(__ LoadLoad); | |
| Address thread_disarmed_addr(rthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset())); | |
| __ ldrw(rscratch2, thread_disarmed_addr); | |
| __ cmpw(rscratch1, rscratch2); | |
| } | |
| __ br(condition, barrier_target); | |
| if (slow_path == nullptr) { | |
| __ lea(rscratch1, RuntimeAddress(StubRoutines::method_entry_barrier())); | |
| __ blr(rscratch1); | |
| __ b(skip_barrier); | |
| __ bind(local_guard); | |
| __ emit_int32(0); // nmethod guard value. Skipped over in common case. | |
| } else { | |
| __ bind(*continuation); | |
| } | |
| __ bind(skip_barrier); | |
| } | |
| void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) { | |
| Label bad_call; | |
| __ cbz(rmethod, bad_call); | |
| // Pointer chase to the method holder to find out if the method is concurrently unloading. | |
| Label method_live; | |
| __ load_method_holder_cld(rscratch1, rmethod); | |
| // Is it a strong CLD? | |
| __ ldrw(rscratch2, Address(rscratch1, ClassLoaderData::keep_alive_ref_count_offset())); | |
| __ cbnz(rscratch2, method_live); | |
| // Is it a weak but alive CLD? | |
| __ push(RegSet::of(r10), sp); | |
| __ ldr(r10, Address(rscratch1, ClassLoaderData::holder_offset())); | |
| __ resolve_weak_handle(r10, rscratch1, rscratch2); | |
| __ mov(rscratch1, r10); | |
| __ pop(RegSet::of(r10), sp); | |
| __ cbnz(rscratch1, method_live); | |
| __ bind(bad_call); | |
| __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); | |
| __ bind(method_live); | |
| } | |
| void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) { | |
| // Check if the oop is in the right area of memory | |
| __ mov(tmp2, (intptr_t) Universe::verify_oop_mask()); | |
| __ andr(tmp1, obj, tmp2); | |
| __ mov(tmp2, (intptr_t) Universe::verify_oop_bits()); | |
| // Compare tmp1 and tmp2. We don't use a compare | |
| // instruction here because the flags register is live. | |
| __ eor(tmp1, tmp1, tmp2); | |
| __ cbnz(tmp1, error); | |
| // make sure klass is 'reasonable', which is not zero. | |
| __ load_klass(obj, obj); // get klass | |
| __ cbz(obj, error); // if klass is null it is broken | |
| } | |
| #ifdef COMPILER2 | |
| OptoReg::Name BarrierSetAssembler::encode_float_vector_register_size(const Node* node, OptoReg::Name opto_reg) { | |
| switch (node->ideal_reg()) { | |
| case Op_RegF: | |
| // No need to refine. The original encoding is already fine to distinguish. | |
| assert(opto_reg % 4 == 0, "Float register should only occupy a single slot"); | |
| break; | |
| // Use different encoding values of the same fp/vector register to help distinguish different sizes. | |
| // Such as V16. The OptoReg::name and its corresponding slot value are | |
| // "V16": 64, "V16_H": 65, "V16_J": 66, "V16_K": 67. | |
| case Op_RegD: | |
| case Op_VecD: | |
| opto_reg &= ~3; | |
| opto_reg |= 1; | |
| break; | |
| case Op_VecX: | |
| opto_reg &= ~3; | |
| opto_reg |= 2; | |
| break; | |
| case Op_VecA: | |
| opto_reg &= ~3; | |
| opto_reg |= 3; | |
| break; | |
| default: | |
| assert(false, "unexpected ideal register"); | |
| ShouldNotReachHere(); | |
| } | |
| return opto_reg; | |
| } | |
| OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) { | |
| if (!OptoReg::is_reg(opto_reg)) { | |
| return OptoReg::Bad; | |
| } | |
| const VMReg vm_reg = OptoReg::as_VMReg(opto_reg); | |
| if (vm_reg->is_FloatRegister()) { | |
| opto_reg = encode_float_vector_register_size(node, opto_reg); | |
| } | |
| return opto_reg; | |
| } | |
| #undef __ | |
| #define __ _masm-> | |
| void SaveLiveRegisters::initialize(BarrierStubC2* stub) { | |
| int index = -1; | |
| GrowableArray<RegisterData> registers; | |
| VMReg prev_vm_reg = VMRegImpl::Bad(); | |
| RegMaskIterator rmi(stub->preserve_set()); | |
| while (rmi.has_next()) { | |
| OptoReg::Name opto_reg = rmi.next(); | |
| VMReg vm_reg = OptoReg::as_VMReg(opto_reg); | |
| if (vm_reg->is_Register()) { | |
| // GPR may have one or two slots in regmask | |
| // Determine whether the current vm_reg is the same physical register as the previous one | |
| if (is_same_register(vm_reg, prev_vm_reg)) { | |
| registers.at(index)._slots++; | |
| } else { | |
| RegisterData reg_data = { vm_reg, 1 }; | |
| index = registers.append(reg_data); | |
| } | |
| } else if (vm_reg->is_FloatRegister()) { | |
| // We have size encoding in OptoReg of stub->preserve_set() | |
| // After encoding, float/neon/sve register has only one slot in regmask | |
| // Decode it to get the actual size | |
| VMReg vm_reg_base = vm_reg->as_FloatRegister()->as_VMReg(); | |
| int slots = decode_float_vector_register_size(opto_reg); | |
| RegisterData reg_data = { vm_reg_base, slots }; | |
| index = registers.append(reg_data); | |
| } else if (vm_reg->is_PRegister()) { | |
| // PRegister has only one slot in regmask | |
| RegisterData reg_data = { vm_reg, 1 }; | |
| index = registers.append(reg_data); | |
| } else { | |
| assert(false, "Unknown register type"); | |
| ShouldNotReachHere(); | |
| } | |
| prev_vm_reg = vm_reg; | |
| } | |
| // Record registers that needs to be saved/restored | |
| for (GrowableArrayIterator<RegisterData> it = registers.begin(); it != registers.end(); ++it) { | |
| RegisterData reg_data = *it; | |
| VMReg vm_reg = reg_data._reg; | |
| int slots = reg_data._slots; | |
| if (vm_reg->is_Register()) { | |
| assert(slots == 1 || slots == 2, "Unexpected register save size"); | |
| _gp_regs += RegSet::of(vm_reg->as_Register()); | |
| } else if (vm_reg->is_FloatRegister()) { | |
| if (slots == 1 || slots == 2) { | |
| _fp_regs += FloatRegSet::of(vm_reg->as_FloatRegister()); | |
| } else if (slots == 4) { | |
| _neon_regs += FloatRegSet::of(vm_reg->as_FloatRegister()); | |
| } else { | |
| assert(slots == Matcher::scalable_vector_reg_size(T_FLOAT), "Unexpected register save size"); | |
| _sve_regs += FloatRegSet::of(vm_reg->as_FloatRegister()); | |
| } | |
| } else { | |
| assert(vm_reg->is_PRegister() && slots == 1, "Unknown register type"); | |
| _p_regs += PRegSet::of(vm_reg->as_PRegister()); | |
| } | |
| } | |
| // Remove C-ABI SOE registers and scratch regs | |
| _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9); | |
| // Remove C-ABI SOE fp registers | |
| _fp_regs -= FloatRegSet::range(v8, v15); | |
| } | |
| enum RC SaveLiveRegisters::rc_class(VMReg reg) { | |
| if (reg->is_reg()) { | |
| if (reg->is_Register()) { | |
| return rc_int; | |
| } else if (reg->is_FloatRegister()) { | |
| return rc_float; | |
| } else if (reg->is_PRegister()) { | |
| return rc_predicate; | |
| } | |
| } | |
| if (reg->is_stack()) { | |
| return rc_stack; | |
| } | |
| return rc_bad; | |
| } | |
| bool SaveLiveRegisters::is_same_register(VMReg reg1, VMReg reg2) { | |
| if (reg1 == reg2) { | |
| return true; | |
| } | |
| if (rc_class(reg1) == rc_class(reg2)) { | |
| if (reg1->is_Register()) { | |
| return reg1->as_Register() == reg2->as_Register(); | |
| } else if (reg1->is_FloatRegister()) { | |
| return reg1->as_FloatRegister() == reg2->as_FloatRegister(); | |
| } else if (reg1->is_PRegister()) { | |
| return reg1->as_PRegister() == reg2->as_PRegister(); | |
| } | |
| } | |
| return false; | |
| } | |
| int SaveLiveRegisters::decode_float_vector_register_size(OptoReg::Name opto_reg) { | |
| switch (opto_reg & 3) { | |
| case 0: | |
| return 1; | |
| case 1: | |
| return 2; | |
| case 2: | |
| return 4; | |
| case 3: | |
| return Matcher::scalable_vector_reg_size(T_FLOAT); | |
| default: | |
| ShouldNotReachHere(); | |
| return 0; | |
| } | |
| } | |
| SaveLiveRegisters::SaveLiveRegisters(MacroAssembler* masm, BarrierStubC2* stub) | |
| : _masm(masm), | |
| _gp_regs(), | |
| _fp_regs(), | |
| _neon_regs(), | |
| _sve_regs(), | |
| _p_regs() { | |
| // Figure out what registers to save/restore | |
| initialize(stub); | |
| // Save registers | |
| __ push(_gp_regs, sp); | |
| __ push_fp(_fp_regs, sp, MacroAssembler::PushPopFp); | |
| __ push_fp(_neon_regs, sp, MacroAssembler::PushPopNeon); | |
| __ push_fp(_sve_regs, sp, MacroAssembler::PushPopSVE); | |
| __ push_p(_p_regs, sp); | |
| } | |
| SaveLiveRegisters::~SaveLiveRegisters() { | |
| // Restore registers | |
| __ pop_p(_p_regs, sp); | |
| __ pop_fp(_sve_regs, sp, MacroAssembler::PushPopSVE); | |
| __ pop_fp(_neon_regs, sp, MacroAssembler::PushPopNeon); | |
| __ pop_fp(_fp_regs, sp, MacroAssembler::PushPopFp); | |
| // External runtime call may clobber ptrue reg | |
| __ reinitialize_ptrue(); | |
| __ pop(_gp_regs, sp); | |
| } | |
| #endif // COMPILER2 |