| /* | |
| * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved. | |
| * Copyright (c) 2018, 2025 SAP SE. All rights reserved. | |
| * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |
| * | |
| * This code is free software; you can redistribute it and/or modify it | |
| * under the terms of the GNU General Public License version 2 only, as | |
| * published by the Free Software Foundation. | |
| * | |
| * This code is distributed in the hope that it will be useful, but WITHOUT | |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| * version 2 for more details (a copy is included in the LICENSE file that | |
| * accompanied this code). | |
| * | |
| * You should have received a copy of the GNU General Public License version | |
| * 2 along with this work; if not, write to the Free Software Foundation, | |
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| * | |
| * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA | |
| * or visit www.oracle.com if you need additional information or have any | |
| * questions. | |
| * | |
| */ | |
| #include "asm/macroAssembler.inline.hpp" | |
| #include "gc/g1/g1BarrierSet.hpp" | |
| #include "gc/g1/g1BarrierSetAssembler.hpp" | |
| #include "gc/g1/g1BarrierSetRuntime.hpp" | |
| #include "gc/g1/g1CardTable.hpp" | |
| #include "gc/g1/g1DirtyCardQueue.hpp" | |
| #include "gc/g1/g1HeapRegion.hpp" | |
| #include "gc/g1/g1SATBMarkQueueSet.hpp" | |
| #include "gc/g1/g1ThreadLocalData.hpp" | |
| #include "interpreter/interp_masm.hpp" | |
| #include "runtime/jniHandles.hpp" | |
| #include "runtime/sharedRuntime.hpp" | |
| #include "utilities/macros.hpp" | |
| #ifdef COMPILER1 | |
| #include "c1/c1_LIRAssembler.hpp" | |
| #include "c1/c1_MacroAssembler.hpp" | |
| #include "gc/g1/c1/g1BarrierSetC1.hpp" | |
| #endif // COMPILER1 | |
| #ifdef COMPILER2 | |
| #include "gc/g1/c2/g1BarrierSetC2.hpp" | |
| #endif // COMPILER2 | |
| #define __ masm-> | |
| static void generate_marking_inactive_test(MacroAssembler* masm) { | |
| int active_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()); | |
| assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); | |
| __ lbz(R0, active_offset, R16_thread); // tmp1 := *(mark queue active address) | |
| __ cmpwi(CR0, R0, 0); | |
| } | |
| void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators, | |
| Register from, Register to, Register count, | |
| Register preserve1, Register preserve2) { | |
| bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; | |
| // With G1, don't generate the call if we statically know that the target in uninitialized | |
| if (!dest_uninitialized) { | |
| int spill_slots = 3; | |
| if (preserve1 != noreg) { spill_slots++; } | |
| if (preserve2 != noreg) { spill_slots++; } | |
| const int frame_size = align_up(frame::native_abi_reg_args_size + spill_slots * BytesPerWord, frame::alignment_in_bytes); | |
| Label filtered; | |
| // Is marking active? | |
| generate_marking_inactive_test(masm); | |
| __ beq(CR0, filtered); | |
| __ save_LR(R0); | |
| __ push_frame(frame_size, R0); | |
| int slot_nr = 0; | |
| __ std(from, frame_size - (++slot_nr) * wordSize, R1_SP); | |
| __ std(to, frame_size - (++slot_nr) * wordSize, R1_SP); | |
| __ std(count, frame_size - (++slot_nr) * wordSize, R1_SP); | |
| if (preserve1 != noreg) { __ std(preserve1, frame_size - (++slot_nr) * wordSize, R1_SP); } | |
| if (preserve2 != noreg) { __ std(preserve2, frame_size - (++slot_nr) * wordSize, R1_SP); } | |
| if (UseCompressedOops) { | |
| __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), to, count); | |
| } else { | |
| __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), to, count); | |
| } | |
| slot_nr = 0; | |
| __ ld(from, frame_size - (++slot_nr) * wordSize, R1_SP); | |
| __ ld(to, frame_size - (++slot_nr) * wordSize, R1_SP); | |
| __ ld(count, frame_size - (++slot_nr) * wordSize, R1_SP); | |
| if (preserve1 != noreg) { __ ld(preserve1, frame_size - (++slot_nr) * wordSize, R1_SP); } | |
| if (preserve2 != noreg) { __ ld(preserve2, frame_size - (++slot_nr) * wordSize, R1_SP); } | |
| __ addi(R1_SP, R1_SP, frame_size); // pop_frame() | |
| __ restore_LR(R0); | |
| __ bind(filtered); | |
| } | |
| } | |
| void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, | |
| Register addr, Register count, Register preserve) { | |
| int spill_slots = (preserve != noreg) ? 1 : 0; | |
| const int frame_size = align_up(frame::native_abi_reg_args_size + spill_slots * BytesPerWord, frame::alignment_in_bytes); | |
| __ save_LR(R0); | |
| __ push_frame(frame_size, R0); | |
| if (preserve != noreg) { __ std(preserve, frame_size - 1 * wordSize, R1_SP); } | |
| __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), addr, count); | |
| if (preserve != noreg) { __ ld(preserve, frame_size - 1 * wordSize, R1_SP); } | |
| __ addi(R1_SP, R1_SP, frame_size); // pop_frame(); | |
| __ restore_LR(R0); | |
| } | |
| static void generate_queue_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime, | |
| const Register value, const Register temp) { | |
| assert_different_registers(value, temp); | |
| // Can we store a value in the given thread's buffer? | |
| // (The index field is typed as size_t.) | |
| __ ld(temp, in_bytes(index_offset), R16_thread); // temp := *(index address) | |
| __ cmpdi(CR0, temp, 0); // jump to runtime if index == 0 (full buffer) | |
| __ beq(CR0, runtime); | |
| // The buffer is not full, store value into it. | |
| __ ld(R0, in_bytes(buffer_offset), R16_thread); // R0 := buffer address | |
| __ addi(temp, temp, -wordSize); // temp := next index | |
| __ std(temp, in_bytes(index_offset), R16_thread); // *(index address) := next index | |
| __ stdx(value, temp, R0); // *(buffer address + next index) := value | |
| } | |
| void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, DecoratorSet decorators, | |
| Register obj, RegisterOrConstant ind_or_offs, Register pre_val, | |
| Register tmp1, Register tmp2, | |
| MacroAssembler::PreservationLevel preservation_level) { | |
| assert_different_registers(pre_val, tmp1, tmp2); | |
| bool not_null = (decorators & IS_NOT_NULL) != 0, | |
| preloaded = obj == noreg; | |
| Register nv_save = noreg; | |
| // Determine necessary runtime invocation preservation measures | |
| const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR; | |
| const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS; | |
| const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS; | |
| int nbytes_save = 0; | |
| if (pre_val->is_volatile() && preloaded && !preserve_gp_registers) { | |
| // We are not loading the previous value so make | |
| // sure that we don't trash the value in pre_val | |
| // with the code below. | |
| nv_save = !tmp1->is_volatile() ? tmp1 : tmp2; | |
| assert(!nv_save->is_volatile(), "need one nv temp register if pre_val lives in volatile register"); | |
| } | |
| Label runtime, filtered; | |
| generate_marking_inactive_test(masm); | |
| __ beq(CR0, filtered); | |
| // Do we need to load the previous value? | |
| if (!preloaded) { | |
| // Load the previous value... | |
| if (UseCompressedOops) { | |
| __ lwz(pre_val, ind_or_offs, obj); | |
| } else { | |
| __ ld(pre_val, ind_or_offs, obj); | |
| } | |
| // Previous value has been loaded into Rpre_val. | |
| } | |
| assert(pre_val != noreg, "must have a real register"); | |
| // Is the previous value null? | |
| if (preloaded && not_null) { | |
| #ifdef ASSERT | |
| __ cmpdi(CR0, pre_val, 0); | |
| __ asm_assert_ne("null oop not allowed (G1 pre)"); // Checked by caller. | |
| #endif | |
| } else { | |
| __ cmpdi(CR0, pre_val, 0); | |
| __ beq(CR0, filtered); | |
| } | |
| if (!preloaded && UseCompressedOops) { | |
| __ decode_heap_oop_not_null(pre_val); | |
| } | |
| // OK, it's not filtered, so we'll need to call enqueue. In the normal | |
| // case, pre_val will be a scratch G-reg, but there are some cases in | |
| // which it's an O-reg. In the first case, do a normal call. In the | |
| // latter, do a save here and call the frameless version. | |
| // Can we store original value in the thread's buffer? | |
| // Is index == 0? | |
| // (The index field is typed as size_t.) | |
| generate_queue_insertion(masm, G1ThreadLocalData::satb_mark_queue_index_offset(), G1ThreadLocalData::satb_mark_queue_buffer_offset(), | |
| runtime, pre_val, tmp1); | |
| __ b(filtered); | |
| __ bind(runtime); | |
| // May need to preserve LR. Also needed if current frame is not compatible with C calling convention. | |
| if (needs_frame) { | |
| if (preserve_gp_registers) { | |
| nbytes_save = (MacroAssembler::num_volatile_gp_regs | |
| + (preserve_fp_registers ? MacroAssembler::num_volatile_fp_regs : 0) | |
| ) * BytesPerWord; | |
| __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers); | |
| } | |
| __ save_LR(tmp1); | |
| __ push_frame_reg_args(nbytes_save, tmp2); | |
| } | |
| if (nv_save != noreg) { | |
| __ mr(nv_save, pre_val); // Save pre_val across C call if it was preloaded. | |
| } | |
| __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, R16_thread); | |
| if (nv_save != noreg) { | |
| __ mr(pre_val, nv_save); // restore | |
| } | |
| if (needs_frame) { | |
| __ pop_frame(); | |
| __ restore_LR(tmp1); | |
| if (preserve_gp_registers) { | |
| __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers); | |
| } | |
| } | |
| __ bind(filtered); | |
| } | |
| static void generate_region_crossing_test(MacroAssembler* masm, const Register store_addr, const Register new_val) { | |
| __ xorr(R0, store_addr, new_val); // tmp1 := store address ^ new value | |
| __ srdi_(R0, R0, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes) | |
| } | |
| static Address generate_card_young_test(MacroAssembler* masm, const Register store_addr, const Register tmp1, const Register tmp2) { | |
| CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set()); | |
| __ load_const_optimized(tmp1, (address)(ct->card_table()->byte_map_base()), tmp2); | |
| __ srdi(tmp2, store_addr, CardTable::card_shift()); // tmp1 := card address relative to card table base | |
| __ lbzx(R0, tmp1, tmp2); // tmp1 := card address | |
| __ cmpwi(CR0, R0, (int)G1CardTable::g1_young_card_val()); | |
| return Address(tmp1, tmp2); // return card address | |
| } | |
| static void generate_card_dirty_test(MacroAssembler* masm, Address card_addr) { | |
| __ membar(Assembler::StoreLoad); // Must reload after StoreLoad membar due to concurrent refinement | |
| __ lbzx(R0, card_addr.base(), card_addr.index()); // tmp2 := card | |
| __ cmpwi(CR0, R0, (int)G1CardTable::dirty_card_val()); // tmp2 := card == dirty_card_val? | |
| } | |
| void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, DecoratorSet decorators, | |
| Register store_addr, Register new_val, | |
| Register tmp1, Register tmp2, Register tmp3, | |
| MacroAssembler::PreservationLevel preservation_level) { | |
| bool not_null = (decorators & IS_NOT_NULL) != 0; | |
| Label runtime, filtered; | |
| assert_different_registers(store_addr, new_val, tmp1, tmp2); | |
| CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set()); | |
| generate_region_crossing_test(masm, store_addr, new_val); | |
| __ beq(CR0, filtered); | |
| // Crosses regions, storing null? | |
| if (not_null) { | |
| #ifdef ASSERT | |
| __ cmpdi(CR0, new_val, 0); | |
| __ asm_assert_ne("null oop not allowed (G1 post)"); // Checked by caller. | |
| #endif | |
| } else { | |
| __ cmpdi(CR0, new_val, 0); | |
| __ beq(CR0, filtered); | |
| } | |
| Address card_addr = generate_card_young_test(masm, store_addr, tmp1, tmp2); | |
| __ beq(CR0, filtered); | |
| generate_card_dirty_test(masm, card_addr); | |
| __ beq(CR0, filtered); | |
| __ li(R0, (int)G1CardTable::dirty_card_val()); | |
| __ stbx(R0, card_addr.base(), card_addr.index()); // *(card address) := dirty_card_val | |
| Register Rcard_addr = tmp3; | |
| __ add(Rcard_addr, card_addr.base(), card_addr.index()); // This is the address which needs to get enqueued. | |
| generate_queue_insertion(masm, | |
| G1ThreadLocalData::dirty_card_queue_index_offset(), | |
| G1ThreadLocalData::dirty_card_queue_buffer_offset(), | |
| runtime, Rcard_addr, tmp1); | |
| __ b(filtered); | |
| __ bind(runtime); | |
| assert(preservation_level == MacroAssembler::PRESERVATION_NONE, | |
| "g1_write_barrier_post doesn't support preservation levels higher than PRESERVATION_NONE"); | |
| // Save the live input values. | |
| __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), Rcard_addr, R16_thread); | |
| __ bind(filtered); | |
| } | |
| void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, | |
| Register base, RegisterOrConstant ind_or_offs, Register val, | |
| Register tmp1, Register tmp2, Register tmp3, | |
| MacroAssembler::PreservationLevel preservation_level) { | |
| bool is_array = (decorators & IS_ARRAY) != 0; | |
| bool on_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0; | |
| bool precise = is_array || on_anonymous; | |
| // Load and record the previous value. | |
| g1_write_barrier_pre(masm, decorators, | |
| base, ind_or_offs, | |
| tmp1, tmp2, tmp3, | |
| preservation_level); | |
| BarrierSetAssembler::store_at(masm, decorators, | |
| type, base, ind_or_offs, val, | |
| tmp1, tmp2, tmp3, | |
| preservation_level); | |
| // No need for post barrier if storing null | |
| if (val != noreg) { | |
| if (precise) { | |
| if (ind_or_offs.is_constant()) { | |
| __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp1); | |
| } else { | |
| __ add(base, ind_or_offs.as_register(), base); | |
| } | |
| } | |
| g1_write_barrier_post(masm, decorators, | |
| base, val, | |
| tmp1, tmp2, tmp3, | |
| preservation_level); | |
| } | |
| } | |
| void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, | |
| Register base, RegisterOrConstant ind_or_offs, Register dst, | |
| Register tmp1, Register tmp2, | |
| MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null) { | |
| bool on_oop = is_reference_type(type); | |
| bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; | |
| bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; | |
| bool on_reference = on_weak || on_phantom; | |
| Label done; | |
| if (on_oop && on_reference && L_handle_null == nullptr) { L_handle_null = &done; } | |
| // Load the value of the referent field. | |
| ModRefBarrierSetAssembler::load_at(masm, decorators, type, | |
| base, ind_or_offs, dst, | |
| tmp1, tmp2, | |
| preservation_level, L_handle_null); | |
| if (on_oop && on_reference) { | |
| // Generate the G1 pre-barrier code to log the value of | |
| // the referent field in an SATB buffer. Note with | |
| // these parameters the pre-barrier does not generate | |
| // the load of the previous value | |
| // We only reach here if value is not null. | |
| g1_write_barrier_pre(masm, decorators | IS_NOT_NULL, | |
| noreg /* obj */, (intptr_t)0, dst /* pre_val */, | |
| tmp1, tmp2, | |
| preservation_level); | |
| } | |
| __ bind(done); | |
| } | |
| void G1BarrierSetAssembler::resolve_jobject(MacroAssembler* masm, Register value, | |
| Register tmp1, Register tmp2, | |
| MacroAssembler::PreservationLevel preservation_level) { | |
| Label done, not_weak; | |
| __ cmpdi(CR0, value, 0); | |
| __ beq(CR0, done); // Use null as-is. | |
| __ clrrdi(tmp1, value, JNIHandles::tag_size); | |
| __ andi_(tmp2, value, JNIHandles::TypeTag::weak_global); | |
| __ ld(value, 0, tmp1); // Resolve (untagged) jobject. | |
| __ beq(CR0, not_weak); // Test for jweak tag. | |
| __ verify_oop(value, FILE_AND_LINE); | |
| g1_write_barrier_pre(masm, IN_NATIVE | ON_PHANTOM_OOP_REF, | |
| noreg, noreg, value, | |
| tmp1, tmp2, | |
| preservation_level); | |
| __ bind(not_weak); | |
| __ verify_oop(value, FILE_AND_LINE); | |
| __ bind(done); | |
| } | |
| #ifdef COMPILER2 | |
| static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) { | |
| SaveLiveRegisters save_registers(masm, stub); | |
| __ call_VM_leaf(runtime_path, arg, R16_thread); | |
| } | |
| void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm, | |
| Register obj, | |
| Register pre_val, | |
| Register tmp1, | |
| Register tmp2, | |
| G1PreBarrierStubC2* stub) { | |
| assert_different_registers(obj, tmp1, tmp2, R0); | |
| assert_different_registers(pre_val, tmp1, R0); | |
| assert(!UseCompressedOops || tmp2 != noreg, "tmp2 needed with CompressedOops"); | |
| stub->initialize_registers(obj, pre_val, R16_thread, tmp1, tmp2); | |
| generate_marking_inactive_test(masm); | |
| __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CR0, Assembler::equal), *stub->entry()); | |
| __ bind(*stub->continuation()); | |
| } | |
| void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm, | |
| G1PreBarrierStubC2* stub) const { | |
| Assembler::InlineSkippedInstructionsCounter skip_counter(masm); | |
| Label runtime; | |
| Register obj = stub->obj(); | |
| Register pre_val = stub->pre_val(); | |
| Register tmp1 = stub->tmp1(); | |
| __ bind(*stub->entry()); | |
| if (obj != noreg) { | |
| // Note: C2 currently doesn't use implicit null checks with barriers. | |
| // Otherwise, obj could be null and the following instruction would raise a SIGSEGV. | |
| if (UseCompressedOops) { | |
| __ lwz(pre_val, 0, obj); | |
| } else { | |
| __ ld(pre_val, 0, obj); | |
| } | |
| } | |
| __ cmpdi(CR0, pre_val, 0); | |
| __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *stub->continuation()); | |
| Register pre_val_decoded = pre_val; | |
| if (UseCompressedOops) { | |
| pre_val_decoded = __ decode_heap_oop_not_null(stub->tmp2(), pre_val); | |
| } | |
| generate_queue_insertion(masm, | |
| G1ThreadLocalData::satb_mark_queue_index_offset(), | |
| G1ThreadLocalData::satb_mark_queue_buffer_offset(), | |
| runtime, pre_val_decoded, tmp1); | |
| __ b(*stub->continuation()); | |
| __ bind(runtime); | |
| generate_c2_barrier_runtime_call(masm, stub, pre_val_decoded, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry)); | |
| __ b(*stub->continuation()); | |
| } | |
| void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm, | |
| Register store_addr, | |
| Register new_val, | |
| Register tmp1, | |
| Register tmp2, | |
| G1PostBarrierStubC2* stub, | |
| bool decode_new_val) { | |
| assert_different_registers(store_addr, new_val, tmp1, R0); | |
| assert_different_registers(store_addr, tmp1, tmp2, R0); | |
| stub->initialize_registers(R16_thread, tmp1, tmp2); | |
| bool null_check_required = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0; | |
| Register new_val_decoded = new_val; | |
| if (decode_new_val) { | |
| assert(UseCompressedOops, "or should not be here"); | |
| if (null_check_required && CompressedOops::base() != nullptr) { | |
| // We prefer doing the null check after the region crossing check. | |
| // Only compressed oop modes with base != null require a null check here. | |
| __ cmpwi(CR0, new_val, 0); | |
| __ beq(CR0, *stub->continuation()); | |
| null_check_required = false; | |
| } | |
| new_val_decoded = __ decode_heap_oop_not_null(tmp2, new_val); | |
| } | |
| generate_region_crossing_test(masm, store_addr, new_val_decoded); | |
| __ beq(CR0, *stub->continuation()); | |
| // crosses regions, storing null? | |
| if (null_check_required) { | |
| __ cmpdi(CR0, new_val_decoded, 0); | |
| __ beq(CR0, *stub->continuation()); | |
| } | |
| Address card_addr = generate_card_young_test(masm, store_addr, tmp1, tmp2); | |
| assert(card_addr.base() == tmp1 && card_addr.index() == tmp2, "needed by post barrier stub"); | |
| __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CR0, Assembler::equal), *stub->entry()); | |
| __ bind(*stub->continuation()); | |
| } | |
| void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm, | |
| G1PostBarrierStubC2* stub) const { | |
| Assembler::InlineSkippedInstructionsCounter skip_counter(masm); | |
| Label runtime; | |
| Address card_addr(stub->tmp1(), stub->tmp2()); // See above. | |
| __ bind(*stub->entry()); | |
| generate_card_dirty_test(masm, card_addr); | |
| __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *stub->continuation()); | |
| __ li(R0, (int)G1CardTable::dirty_card_val()); | |
| __ stbx(R0, card_addr.base(), card_addr.index()); // *(card address) := dirty_card_val | |
| Register Rcard_addr = stub->tmp1(); | |
| __ add(Rcard_addr, card_addr.base(), card_addr.index()); // This is the address which needs to get enqueued. | |
| generate_queue_insertion(masm, | |
| G1ThreadLocalData::dirty_card_queue_index_offset(), | |
| G1ThreadLocalData::dirty_card_queue_buffer_offset(), | |
| runtime, Rcard_addr, stub->tmp2()); | |
| __ b(*stub->continuation()); | |
| __ bind(runtime); | |
| generate_c2_barrier_runtime_call(masm, stub, Rcard_addr, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry)); | |
| __ b(*stub->continuation()); | |
| } | |
| #endif // COMPILER2 | |
| #ifdef COMPILER1 | |
| #undef __ | |
| #define __ ce->masm()-> | |
| void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) { | |
| G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); | |
| // At this point we know that marking is in progress. | |
| // If do_load() is true then we have to emit the | |
| // load of the previous value; otherwise it has already | |
| // been loaded into _pre_val. | |
| __ bind(*stub->entry()); | |
| assert(stub->pre_val()->is_register(), "Precondition."); | |
| Register pre_val_reg = stub->pre_val()->as_register(); | |
| if (stub->do_load()) { | |
| ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/); | |
| } | |
| __ cmpdi(CR0, pre_val_reg, 0); | |
| __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *stub->continuation()); | |
| address c_code = bs->pre_barrier_c1_runtime_code_blob()->code_begin(); | |
| //__ load_const_optimized(R0, c_code); | |
| __ add_const_optimized(R0, R29_TOC, MacroAssembler::offset_to_global_toc(c_code)); | |
| __ std(pre_val_reg, -8, R1_SP); // Pass pre_val on stack. | |
| __ mtctr(R0); | |
| __ bctrl(); | |
| __ b(*stub->continuation()); | |
| } | |
| void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) { | |
| G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); | |
| __ bind(*stub->entry()); | |
| assert(stub->addr()->is_register(), "Precondition."); | |
| assert(stub->new_val()->is_register(), "Precondition."); | |
| Register addr_reg = stub->addr()->as_pointer_register(); | |
| Register new_val_reg = stub->new_val()->as_register(); | |
| __ cmpdi(CR0, new_val_reg, 0); | |
| __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *stub->continuation()); | |
| address c_code = bs->post_barrier_c1_runtime_code_blob()->code_begin(); | |
| //__ load_const_optimized(R0, c_code); | |
| __ add_const_optimized(R0, R29_TOC, MacroAssembler::offset_to_global_toc(c_code)); | |
| __ mtctr(R0); | |
| __ mr(R0, addr_reg); // Pass addr in R0. | |
| __ bctrl(); | |
| __ b(*stub->continuation()); | |
| } | |
| #undef __ | |
| #define __ sasm-> | |
| void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { | |
| BarrierSet* bs = BarrierSet::barrier_set(); | |
| __ set_info("g1_pre_barrier_slow_id", false); | |
| // Using stack slots: pre_val (pre-pushed), spill tmp, spill tmp2. | |
| const int stack_slots = 3; | |
| Register pre_val = R0; // previous value of memory | |
| Register tmp = R14; | |
| Register tmp2 = R15; | |
| Label refill, restart, marking_not_active; | |
| int satb_q_active_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()); | |
| int satb_q_index_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()); | |
| int satb_q_buf_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()); | |
| // Spill | |
| __ std(tmp, -16, R1_SP); | |
| __ std(tmp2, -24, R1_SP); | |
| // Is marking still active? | |
| generate_marking_inactive_test(sasm); | |
| __ beq(CR0, marking_not_active); | |
| __ bind(restart); | |
| // Load the index into the SATB buffer. SATBMarkQueue::_index is a | |
| // size_t so ld_ptr is appropriate. | |
| __ ld(tmp, satb_q_index_byte_offset, R16_thread); | |
| // index == 0? | |
| __ cmpdi(CR0, tmp, 0); | |
| __ beq(CR0, refill); | |
| __ ld(tmp2, satb_q_buf_byte_offset, R16_thread); | |
| __ ld(pre_val, -8, R1_SP); // Load from stack. | |
| __ addi(tmp, tmp, -oopSize); | |
| __ std(tmp, satb_q_index_byte_offset, R16_thread); | |
| __ stdx(pre_val, tmp2, tmp); // [_buf + index] := <address_of_card> | |
| __ bind(marking_not_active); | |
| // Restore temp registers and return-from-leaf. | |
| __ ld(tmp2, -24, R1_SP); | |
| __ ld(tmp, -16, R1_SP); | |
| __ blr(); | |
| __ bind(refill); | |
| const int nbytes_save = (MacroAssembler::num_volatile_regs + stack_slots) * BytesPerWord; | |
| __ save_volatile_gprs(R1_SP, -nbytes_save); // except R0 | |
| __ mflr(R0); | |
| __ std(R0, _abi0(lr), R1_SP); | |
| __ push_frame_reg_args(nbytes_save, R0); // dummy frame for C call | |
| __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1SATBMarkQueueSet::handle_zero_index_for_thread), R16_thread); | |
| __ pop_frame(); | |
| __ ld(R0, _abi0(lr), R1_SP); | |
| __ mtlr(R0); | |
| __ restore_volatile_gprs(R1_SP, -nbytes_save); // except R0 | |
| __ b(restart); | |
| } | |
| void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) { | |
| G1BarrierSet* bs = barrier_set_cast<G1BarrierSet>(BarrierSet::barrier_set()); | |
| __ set_info("g1_post_barrier_slow_id", false); | |
| // Using stack slots: spill addr, spill tmp2 | |
| const int stack_slots = 2; | |
| Register tmp = R0; | |
| Register addr = R14; | |
| Register tmp2 = R15; | |
| CardTable::CardValue* byte_map_base = bs->card_table()->byte_map_base(); | |
| Label restart, refill, ret; | |
| // Spill | |
| __ std(addr, -8, R1_SP); | |
| __ std(tmp2, -16, R1_SP); | |
| __ srdi(addr, R0, CardTable::card_shift()); // Addr is passed in R0. | |
| __ load_const_optimized(/*cardtable*/ tmp2, byte_map_base, tmp); | |
| __ add(addr, tmp2, addr); | |
| __ lbz(tmp, 0, addr); // tmp := [addr + cardtable] | |
| // Return if young card. | |
| __ cmpwi(CR0, tmp, G1CardTable::g1_young_card_val()); | |
| __ beq(CR0, ret); | |
| // Return if sequential consistent value is already dirty. | |
| __ membar(Assembler::StoreLoad); | |
| __ lbz(tmp, 0, addr); // tmp := [addr + cardtable] | |
| __ cmpwi(CR0, tmp, G1CardTable::dirty_card_val()); | |
| __ beq(CR0, ret); | |
| // Not dirty. | |
| // First, dirty it. | |
| __ li(tmp, G1CardTable::dirty_card_val()); | |
| __ stb(tmp, 0, addr); | |
| int dirty_card_q_index_byte_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()); | |
| int dirty_card_q_buf_byte_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()); | |
| __ bind(restart); | |
| // Get the index into the update buffer. G1DirtyCardQueue::_index is | |
| // a size_t so ld_ptr is appropriate here. | |
| __ ld(tmp2, dirty_card_q_index_byte_offset, R16_thread); | |
| // index == 0? | |
| __ cmpdi(CR0, tmp2, 0); | |
| __ beq(CR0, refill); | |
| __ ld(tmp, dirty_card_q_buf_byte_offset, R16_thread); | |
| __ addi(tmp2, tmp2, -oopSize); | |
| __ std(tmp2, dirty_card_q_index_byte_offset, R16_thread); | |
| __ add(tmp2, tmp, tmp2); | |
| __ std(addr, 0, tmp2); // [_buf + index] := <address_of_card> | |
| // Restore temp registers and return-from-leaf. | |
| __ bind(ret); | |
| __ ld(tmp2, -16, R1_SP); | |
| __ ld(addr, -8, R1_SP); | |
| __ blr(); | |
| __ bind(refill); | |
| const int nbytes_save = (MacroAssembler::num_volatile_regs + stack_slots) * BytesPerWord; | |
| __ save_volatile_gprs(R1_SP, -nbytes_save); // except R0 | |
| __ mflr(R0); | |
| __ std(R0, _abi0(lr), R1_SP); | |
| __ push_frame_reg_args(nbytes_save, R0); // dummy frame for C call | |
| __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1DirtyCardQueueSet::handle_zero_index_for_thread), R16_thread); | |
| __ pop_frame(); | |
| __ ld(R0, _abi0(lr), R1_SP); | |
| __ mtlr(R0); | |
| __ restore_volatile_gprs(R1_SP, -nbytes_save); // except R0 | |
| __ b(restart); | |
| } | |
| #undef __ | |
| #endif // COMPILER1 |