| /* |
| * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved. |
| * Copyright (c) 2018, 2025 SAP SE. All rights reserved. |
| * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| * |
| * This code is free software; you can redistribute it and/or modify it |
| * under the terms of the GNU General Public License version 2 only, as |
| * published by the Free Software Foundation. |
| * |
| * This code is distributed in the hope that it will be useful, but WITHOUT |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| * version 2 for more details (a copy is included in the LICENSE file that |
| * accompanied this code). |
| * |
| * You should have received a copy of the GNU General Public License version |
| * 2 along with this work; if not, write to the Free Software Foundation, |
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| * |
| * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| * or visit www.oracle.com if you need additional information or have any |
| * questions. |
| * |
| */ |
| |
| #include "asm/macroAssembler.inline.hpp" |
| #include "gc/g1/g1BarrierSet.hpp" |
| #include "gc/g1/g1BarrierSetAssembler.hpp" |
| #include "gc/g1/g1BarrierSetRuntime.hpp" |
| #include "gc/g1/g1CardTable.hpp" |
| #include "gc/g1/g1DirtyCardQueue.hpp" |
| #include "gc/g1/g1HeapRegion.hpp" |
| #include "gc/g1/g1SATBMarkQueueSet.hpp" |
| #include "gc/g1/g1ThreadLocalData.hpp" |
| #include "interpreter/interp_masm.hpp" |
| #include "runtime/jniHandles.hpp" |
| #include "runtime/sharedRuntime.hpp" |
| #include "utilities/macros.hpp" |
| #ifdef COMPILER1 |
| #include "c1/c1_LIRAssembler.hpp" |
| #include "c1/c1_MacroAssembler.hpp" |
| #include "gc/g1/c1/g1BarrierSetC1.hpp" |
| #endif // COMPILER1 |
| #ifdef COMPILER2 |
| #include "gc/g1/c2/g1BarrierSetC2.hpp" |
| #endif // COMPILER2 |
| |
| #define __ masm-> |
| |
| static void generate_marking_inactive_test(MacroAssembler* masm) { |
| int active_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()); |
| assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); |
| __ lbz(R0, active_offset, R16_thread); // tmp1 := *(mark queue active address) |
| __ cmpwi(CR0, R0, 0); |
| } |
| |
| void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators, |
| Register from, Register to, Register count, |
| Register preserve1, Register preserve2) { |
| bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; |
| // With G1, don't generate the call if we statically know that the target in uninitialized |
| if (!dest_uninitialized) { |
| int spill_slots = 3; |
| if (preserve1 != noreg) { spill_slots++; } |
| if (preserve2 != noreg) { spill_slots++; } |
| const int frame_size = align_up(frame::native_abi_reg_args_size + spill_slots * BytesPerWord, frame::alignment_in_bytes); |
| Label filtered; |
| |
| // Is marking active? |
| generate_marking_inactive_test(masm); |
| __ beq(CR0, filtered); |
| |
| __ save_LR(R0); |
| __ push_frame(frame_size, R0); |
| int slot_nr = 0; |
| __ std(from, frame_size - (++slot_nr) * wordSize, R1_SP); |
| __ std(to, frame_size - (++slot_nr) * wordSize, R1_SP); |
| __ std(count, frame_size - (++slot_nr) * wordSize, R1_SP); |
| if (preserve1 != noreg) { __ std(preserve1, frame_size - (++slot_nr) * wordSize, R1_SP); } |
| if (preserve2 != noreg) { __ std(preserve2, frame_size - (++slot_nr) * wordSize, R1_SP); } |
| |
| if (UseCompressedOops) { |
| __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), to, count); |
| } else { |
| __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), to, count); |
| } |
| |
| slot_nr = 0; |
| __ ld(from, frame_size - (++slot_nr) * wordSize, R1_SP); |
| __ ld(to, frame_size - (++slot_nr) * wordSize, R1_SP); |
| __ ld(count, frame_size - (++slot_nr) * wordSize, R1_SP); |
| if (preserve1 != noreg) { __ ld(preserve1, frame_size - (++slot_nr) * wordSize, R1_SP); } |
| if (preserve2 != noreg) { __ ld(preserve2, frame_size - (++slot_nr) * wordSize, R1_SP); } |
| __ addi(R1_SP, R1_SP, frame_size); // pop_frame() |
| __ restore_LR(R0); |
| |
| __ bind(filtered); |
| } |
| } |
| |
| void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, |
| Register addr, Register count, Register preserve) { |
| int spill_slots = (preserve != noreg) ? 1 : 0; |
| const int frame_size = align_up(frame::native_abi_reg_args_size + spill_slots * BytesPerWord, frame::alignment_in_bytes); |
| |
| __ save_LR(R0); |
| __ push_frame(frame_size, R0); |
| if (preserve != noreg) { __ std(preserve, frame_size - 1 * wordSize, R1_SP); } |
| __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), addr, count); |
| if (preserve != noreg) { __ ld(preserve, frame_size - 1 * wordSize, R1_SP); } |
| __ addi(R1_SP, R1_SP, frame_size); // pop_frame(); |
| __ restore_LR(R0); |
| } |
| |
| static void generate_queue_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime, |
| const Register value, const Register temp) { |
| assert_different_registers(value, temp); |
| // Can we store a value in the given thread's buffer? |
| // (The index field is typed as size_t.) |
| __ ld(temp, in_bytes(index_offset), R16_thread); // temp := *(index address) |
| __ cmpdi(CR0, temp, 0); // jump to runtime if index == 0 (full buffer) |
| __ beq(CR0, runtime); |
| // The buffer is not full, store value into it. |
| __ ld(R0, in_bytes(buffer_offset), R16_thread); // R0 := buffer address |
| __ addi(temp, temp, -wordSize); // temp := next index |
| __ std(temp, in_bytes(index_offset), R16_thread); // *(index address) := next index |
| __ stdx(value, temp, R0); // *(buffer address + next index) := value |
| } |
| |
| void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, DecoratorSet decorators, |
| Register obj, RegisterOrConstant ind_or_offs, Register pre_val, |
| Register tmp1, Register tmp2, |
| MacroAssembler::PreservationLevel preservation_level) { |
| assert_different_registers(pre_val, tmp1, tmp2); |
| |
| bool not_null = (decorators & IS_NOT_NULL) != 0, |
| preloaded = obj == noreg; |
| Register nv_save = noreg; |
| |
| // Determine necessary runtime invocation preservation measures |
| const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR; |
| const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS; |
| const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS; |
| int nbytes_save = 0; |
| |
| if (pre_val->is_volatile() && preloaded && !preserve_gp_registers) { |
| // We are not loading the previous value so make |
| // sure that we don't trash the value in pre_val |
| // with the code below. |
| nv_save = !tmp1->is_volatile() ? tmp1 : tmp2; |
| assert(!nv_save->is_volatile(), "need one nv temp register if pre_val lives in volatile register"); |
| } |
| |
| Label runtime, filtered; |
| |
| generate_marking_inactive_test(masm); |
| __ beq(CR0, filtered); |
| |
| // Do we need to load the previous value? |
| if (!preloaded) { |
| // Load the previous value... |
| if (UseCompressedOops) { |
| __ lwz(pre_val, ind_or_offs, obj); |
| } else { |
| __ ld(pre_val, ind_or_offs, obj); |
| } |
| // Previous value has been loaded into Rpre_val. |
| } |
| assert(pre_val != noreg, "must have a real register"); |
| |
| // Is the previous value null? |
| if (preloaded && not_null) { |
| #ifdef ASSERT |
| __ cmpdi(CR0, pre_val, 0); |
| __ asm_assert_ne("null oop not allowed (G1 pre)"); // Checked by caller. |
| #endif |
| } else { |
| __ cmpdi(CR0, pre_val, 0); |
| __ beq(CR0, filtered); |
| } |
| |
| if (!preloaded && UseCompressedOops) { |
| __ decode_heap_oop_not_null(pre_val); |
| } |
| |
| // OK, it's not filtered, so we'll need to call enqueue. In the normal |
| // case, pre_val will be a scratch G-reg, but there are some cases in |
| // which it's an O-reg. In the first case, do a normal call. In the |
| // latter, do a save here and call the frameless version. |
| |
| // Can we store original value in the thread's buffer? |
| // Is index == 0? |
| // (The index field is typed as size_t.) |
| generate_queue_insertion(masm, G1ThreadLocalData::satb_mark_queue_index_offset(), G1ThreadLocalData::satb_mark_queue_buffer_offset(), |
| runtime, pre_val, tmp1); |
| __ b(filtered); |
| |
| __ bind(runtime); |
| |
| // May need to preserve LR. Also needed if current frame is not compatible with C calling convention. |
| if (needs_frame) { |
| if (preserve_gp_registers) { |
| nbytes_save = (MacroAssembler::num_volatile_gp_regs |
| + (preserve_fp_registers ? MacroAssembler::num_volatile_fp_regs : 0) |
| ) * BytesPerWord; |
| __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers); |
| } |
| |
| __ save_LR(tmp1); |
| __ push_frame_reg_args(nbytes_save, tmp2); |
| } |
| |
| if (nv_save != noreg) { |
| __ mr(nv_save, pre_val); // Save pre_val across C call if it was preloaded. |
| } |
| __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, R16_thread); |
| if (nv_save != noreg) { |
| __ mr(pre_val, nv_save); // restore |
| } |
| |
| if (needs_frame) { |
| __ pop_frame(); |
| __ restore_LR(tmp1); |
| |
| if (preserve_gp_registers) { |
| __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers); |
| } |
| } |
| |
| __ bind(filtered); |
| } |
| |
| static void generate_region_crossing_test(MacroAssembler* masm, const Register store_addr, const Register new_val) { |
| __ xorr(R0, store_addr, new_val); // tmp1 := store address ^ new value |
| __ srdi_(R0, R0, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes) |
| } |
| |
| static Address generate_card_young_test(MacroAssembler* masm, const Register store_addr, const Register tmp1, const Register tmp2) { |
| CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set()); |
| __ load_const_optimized(tmp1, (address)(ct->card_table()->byte_map_base()), tmp2); |
| __ srdi(tmp2, store_addr, CardTable::card_shift()); // tmp1 := card address relative to card table base |
| __ lbzx(R0, tmp1, tmp2); // tmp1 := card address |
| __ cmpwi(CR0, R0, (int)G1CardTable::g1_young_card_val()); |
| return Address(tmp1, tmp2); // return card address |
| } |
| |
| static void generate_card_dirty_test(MacroAssembler* masm, Address card_addr) { |
| __ membar(Assembler::StoreLoad); // Must reload after StoreLoad membar due to concurrent refinement |
| __ lbzx(R0, card_addr.base(), card_addr.index()); // tmp2 := card |
| __ cmpwi(CR0, R0, (int)G1CardTable::dirty_card_val()); // tmp2 := card == dirty_card_val? |
| } |
| |
| void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, DecoratorSet decorators, |
| Register store_addr, Register new_val, |
| Register tmp1, Register tmp2, Register tmp3, |
| MacroAssembler::PreservationLevel preservation_level) { |
| bool not_null = (decorators & IS_NOT_NULL) != 0; |
| |
| Label runtime, filtered; |
| assert_different_registers(store_addr, new_val, tmp1, tmp2); |
| |
| CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set()); |
| |
| generate_region_crossing_test(masm, store_addr, new_val); |
| __ beq(CR0, filtered); |
| |
| // Crosses regions, storing null? |
| if (not_null) { |
| #ifdef ASSERT |
| __ cmpdi(CR0, new_val, 0); |
| __ asm_assert_ne("null oop not allowed (G1 post)"); // Checked by caller. |
| #endif |
| } else { |
| __ cmpdi(CR0, new_val, 0); |
| __ beq(CR0, filtered); |
| } |
| |
| Address card_addr = generate_card_young_test(masm, store_addr, tmp1, tmp2); |
| __ beq(CR0, filtered); |
| |
| generate_card_dirty_test(masm, card_addr); |
| __ beq(CR0, filtered); |
| |
| __ li(R0, (int)G1CardTable::dirty_card_val()); |
| __ stbx(R0, card_addr.base(), card_addr.index()); // *(card address) := dirty_card_val |
| |
| Register Rcard_addr = tmp3; |
| __ add(Rcard_addr, card_addr.base(), card_addr.index()); // This is the address which needs to get enqueued. |
| |
| generate_queue_insertion(masm, |
| G1ThreadLocalData::dirty_card_queue_index_offset(), |
| G1ThreadLocalData::dirty_card_queue_buffer_offset(), |
| runtime, Rcard_addr, tmp1); |
| __ b(filtered); |
| |
| __ bind(runtime); |
| |
| assert(preservation_level == MacroAssembler::PRESERVATION_NONE, |
| "g1_write_barrier_post doesn't support preservation levels higher than PRESERVATION_NONE"); |
| |
| // Save the live input values. |
| __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), Rcard_addr, R16_thread); |
| |
| __ bind(filtered); |
| } |
| |
| void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, |
| Register base, RegisterOrConstant ind_or_offs, Register val, |
| Register tmp1, Register tmp2, Register tmp3, |
| MacroAssembler::PreservationLevel preservation_level) { |
| bool is_array = (decorators & IS_ARRAY) != 0; |
| bool on_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0; |
| bool precise = is_array || on_anonymous; |
| // Load and record the previous value. |
| g1_write_barrier_pre(masm, decorators, |
| base, ind_or_offs, |
| tmp1, tmp2, tmp3, |
| preservation_level); |
| |
| BarrierSetAssembler::store_at(masm, decorators, |
| type, base, ind_or_offs, val, |
| tmp1, tmp2, tmp3, |
| preservation_level); |
| |
| // No need for post barrier if storing null |
| if (val != noreg) { |
| if (precise) { |
| if (ind_or_offs.is_constant()) { |
| __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp1); |
| } else { |
| __ add(base, ind_or_offs.as_register(), base); |
| } |
| } |
| g1_write_barrier_post(masm, decorators, |
| base, val, |
| tmp1, tmp2, tmp3, |
| preservation_level); |
| } |
| } |
| |
| void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, |
| Register base, RegisterOrConstant ind_or_offs, Register dst, |
| Register tmp1, Register tmp2, |
| MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null) { |
| bool on_oop = is_reference_type(type); |
| bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; |
| bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; |
| bool on_reference = on_weak || on_phantom; |
| Label done; |
| if (on_oop && on_reference && L_handle_null == nullptr) { L_handle_null = &done; } |
| // Load the value of the referent field. |
| ModRefBarrierSetAssembler::load_at(masm, decorators, type, |
| base, ind_or_offs, dst, |
| tmp1, tmp2, |
| preservation_level, L_handle_null); |
| if (on_oop && on_reference) { |
| // Generate the G1 pre-barrier code to log the value of |
| // the referent field in an SATB buffer. Note with |
| // these parameters the pre-barrier does not generate |
| // the load of the previous value |
| // We only reach here if value is not null. |
| g1_write_barrier_pre(masm, decorators | IS_NOT_NULL, |
| noreg /* obj */, (intptr_t)0, dst /* pre_val */, |
| tmp1, tmp2, |
| preservation_level); |
| } |
| __ bind(done); |
| } |
| |
| void G1BarrierSetAssembler::resolve_jobject(MacroAssembler* masm, Register value, |
| Register tmp1, Register tmp2, |
| MacroAssembler::PreservationLevel preservation_level) { |
| Label done, not_weak; |
| __ cmpdi(CR0, value, 0); |
| __ beq(CR0, done); // Use null as-is. |
| |
| __ clrrdi(tmp1, value, JNIHandles::tag_size); |
| __ andi_(tmp2, value, JNIHandles::TypeTag::weak_global); |
| __ ld(value, 0, tmp1); // Resolve (untagged) jobject. |
| |
| __ beq(CR0, not_weak); // Test for jweak tag. |
| __ verify_oop(value, FILE_AND_LINE); |
| g1_write_barrier_pre(masm, IN_NATIVE | ON_PHANTOM_OOP_REF, |
| noreg, noreg, value, |
| tmp1, tmp2, |
| preservation_level); |
| __ bind(not_weak); |
| __ verify_oop(value, FILE_AND_LINE); |
| __ bind(done); |
| } |
| |
| #ifdef COMPILER2 |
| |
| static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) { |
| SaveLiveRegisters save_registers(masm, stub); |
| __ call_VM_leaf(runtime_path, arg, R16_thread); |
| } |
| |
| void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm, |
| Register obj, |
| Register pre_val, |
| Register tmp1, |
| Register tmp2, |
| G1PreBarrierStubC2* stub) { |
| assert_different_registers(obj, tmp1, tmp2, R0); |
| assert_different_registers(pre_val, tmp1, R0); |
| assert(!UseCompressedOops || tmp2 != noreg, "tmp2 needed with CompressedOops"); |
| |
| stub->initialize_registers(obj, pre_val, R16_thread, tmp1, tmp2); |
| |
| generate_marking_inactive_test(masm); |
| __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CR0, Assembler::equal), *stub->entry()); |
| |
| __ bind(*stub->continuation()); |
| } |
| |
| void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm, |
| G1PreBarrierStubC2* stub) const { |
| Assembler::InlineSkippedInstructionsCounter skip_counter(masm); |
| Label runtime; |
| Register obj = stub->obj(); |
| Register pre_val = stub->pre_val(); |
| Register tmp1 = stub->tmp1(); |
| |
| __ bind(*stub->entry()); |
| |
| if (obj != noreg) { |
| // Note: C2 currently doesn't use implicit null checks with barriers. |
| // Otherwise, obj could be null and the following instruction would raise a SIGSEGV. |
| if (UseCompressedOops) { |
| __ lwz(pre_val, 0, obj); |
| } else { |
| __ ld(pre_val, 0, obj); |
| } |
| } |
| __ cmpdi(CR0, pre_val, 0); |
| __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *stub->continuation()); |
| |
| Register pre_val_decoded = pre_val; |
| if (UseCompressedOops) { |
| pre_val_decoded = __ decode_heap_oop_not_null(stub->tmp2(), pre_val); |
| } |
| |
| generate_queue_insertion(masm, |
| G1ThreadLocalData::satb_mark_queue_index_offset(), |
| G1ThreadLocalData::satb_mark_queue_buffer_offset(), |
| runtime, pre_val_decoded, tmp1); |
| __ b(*stub->continuation()); |
| |
| __ bind(runtime); |
| generate_c2_barrier_runtime_call(masm, stub, pre_val_decoded, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry)); |
| __ b(*stub->continuation()); |
| } |
| |
| void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm, |
| Register store_addr, |
| Register new_val, |
| Register tmp1, |
| Register tmp2, |
| G1PostBarrierStubC2* stub, |
| bool decode_new_val) { |
| assert_different_registers(store_addr, new_val, tmp1, R0); |
| assert_different_registers(store_addr, tmp1, tmp2, R0); |
| |
| stub->initialize_registers(R16_thread, tmp1, tmp2); |
| |
| bool null_check_required = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0; |
| Register new_val_decoded = new_val; |
| |
| if (decode_new_val) { |
| assert(UseCompressedOops, "or should not be here"); |
| if (null_check_required && CompressedOops::base() != nullptr) { |
| // We prefer doing the null check after the region crossing check. |
| // Only compressed oop modes with base != null require a null check here. |
| __ cmpwi(CR0, new_val, 0); |
| __ beq(CR0, *stub->continuation()); |
| null_check_required = false; |
| } |
| new_val_decoded = __ decode_heap_oop_not_null(tmp2, new_val); |
| } |
| |
| generate_region_crossing_test(masm, store_addr, new_val_decoded); |
| __ beq(CR0, *stub->continuation()); |
| |
| // crosses regions, storing null? |
| if (null_check_required) { |
| __ cmpdi(CR0, new_val_decoded, 0); |
| __ beq(CR0, *stub->continuation()); |
| } |
| |
| Address card_addr = generate_card_young_test(masm, store_addr, tmp1, tmp2); |
| assert(card_addr.base() == tmp1 && card_addr.index() == tmp2, "needed by post barrier stub"); |
| __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CR0, Assembler::equal), *stub->entry()); |
| |
| __ bind(*stub->continuation()); |
| } |
| |
| void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm, |
| G1PostBarrierStubC2* stub) const { |
| Assembler::InlineSkippedInstructionsCounter skip_counter(masm); |
| Label runtime; |
| Address card_addr(stub->tmp1(), stub->tmp2()); // See above. |
| |
| __ bind(*stub->entry()); |
| |
| generate_card_dirty_test(masm, card_addr); |
| __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *stub->continuation()); |
| |
| __ li(R0, (int)G1CardTable::dirty_card_val()); |
| __ stbx(R0, card_addr.base(), card_addr.index()); // *(card address) := dirty_card_val |
| |
| Register Rcard_addr = stub->tmp1(); |
| __ add(Rcard_addr, card_addr.base(), card_addr.index()); // This is the address which needs to get enqueued. |
| |
| generate_queue_insertion(masm, |
| G1ThreadLocalData::dirty_card_queue_index_offset(), |
| G1ThreadLocalData::dirty_card_queue_buffer_offset(), |
| runtime, Rcard_addr, stub->tmp2()); |
| __ b(*stub->continuation()); |
| |
| __ bind(runtime); |
| generate_c2_barrier_runtime_call(masm, stub, Rcard_addr, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry)); |
| __ b(*stub->continuation()); |
| } |
| |
| #endif // COMPILER2 |
| |
| #ifdef COMPILER1 |
| |
| #undef __ |
| #define __ ce->masm()-> |
| |
| void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) { |
| G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); |
| // At this point we know that marking is in progress. |
| // If do_load() is true then we have to emit the |
| // load of the previous value; otherwise it has already |
| // been loaded into _pre_val. |
| |
| __ bind(*stub->entry()); |
| |
| assert(stub->pre_val()->is_register(), "Precondition."); |
| Register pre_val_reg = stub->pre_val()->as_register(); |
| |
| if (stub->do_load()) { |
| ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/); |
| } |
| |
| __ cmpdi(CR0, pre_val_reg, 0); |
| __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *stub->continuation()); |
| |
| address c_code = bs->pre_barrier_c1_runtime_code_blob()->code_begin(); |
| //__ load_const_optimized(R0, c_code); |
| __ add_const_optimized(R0, R29_TOC, MacroAssembler::offset_to_global_toc(c_code)); |
| __ std(pre_val_reg, -8, R1_SP); // Pass pre_val on stack. |
| __ mtctr(R0); |
| __ bctrl(); |
| __ b(*stub->continuation()); |
| } |
| |
| void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) { |
| G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); |
| __ bind(*stub->entry()); |
| |
| assert(stub->addr()->is_register(), "Precondition."); |
| assert(stub->new_val()->is_register(), "Precondition."); |
| Register addr_reg = stub->addr()->as_pointer_register(); |
| Register new_val_reg = stub->new_val()->as_register(); |
| |
| __ cmpdi(CR0, new_val_reg, 0); |
| __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *stub->continuation()); |
| |
| address c_code = bs->post_barrier_c1_runtime_code_blob()->code_begin(); |
| //__ load_const_optimized(R0, c_code); |
| __ add_const_optimized(R0, R29_TOC, MacroAssembler::offset_to_global_toc(c_code)); |
| __ mtctr(R0); |
| __ mr(R0, addr_reg); // Pass addr in R0. |
| __ bctrl(); |
| __ b(*stub->continuation()); |
| } |
| |
| #undef __ |
| #define __ sasm-> |
| |
| void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { |
| BarrierSet* bs = BarrierSet::barrier_set(); |
| |
| __ set_info("g1_pre_barrier_slow_id", false); |
| |
| // Using stack slots: pre_val (pre-pushed), spill tmp, spill tmp2. |
| const int stack_slots = 3; |
| Register pre_val = R0; // previous value of memory |
| Register tmp = R14; |
| Register tmp2 = R15; |
| |
| Label refill, restart, marking_not_active; |
| int satb_q_active_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()); |
| int satb_q_index_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()); |
| int satb_q_buf_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()); |
| |
| // Spill |
| __ std(tmp, -16, R1_SP); |
| __ std(tmp2, -24, R1_SP); |
| |
| // Is marking still active? |
| generate_marking_inactive_test(sasm); |
| __ beq(CR0, marking_not_active); |
| |
| __ bind(restart); |
| // Load the index into the SATB buffer. SATBMarkQueue::_index is a |
| // size_t so ld_ptr is appropriate. |
| __ ld(tmp, satb_q_index_byte_offset, R16_thread); |
| |
| // index == 0? |
| __ cmpdi(CR0, tmp, 0); |
| __ beq(CR0, refill); |
| |
| __ ld(tmp2, satb_q_buf_byte_offset, R16_thread); |
| __ ld(pre_val, -8, R1_SP); // Load from stack. |
| __ addi(tmp, tmp, -oopSize); |
| |
| __ std(tmp, satb_q_index_byte_offset, R16_thread); |
| __ stdx(pre_val, tmp2, tmp); // [_buf + index] := <address_of_card> |
| |
| __ bind(marking_not_active); |
| // Restore temp registers and return-from-leaf. |
| __ ld(tmp2, -24, R1_SP); |
| __ ld(tmp, -16, R1_SP); |
| __ blr(); |
| |
| __ bind(refill); |
| const int nbytes_save = (MacroAssembler::num_volatile_regs + stack_slots) * BytesPerWord; |
| __ save_volatile_gprs(R1_SP, -nbytes_save); // except R0 |
| __ mflr(R0); |
| __ std(R0, _abi0(lr), R1_SP); |
| __ push_frame_reg_args(nbytes_save, R0); // dummy frame for C call |
| __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1SATBMarkQueueSet::handle_zero_index_for_thread), R16_thread); |
| __ pop_frame(); |
| __ ld(R0, _abi0(lr), R1_SP); |
| __ mtlr(R0); |
| __ restore_volatile_gprs(R1_SP, -nbytes_save); // except R0 |
| __ b(restart); |
| } |
| |
| void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) { |
| G1BarrierSet* bs = barrier_set_cast<G1BarrierSet>(BarrierSet::barrier_set()); |
| |
| __ set_info("g1_post_barrier_slow_id", false); |
| |
| // Using stack slots: spill addr, spill tmp2 |
| const int stack_slots = 2; |
| Register tmp = R0; |
| Register addr = R14; |
| Register tmp2 = R15; |
| CardTable::CardValue* byte_map_base = bs->card_table()->byte_map_base(); |
| |
| Label restart, refill, ret; |
| |
| // Spill |
| __ std(addr, -8, R1_SP); |
| __ std(tmp2, -16, R1_SP); |
| |
| __ srdi(addr, R0, CardTable::card_shift()); // Addr is passed in R0. |
| __ load_const_optimized(/*cardtable*/ tmp2, byte_map_base, tmp); |
| __ add(addr, tmp2, addr); |
| __ lbz(tmp, 0, addr); // tmp := [addr + cardtable] |
| |
| // Return if young card. |
| __ cmpwi(CR0, tmp, G1CardTable::g1_young_card_val()); |
| __ beq(CR0, ret); |
| |
| // Return if sequential consistent value is already dirty. |
| __ membar(Assembler::StoreLoad); |
| __ lbz(tmp, 0, addr); // tmp := [addr + cardtable] |
| |
| __ cmpwi(CR0, tmp, G1CardTable::dirty_card_val()); |
| __ beq(CR0, ret); |
| |
| // Not dirty. |
| |
| // First, dirty it. |
| __ li(tmp, G1CardTable::dirty_card_val()); |
| __ stb(tmp, 0, addr); |
| |
| int dirty_card_q_index_byte_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()); |
| int dirty_card_q_buf_byte_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()); |
| |
| __ bind(restart); |
| |
| // Get the index into the update buffer. G1DirtyCardQueue::_index is |
| // a size_t so ld_ptr is appropriate here. |
| __ ld(tmp2, dirty_card_q_index_byte_offset, R16_thread); |
| |
| // index == 0? |
| __ cmpdi(CR0, tmp2, 0); |
| __ beq(CR0, refill); |
| |
| __ ld(tmp, dirty_card_q_buf_byte_offset, R16_thread); |
| __ addi(tmp2, tmp2, -oopSize); |
| |
| __ std(tmp2, dirty_card_q_index_byte_offset, R16_thread); |
| __ add(tmp2, tmp, tmp2); |
| __ std(addr, 0, tmp2); // [_buf + index] := <address_of_card> |
| |
| // Restore temp registers and return-from-leaf. |
| __ bind(ret); |
| __ ld(tmp2, -16, R1_SP); |
| __ ld(addr, -8, R1_SP); |
| __ blr(); |
| |
| __ bind(refill); |
| const int nbytes_save = (MacroAssembler::num_volatile_regs + stack_slots) * BytesPerWord; |
| __ save_volatile_gprs(R1_SP, -nbytes_save); // except R0 |
| __ mflr(R0); |
| __ std(R0, _abi0(lr), R1_SP); |
| __ push_frame_reg_args(nbytes_save, R0); // dummy frame for C call |
| __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1DirtyCardQueueSet::handle_zero_index_for_thread), R16_thread); |
| __ pop_frame(); |
| __ ld(R0, _abi0(lr), R1_SP); |
| __ mtlr(R0); |
| __ restore_volatile_gprs(R1_SP, -nbytes_save); // except R0 |
| __ b(restart); |
| } |
| |
| #undef __ |
| |
| #endif // COMPILER1 |
