| /* | |
| * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved. | |
| * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |
| * | |
| * This code is free software; you can redistribute it and/or modify it | |
| * under the terms of the GNU General Public License version 2 only, as | |
| * published by the Free Software Foundation. | |
| * | |
| * This code is distributed in the hope that it will be useful, but WITHOUT | |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| * version 2 for more details (a copy is included in the LICENSE file that | |
| * accompanied this code). | |
| * | |
| * You should have received a copy of the GNU General Public License version | |
| * 2 along with this work; if not, write to the Free Software Foundation, | |
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| * | |
| * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA | |
| * or visit www.oracle.com if you need additional information or have any | |
| * questions. | |
| * | |
| */ | |
| #include "asm/macroAssembler.hpp" | |
| #include "gc/shared/barrierSet.hpp" | |
| #include "gc/shared/barrierSetAssembler.hpp" | |
| #include "oops/objArrayKlass.hpp" | |
| #include "runtime/sharedRuntime.hpp" | |
| #include "runtime/stubRoutines.hpp" | |
| #include "stubGenerator_x86_64.hpp" | |
| #ifdef COMPILER2 | |
| #include "opto/c2_globals.hpp" | |
| #endif | |
| #if INCLUDE_JVMCI | |
| #include "jvmci/jvmci_globals.hpp" | |
| #endif | |
| #define __ _masm-> | |
| #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8) | |
| #ifdef PRODUCT | |
| #define BLOCK_COMMENT(str) /* nothing */ | |
| #else | |
| #define BLOCK_COMMENT(str) __ block_comment(str) | |
| #endif // PRODUCT | |
| #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") | |
| #ifdef PRODUCT | |
| #define INC_COUNTER_NP(counter, rscratch) ((void)0) | |
| #else | |
| #define INC_COUNTER_NP(counter, rscratch) \ | |
| BLOCK_COMMENT("inc_counter " #counter); \ | |
| inc_counter_np(_masm, counter, rscratch); | |
| static void inc_counter_np(MacroAssembler* _masm, uint& counter, Register rscratch) { | |
| __ incrementl(ExternalAddress((address)&counter), rscratch); | |
| } | |
| #if COMPILER2_OR_JVMCI | |
| static uint& get_profile_ctr(int shift) { | |
| if (shift == 0) { | |
| return SharedRuntime::_jbyte_array_copy_ctr; | |
| } else if (shift == 1) { | |
| return SharedRuntime::_jshort_array_copy_ctr; | |
| } else if (shift == 2) { | |
| return SharedRuntime::_jint_array_copy_ctr; | |
| } else { | |
| assert(shift == 3, ""); | |
| return SharedRuntime::_jlong_array_copy_ctr; | |
| } | |
| } | |
| #endif // COMPILER2_OR_JVMCI | |
| #endif // !PRODUCT | |
| void StubGenerator::generate_arraycopy_stubs() { | |
| address entry; | |
| address entry_jbyte_arraycopy; | |
| address entry_jshort_arraycopy; | |
| address entry_jint_arraycopy; | |
| address entry_oop_arraycopy; | |
| address entry_jlong_arraycopy; | |
| address entry_checkcast_arraycopy; | |
| StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(&entry); | |
| StubRoutines::_jbyte_arraycopy = generate_conjoint_byte_copy(entry, &entry_jbyte_arraycopy); | |
| StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(&entry); | |
| StubRoutines::_jshort_arraycopy = generate_conjoint_short_copy(entry, &entry_jshort_arraycopy); | |
| StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_int_oop_copy(StubGenStubId::jint_disjoint_arraycopy_id, &entry); | |
| StubRoutines::_jint_arraycopy = generate_conjoint_int_oop_copy(StubGenStubId::jint_arraycopy_id, entry, &entry_jint_arraycopy); | |
| StubRoutines::_jlong_disjoint_arraycopy = generate_disjoint_long_oop_copy(StubGenStubId::jlong_disjoint_arraycopy_id, &entry); | |
| StubRoutines::_jlong_arraycopy = generate_conjoint_long_oop_copy(StubGenStubId::jlong_arraycopy_id, entry, &entry_jlong_arraycopy); | |
| if (UseCompressedOops) { | |
| StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_int_oop_copy(StubGenStubId::oop_disjoint_arraycopy_id, &entry); | |
| StubRoutines::_oop_arraycopy = generate_conjoint_int_oop_copy(StubGenStubId::oop_arraycopy_id, entry, &entry_oop_arraycopy); | |
| StubRoutines::_oop_disjoint_arraycopy_uninit = generate_disjoint_int_oop_copy(StubGenStubId::oop_disjoint_arraycopy_uninit_id, &entry); | |
| StubRoutines::_oop_arraycopy_uninit = generate_conjoint_int_oop_copy(StubGenStubId::oop_arraycopy_uninit_id, entry, nullptr); | |
| } else { | |
| StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_long_oop_copy(StubGenStubId::oop_disjoint_arraycopy_id, &entry); | |
| StubRoutines::_oop_arraycopy = generate_conjoint_long_oop_copy(StubGenStubId::oop_arraycopy_id, entry, &entry_oop_arraycopy); | |
| StubRoutines::_oop_disjoint_arraycopy_uninit = generate_disjoint_long_oop_copy(StubGenStubId::oop_disjoint_arraycopy_uninit_id, &entry); | |
| StubRoutines::_oop_arraycopy_uninit = generate_conjoint_long_oop_copy(StubGenStubId::oop_arraycopy_uninit_id, entry, nullptr); | |
| } | |
| StubRoutines::_checkcast_arraycopy = generate_checkcast_copy(StubGenStubId::checkcast_arraycopy_id, &entry_checkcast_arraycopy); | |
| StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy(StubGenStubId::checkcast_arraycopy_uninit_id, nullptr); | |
| StubRoutines::_unsafe_arraycopy = generate_unsafe_copy(entry_jbyte_arraycopy, | |
| entry_jshort_arraycopy, | |
| entry_jint_arraycopy, | |
| entry_jlong_arraycopy); | |
| StubRoutines::_generic_arraycopy = generate_generic_copy(entry_jbyte_arraycopy, | |
| entry_jshort_arraycopy, | |
| entry_jint_arraycopy, | |
| entry_oop_arraycopy, | |
| entry_jlong_arraycopy, | |
| entry_checkcast_arraycopy); | |
| StubRoutines::_jbyte_fill = generate_fill(StubGenStubId::jbyte_fill_id); | |
| StubRoutines::_jshort_fill = generate_fill(StubGenStubId::jshort_fill_id); | |
| StubRoutines::_jint_fill = generate_fill(StubGenStubId::jint_fill_id); | |
| StubRoutines::_arrayof_jbyte_fill = generate_fill(StubGenStubId::arrayof_jbyte_fill_id); | |
| StubRoutines::_arrayof_jshort_fill = generate_fill(StubGenStubId::arrayof_jshort_fill_id); | |
| StubRoutines::_arrayof_jint_fill = generate_fill(StubGenStubId::arrayof_jint_fill_id); | |
| StubRoutines::_unsafe_setmemory = generate_unsafe_setmemory(StubRoutines::_jbyte_fill); | |
| // We don't generate specialized code for HeapWord-aligned source | |
| // arrays, so just use the code we've already generated | |
| StubRoutines::_arrayof_jbyte_disjoint_arraycopy = StubRoutines::_jbyte_disjoint_arraycopy; | |
| StubRoutines::_arrayof_jbyte_arraycopy = StubRoutines::_jbyte_arraycopy; | |
| StubRoutines::_arrayof_jshort_disjoint_arraycopy = StubRoutines::_jshort_disjoint_arraycopy; | |
| StubRoutines::_arrayof_jshort_arraycopy = StubRoutines::_jshort_arraycopy; | |
| StubRoutines::_arrayof_jint_disjoint_arraycopy = StubRoutines::_jint_disjoint_arraycopy; | |
| StubRoutines::_arrayof_jint_arraycopy = StubRoutines::_jint_arraycopy; | |
| StubRoutines::_arrayof_jlong_disjoint_arraycopy = StubRoutines::_jlong_disjoint_arraycopy; | |
| StubRoutines::_arrayof_jlong_arraycopy = StubRoutines::_jlong_arraycopy; | |
| StubRoutines::_arrayof_oop_disjoint_arraycopy = StubRoutines::_oop_disjoint_arraycopy; | |
| StubRoutines::_arrayof_oop_arraycopy = StubRoutines::_oop_arraycopy; | |
| StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit = StubRoutines::_oop_disjoint_arraycopy_uninit; | |
| StubRoutines::_arrayof_oop_arraycopy_uninit = StubRoutines::_oop_arraycopy_uninit; | |
| } | |
| // Verify that a register contains clean 32-bits positive value | |
| // (high 32-bits are 0) so it could be used in 64-bits shifts. | |
| // | |
| // Input: | |
| // Rint - 32-bits value | |
| // Rtmp - scratch | |
| // | |
| void StubGenerator::assert_clean_int(Register Rint, Register Rtmp) { | |
| #ifdef ASSERT | |
| Label L; | |
| assert_different_registers(Rtmp, Rint); | |
| __ movslq(Rtmp, Rint); | |
| __ cmpq(Rtmp, Rint); | |
| __ jcc(Assembler::equal, L); | |
| __ stop("high 32-bits of int value are not 0"); | |
| __ bind(L); | |
| #endif | |
| } | |
| // Generate overlap test for array copy stubs | |
| // | |
| // Input: | |
| // c_rarg0 - from | |
| // c_rarg1 - to | |
| // c_rarg2 - element count | |
| // | |
| // Output: | |
| // rax - &from[element count - 1] | |
| // | |
| void StubGenerator::array_overlap_test(address no_overlap_target, Label* NOLp, Address::ScaleFactor sf) { | |
| const Register from = c_rarg0; | |
| const Register to = c_rarg1; | |
| const Register count = c_rarg2; | |
| const Register end_from = rax; | |
| __ cmpptr(to, from); | |
| __ lea(end_from, Address(from, count, sf, 0)); | |
| if (NOLp == nullptr) { | |
| RuntimeAddress no_overlap(no_overlap_target); | |
| __ jump_cc(Assembler::belowEqual, no_overlap); | |
| __ cmpptr(to, end_from); | |
| __ jump_cc(Assembler::aboveEqual, no_overlap); | |
| } else { | |
| __ jcc(Assembler::belowEqual, (*NOLp)); | |
| __ cmpptr(to, end_from); | |
| __ jcc(Assembler::aboveEqual, (*NOLp)); | |
| } | |
| } | |
| // Copy big chunks forward | |
| // | |
| // Inputs: | |
| // end_from - source arrays end address | |
| // end_to - destination array end address | |
| // qword_count - 64-bits element count, negative | |
| // tmp1 - scratch | |
| // L_copy_bytes - entry label | |
| // L_copy_8_bytes - exit label | |
| // | |
| void StubGenerator::copy_bytes_forward(Register end_from, Register end_to, | |
| Register qword_count, Register tmp1, | |
| Register tmp2, Label& L_copy_bytes, | |
| Label& L_copy_8_bytes, DecoratorSet decorators, | |
| BasicType type) { | |
| BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); | |
| DEBUG_ONLY(__ stop("enter at entry label, not here")); | |
| Label L_loop; | |
| __ align(OptoLoopAlignment); | |
| if (UseUnalignedLoadStores) { | |
| Label L_end; | |
| __ BIND(L_loop); | |
| if (UseAVX >= 2) { | |
| bs->copy_load_at(_masm, decorators, type, 32, | |
| xmm0, Address(end_from, qword_count, Address::times_8, -56), | |
| tmp1, xmm1); | |
| bs->copy_store_at(_masm, decorators, type, 32, | |
| Address(end_to, qword_count, Address::times_8, -56), xmm0, | |
| tmp1, tmp2, xmm1); | |
| bs->copy_load_at(_masm, decorators, type, 32, | |
| xmm0, Address(end_from, qword_count, Address::times_8, -24), | |
| tmp1, xmm1); | |
| bs->copy_store_at(_masm, decorators, type, 32, | |
| Address(end_to, qword_count, Address::times_8, -24), xmm0, | |
| tmp1, tmp2, xmm1); | |
| } else { | |
| bs->copy_load_at(_masm, decorators, type, 16, | |
| xmm0, Address(end_from, qword_count, Address::times_8, -56), | |
| tmp1, xmm1); | |
| bs->copy_store_at(_masm, decorators, type, 16, | |
| Address(end_to, qword_count, Address::times_8, -56), xmm0, | |
| tmp1, tmp2, xmm1); | |
| bs->copy_load_at(_masm, decorators, type, 16, | |
| xmm0, Address(end_from, qword_count, Address::times_8, -40), | |
| tmp1, xmm1); | |
| bs->copy_store_at(_masm, decorators, type, 16, | |
| Address(end_to, qword_count, Address::times_8, -40), xmm0, | |
| tmp1, tmp2, xmm1); | |
| bs->copy_load_at(_masm, decorators, type, 16, | |
| xmm0, Address(end_from, qword_count, Address::times_8, -24), | |
| tmp1, xmm1); | |
| bs->copy_store_at(_masm, decorators, type, 16, | |
| Address(end_to, qword_count, Address::times_8, -24), xmm0, | |
| tmp1, tmp2, xmm1); | |
| bs->copy_load_at(_masm, decorators, type, 16, | |
| xmm0, Address(end_from, qword_count, Address::times_8, -8), | |
| tmp1, xmm1); | |
| bs->copy_store_at(_masm, decorators, type, 16, | |
| Address(end_to, qword_count, Address::times_8, -8), xmm0, | |
| tmp1, tmp2, xmm1); | |
| } | |
| __ BIND(L_copy_bytes); | |
| __ addptr(qword_count, 8); | |
| __ jcc(Assembler::lessEqual, L_loop); | |
| __ subptr(qword_count, 4); // sub(8) and add(4) | |
| __ jcc(Assembler::greater, L_end); | |
| // Copy trailing 32 bytes | |
| if (UseAVX >= 2) { | |
| bs->copy_load_at(_masm, decorators, type, 32, | |
| xmm0, Address(end_from, qword_count, Address::times_8, -24), | |
| tmp1, xmm1); | |
| bs->copy_store_at(_masm, decorators, type, 32, | |
| Address(end_to, qword_count, Address::times_8, -24), xmm0, | |
| tmp1, tmp2, xmm1); | |
| } else { | |
| bs->copy_load_at(_masm, decorators, type, 16, | |
| xmm0, Address(end_from, qword_count, Address::times_8, -24), | |
| tmp1, xmm1); | |
| bs->copy_store_at(_masm, decorators, type, 16, | |
| Address(end_to, qword_count, Address::times_8, -24), xmm0, | |
| tmp1, tmp2, xmm1); | |
| bs->copy_load_at(_masm, decorators, type, 16, | |
| xmm0, Address(end_from, qword_count, Address::times_8, -8), | |
| tmp1, xmm1); | |
| bs->copy_store_at(_masm, decorators, type, 16, | |
| Address(end_to, qword_count, Address::times_8, -8), xmm0, | |
| tmp1, tmp2, xmm1); | |
| } | |
| __ addptr(qword_count, 4); | |
| __ BIND(L_end); | |
| } else { | |
| // Copy 32-bytes per iteration | |
| __ BIND(L_loop); | |
| bs->copy_load_at(_masm, decorators, type, 8, | |
| tmp1, Address(end_from, qword_count, Address::times_8, -24), | |
| tmp2); | |
| bs->copy_store_at(_masm, decorators, type, 8, | |
| Address(end_to, qword_count, Address::times_8, -24), tmp1, | |
| tmp2); | |
| bs->copy_load_at(_masm, decorators, type, 8, | |
| tmp1, Address(end_from, qword_count, Address::times_8, -16), | |
| tmp2); | |
| bs->copy_store_at(_masm, decorators, type, 8, | |
| Address(end_to, qword_count, Address::times_8, -16), tmp1, | |
| tmp2); | |
| bs->copy_load_at(_masm, decorators, type, 8, | |
| tmp1, Address(end_from, qword_count, Address::times_8, -8), | |
| tmp2); | |
| bs->copy_store_at(_masm, decorators, type, 8, | |
| Address(end_to, qword_count, Address::times_8, -8), tmp1, | |
| tmp2); | |
| bs->copy_load_at(_masm, decorators, type, 8, | |
| tmp1, Address(end_from, qword_count, Address::times_8, 0), | |
| tmp2); | |
| bs->copy_store_at(_masm, decorators, type, 8, | |
| Address(end_to, qword_count, Address::times_8, 0), tmp1, | |
| tmp2); | |
| __ BIND(L_copy_bytes); | |
| __ addptr(qword_count, 4); | |
| __ jcc(Assembler::lessEqual, L_loop); | |
| } | |
| __ subptr(qword_count, 4); | |
| __ jcc(Assembler::less, L_copy_8_bytes); // Copy trailing qwords | |
| } | |
| // Copy big chunks backward | |
| // | |
| // Inputs: | |
| // from - source arrays address | |
| // dest - destination array address | |
| // qword_count - 64-bits element count | |
| // tmp1 - scratch | |
| // L_copy_bytes - entry label | |
| // L_copy_8_bytes - exit label | |
| // | |
| void StubGenerator::copy_bytes_backward(Register from, Register dest, | |
| Register qword_count, Register tmp1, | |
| Register tmp2, Label& L_copy_bytes, | |
| Label& L_copy_8_bytes, DecoratorSet decorators, | |
| BasicType type) { | |
| BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); | |
| DEBUG_ONLY(__ stop("enter at entry label, not here")); | |
| Label L_loop; | |
| __ align(OptoLoopAlignment); | |
| if (UseUnalignedLoadStores) { | |
| Label L_end; | |
| __ BIND(L_loop); | |
| if (UseAVX >= 2) { | |
| bs->copy_load_at(_masm, decorators, type, 32, | |
| xmm0, Address(from, qword_count, Address::times_8, 32), | |
| tmp1, xmm1); | |
| bs->copy_store_at(_masm, decorators, type, 32, | |
| Address(dest, qword_count, Address::times_8, 32), xmm0, | |
| tmp1, tmp2, xmm1); | |
| bs->copy_load_at(_masm, decorators, type, 32, | |
| xmm0, Address(from, qword_count, Address::times_8, 0), | |
| tmp1, xmm1); | |
| bs->copy_store_at(_masm, decorators, type, 32, | |
| Address(dest, qword_count, Address::times_8, 0), xmm0, | |
| tmp1, tmp2, xmm1); | |
| } else { | |
| bs->copy_load_at(_masm, decorators, type, 16, | |
| xmm0, Address(from, qword_count, Address::times_8, 48), | |
| tmp1, xmm1); | |
| bs->copy_store_at(_masm, decorators, type, 16, | |
| Address(dest, qword_count, Address::times_8, 48), xmm0, | |
| tmp1, tmp2, xmm1); | |
| bs->copy_load_at(_masm, decorators, type, 16, | |
| xmm0, Address(from, qword_count, Address::times_8, 32), | |
| tmp1, xmm1); | |
| bs->copy_store_at(_masm, decorators, type, 16, | |
| Address(dest, qword_count, Address::times_8, 32), xmm0, | |
| tmp1, tmp2, xmm1); | |
| bs->copy_load_at(_masm, decorators, type, 16, | |
| xmm0, Address(from, qword_count, Address::times_8, 16), | |
| tmp1, xmm1); | |
| bs->copy_store_at(_masm, decorators, type, 16, | |
| Address(dest, qword_count, Address::times_8, 16), xmm0, | |
| tmp1, tmp2, xmm1); | |
| bs->copy_load_at(_masm, decorators, type, 16, | |
| xmm0, Address(from, qword_count, Address::times_8, 0), | |
| tmp1, xmm1); | |
| bs->copy_store_at(_masm, decorators, type, 16, | |
| Address(dest, qword_count, Address::times_8, 0), xmm0, | |
| tmp1, tmp2, xmm1); | |
| } | |
| __ BIND(L_copy_bytes); | |
| __ subptr(qword_count, 8); | |
| __ jcc(Assembler::greaterEqual, L_loop); | |
| __ addptr(qword_count, 4); // add(8) and sub(4) | |
| __ jcc(Assembler::less, L_end); | |
| // Copy trailing 32 bytes | |
| if (UseAVX >= 2) { | |
| bs->copy_load_at(_masm, decorators, type, 32, | |
| xmm0, Address(from, qword_count, Address::times_8, 0), | |
| tmp1, xmm1); | |
| bs->copy_store_at(_masm, decorators, type, 32, | |
| Address(dest, qword_count, Address::times_8, 0), xmm0, | |
| tmp1, tmp2, xmm1); | |
| } else { | |
| bs->copy_load_at(_masm, decorators, type, 16, | |
| xmm0, Address(from, qword_count, Address::times_8, 16), | |
| tmp1, xmm1); | |
| bs->copy_store_at(_masm, decorators, type, 16, | |
| Address(dest, qword_count, Address::times_8, 16), xmm0, | |
| tmp1, tmp2, xmm1); | |
| bs->copy_load_at(_masm, decorators, type, 16, | |
| xmm0, Address(from, qword_count, Address::times_8, 0), | |
| tmp1, xmm1); | |
| bs->copy_store_at(_masm, decorators, type, 16, | |
| Address(dest, qword_count, Address::times_8, 0), xmm0, | |
| tmp1, tmp2, xmm1); | |
| } | |
| __ subptr(qword_count, 4); | |
| __ BIND(L_end); | |
| } else { | |
| // Copy 32-bytes per iteration | |
| __ BIND(L_loop); | |
| bs->copy_load_at(_masm, decorators, type, 8, | |
| tmp1, Address(from, qword_count, Address::times_8, 24), | |
| tmp2); | |
| bs->copy_store_at(_masm, decorators, type, 8, | |
| Address(dest, qword_count, Address::times_8, 24), tmp1, | |
| tmp2); | |
| bs->copy_load_at(_masm, decorators, type, 8, | |
| tmp1, Address(from, qword_count, Address::times_8, 16), | |
| tmp2); | |
| bs->copy_store_at(_masm, decorators, type, 8, | |
| Address(dest, qword_count, Address::times_8, 16), tmp1, | |
| tmp2); | |
| bs->copy_load_at(_masm, decorators, type, 8, | |
| tmp1, Address(from, qword_count, Address::times_8, 8), | |
| tmp2); | |
| bs->copy_store_at(_masm, decorators, type, 8, | |
| Address(dest, qword_count, Address::times_8, 8), tmp1, | |
| tmp2); | |
| bs->copy_load_at(_masm, decorators, type, 8, | |
| tmp1, Address(from, qword_count, Address::times_8, 0), | |
| tmp2); | |
| bs->copy_store_at(_masm, decorators, type, 8, | |
| Address(dest, qword_count, Address::times_8, 0), tmp1, | |
| tmp2); | |
| __ BIND(L_copy_bytes); | |
| __ subptr(qword_count, 4); | |
| __ jcc(Assembler::greaterEqual, L_loop); | |
| } | |
| __ addptr(qword_count, 4); | |
| __ jcc(Assembler::greater, L_copy_8_bytes); // Copy trailing qwords | |
| } | |
| #if COMPILER2_OR_JVMCI | |
| // Note: Following rules apply to AVX3 optimized arraycopy stubs:- | |
| // - If target supports AVX3 features (BW+VL+F) then implementation uses 32 byte vectors (YMMs) | |
| // for both special cases (various small block sizes) and aligned copy loop. This is the | |
| // default configuration. | |
| // - If copy length is above AVX3Threshold, then implementation use 64 byte vectors (ZMMs) | |
| // for main copy loop (and subsequent tail) since bulk of the cycles will be consumed in it. | |
| // - If user forces MaxVectorSize=32 then above 4096 bytes its seen that REP MOVs shows a | |
| // better performance for disjoint copies. For conjoint/backward copy vector based | |
| // copy performs better. | |
| // - If user sets AVX3Threshold=0, then special cases for small blocks sizes operate over | |
| // 64 byte vector registers (ZMMs). | |
| // Inputs: | |
| // c_rarg0 - source array address | |
| // c_rarg1 - destination array address | |
| // c_rarg2 - element count, treated as ssize_t, can be zero | |
| // | |
| // | |
| // Side Effects: | |
| // disjoint_copy_avx3_masked is set to the no-overlap entry point | |
| // used by generate_conjoint_[byte/int/short/long]_copy(). | |
| // | |
| address StubGenerator::generate_disjoint_copy_avx3_masked(StubGenStubId stub_id, address* entry) { | |
| // aligned is always false -- x86_64 always uses the unaligned code | |
| const bool aligned = false; | |
| int shift; | |
| bool is_oop; | |
| bool dest_uninitialized; | |
| switch (stub_id) { | |
| case jbyte_disjoint_arraycopy_id: | |
| shift = 0; | |
| is_oop = false; | |
| dest_uninitialized = false; | |
| break; | |
| case jshort_disjoint_arraycopy_id: | |
| shift = 1; | |
| is_oop = false; | |
| dest_uninitialized = false; | |
| break; | |
| case jint_disjoint_arraycopy_id: | |
| shift = 2; | |
| is_oop = false; | |
| dest_uninitialized = false; | |
| break; | |
| case jlong_disjoint_arraycopy_id: | |
| shift = 3; | |
| is_oop = false; | |
| dest_uninitialized = false; | |
| break; | |
| case oop_disjoint_arraycopy_id: | |
| shift = (UseCompressedOops ? 2 : 3); | |
| is_oop = true; | |
| dest_uninitialized = false; | |
| break; | |
| case oop_disjoint_arraycopy_uninit_id: | |
| shift = (UseCompressedOops ? 2 : 3); | |
| is_oop = true; | |
| dest_uninitialized = true; | |
| break; | |
| default: | |
| ShouldNotReachHere(); | |
| } | |
| __ align(CodeEntryAlignment); | |
| StubCodeMark mark(this, stub_id); | |
| address start = __ pc(); | |
| int avx3threshold = VM_Version::avx3_threshold(); | |
| bool use64byteVector = (MaxVectorSize > 32) && (avx3threshold == 0); | |
| const int large_threshold = 2621440; // 2.5 MB | |
| Label L_main_loop, L_main_loop_64bytes, L_tail, L_tail64, L_exit, L_entry; | |
| Label L_repmovs, L_main_pre_loop, L_main_pre_loop_64bytes, L_pre_main_post_64; | |
| Label L_copy_large, L_finish; | |
| const Register from = rdi; // source array address | |
| const Register to = rsi; // destination array address | |
| const Register count = rdx; // elements count | |
| const Register temp1 = r8; | |
| const Register temp2 = r11; | |
| const Register temp3 = rax; | |
| const Register temp4 = rcx; | |
| // End pointers are inclusive, and if count is not zero they point | |
| // to the last unit copied: end_to[0] := end_from[0] | |
| __ enter(); // required for proper stackwalking of RuntimeStub frame | |
| assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | |
| if (entry != nullptr) { | |
| *entry = __ pc(); | |
| // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | |
| BLOCK_COMMENT("Entry:"); | |
| } | |
| BasicType type_vec[] = { T_BYTE, T_SHORT, T_INT, T_LONG}; | |
| BasicType type = is_oop ? T_OBJECT : type_vec[shift]; | |
| setup_argument_regs(type); | |
| DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT; | |
| if (dest_uninitialized) { | |
| decorators |= IS_DEST_UNINITIALIZED; | |
| } | |
| if (aligned) { | |
| decorators |= ARRAYCOPY_ALIGNED; | |
| } | |
| BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); | |
| bs->arraycopy_prologue(_masm, decorators, type, from, to, count); | |
| { | |
| // Type(shift) byte(0), short(1), int(2), long(3) | |
| int loop_size[] = { 192, 96, 48, 24}; | |
| int threshold[] = { 4096, 2048, 1024, 512}; | |
| // UnsafeMemoryAccess page error: continue after unsafe access | |
| UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); | |
| // 'from', 'to' and 'count' are now valid | |
| // temp1 holds remaining count and temp4 holds running count used to compute | |
| // next address offset for start of to/from addresses (temp4 * scale). | |
| __ mov64(temp4, 0); | |
| __ movq(temp1, count); | |
| // Zero length check. | |
| __ BIND(L_tail); | |
| __ cmpq(temp1, 0); | |
| __ jcc(Assembler::lessEqual, L_exit); | |
| // Special cases using 32 byte [masked] vector copy operations. | |
| arraycopy_avx3_special_cases(xmm1, k2, from, to, temp1, shift, | |
| temp4, temp3, use64byteVector, L_entry, L_exit); | |
| // PRE-MAIN-POST loop for aligned copy. | |
| __ BIND(L_entry); | |
| if (MaxVectorSize == 64) { | |
| __ movq(temp2, temp1); | |
| __ shlq(temp2, shift); | |
| __ cmpq(temp2, large_threshold); | |
| __ jcc(Assembler::greaterEqual, L_copy_large); | |
| } | |
| if (avx3threshold != 0) { | |
| __ cmpq(count, threshold[shift]); | |
| if (MaxVectorSize == 64) { | |
| // Copy using 64 byte vectors. | |
| __ jcc(Assembler::greaterEqual, L_pre_main_post_64); | |
| } else { | |
| assert(MaxVectorSize < 64, "vector size should be < 64 bytes"); | |
| // REP MOVS offer a faster copy path. | |
| __ jcc(Assembler::greaterEqual, L_repmovs); | |
| } | |
| } | |
| if ((MaxVectorSize < 64) || (avx3threshold != 0)) { | |
| // Partial copy to make dst address 32 byte aligned. | |
| __ movq(temp2, to); | |
| __ andq(temp2, 31); | |
| __ jcc(Assembler::equal, L_main_pre_loop); | |
| __ negptr(temp2); | |
| __ addq(temp2, 32); | |
| if (shift) { | |
| __ shrq(temp2, shift); | |
| } | |
| __ movq(temp3, temp2); | |
| copy32_masked_avx(to, from, xmm1, k2, temp3, temp4, temp1, shift); | |
| __ movq(temp4, temp2); | |
| __ movq(temp1, count); | |
| __ subq(temp1, temp2); | |
| __ cmpq(temp1, loop_size[shift]); | |
| __ jcc(Assembler::less, L_tail); | |
| __ BIND(L_main_pre_loop); | |
| __ subq(temp1, loop_size[shift]); | |
| // Main loop with aligned copy block size of 192 bytes at 32 byte granularity. | |
| __ align32(); | |
| __ BIND(L_main_loop); | |
| copy64_avx(to, from, temp4, xmm1, false, shift, 0); | |
| copy64_avx(to, from, temp4, xmm1, false, shift, 64); | |
| copy64_avx(to, from, temp4, xmm1, false, shift, 128); | |
| __ addptr(temp4, loop_size[shift]); | |
| __ subq(temp1, loop_size[shift]); | |
| __ jcc(Assembler::greater, L_main_loop); | |
| __ addq(temp1, loop_size[shift]); | |
| // Tail loop. | |
| __ jmp(L_tail); | |
| __ BIND(L_repmovs); | |
| __ movq(temp2, temp1); | |
| // Swap to(RSI) and from(RDI) addresses to comply with REP MOVs semantics. | |
| __ movq(temp3, to); | |
| __ movq(to, from); | |
| __ movq(from, temp3); | |
| // Save to/from for restoration post rep_mov. | |
| __ movq(temp1, to); | |
| __ movq(temp3, from); | |
| if(shift < 3) { | |
| __ shrq(temp2, 3-shift); // quad word count | |
| } | |
| __ movq(temp4 , temp2); // move quad ward count into temp4(RCX). | |
| __ rep_mov(); | |
| __ shlq(temp2, 3); // convert quad words into byte count. | |
| if(shift) { | |
| __ shrq(temp2, shift); // type specific count. | |
| } | |
| // Restore original addresses in to/from. | |
| __ movq(to, temp3); | |
| __ movq(from, temp1); | |
| __ movq(temp4, temp2); | |
| __ movq(temp1, count); | |
| __ subq(temp1, temp2); // tailing part (less than a quad ward size). | |
| __ jmp(L_tail); | |
| } | |
| if (MaxVectorSize > 32) { | |
| __ BIND(L_pre_main_post_64); | |
| // Partial copy to make dst address 64 byte aligned. | |
| __ movq(temp2, to); | |
| __ andq(temp2, 63); | |
| __ jcc(Assembler::equal, L_main_pre_loop_64bytes); | |
| __ negptr(temp2); | |
| __ addq(temp2, 64); | |
| if (shift) { | |
| __ shrq(temp2, shift); | |
| } | |
| __ movq(temp3, temp2); | |
| copy64_masked_avx(to, from, xmm1, k2, temp3, temp4, temp1, shift, 0 , true); | |
| __ movq(temp4, temp2); | |
| __ movq(temp1, count); | |
| __ subq(temp1, temp2); | |
| __ cmpq(temp1, loop_size[shift]); | |
| __ jcc(Assembler::less, L_tail64); | |
| __ BIND(L_main_pre_loop_64bytes); | |
| __ subq(temp1, loop_size[shift]); | |
| // Main loop with aligned copy block size of 192 bytes at | |
| // 64 byte copy granularity. | |
| __ align32(); | |
| __ BIND(L_main_loop_64bytes); | |
| copy64_avx(to, from, temp4, xmm1, false, shift, 0 , true); | |
| copy64_avx(to, from, temp4, xmm1, false, shift, 64, true); | |
| copy64_avx(to, from, temp4, xmm1, false, shift, 128, true); | |
| __ addptr(temp4, loop_size[shift]); | |
| __ subq(temp1, loop_size[shift]); | |
| __ jcc(Assembler::greater, L_main_loop_64bytes); | |
| __ addq(temp1, loop_size[shift]); | |
| // Zero length check. | |
| __ jcc(Assembler::lessEqual, L_exit); | |
| __ BIND(L_tail64); | |
| // Tail handling using 64 byte [masked] vector copy operations. | |
| use64byteVector = true; | |
| arraycopy_avx3_special_cases(xmm1, k2, from, to, temp1, shift, | |
| temp4, temp3, use64byteVector, L_entry, L_exit); | |
| } | |
| __ BIND(L_exit); | |
| } | |
| __ BIND(L_finish); | |
| address ucme_exit_pc = __ pc(); | |
| // When called from generic_arraycopy r11 contains specific values | |
| // used during arraycopy epilogue, re-initializing r11. | |
| if (is_oop) { | |
| __ movq(r11, shift == 3 ? count : to); | |
| } | |
| bs->arraycopy_epilogue(_masm, decorators, type, from, to, count); | |
| restore_argument_regs(type); | |
| INC_COUNTER_NP(get_profile_ctr(shift), rscratch1); // Update counter after rscratch1 is free | |
| __ xorptr(rax, rax); // return 0 | |
| __ vzeroupper(); | |
| __ leave(); // required for proper stackwalking of RuntimeStub frame | |
| __ ret(0); | |
| if (MaxVectorSize == 64) { | |
| __ BIND(L_copy_large); | |
| UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, false, ucme_exit_pc); | |
| arraycopy_avx3_large(to, from, temp1, temp2, temp3, temp4, count, xmm1, xmm2, xmm3, xmm4, shift); | |
| __ jmp(L_finish); | |
| } | |
| return start; | |
| } | |
| void StubGenerator::arraycopy_avx3_large(Register to, Register from, Register temp1, Register temp2, | |
| Register temp3, Register temp4, Register count, | |
| XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3, | |
| XMMRegister xmm4, int shift) { | |
| // Type(shift) byte(0), short(1), int(2), long(3) | |
| int loop_size[] = { 256, 128, 64, 32}; | |
| int threshold[] = { 4096, 2048, 1024, 512}; | |
| Label L_main_loop_large; | |
| Label L_tail_large; | |
| Label L_exit_large; | |
| Label L_entry_large; | |
| Label L_main_pre_loop_large; | |
| Label L_pre_main_post_large; | |
| assert(MaxVectorSize == 64, "vector length != 64"); | |
| __ BIND(L_entry_large); | |
| __ BIND(L_pre_main_post_large); | |
| // Partial copy to make dst address 64 byte aligned. | |
| __ movq(temp2, to); | |
| __ andq(temp2, 63); | |
| __ jcc(Assembler::equal, L_main_pre_loop_large); | |
| __ negptr(temp2); | |
| __ addq(temp2, 64); | |
| if (shift) { | |
| __ shrq(temp2, shift); | |
| } | |
| __ movq(temp3, temp2); | |
| copy64_masked_avx(to, from, xmm1, k2, temp3, temp4, temp1, shift, 0, true); | |
| __ movq(temp4, temp2); | |
| __ movq(temp1, count); | |
| __ subq(temp1, temp2); | |
| __ cmpq(temp1, loop_size[shift]); | |
| __ jcc(Assembler::less, L_tail_large); | |
| __ BIND(L_main_pre_loop_large); | |
| __ subq(temp1, loop_size[shift]); | |
| // Main loop with aligned copy block size of 256 bytes at 64 byte copy granularity. | |
| __ align32(); | |
| __ BIND(L_main_loop_large); | |
| copy256_avx3(to, from, temp4, xmm1, xmm2, xmm3, xmm4, shift, 0); | |
| __ addptr(temp4, loop_size[shift]); | |
| __ subq(temp1, loop_size[shift]); | |
| __ jcc(Assembler::greater, L_main_loop_large); | |
| // fence needed because copy256_avx3 uses non-temporal stores | |
| __ sfence(); | |
| __ addq(temp1, loop_size[shift]); | |
| // Zero length check. | |
| __ jcc(Assembler::lessEqual, L_exit_large); | |
| __ BIND(L_tail_large); | |
| // Tail handling using 64 byte [masked] vector copy operations. | |
| __ cmpq(temp1, 0); | |
| __ jcc(Assembler::lessEqual, L_exit_large); | |
| arraycopy_avx3_special_cases_256(xmm1, k2, from, to, temp1, shift, | |
| temp4, temp3, L_exit_large); | |
| __ BIND(L_exit_large); | |
| } | |
| // Inputs: | |
| // c_rarg0 - source array address | |
| // c_rarg1 - destination array address | |
| // c_rarg2 - element count, treated as ssize_t, can be zero | |
| // | |
| // | |
| address StubGenerator::generate_conjoint_copy_avx3_masked(StubGenStubId stub_id, address* entry, address nooverlap_target) { | |
| // aligned is always false -- x86_64 always uses the unaligned code | |
| const bool aligned = false; | |
| int shift; | |
| bool is_oop; | |
| bool dest_uninitialized; | |
| switch (stub_id) { | |
| case jbyte_arraycopy_id: | |
| shift = 0; | |
| is_oop = false; | |
| dest_uninitialized = false; | |
| break; | |
| case jshort_arraycopy_id: | |
| shift = 1; | |
| is_oop = false; | |
| dest_uninitialized = false; | |
| break; | |
| case jint_arraycopy_id: | |
| shift = 2; | |
| is_oop = false; | |
| dest_uninitialized = false; | |
| break; | |
| case jlong_arraycopy_id: | |
| shift = 3; | |
| is_oop = false; | |
| dest_uninitialized = false; | |
| break; | |
| case oop_arraycopy_id: | |
| shift = (UseCompressedOops ? 2 : 3); | |
| is_oop = true; | |
| dest_uninitialized = false; | |
| break; | |
| case oop_arraycopy_uninit_id: | |
| shift = (UseCompressedOops ? 2 : 3); | |
| is_oop = true; | |
| dest_uninitialized = true; | |
| break; | |
| default: | |
| ShouldNotReachHere(); | |
| } | |
| __ align(CodeEntryAlignment); | |
| StubCodeMark mark(this, stub_id); | |
| address start = __ pc(); | |
| int avx3threshold = VM_Version::avx3_threshold(); | |
| bool use64byteVector = (MaxVectorSize > 32) && (avx3threshold == 0); | |
| Label L_main_pre_loop, L_main_pre_loop_64bytes, L_pre_main_post_64; | |
| Label L_main_loop, L_main_loop_64bytes, L_tail, L_tail64, L_exit, L_entry; | |
| const Register from = rdi; // source array address | |
| const Register to = rsi; // destination array address | |
| const Register count = rdx; // elements count | |
| const Register temp1 = r8; | |
| const Register temp2 = rcx; | |
| const Register temp3 = r11; | |
| const Register temp4 = rax; | |
| // End pointers are inclusive, and if count is not zero they point | |
| // to the last unit copied: end_to[0] := end_from[0] | |
| __ enter(); // required for proper stackwalking of RuntimeStub frame | |
| assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | |
| if (entry != nullptr) { | |
| *entry = __ pc(); | |
| // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | |
| BLOCK_COMMENT("Entry:"); | |
| } | |
| array_overlap_test(nooverlap_target, (Address::ScaleFactor)(shift)); | |
| BasicType type_vec[] = { T_BYTE, T_SHORT, T_INT, T_LONG}; | |
| BasicType type = is_oop ? T_OBJECT : type_vec[shift]; | |
| setup_argument_regs(type); | |
| DecoratorSet decorators = IN_HEAP | IS_ARRAY; | |
| if (dest_uninitialized) { | |
| decorators |= IS_DEST_UNINITIALIZED; | |
| } | |
| if (aligned) { | |
| decorators |= ARRAYCOPY_ALIGNED; | |
| } | |
| BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); | |
| bs->arraycopy_prologue(_masm, decorators, type, from, to, count); | |
| { | |
| // Type(shift) byte(0), short(1), int(2), long(3) | |
| int loop_size[] = { 192, 96, 48, 24}; | |
| int threshold[] = { 4096, 2048, 1024, 512}; | |
| // UnsafeMemoryAccess page error: continue after unsafe access | |
| UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); | |
| // 'from', 'to' and 'count' are now valid | |
| // temp1 holds remaining count. | |
| __ movq(temp1, count); | |
| // Zero length check. | |
| __ BIND(L_tail); | |
| __ cmpq(temp1, 0); | |
| __ jcc(Assembler::lessEqual, L_exit); | |
| __ mov64(temp2, 0); | |
| __ movq(temp3, temp1); | |
| // Special cases using 32 byte [masked] vector copy operations. | |
| arraycopy_avx3_special_cases_conjoint(xmm1, k2, from, to, temp2, temp3, temp1, shift, | |
| temp4, use64byteVector, L_entry, L_exit); | |
| // PRE-MAIN-POST loop for aligned copy. | |
| __ BIND(L_entry); | |
| if ((MaxVectorSize > 32) && (avx3threshold != 0)) { | |
| __ cmpq(temp1, threshold[shift]); | |
| __ jcc(Assembler::greaterEqual, L_pre_main_post_64); | |
| } | |
| if ((MaxVectorSize < 64) || (avx3threshold != 0)) { | |
| // Partial copy to make dst address 32 byte aligned. | |
| __ leaq(temp2, Address(to, temp1, (Address::ScaleFactor)(shift), 0)); | |
| __ andq(temp2, 31); | |
| __ jcc(Assembler::equal, L_main_pre_loop); | |
| if (shift) { | |
| __ shrq(temp2, shift); | |
| } | |
| __ subq(temp1, temp2); | |
| copy32_masked_avx(to, from, xmm1, k2, temp2, temp1, temp3, shift); | |
| __ cmpq(temp1, loop_size[shift]); | |
| __ jcc(Assembler::less, L_tail); | |
| __ BIND(L_main_pre_loop); | |
| // Main loop with aligned copy block size of 192 bytes at 32 byte granularity. | |
| __ align32(); | |
| __ BIND(L_main_loop); | |
| copy64_avx(to, from, temp1, xmm1, true, shift, -64); | |
| copy64_avx(to, from, temp1, xmm1, true, shift, -128); | |
| copy64_avx(to, from, temp1, xmm1, true, shift, -192); | |
| __ subptr(temp1, loop_size[shift]); | |
| __ cmpq(temp1, loop_size[shift]); | |
| __ jcc(Assembler::greater, L_main_loop); | |
| // Tail loop. | |
| __ jmp(L_tail); | |
| } | |
| if (MaxVectorSize > 32) { | |
| __ BIND(L_pre_main_post_64); | |
| // Partial copy to make dst address 64 byte aligned. | |
| __ leaq(temp2, Address(to, temp1, (Address::ScaleFactor)(shift), 0)); | |
| __ andq(temp2, 63); | |
| __ jcc(Assembler::equal, L_main_pre_loop_64bytes); | |
| if (shift) { | |
| __ shrq(temp2, shift); | |
| } | |
| __ subq(temp1, temp2); | |
| copy64_masked_avx(to, from, xmm1, k2, temp2, temp1, temp3, shift, 0 , true); | |
| __ cmpq(temp1, loop_size[shift]); | |
| __ jcc(Assembler::less, L_tail64); | |
| __ BIND(L_main_pre_loop_64bytes); | |
| // Main loop with aligned copy block size of 192 bytes at | |
| // 64 byte copy granularity. | |
| __ align32(); | |
| __ BIND(L_main_loop_64bytes); | |
| copy64_avx(to, from, temp1, xmm1, true, shift, -64 , true); | |
| copy64_avx(to, from, temp1, xmm1, true, shift, -128, true); | |
| copy64_avx(to, from, temp1, xmm1, true, shift, -192, true); | |
| __ subq(temp1, loop_size[shift]); | |
| __ cmpq(temp1, loop_size[shift]); | |
| __ jcc(Assembler::greater, L_main_loop_64bytes); | |
| // Zero length check. | |
| __ cmpq(temp1, 0); | |
| __ jcc(Assembler::lessEqual, L_exit); | |
| __ BIND(L_tail64); | |
| // Tail handling using 64 byte [masked] vector copy operations. | |
| use64byteVector = true; | |
| __ mov64(temp2, 0); | |
| __ movq(temp3, temp1); | |
| arraycopy_avx3_special_cases_conjoint(xmm1, k2, from, to, temp2, temp3, temp1, shift, | |
| temp4, use64byteVector, L_entry, L_exit); | |
| } | |
| __ BIND(L_exit); | |
| } | |
| address ucme_exit_pc = __ pc(); | |
| // When called from generic_arraycopy r11 contains specific values | |
| // used during arraycopy epilogue, re-initializing r11. | |
| if(is_oop) { | |
| __ movq(r11, count); | |
| } | |
| bs->arraycopy_epilogue(_masm, decorators, type, from, to, count); | |
| restore_argument_regs(type); | |
| INC_COUNTER_NP(get_profile_ctr(shift), rscratch1); // Update counter after rscratch1 is free | |
| __ xorptr(rax, rax); // return 0 | |
| __ vzeroupper(); | |
| __ leave(); // required for proper stackwalking of RuntimeStub frame | |
| __ ret(0); | |
| return start; | |
| } | |
| void StubGenerator::arraycopy_avx3_special_cases(XMMRegister xmm, KRegister mask, Register from, | |
| Register to, Register count, int shift, | |
| Register index, Register temp, | |
| bool use64byteVector, Label& L_entry, Label& L_exit) { | |
| Label L_entry_64, L_entry_96, L_entry_128; | |
| Label L_entry_160, L_entry_192; | |
| int size_mat[][6] = { | |
| /* T_BYTE */ {32 , 64, 96 , 128 , 160 , 192 }, | |
| /* T_SHORT*/ {16 , 32, 48 , 64 , 80 , 96 }, | |
| /* T_INT */ {8 , 16, 24 , 32 , 40 , 48 }, | |
| /* T_LONG */ {4 , 8, 12 , 16 , 20 , 24 } | |
| }; | |
| // Case A) Special case for length less than equal to 32 bytes. | |
| __ cmpq(count, size_mat[shift][0]); | |
| __ jccb(Assembler::greater, L_entry_64); | |
| copy32_masked_avx(to, from, xmm, mask, count, index, temp, shift); | |
| __ jmp(L_exit); | |
| // Case B) Special case for length less than equal to 64 bytes. | |
| __ BIND(L_entry_64); | |
| __ cmpq(count, size_mat[shift][1]); | |
| __ jccb(Assembler::greater, L_entry_96); | |
| copy64_masked_avx(to, from, xmm, mask, count, index, temp, shift, 0, use64byteVector); | |
| __ jmp(L_exit); | |
| // Case C) Special case for length less than equal to 96 bytes. | |
| __ BIND(L_entry_96); | |
| __ cmpq(count, size_mat[shift][2]); | |
| __ jccb(Assembler::greater, L_entry_128); | |
| copy64_avx(to, from, index, xmm, false, shift, 0, use64byteVector); | |
| __ subq(count, 64 >> shift); | |
| copy32_masked_avx(to, from, xmm, mask, count, index, temp, shift, 64); | |
| __ jmp(L_exit); | |
| // Case D) Special case for length less than equal to 128 bytes. | |
| __ BIND(L_entry_128); | |
| __ cmpq(count, size_mat[shift][3]); | |
| __ jccb(Assembler::greater, L_entry_160); | |
| copy64_avx(to, from, index, xmm, false, shift, 0, use64byteVector); | |
| copy32_avx(to, from, index, xmm, shift, 64); | |
| __ subq(count, 96 >> shift); | |
| copy32_masked_avx(to, from, xmm, mask, count, index, temp, shift, 96); | |
| __ jmp(L_exit); | |
| // Case E) Special case for length less than equal to 160 bytes. | |
| __ BIND(L_entry_160); | |
| __ cmpq(count, size_mat[shift][4]); | |
| __ jccb(Assembler::greater, L_entry_192); | |
| copy64_avx(to, from, index, xmm, false, shift, 0, use64byteVector); | |
| copy64_avx(to, from, index, xmm, false, shift, 64, use64byteVector); | |
| __ subq(count, 128 >> shift); | |
| copy32_masked_avx(to, from, xmm, mask, count, index, temp, shift, 128); | |
| __ jmp(L_exit); | |
| // Case F) Special case for length less than equal to 192 bytes. | |
| __ BIND(L_entry_192); | |
| __ cmpq(count, size_mat[shift][5]); | |
| __ jcc(Assembler::greater, L_entry); | |
| copy64_avx(to, from, index, xmm, false, shift, 0, use64byteVector); | |
| copy64_avx(to, from, index, xmm, false, shift, 64, use64byteVector); | |
| copy32_avx(to, from, index, xmm, shift, 128); | |
| __ subq(count, 160 >> shift); | |
| copy32_masked_avx(to, from, xmm, mask, count, index, temp, shift, 160); | |
| __ jmp(L_exit); | |
| } | |
| void StubGenerator::arraycopy_avx3_special_cases_256(XMMRegister xmm, KRegister mask, Register from, | |
| Register to, Register count, int shift, Register index, | |
| Register temp, Label& L_exit) { | |
| Label L_entry_64, L_entry_128, L_entry_192, L_entry_256; | |
| int size_mat[][4] = { | |
| /* T_BYTE */ {64, 128, 192, 256}, | |
| /* T_SHORT*/ {32, 64 , 96 , 128}, | |
| /* T_INT */ {16, 32 , 48 , 64}, | |
| /* T_LONG */ { 8, 16 , 24 , 32} | |
| }; | |
| assert(MaxVectorSize == 64, "vector length != 64"); | |
| // Case A) Special case for length less than or equal to 64 bytes. | |
| __ BIND(L_entry_64); | |
| __ cmpq(count, size_mat[shift][0]); | |
| __ jccb(Assembler::greater, L_entry_128); | |
| copy64_masked_avx(to, from, xmm, mask, count, index, temp, shift, 0, true); | |
| __ jmp(L_exit); | |
| // Case B) Special case for length less than or equal to 128 bytes. | |
| __ BIND(L_entry_128); | |
| __ cmpq(count, size_mat[shift][1]); | |
| __ jccb(Assembler::greater, L_entry_192); | |
| copy64_avx(to, from, index, xmm, false, shift, 0, true); | |
| __ subq(count, 64 >> shift); | |
| copy64_masked_avx(to, from, xmm, mask, count, index, temp, shift, 64, true); | |
| __ jmp(L_exit); | |
| // Case C) Special case for length less than or equal to 192 bytes. | |
| __ BIND(L_entry_192); | |
| __ cmpq(count, size_mat[shift][2]); | |
| __ jcc(Assembler::greater, L_entry_256); | |
| copy64_avx(to, from, index, xmm, false, shift, 0, true); | |
| copy64_avx(to, from, index, xmm, false, shift, 64, true); | |
| __ subq(count, 128 >> shift); | |
| copy64_masked_avx(to, from, xmm, mask, count, index, temp, shift, 128, true); | |
| __ jmp(L_exit); | |
| // Case D) Special case for length less than or equal to 256 bytes. | |
| __ BIND(L_entry_256); | |
| copy64_avx(to, from, index, xmm, false, shift, 0, true); | |
| copy64_avx(to, from, index, xmm, false, shift, 64, true); | |
| copy64_avx(to, from, index, xmm, false, shift, 128, true); | |
| __ subq(count, 192 >> shift); | |
| copy64_masked_avx(to, from, xmm, mask, count, index, temp, shift, 192, true); | |
| __ jmp(L_exit); | |
| } | |
| void StubGenerator::arraycopy_avx3_special_cases_conjoint(XMMRegister xmm, KRegister mask, Register from, | |
| Register to, Register start_index, Register end_index, | |
| Register count, int shift, Register temp, | |
| bool use64byteVector, Label& L_entry, Label& L_exit) { | |
| Label L_entry_64, L_entry_96, L_entry_128; | |
| Label L_entry_160, L_entry_192; | |
| bool avx3 = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0); | |
| int size_mat[][6] = { | |
| /* T_BYTE */ {32 , 64, 96 , 128 , 160 , 192 }, | |
| /* T_SHORT*/ {16 , 32, 48 , 64 , 80 , 96 }, | |
| /* T_INT */ {8 , 16, 24 , 32 , 40 , 48 }, | |
| /* T_LONG */ {4 , 8, 12 , 16 , 20 , 24 } | |
| }; | |
| // Case A) Special case for length less than equal to 32 bytes. | |
| __ cmpq(count, size_mat[shift][0]); | |
| __ jccb(Assembler::greater, L_entry_64); | |
| copy32_masked_avx(to, from, xmm, mask, count, start_index, temp, shift); | |
| __ jmp(L_exit); | |
| // Case B) Special case for length less than equal to 64 bytes. | |
| __ BIND(L_entry_64); | |
| __ cmpq(count, size_mat[shift][1]); | |
| __ jccb(Assembler::greater, L_entry_96); | |
| if (avx3) { | |
| copy64_masked_avx(to, from, xmm, mask, count, start_index, temp, shift, 0, true); | |
| } else { | |
| copy32_avx(to, from, end_index, xmm, shift, -32); | |
| __ subq(count, 32 >> shift); | |
| copy32_masked_avx(to, from, xmm, mask, count, start_index, temp, shift); | |
| } | |
| __ jmp(L_exit); | |
| // Case C) Special case for length less than equal to 96 bytes. | |
| __ BIND(L_entry_96); | |
| __ cmpq(count, size_mat[shift][2]); | |
| __ jccb(Assembler::greater, L_entry_128); | |
| copy64_avx(to, from, end_index, xmm, true, shift, -64, use64byteVector); | |
| __ subq(count, 64 >> shift); | |
| copy32_masked_avx(to, from, xmm, mask, count, start_index, temp, shift); | |
| __ jmp(L_exit); | |
| // Case D) Special case for length less than equal to 128 bytes. | |
| __ BIND(L_entry_128); | |
| __ cmpq(count, size_mat[shift][3]); | |
| __ jccb(Assembler::greater, L_entry_160); | |
| copy64_avx(to, from, end_index, xmm, true, shift, -64, use64byteVector); | |
| copy32_avx(to, from, end_index, xmm, shift, -96); | |
| __ subq(count, 96 >> shift); | |
| copy32_masked_avx(to, from, xmm, mask, count, start_index, temp, shift); | |
| __ jmp(L_exit); | |
| // Case E) Special case for length less than equal to 160 bytes. | |
| __ BIND(L_entry_160); | |
| __ cmpq(count, size_mat[shift][4]); | |
| __ jccb(Assembler::greater, L_entry_192); | |
| copy64_avx(to, from, end_index, xmm, true, shift, -64, use64byteVector); | |
| copy64_avx(to, from, end_index, xmm, true, shift, -128, use64byteVector); | |
| __ subq(count, 128 >> shift); | |
| copy32_masked_avx(to, from, xmm, mask, count, start_index, temp, shift); | |
| __ jmp(L_exit); | |
| // Case F) Special case for length less than equal to 192 bytes. | |
| __ BIND(L_entry_192); | |
| __ cmpq(count, size_mat[shift][5]); | |
| __ jcc(Assembler::greater, L_entry); | |
| copy64_avx(to, from, end_index, xmm, true, shift, -64, use64byteVector); | |
| copy64_avx(to, from, end_index, xmm, true, shift, -128, use64byteVector); | |
| copy32_avx(to, from, end_index, xmm, shift, -160); | |
| __ subq(count, 160 >> shift); | |
| copy32_masked_avx(to, from, xmm, mask, count, start_index, temp, shift); | |
| __ jmp(L_exit); | |
| } | |
| void StubGenerator::copy256_avx3(Register dst, Register src, Register index, XMMRegister xmm1, | |
| XMMRegister xmm2, XMMRegister xmm3, XMMRegister xmm4, | |
| int shift, int offset) { | |
| if (MaxVectorSize == 64) { | |
| Address::ScaleFactor scale = (Address::ScaleFactor)(shift); | |
| __ prefetcht0(Address(src, index, scale, offset + 0x200)); | |
| __ prefetcht0(Address(src, index, scale, offset + 0x240)); | |
| __ prefetcht0(Address(src, index, scale, offset + 0x280)); | |
| __ prefetcht0(Address(src, index, scale, offset + 0x2C0)); | |
| __ prefetcht0(Address(src, index, scale, offset + 0x400)); | |
| __ prefetcht0(Address(src, index, scale, offset + 0x440)); | |
| __ prefetcht0(Address(src, index, scale, offset + 0x480)); | |
| __ prefetcht0(Address(src, index, scale, offset + 0x4C0)); | |
| __ evmovdquq(xmm1, Address(src, index, scale, offset), Assembler::AVX_512bit); | |
| __ evmovdquq(xmm2, Address(src, index, scale, offset + 0x40), Assembler::AVX_512bit); | |
| __ evmovdquq(xmm3, Address(src, index, scale, offset + 0x80), Assembler::AVX_512bit); | |
| __ evmovdquq(xmm4, Address(src, index, scale, offset + 0xC0), Assembler::AVX_512bit); | |
| __ evmovntdquq(Address(dst, index, scale, offset), xmm1, Assembler::AVX_512bit); | |
| __ evmovntdquq(Address(dst, index, scale, offset + 0x40), xmm2, Assembler::AVX_512bit); | |
| __ evmovntdquq(Address(dst, index, scale, offset + 0x80), xmm3, Assembler::AVX_512bit); | |
| __ evmovntdquq(Address(dst, index, scale, offset + 0xC0), xmm4, Assembler::AVX_512bit); | |
| } | |
| } | |
| void StubGenerator::copy64_masked_avx(Register dst, Register src, XMMRegister xmm, | |
| KRegister mask, Register length, Register index, | |
| Register temp, int shift, int offset, | |
| bool use64byteVector) { | |
| BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG}; | |
| assert(MaxVectorSize >= 32, "vector length should be >= 32"); | |
| if (!use64byteVector) { | |
| copy32_avx(dst, src, index, xmm, shift, offset); | |
| __ subptr(length, 32 >> shift); | |
| copy32_masked_avx(dst, src, xmm, mask, length, index, temp, shift, offset+32); | |
| } else { | |
| Address::ScaleFactor scale = (Address::ScaleFactor)(shift); | |
| assert(MaxVectorSize == 64, "vector length != 64"); | |
| __ mov64(temp, -1L); | |
| __ bzhiq(temp, temp, length); | |
| __ kmovql(mask, temp); | |
| __ evmovdqu(type[shift], mask, xmm, Address(src, index, scale, offset), false, Assembler::AVX_512bit); | |
| __ evmovdqu(type[shift], mask, Address(dst, index, scale, offset), xmm, true, Assembler::AVX_512bit); | |
| } | |
| } | |
| void StubGenerator::copy32_masked_avx(Register dst, Register src, XMMRegister xmm, | |
| KRegister mask, Register length, Register index, | |
| Register temp, int shift, int offset) { | |
| assert(MaxVectorSize >= 32, "vector length should be >= 32"); | |
| BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG}; | |
| Address::ScaleFactor scale = (Address::ScaleFactor)(shift); | |
| __ mov64(temp, -1L); | |
| __ bzhiq(temp, temp, length); | |
| __ kmovql(mask, temp); | |
| __ evmovdqu(type[shift], mask, xmm, Address(src, index, scale, offset), false, Assembler::AVX_256bit); | |
| __ evmovdqu(type[shift], mask, Address(dst, index, scale, offset), xmm, true, Assembler::AVX_256bit); | |
| } | |
| void StubGenerator::copy32_avx(Register dst, Register src, Register index, XMMRegister xmm, | |
| int shift, int offset) { | |
| assert(MaxVectorSize >= 32, "vector length should be >= 32"); | |
| Address::ScaleFactor scale = (Address::ScaleFactor)(shift); | |
| __ vmovdqu(xmm, Address(src, index, scale, offset)); | |
| __ vmovdqu(Address(dst, index, scale, offset), xmm); | |
| } | |
| void StubGenerator::copy64_avx(Register dst, Register src, Register index, XMMRegister xmm, | |
| bool conjoint, int shift, int offset, bool use64byteVector) { | |
| assert(MaxVectorSize == 64 || MaxVectorSize == 32, "vector length mismatch"); | |
| if (!use64byteVector) { | |
| if (conjoint) { | |
| copy32_avx(dst, src, index, xmm, shift, offset+32); | |
| copy32_avx(dst, src, index, xmm, shift, offset); | |
| } else { | |
| copy32_avx(dst, src, index, xmm, shift, offset); | |
| copy32_avx(dst, src, index, xmm, shift, offset+32); | |
| } | |
| } else { | |
| Address::ScaleFactor scale = (Address::ScaleFactor)(shift); | |
| __ evmovdquq(xmm, Address(src, index, scale, offset), Assembler::AVX_512bit); | |
| __ evmovdquq(Address(dst, index, scale, offset), xmm, Assembler::AVX_512bit); | |
| } | |
| } | |
| #endif // COMPILER2_OR_JVMCI | |
| // Arguments: | |
| // entry - location for return of (post-push) entry | |
| // | |
| // Inputs: | |
| // c_rarg0 - source array address | |
| // c_rarg1 - destination array address | |
| // c_rarg2 - element count, treated as ssize_t, can be zero | |
| // | |
| // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries, | |
| // we let the hardware handle it. The one to eight bytes within words, | |
| // dwords or qwords that span cache line boundaries will still be loaded | |
| // and stored atomically. | |
| // | |
| // Side Effects: | |
| // entry is set to the no-overlap entry point | |
| // used by generate_conjoint_byte_copy(). | |
| // | |
| address StubGenerator::generate_disjoint_byte_copy(address* entry) { | |
| StubGenStubId stub_id = StubGenStubId::jbyte_disjoint_arraycopy_id; | |
| // aligned is always false -- x86_64 always uses the unaligned code | |
| const bool aligned = false; | |
| #if COMPILER2_OR_JVMCI | |
| if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) { | |
| return generate_disjoint_copy_avx3_masked(stub_id, entry); | |
| } | |
| #endif | |
| __ align(CodeEntryAlignment); | |
| StubCodeMark mark(this, stub_id); | |
| address start = __ pc(); | |
| DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT; | |
| Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes; | |
| Label L_copy_byte, L_exit; | |
| const Register from = rdi; // source array address | |
| const Register to = rsi; // destination array address | |
| const Register count = rdx; // elements count | |
| const Register byte_count = rcx; | |
| const Register qword_count = count; | |
| const Register end_from = from; // source array end address | |
| const Register end_to = to; // destination array end address | |
| // End pointers are inclusive, and if count is not zero they point | |
| // to the last unit copied: end_to[0] := end_from[0] | |
| __ enter(); // required for proper stackwalking of RuntimeStub frame | |
| assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | |
| if (entry != nullptr) { | |
| *entry = __ pc(); | |
| // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | |
| BLOCK_COMMENT("Entry:"); | |
| } | |
| setup_arg_regs(); // from => rdi, to => rsi, count => rdx | |
| // r9 and r10 may be used to save non-volatile registers | |
| { | |
| // UnsafeMemoryAccess page error: continue after unsafe access | |
| UnsafeMemoryAccessMark umam(this, !aligned, true); | |
| // 'from', 'to' and 'count' are now valid | |
| __ movptr(byte_count, count); | |
| __ shrptr(count, 3); // count => qword_count | |
| // Copy from low to high addresses. Use 'to' as scratch. | |
| __ lea(end_from, Address(from, qword_count, Address::times_8, -8)); | |
| __ lea(end_to, Address(to, qword_count, Address::times_8, -8)); | |
| __ negptr(qword_count); // make the count negative | |
| __ jmp(L_copy_bytes); | |
| // Copy trailing qwords | |
| __ BIND(L_copy_8_bytes); | |
| __ movq(rax, Address(end_from, qword_count, Address::times_8, 8)); | |
| __ movq(Address(end_to, qword_count, Address::times_8, 8), rax); | |
| __ increment(qword_count); | |
| __ jcc(Assembler::notZero, L_copy_8_bytes); | |
| // Check for and copy trailing dword | |
| __ BIND(L_copy_4_bytes); | |
| __ testl(byte_count, 4); | |
| __ jccb(Assembler::zero, L_copy_2_bytes); | |
| __ movl(rax, Address(end_from, 8)); | |
| __ movl(Address(end_to, 8), rax); | |
| __ addptr(end_from, 4); | |
| __ addptr(end_to, 4); | |
| // Check for and copy trailing word | |
| __ BIND(L_copy_2_bytes); | |
| __ testl(byte_count, 2); | |
| __ jccb(Assembler::zero, L_copy_byte); | |
| __ movw(rax, Address(end_from, 8)); | |
| __ movw(Address(end_to, 8), rax); | |
| __ addptr(end_from, 2); | |
| __ addptr(end_to, 2); | |
| // Check for and copy trailing byte | |
| __ BIND(L_copy_byte); | |
| __ testl(byte_count, 1); | |
| __ jccb(Assembler::zero, L_exit); | |
| __ movb(rax, Address(end_from, 8)); | |
| __ movb(Address(end_to, 8), rax); | |
| } | |
| __ BIND(L_exit); | |
| address ucme_exit_pc = __ pc(); | |
| restore_arg_regs(); | |
| INC_COUNTER_NP(SharedRuntime::_jbyte_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free | |
| __ xorptr(rax, rax); // return 0 | |
| __ vzeroupper(); | |
| __ leave(); // required for proper stackwalking of RuntimeStub frame | |
| __ ret(0); | |
| { | |
| UnsafeMemoryAccessMark umam(this, !aligned, false, ucme_exit_pc); | |
| // Copy in multi-bytes chunks | |
| copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, T_BYTE); | |
| __ jmp(L_copy_4_bytes); | |
| } | |
| return start; | |
| } | |
| // Arguments: | |
| // entry - location for return of (post-push) entry | |
| // nooverlap_target - entry to branch to if no overlap detected | |
| // | |
| // Inputs: | |
| // c_rarg0 - source array address | |
| // c_rarg1 - destination array address | |
| // c_rarg2 - element count, treated as ssize_t, can be zero | |
| // | |
| // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries, | |
| // we let the hardware handle it. The one to eight bytes within words, | |
| // dwords or qwords that span cache line boundaries will still be loaded | |
| // and stored atomically. | |
| // | |
| address StubGenerator::generate_conjoint_byte_copy(address nooverlap_target, address* entry) { | |
| StubGenStubId stub_id = StubGenStubId::jbyte_arraycopy_id; | |
| // aligned is always false -- x86_64 always uses the unaligned code | |
| const bool aligned = false; | |
| #if COMPILER2_OR_JVMCI | |
| if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) { | |
| return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target); | |
| } | |
| #endif | |
| __ align(CodeEntryAlignment); | |
| StubCodeMark mark(this, stub_id); | |
| address start = __ pc(); | |
| DecoratorSet decorators = IN_HEAP | IS_ARRAY; | |
| Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes; | |
| const Register from = rdi; // source array address | |
| const Register to = rsi; // destination array address | |
| const Register count = rdx; // elements count | |
| const Register byte_count = rcx; | |
| const Register qword_count = count; | |
| __ enter(); // required for proper stackwalking of RuntimeStub frame | |
| assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | |
| if (entry != nullptr) { | |
| *entry = __ pc(); | |
| // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | |
| BLOCK_COMMENT("Entry:"); | |
| } | |
| array_overlap_test(nooverlap_target, Address::times_1); | |
| setup_arg_regs(); // from => rdi, to => rsi, count => rdx | |
| // r9 and r10 may be used to save non-volatile registers | |
| { | |
| // UnsafeMemoryAccess page error: continue after unsafe access | |
| UnsafeMemoryAccessMark umam(this, !aligned, true); | |
| // 'from', 'to' and 'count' are now valid | |
| __ movptr(byte_count, count); | |
| __ shrptr(count, 3); // count => qword_count | |
| // Copy from high to low addresses. | |
| // Check for and copy trailing byte | |
| __ testl(byte_count, 1); | |
| __ jcc(Assembler::zero, L_copy_2_bytes); | |
| __ movb(rax, Address(from, byte_count, Address::times_1, -1)); | |
| __ movb(Address(to, byte_count, Address::times_1, -1), rax); | |
| __ decrement(byte_count); // Adjust for possible trailing word | |
| // Check for and copy trailing word | |
| __ BIND(L_copy_2_bytes); | |
| __ testl(byte_count, 2); | |
| __ jcc(Assembler::zero, L_copy_4_bytes); | |
| __ movw(rax, Address(from, byte_count, Address::times_1, -2)); | |
| __ movw(Address(to, byte_count, Address::times_1, -2), rax); | |
| // Check for and copy trailing dword | |
| __ BIND(L_copy_4_bytes); | |
| __ testl(byte_count, 4); | |
| __ jcc(Assembler::zero, L_copy_bytes); | |
| __ movl(rax, Address(from, qword_count, Address::times_8)); | |
| __ movl(Address(to, qword_count, Address::times_8), rax); | |
| __ jmp(L_copy_bytes); | |
| // Copy trailing qwords | |
| __ BIND(L_copy_8_bytes); | |
| __ movq(rax, Address(from, qword_count, Address::times_8, -8)); | |
| __ movq(Address(to, qword_count, Address::times_8, -8), rax); | |
| __ decrement(qword_count); | |
| __ jcc(Assembler::notZero, L_copy_8_bytes); | |
| } | |
| restore_arg_regs(); | |
| INC_COUNTER_NP(SharedRuntime::_jbyte_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free | |
| __ xorptr(rax, rax); // return 0 | |
| __ vzeroupper(); | |
| __ leave(); // required for proper stackwalking of RuntimeStub frame | |
| __ ret(0); | |
| { | |
| // UnsafeMemoryAccess page error: continue after unsafe access | |
| UnsafeMemoryAccessMark umam(this, !aligned, true); | |
| // Copy in multi-bytes chunks | |
| copy_bytes_backward(from, to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, T_BYTE); | |
| } | |
| restore_arg_regs(); | |
| INC_COUNTER_NP(SharedRuntime::_jbyte_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free | |
| __ xorptr(rax, rax); // return 0 | |
| __ vzeroupper(); | |
| __ leave(); // required for proper stackwalking of RuntimeStub frame | |
| __ ret(0); | |
| return start; | |
| } | |
| // Arguments: | |
| // entry - location for return of (post-push) entry | |
| // | |
| // Inputs: | |
| // c_rarg0 - source array address | |
| // c_rarg1 - destination array address | |
| // c_rarg2 - element count, treated as ssize_t, can be zero | |
| // | |
| // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we | |
| // let the hardware handle it. The two or four words within dwords | |
| // or qwords that span cache line boundaries will still be loaded | |
| // and stored atomically. | |
| // | |
| // Side Effects: | |
| // entry is set to the no-overlap entry point | |
| // used by generate_conjoint_short_copy(). | |
| // | |
| address StubGenerator::generate_disjoint_short_copy(address *entry) { | |
| StubGenStubId stub_id = StubGenStubId::jshort_disjoint_arraycopy_id; | |
| // aligned is always false -- x86_64 always uses the unaligned code | |
| const bool aligned = false; | |
| #if COMPILER2_OR_JVMCI | |
| if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) { | |
| return generate_disjoint_copy_avx3_masked(stub_id, entry); | |
| } | |
| #endif | |
| __ align(CodeEntryAlignment); | |
| StubCodeMark mark(this, stub_id); | |
| address start = __ pc(); | |
| DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT; | |
| Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes,L_copy_2_bytes,L_exit; | |
| const Register from = rdi; // source array address | |
| const Register to = rsi; // destination array address | |
| const Register count = rdx; // elements count | |
| const Register word_count = rcx; | |
| const Register qword_count = count; | |
| const Register end_from = from; // source array end address | |
| const Register end_to = to; // destination array end address | |
| // End pointers are inclusive, and if count is not zero they point | |
| // to the last unit copied: end_to[0] := end_from[0] | |
| __ enter(); // required for proper stackwalking of RuntimeStub frame | |
| assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | |
| if (entry != nullptr) { | |
| *entry = __ pc(); | |
| // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | |
| BLOCK_COMMENT("Entry:"); | |
| } | |
| setup_arg_regs(); // from => rdi, to => rsi, count => rdx | |
| // r9 and r10 may be used to save non-volatile registers | |
| { | |
| // UnsafeMemoryAccess page error: continue after unsafe access | |
| UnsafeMemoryAccessMark umam(this, !aligned, true); | |
| // 'from', 'to' and 'count' are now valid | |
| __ movptr(word_count, count); | |
| __ shrptr(count, 2); // count => qword_count | |
| // Copy from low to high addresses. Use 'to' as scratch. | |
| __ lea(end_from, Address(from, qword_count, Address::times_8, -8)); | |
| __ lea(end_to, Address(to, qword_count, Address::times_8, -8)); | |
| __ negptr(qword_count); | |
| __ jmp(L_copy_bytes); | |
| // Copy trailing qwords | |
| __ BIND(L_copy_8_bytes); | |
| __ movq(rax, Address(end_from, qword_count, Address::times_8, 8)); | |
| __ movq(Address(end_to, qword_count, Address::times_8, 8), rax); | |
| __ increment(qword_count); | |
| __ jcc(Assembler::notZero, L_copy_8_bytes); | |
| // Original 'dest' is trashed, so we can't use it as a | |
| // base register for a possible trailing word copy | |
| // Check for and copy trailing dword | |
| __ BIND(L_copy_4_bytes); | |
| __ testl(word_count, 2); | |
| __ jccb(Assembler::zero, L_copy_2_bytes); | |
| __ movl(rax, Address(end_from, 8)); | |
| __ movl(Address(end_to, 8), rax); | |
| __ addptr(end_from, 4); | |
| __ addptr(end_to, 4); | |
| // Check for and copy trailing word | |
| __ BIND(L_copy_2_bytes); | |
| __ testl(word_count, 1); | |
| __ jccb(Assembler::zero, L_exit); | |
| __ movw(rax, Address(end_from, 8)); | |
| __ movw(Address(end_to, 8), rax); | |
| } | |
| __ BIND(L_exit); | |
| address ucme_exit_pc = __ pc(); | |
| restore_arg_regs(); | |
| INC_COUNTER_NP(SharedRuntime::_jshort_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free | |
| __ xorptr(rax, rax); // return 0 | |
| __ vzeroupper(); | |
| __ leave(); // required for proper stackwalking of RuntimeStub frame | |
| __ ret(0); | |
| { | |
| UnsafeMemoryAccessMark umam(this, !aligned, false, ucme_exit_pc); | |
| // Copy in multi-bytes chunks | |
| copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, T_SHORT); | |
| __ jmp(L_copy_4_bytes); | |
| } | |
| return start; | |
| } | |
| address StubGenerator::generate_fill(StubGenStubId stub_id) { | |
| BasicType t; | |
| bool aligned; | |
| switch (stub_id) { | |
| case jbyte_fill_id: | |
| t = T_BYTE; | |
| aligned = false; | |
| break; | |
| case jshort_fill_id: | |
| t = T_SHORT; | |
| aligned = false; | |
| break; | |
| case jint_fill_id: | |
| t = T_INT; | |
| aligned = false; | |
| break; | |
| case arrayof_jbyte_fill_id: | |
| t = T_BYTE; | |
| aligned = true; | |
| break; | |
| case arrayof_jshort_fill_id: | |
| t = T_SHORT; | |
| aligned = true; | |
| break; | |
| case arrayof_jint_fill_id: | |
| t = T_INT; | |
| aligned = true; | |
| break; | |
| default: | |
| ShouldNotReachHere(); | |
| } | |
| __ align(CodeEntryAlignment); | |
| StubCodeMark mark(this, stub_id); | |
| address start = __ pc(); | |
| BLOCK_COMMENT("Entry:"); | |
| const Register to = c_rarg0; // destination array address | |
| const Register value = c_rarg1; // value | |
| const Register count = c_rarg2; // elements count | |
| __ mov(r11, count); | |
| __ enter(); // required for proper stackwalking of RuntimeStub frame | |
| { | |
| // Add set memory mark to protect against unsafe accesses faulting | |
| UnsafeMemoryAccessMark umam(this, ((t == T_BYTE) && !aligned), true); | |
| __ generate_fill(t, aligned, to, value, r11, rax, xmm0); | |
| } | |
| __ vzeroupper(); | |
| __ leave(); // required for proper stackwalking of RuntimeStub frame | |
| __ ret(0); | |
| return start; | |
| } | |
| // Arguments: | |
| // entry - location for return of (post-push) entry | |
| // nooverlap_target - entry to branch to if no overlap detected | |
| // | |
| // Inputs: | |
| // c_rarg0 - source array address | |
| // c_rarg1 - destination array address | |
| // c_rarg2 - element count, treated as ssize_t, can be zero | |
| // | |
| // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we | |
| // let the hardware handle it. The two or four words within dwords | |
| // or qwords that span cache line boundaries will still be loaded | |
| // and stored atomically. | |
| // | |
| address StubGenerator::generate_conjoint_short_copy(address nooverlap_target, address *entry) { | |
| StubGenStubId stub_id = StubGenStubId::jshort_arraycopy_id; | |
| // aligned is always false -- x86_64 always uses the unaligned code | |
| const bool aligned = false; | |
| #if COMPILER2_OR_JVMCI | |
| if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) { | |
| return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target); | |
| } | |
| #endif | |
| __ align(CodeEntryAlignment); | |
| StubCodeMark mark(this, stub_id); | |
| address start = __ pc(); | |
| DecoratorSet decorators = IN_HEAP | IS_ARRAY; | |
| Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes; | |
| const Register from = rdi; // source array address | |
| const Register to = rsi; // destination array address | |
| const Register count = rdx; // elements count | |
| const Register word_count = rcx; | |
| const Register qword_count = count; | |
| __ enter(); // required for proper stackwalking of RuntimeStub frame | |
| assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | |
| if (entry != nullptr) { | |
| *entry = __ pc(); | |
| // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | |
| BLOCK_COMMENT("Entry:"); | |
| } | |
| array_overlap_test(nooverlap_target, Address::times_2); | |
| setup_arg_regs(); // from => rdi, to => rsi, count => rdx | |
| // r9 and r10 may be used to save non-volatile registers | |
| { | |
| // UnsafeMemoryAccess page error: continue after unsafe access | |
| UnsafeMemoryAccessMark umam(this, !aligned, true); | |
| // 'from', 'to' and 'count' are now valid | |
| __ movptr(word_count, count); | |
| __ shrptr(count, 2); // count => qword_count | |
| // Copy from high to low addresses. Use 'to' as scratch. | |
| // Check for and copy trailing word | |
| __ testl(word_count, 1); | |
| __ jccb(Assembler::zero, L_copy_4_bytes); | |
| __ movw(rax, Address(from, word_count, Address::times_2, -2)); | |
| __ movw(Address(to, word_count, Address::times_2, -2), rax); | |
| // Check for and copy trailing dword | |
| __ BIND(L_copy_4_bytes); | |
| __ testl(word_count, 2); | |
| __ jcc(Assembler::zero, L_copy_bytes); | |
| __ movl(rax, Address(from, qword_count, Address::times_8)); | |
| __ movl(Address(to, qword_count, Address::times_8), rax); | |
| __ jmp(L_copy_bytes); | |
| // Copy trailing qwords | |
| __ BIND(L_copy_8_bytes); | |
| __ movq(rax, Address(from, qword_count, Address::times_8, -8)); | |
| __ movq(Address(to, qword_count, Address::times_8, -8), rax); | |
| __ decrement(qword_count); | |
| __ jcc(Assembler::notZero, L_copy_8_bytes); | |
| } | |
| restore_arg_regs(); | |
| INC_COUNTER_NP(SharedRuntime::_jshort_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free | |
| __ xorptr(rax, rax); // return 0 | |
| __ vzeroupper(); | |
| __ leave(); // required for proper stackwalking of RuntimeStub frame | |
| __ ret(0); | |
| { | |
| // UnsafeMemoryAccess page error: continue after unsafe access | |
| UnsafeMemoryAccessMark umam(this, !aligned, true); | |
| // Copy in multi-bytes chunks | |
| copy_bytes_backward(from, to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, T_SHORT); | |
| } | |
| restore_arg_regs(); | |
| INC_COUNTER_NP(SharedRuntime::_jshort_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free | |
| __ xorptr(rax, rax); // return 0 | |
| __ vzeroupper(); | |
| __ leave(); // required for proper stackwalking of RuntimeStub frame | |
| __ ret(0); | |
| return start; | |
| } | |
| // Arguments: | |
| // stub_id - unqiue id for stub to generate | |
| // entry - location for return of (post-push) entry | |
| // is_oop - true => oop array, so generate store check code | |
| // | |
| // Inputs: | |
| // c_rarg0 - source array address | |
| // c_rarg1 - destination array address | |
| // c_rarg2 - element count, treated as ssize_t, can be zero | |
| // | |
| // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let | |
| // the hardware handle it. The two dwords within qwords that span | |
| // cache line boundaries will still be loaded and stored atomically. | |
| // | |
| // Side Effects: | |
| // disjoint_int_copy_entry is set to the no-overlap entry point | |
| // used by generate_conjoint_int_oop_copy(). | |
| // | |
| address StubGenerator::generate_disjoint_int_oop_copy(StubGenStubId stub_id, address* entry) { | |
| // aligned is always false -- x86_64 always uses the unaligned code | |
| const bool aligned = false; | |
| bool is_oop; | |
| bool dest_uninitialized; | |
| switch (stub_id) { | |
| case StubGenStubId::jint_disjoint_arraycopy_id: | |
| is_oop = false; | |
| dest_uninitialized = false; | |
| break; | |
| case StubGenStubId::oop_disjoint_arraycopy_id: | |
| assert(UseCompressedOops, "inconsistent oop copy size!"); | |
| is_oop = true; | |
| dest_uninitialized = false; | |
| break; | |
| case StubGenStubId::oop_disjoint_arraycopy_uninit_id: | |
| assert(UseCompressedOops, "inconsistent oop copy size!"); | |
| is_oop = true; | |
| dest_uninitialized = true; | |
| break; | |
| default: | |
| ShouldNotReachHere(); | |
| } | |
| BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); | |
| #if COMPILER2_OR_JVMCI | |
| if ((!is_oop || bs->supports_avx3_masked_arraycopy()) && VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) { | |
| return generate_disjoint_copy_avx3_masked(stub_id, entry); | |
| } | |
| #endif | |
| __ align(CodeEntryAlignment); | |
| StubCodeMark mark(this, stub_id); | |
| address start = __ pc(); | |
| Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_exit; | |
| const Register from = rdi; // source array address | |
| const Register to = rsi; // destination array address | |
| const Register count = rdx; // elements count | |
| const Register dword_count = rcx; | |
| const Register qword_count = count; | |
| const Register end_from = from; // source array end address | |
| const Register end_to = to; // destination array end address | |
| // End pointers are inclusive, and if count is not zero they point | |
| // to the last unit copied: end_to[0] := end_from[0] | |
| __ enter(); // required for proper stackwalking of RuntimeStub frame | |
| assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | |
| if (entry != nullptr) { | |
| *entry = __ pc(); | |
| // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | |
| BLOCK_COMMENT("Entry:"); | |
| } | |
| setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx | |
| // r9 is used to save r15_thread | |
| DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT; | |
| if (dest_uninitialized) { | |
| decorators |= IS_DEST_UNINITIALIZED; | |
| } | |
| if (aligned) { | |
| decorators |= ARRAYCOPY_ALIGNED; | |
| } | |
| BasicType type = is_oop ? T_OBJECT : T_INT; | |
| bs->arraycopy_prologue(_masm, decorators, type, from, to, count); | |
| { | |
| // UnsafeMemoryAccess page error: continue after unsafe access | |
| UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); | |
| // 'from', 'to' and 'count' are now valid | |
| __ movptr(dword_count, count); | |
| __ shrptr(count, 1); // count => qword_count | |
| // Copy from low to high addresses. Use 'to' as scratch. | |
| __ lea(end_from, Address(from, qword_count, Address::times_8, -8)); | |
| __ lea(end_to, Address(to, qword_count, Address::times_8, -8)); | |
| __ negptr(qword_count); | |
| __ jmp(L_copy_bytes); | |
| // Copy trailing qwords | |
| __ BIND(L_copy_8_bytes); | |
| __ movq(rax, Address(end_from, qword_count, Address::times_8, 8)); | |
| __ movq(Address(end_to, qword_count, Address::times_8, 8), rax); | |
| __ increment(qword_count); | |
| __ jcc(Assembler::notZero, L_copy_8_bytes); | |
| // Check for and copy trailing dword | |
| __ BIND(L_copy_4_bytes); | |
| __ testl(dword_count, 1); // Only byte test since the value is 0 or 1 | |
| __ jccb(Assembler::zero, L_exit); | |
| __ movl(rax, Address(end_from, 8)); | |
| __ movl(Address(end_to, 8), rax); | |
| } | |
| __ BIND(L_exit); | |
| address ucme_exit_pc = __ pc(); | |
| bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count); | |
| restore_arg_regs_using_thread(); | |
| INC_COUNTER_NP(SharedRuntime::_jint_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free | |
| __ vzeroupper(); | |
| __ xorptr(rax, rax); // return 0 | |
| __ leave(); // required for proper stackwalking of RuntimeStub frame | |
| __ ret(0); | |
| { | |
| UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, false, ucme_exit_pc); | |
| // Copy in multi-bytes chunks | |
| copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_INT); | |
| __ jmp(L_copy_4_bytes); | |
| } | |
| return start; | |
| } | |
| // Arguments: | |
| // entry - location for return of (post-push) entry | |
| // nooverlap_target - entry to branch to if no overlap detected | |
| // is_oop - true => oop array, so generate store check code | |
| // | |
| // Inputs: | |
| // c_rarg0 - source array address | |
| // c_rarg1 - destination array address | |
| // c_rarg2 - element count, treated as ssize_t, can be zero | |
| // | |
| // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let | |
| // the hardware handle it. The two dwords within qwords that span | |
| // cache line boundaries will still be loaded and stored atomically. | |
| // | |
| address StubGenerator::generate_conjoint_int_oop_copy(StubGenStubId stub_id, address nooverlap_target, address *entry) { | |
| // aligned is always false -- x86_64 always uses the unaligned code | |
| const bool aligned = false; | |
| bool is_oop; | |
| bool dest_uninitialized; | |
| switch (stub_id) { | |
| case StubGenStubId::jint_arraycopy_id: | |
| is_oop = false; | |
| dest_uninitialized = false; | |
| break; | |
| case StubGenStubId::oop_arraycopy_id: | |
| assert(UseCompressedOops, "inconsistent oop copy size!"); | |
| is_oop = true; | |
| dest_uninitialized = false; | |
| break; | |
| case StubGenStubId::oop_arraycopy_uninit_id: | |
| assert(UseCompressedOops, "inconsistent oop copy size!"); | |
| is_oop = true; | |
| dest_uninitialized = true; | |
| break; | |
| default: | |
| ShouldNotReachHere(); | |
| } | |
| BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); | |
| #if COMPILER2_OR_JVMCI | |
| if ((!is_oop || bs->supports_avx3_masked_arraycopy()) && VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) { | |
| return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target); | |
| } | |
| #endif | |
| __ align(CodeEntryAlignment); | |
| StubCodeMark mark(this, stub_id); | |
| address start = __ pc(); | |
| Label L_copy_bytes, L_copy_8_bytes, L_exit; | |
| const Register from = rdi; // source array address | |
| const Register to = rsi; // destination array address | |
| const Register count = rdx; // elements count | |
| const Register dword_count = rcx; | |
| const Register qword_count = count; | |
| __ enter(); // required for proper stackwalking of RuntimeStub frame | |
| assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | |
| if (entry != nullptr) { | |
| *entry = __ pc(); | |
| // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | |
| BLOCK_COMMENT("Entry:"); | |
| } | |
| array_overlap_test(nooverlap_target, Address::times_4); | |
| setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx | |
| // r9 is used to save r15_thread | |
| DecoratorSet decorators = IN_HEAP | IS_ARRAY; | |
| if (dest_uninitialized) { | |
| decorators |= IS_DEST_UNINITIALIZED; | |
| } | |
| if (aligned) { | |
| decorators |= ARRAYCOPY_ALIGNED; | |
| } | |
| BasicType type = is_oop ? T_OBJECT : T_INT; | |
| // no registers are destroyed by this call | |
| bs->arraycopy_prologue(_masm, decorators, type, from, to, count); | |
| assert_clean_int(count, rax); // Make sure 'count' is clean int. | |
| { | |
| // UnsafeMemoryAccess page error: continue after unsafe access | |
| UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); | |
| // 'from', 'to' and 'count' are now valid | |
| __ movptr(dword_count, count); | |
| __ shrptr(count, 1); // count => qword_count | |
| // Copy from high to low addresses. Use 'to' as scratch. | |
| // Check for and copy trailing dword | |
| __ testl(dword_count, 1); | |
| __ jcc(Assembler::zero, L_copy_bytes); | |
| __ movl(rax, Address(from, dword_count, Address::times_4, -4)); | |
| __ movl(Address(to, dword_count, Address::times_4, -4), rax); | |
| __ jmp(L_copy_bytes); | |
| // Copy trailing qwords | |
| __ BIND(L_copy_8_bytes); | |
| __ movq(rax, Address(from, qword_count, Address::times_8, -8)); | |
| __ movq(Address(to, qword_count, Address::times_8, -8), rax); | |
| __ decrement(qword_count); | |
| __ jcc(Assembler::notZero, L_copy_8_bytes); | |
| } | |
| if (is_oop) { | |
| __ jmp(L_exit); | |
| } | |
| restore_arg_regs_using_thread(); | |
| INC_COUNTER_NP(SharedRuntime::_jint_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free | |
| __ xorptr(rax, rax); // return 0 | |
| __ vzeroupper(); | |
| __ leave(); // required for proper stackwalking of RuntimeStub frame | |
| __ ret(0); | |
| { | |
| // UnsafeMemoryAccess page error: continue after unsafe access | |
| UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); | |
| // Copy in multi-bytes chunks | |
| copy_bytes_backward(from, to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_INT); | |
| } | |
| __ BIND(L_exit); | |
| bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count); | |
| restore_arg_regs_using_thread(); | |
| INC_COUNTER_NP(SharedRuntime::_jint_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free | |
| __ xorptr(rax, rax); // return 0 | |
| __ vzeroupper(); | |
| __ leave(); // required for proper stackwalking of RuntimeStub frame | |
| __ ret(0); | |
| return start; | |
| } | |
| // Arguments: | |
| // entry - location for return of (post-push) entry | |
| // | |
| // Inputs: | |
| // c_rarg0 - source array address | |
| // c_rarg1 - destination array address | |
| // c_rarg2 - element count, treated as ssize_t, can be zero | |
| // | |
| // Side Effects: | |
| // disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the | |
| // no-overlap entry point used by generate_conjoint_long_oop_copy(). | |
| // | |
| address StubGenerator::generate_disjoint_long_oop_copy(StubGenStubId stub_id, address *entry) { | |
| // aligned is always false -- x86_64 always uses the unaligned code | |
| const bool aligned = false; | |
| bool is_oop; | |
| bool dest_uninitialized; | |
| switch (stub_id) { | |
| case StubGenStubId::jlong_disjoint_arraycopy_id: | |
| is_oop = false; | |
| dest_uninitialized = false; | |
| break; | |
| case StubGenStubId::oop_disjoint_arraycopy_id: | |
| assert(!UseCompressedOops, "inconsistent oop copy size!"); | |
| is_oop = true; | |
| dest_uninitialized = false; | |
| break; | |
| case StubGenStubId::oop_disjoint_arraycopy_uninit_id: | |
| assert(!UseCompressedOops, "inconsistent oop copy size!"); | |
| is_oop = true; | |
| dest_uninitialized = true; | |
| break; | |
| default: | |
| ShouldNotReachHere(); | |
| } | |
| BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); | |
| #if COMPILER2_OR_JVMCI | |
| if ((!is_oop || bs->supports_avx3_masked_arraycopy()) && VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) { | |
| return generate_disjoint_copy_avx3_masked(stub_id, entry); | |
| } | |
| #endif | |
| __ align(CodeEntryAlignment); | |
| StubCodeMark mark(this, stub_id); | |
| address start = __ pc(); | |
| Label L_copy_bytes, L_copy_8_bytes, L_exit; | |
| const Register from = rdi; // source array address | |
| const Register to = rsi; // destination array address | |
| const Register qword_count = rdx; // elements count | |
| const Register end_from = from; // source array end address | |
| const Register end_to = rcx; // destination array end address | |
| const Register saved_count = r11; | |
| // End pointers are inclusive, and if count is not zero they point | |
| // to the last unit copied: end_to[0] := end_from[0] | |
| __ enter(); // required for proper stackwalking of RuntimeStub frame | |
| // Save no-overlap entry point for generate_conjoint_long_oop_copy() | |
| assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | |
| if (entry != nullptr) { | |
| *entry = __ pc(); | |
| // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | |
| BLOCK_COMMENT("Entry:"); | |
| } | |
| setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx | |
| // r9 is used to save r15_thread | |
| // 'from', 'to' and 'qword_count' are now valid | |
| DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT; | |
| if (dest_uninitialized) { | |
| decorators |= IS_DEST_UNINITIALIZED; | |
| } | |
| if (aligned) { | |
| decorators |= ARRAYCOPY_ALIGNED; | |
| } | |
| BasicType type = is_oop ? T_OBJECT : T_LONG; | |
| bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count); | |
| { | |
| // UnsafeMemoryAccess page error: continue after unsafe access | |
| UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); | |
| // Copy from low to high addresses. Use 'to' as scratch. | |
| __ lea(end_from, Address(from, qword_count, Address::times_8, -8)); | |
| __ lea(end_to, Address(to, qword_count, Address::times_8, -8)); | |
| __ negptr(qword_count); | |
| __ jmp(L_copy_bytes); | |
| // Copy trailing qwords | |
| __ BIND(L_copy_8_bytes); | |
| bs->copy_load_at(_masm, decorators, type, 8, | |
| rax, Address(end_from, qword_count, Address::times_8, 8), | |
| r10); | |
| bs->copy_store_at(_masm, decorators, type, 8, | |
| Address(end_to, qword_count, Address::times_8, 8), rax, | |
| r10); | |
| __ increment(qword_count); | |
| __ jcc(Assembler::notZero, L_copy_8_bytes); | |
| } | |
| if (is_oop) { | |
| __ jmp(L_exit); | |
| } else { | |
| restore_arg_regs_using_thread(); | |
| INC_COUNTER_NP(SharedRuntime::_jlong_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free | |
| __ xorptr(rax, rax); // return 0 | |
| __ vzeroupper(); | |
| __ leave(); // required for proper stackwalking of RuntimeStub frame | |
| __ ret(0); | |
| } | |
| { | |
| // UnsafeMemoryAccess page error: continue after unsafe access | |
| UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); | |
| // Copy in multi-bytes chunks | |
| copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_LONG); | |
| } | |
| __ BIND(L_exit); | |
| bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count); | |
| restore_arg_regs_using_thread(); | |
| INC_COUNTER_NP(is_oop ? SharedRuntime::_oop_array_copy_ctr : | |
| SharedRuntime::_jlong_array_copy_ctr, | |
| rscratch1); // Update counter after rscratch1 is free | |
| __ vzeroupper(); | |
| __ xorptr(rax, rax); // return 0 | |
| __ leave(); // required for proper stackwalking of RuntimeStub frame | |
| __ ret(0); | |
| return start; | |
| } | |
| // Arguments: | |
| // entry - location for return of (post-push) entry | |
| // nooverlap_target - entry to branch to if no overlap detected | |
| // is_oop - true => oop array, so generate store check code | |
| // | |
| // Inputs: | |
| // c_rarg0 - source array address | |
| // c_rarg1 - destination array address | |
| // c_rarg2 - element count, treated as ssize_t, can be zero | |
| // | |
| address StubGenerator::generate_conjoint_long_oop_copy(StubGenStubId stub_id, address nooverlap_target, address *entry) { | |
| // aligned is always false -- x86_64 always uses the unaligned code | |
| const bool aligned = false; | |
| bool is_oop; | |
| bool dest_uninitialized; | |
| switch (stub_id) { | |
| case StubGenStubId::jlong_arraycopy_id: | |
| is_oop = false; | |
| dest_uninitialized = false; | |
| break; | |
| case StubGenStubId::oop_arraycopy_id: | |
| assert(!UseCompressedOops, "inconsistent oop copy size!"); | |
| is_oop = true; | |
| dest_uninitialized = false; | |
| break; | |
| case StubGenStubId::oop_arraycopy_uninit_id: | |
| assert(!UseCompressedOops, "inconsistent oop copy size!"); | |
| is_oop = true; | |
| dest_uninitialized = true; | |
| break; | |
| default: | |
| ShouldNotReachHere(); | |
| } | |
| BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); | |
| #if COMPILER2_OR_JVMCI | |
| if ((!is_oop || bs->supports_avx3_masked_arraycopy()) && VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) { | |
| return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target); | |
| } | |
| #endif | |
| __ align(CodeEntryAlignment); | |
| StubCodeMark mark(this, stub_id); | |
| address start = __ pc(); | |
| Label L_copy_bytes, L_copy_8_bytes, L_exit; | |
| const Register from = rdi; // source array address | |
| const Register to = rsi; // destination array address | |
| const Register qword_count = rdx; // elements count | |
| const Register saved_count = rcx; | |
| __ enter(); // required for proper stackwalking of RuntimeStub frame | |
| assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | |
| if (entry != nullptr) { | |
| *entry = __ pc(); | |
| // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | |
| BLOCK_COMMENT("Entry:"); | |
| } | |
| array_overlap_test(nooverlap_target, Address::times_8); | |
| setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx | |
| // r9 is used to save r15_thread | |
| // 'from', 'to' and 'qword_count' are now valid | |
| DecoratorSet decorators = IN_HEAP | IS_ARRAY; | |
| if (dest_uninitialized) { | |
| decorators |= IS_DEST_UNINITIALIZED; | |
| } | |
| if (aligned) { | |
| decorators |= ARRAYCOPY_ALIGNED; | |
| } | |
| BasicType type = is_oop ? T_OBJECT : T_LONG; | |
| bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count); | |
| { | |
| // UnsafeMemoryAccess page error: continue after unsafe access | |
| UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); | |
| __ jmp(L_copy_bytes); | |
| // Copy trailing qwords | |
| __ BIND(L_copy_8_bytes); | |
| bs->copy_load_at(_masm, decorators, type, 8, | |
| rax, Address(from, qword_count, Address::times_8, -8), | |
| r10); | |
| bs->copy_store_at(_masm, decorators, type, 8, | |
| Address(to, qword_count, Address::times_8, -8), rax, | |
| r10); | |
| __ decrement(qword_count); | |
| __ jcc(Assembler::notZero, L_copy_8_bytes); | |
| } | |
| if (is_oop) { | |
| __ jmp(L_exit); | |
| } else { | |
| restore_arg_regs_using_thread(); | |
| INC_COUNTER_NP(SharedRuntime::_jlong_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free | |
| __ xorptr(rax, rax); // return 0 | |
| __ vzeroupper(); | |
| __ leave(); // required for proper stackwalking of RuntimeStub frame | |
| __ ret(0); | |
| } | |
| { | |
| // UnsafeMemoryAccess page error: continue after unsafe access | |
| UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); | |
| // Copy in multi-bytes chunks | |
| copy_bytes_backward(from, to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_LONG); | |
| } | |
| __ BIND(L_exit); | |
| bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count); | |
| restore_arg_regs_using_thread(); | |
| INC_COUNTER_NP(is_oop ? SharedRuntime::_oop_array_copy_ctr : | |
| SharedRuntime::_jlong_array_copy_ctr, | |
| rscratch1); // Update counter after rscratch1 is free | |
| __ vzeroupper(); | |
| __ xorptr(rax, rax); // return 0 | |
| __ leave(); // required for proper stackwalking of RuntimeStub frame | |
| __ ret(0); | |
| return start; | |
| } | |
| // Helper for generating a dynamic type check. | |
| // Smashes no registers. | |
| void StubGenerator::generate_type_check(Register sub_klass, | |
| Register super_check_offset, | |
| Register super_klass, | |
| Label& L_success) { | |
| assert_different_registers(sub_klass, super_check_offset, super_klass); | |
| BLOCK_COMMENT("type_check:"); | |
| Label L_miss; | |
| __ check_klass_subtype_fast_path(sub_klass, super_klass, noreg, &L_success, &L_miss, nullptr, | |
| super_check_offset); | |
| __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg, &L_success, nullptr); | |
| // Fall through on failure! | |
| __ BIND(L_miss); | |
| } | |
| // | |
| // Generate checkcasting array copy stub | |
| // | |
| // Input: | |
| // c_rarg0 - source array address | |
| // c_rarg1 - destination array address | |
| // c_rarg2 - element count, treated as ssize_t, can be zero | |
| // c_rarg3 - size_t ckoff (super_check_offset) | |
| // not Win64 | |
| // c_rarg4 - oop ckval (super_klass) | |
| // Win64 | |
| // rsp+40 - oop ckval (super_klass) | |
| // | |
| // Output: | |
| // rax == 0 - success | |
| // rax == -1^K - failure, where K is partial transfer count | |
| // | |
| address StubGenerator::generate_checkcast_copy(StubGenStubId stub_id, address *entry) { | |
| bool dest_uninitialized; | |
| switch (stub_id) { | |
| case StubGenStubId::checkcast_arraycopy_id: | |
| dest_uninitialized = false; | |
| break; | |
| case StubGenStubId::checkcast_arraycopy_uninit_id: | |
| dest_uninitialized = true; | |
| break; | |
| default: | |
| ShouldNotReachHere(); | |
| } | |
| Label L_load_element, L_store_element, L_do_card_marks, L_done; | |
| // Input registers (after setup_arg_regs) | |
| const Register from = rdi; // source array address | |
| const Register to = rsi; // destination array address | |
| const Register length = rdx; // elements count | |
| const Register ckoff = rcx; // super_check_offset | |
| const Register ckval = r8; // super_klass | |
| // Registers used as temps (r13, r14 are save-on-entry) | |
| const Register end_from = from; // source array end address | |
| const Register end_to = r13; // destination array end address | |
| const Register count = rdx; // -(count_remaining) | |
| const Register r14_length = r14; // saved copy of length | |
| // End pointers are inclusive, and if length is not zero they point | |
| // to the last unit copied: end_to[0] := end_from[0] | |
| const Register rax_oop = rax; // actual oop copied | |
| const Register r11_klass = r11; // oop._klass | |
| //--------------------------------------------------------------- | |
| // Assembler stub will be used for this call to arraycopy | |
| // if the two arrays are subtypes of Object[] but the | |
| // destination array type is not equal to or a supertype | |
| // of the source type. Each element must be separately | |
| // checked. | |
| __ align(CodeEntryAlignment); | |
| StubCodeMark mark(this, stub_id); | |
| address start = __ pc(); | |
| __ enter(); // required for proper stackwalking of RuntimeStub frame | |
| #ifdef ASSERT | |
| // caller guarantees that the arrays really are different | |
| // otherwise, we would have to make conjoint checks | |
| { Label L; | |
| array_overlap_test(L, TIMES_OOP); | |
| __ stop("checkcast_copy within a single array"); | |
| __ bind(L); | |
| } | |
| #endif //ASSERT | |
| setup_arg_regs_using_thread(4); // from => rdi, to => rsi, length => rdx | |
| // ckoff => rcx, ckval => r8 | |
| // r9 is used to save r15_thread | |
| #ifdef _WIN64 | |
| // last argument (#4) is on stack on Win64 | |
| __ movptr(ckval, Address(rsp, 6 * wordSize)); | |
| #endif | |
| // Caller of this entry point must set up the argument registers. | |
| if (entry != nullptr) { | |
| *entry = __ pc(); | |
| BLOCK_COMMENT("Entry:"); | |
| } | |
| // allocate spill slots for r13, r14 | |
| enum { | |
| saved_r13_offset, | |
| saved_r14_offset, | |
| saved_r10_offset, | |
| saved_rbp_offset | |
| }; | |
| __ subptr(rsp, saved_rbp_offset * wordSize); | |
| __ movptr(Address(rsp, saved_r13_offset * wordSize), r13); | |
| __ movptr(Address(rsp, saved_r14_offset * wordSize), r14); | |
| __ movptr(Address(rsp, saved_r10_offset * wordSize), r10); | |
| #ifdef ASSERT | |
| Label L2; | |
| __ get_thread_slow(r14); | |
| __ cmpptr(r15_thread, r14); | |
| __ jcc(Assembler::equal, L2); | |
| __ stop("StubRoutines::call_stub: r15_thread is modified by call"); | |
| __ bind(L2); | |
| #endif // ASSERT | |
| // check that int operands are properly extended to size_t | |
| assert_clean_int(length, rax); | |
| assert_clean_int(ckoff, rax); | |
| #ifdef ASSERT | |
| BLOCK_COMMENT("assert consistent ckoff/ckval"); | |
| // The ckoff and ckval must be mutually consistent, | |
| // even though caller generates both. | |
| { Label L; | |
| int sco_offset = in_bytes(Klass::super_check_offset_offset()); | |
| __ cmpl(ckoff, Address(ckval, sco_offset)); | |
| __ jcc(Assembler::equal, L); | |
| __ stop("super_check_offset inconsistent"); | |
| __ bind(L); | |
| } | |
| #endif //ASSERT | |
| // Loop-invariant addresses. They are exclusive end pointers. | |
| Address end_from_addr(from, length, TIMES_OOP, 0); | |
| Address end_to_addr(to, length, TIMES_OOP, 0); | |
| // Loop-variant addresses. They assume post-incremented count < 0. | |
| Address from_element_addr(end_from, count, TIMES_OOP, 0); | |
| Address to_element_addr(end_to, count, TIMES_OOP, 0); | |
| DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST | ARRAYCOPY_DISJOINT; | |
| if (dest_uninitialized) { | |
| decorators |= IS_DEST_UNINITIALIZED; | |
| } | |
| BasicType type = T_OBJECT; | |
| size_t element_size = UseCompressedOops ? 4 : 8; | |
| BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); | |
| bs->arraycopy_prologue(_masm, decorators, type, from, to, count); | |
| // Copy from low to high addresses, indexed from the end of each array. | |
| __ lea(end_from, end_from_addr); | |
| __ lea(end_to, end_to_addr); | |
| __ movptr(r14_length, length); // save a copy of the length | |
| assert(length == count, ""); // else fix next line: | |
| __ negptr(count); // negate and test the length | |
| __ jcc(Assembler::notZero, L_load_element); | |
| // Empty array: Nothing to do. | |
| __ xorptr(rax, rax); // return 0 on (trivial) success | |
| __ jmp(L_done); | |
| // ======== begin loop ======== | |
| // (Loop is rotated; its entry is L_load_element.) | |
| // Loop control: | |
| // for (count = -count; count != 0; count++) | |
| // Base pointers src, dst are biased by 8*(count-1),to last element. | |
| __ align(OptoLoopAlignment); | |
| __ BIND(L_store_element); | |
| bs->copy_store_at(_masm, | |
| decorators, | |
| type, | |
| element_size, | |
| to_element_addr, | |
| rax_oop, | |
| r10); | |
| __ increment(count); // increment the count toward zero | |
| __ jcc(Assembler::zero, L_do_card_marks); | |
| // ======== loop entry is here ======== | |
| __ BIND(L_load_element); | |
| bs->copy_load_at(_masm, | |
| decorators, | |
| type, | |
| element_size, | |
| rax_oop, | |
| from_element_addr, | |
| r10); | |
| __ testptr(rax_oop, rax_oop); | |
| __ jcc(Assembler::zero, L_store_element); | |
| __ load_klass(r11_klass, rax_oop, rscratch1);// query the object klass | |
| generate_type_check(r11_klass, ckoff, ckval, L_store_element); | |
| // ======== end loop ======== | |
| // It was a real error; we must depend on the caller to finish the job. | |
| // Register rdx = -1 * number of *remaining* oops, r14 = *total* oops. | |
| // Emit GC store barriers for the oops we have copied (r14 + rdx), | |
| // and report their number to the caller. | |
| assert_different_registers(rax, r14_length, count, to, end_to, rcx, rscratch1); | |
| Label L_post_barrier; | |
| __ addptr(r14_length, count); // K = (original - remaining) oops | |
| __ movptr(rax, r14_length); // save the value | |
| __ notptr(rax); // report (-1^K) to caller (does not affect flags) | |
| __ jccb(Assembler::notZero, L_post_barrier); | |
| __ jmp(L_done); // K == 0, nothing was copied, skip post barrier | |
| // Come here on success only. | |
| __ BIND(L_do_card_marks); | |
| __ xorptr(rax, rax); // return 0 on success | |
| __ BIND(L_post_barrier); | |
| bs->arraycopy_epilogue(_masm, decorators, type, from, to, r14_length); | |
| // Common exit point (success or failure). | |
| __ BIND(L_done); | |
| __ movptr(r13, Address(rsp, saved_r13_offset * wordSize)); | |
| __ movptr(r14, Address(rsp, saved_r14_offset * wordSize)); | |
| __ movptr(r10, Address(rsp, saved_r10_offset * wordSize)); | |
| restore_arg_regs_using_thread(); | |
| INC_COUNTER_NP(SharedRuntime::_checkcast_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free | |
| __ leave(); // required for proper stackwalking of RuntimeStub frame | |
| __ ret(0); | |
| return start; | |
| } | |
| // Generate 'unsafe' array copy stub | |
| // Though just as safe as the other stubs, it takes an unscaled | |
| // size_t argument instead of an element count. | |
| // | |
| // Input: | |
| // c_rarg0 - source array address | |
| // c_rarg1 - destination array address | |
| // c_rarg2 - byte count, treated as ssize_t, can be zero | |
| // | |
| // Examines the alignment of the operands and dispatches | |
| // to a long, int, short, or byte copy loop. | |
| // | |
| address StubGenerator::generate_unsafe_copy(address byte_copy_entry, address short_copy_entry, | |
| address int_copy_entry, address long_copy_entry) { | |
| Label L_long_aligned, L_int_aligned, L_short_aligned; | |
| // Input registers (before setup_arg_regs) | |
| const Register from = c_rarg0; // source array address | |
| const Register to = c_rarg1; // destination array address | |
| const Register size = c_rarg2; // byte count (size_t) | |
| // Register used as a temp | |
| const Register bits = rax; // test copy of low bits | |
| __ align(CodeEntryAlignment); | |
| StubGenStubId stub_id = StubGenStubId::unsafe_arraycopy_id; | |
| StubCodeMark mark(this, stub_id); | |
| address start = __ pc(); | |
| __ enter(); // required for proper stackwalking of RuntimeStub frame | |
| // bump this on entry, not on exit: | |
| INC_COUNTER_NP(SharedRuntime::_unsafe_array_copy_ctr, rscratch1); | |
| __ mov(bits, from); | |
| __ orptr(bits, to); | |
| __ orptr(bits, size); | |
| __ testb(bits, BytesPerLong-1); | |
| __ jccb(Assembler::zero, L_long_aligned); | |
| __ testb(bits, BytesPerInt-1); | |
| __ jccb(Assembler::zero, L_int_aligned); | |
| __ testb(bits, BytesPerShort-1); | |
| __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry)); | |
| __ BIND(L_short_aligned); | |
| __ shrptr(size, LogBytesPerShort); // size => short_count | |
| __ jump(RuntimeAddress(short_copy_entry)); | |
| __ BIND(L_int_aligned); | |
| __ shrptr(size, LogBytesPerInt); // size => int_count | |
| __ jump(RuntimeAddress(int_copy_entry)); | |
| __ BIND(L_long_aligned); | |
| __ shrptr(size, LogBytesPerLong); // size => qword_count | |
| __ jump(RuntimeAddress(long_copy_entry)); | |
| return start; | |
| } | |
| // Static enum for helper | |
| enum USM_TYPE {USM_SHORT, USM_DWORD, USM_QUADWORD}; | |
| // Helper for generate_unsafe_setmemory | |
| // | |
| // Atomically fill an array of memory using 2-, 4-, or 8-byte chunks | |
| static void do_setmemory_atomic_loop(USM_TYPE type, Register dest, | |
| Register size, Register wide_value, | |
| Register tmp, Label& L_exit, | |
| MacroAssembler *_masm) { | |
| Label L_Loop, L_Tail, L_TailLoop; | |
| int shiftval = 0; | |
| int incr = 0; | |
| switch (type) { | |
| case USM_SHORT: | |
| shiftval = 1; | |
| incr = 16; | |
| break; | |
| case USM_DWORD: | |
| shiftval = 2; | |
| incr = 32; | |
| break; | |
| case USM_QUADWORD: | |
| shiftval = 3; | |
| incr = 64; | |
| break; | |
| } | |
| // At this point, we know the lower bits of size are zero | |
| __ shrq(size, shiftval); | |
| // size now has number of X-byte chunks (2, 4 or 8) | |
| // Number of (8*X)-byte chunks into tmp | |
| __ movq(tmp, size); | |
| __ shrq(tmp, 3); | |
| __ jccb(Assembler::zero, L_Tail); | |
| __ BIND(L_Loop); | |
| // Unroll 8 stores | |
| for (int i = 0; i < 8; i++) { | |
| switch (type) { | |
| case USM_SHORT: | |
| __ movw(Address(dest, (2 * i)), wide_value); | |
| break; | |
| case USM_DWORD: | |
| __ movl(Address(dest, (4 * i)), wide_value); | |
| break; | |
| case USM_QUADWORD: | |
| __ movq(Address(dest, (8 * i)), wide_value); | |
| break; | |
| } | |
| } | |
| __ addq(dest, incr); | |
| __ decrementq(tmp); | |
| __ jccb(Assembler::notZero, L_Loop); | |
| __ BIND(L_Tail); | |
| // Find number of remaining X-byte chunks | |
| __ andq(size, 0x7); | |
| // If zero, then we're done | |
| __ jccb(Assembler::zero, L_exit); | |
| __ BIND(L_TailLoop); | |
| switch (type) { | |
| case USM_SHORT: | |
| __ movw(Address(dest, 0), wide_value); | |
| break; | |
| case USM_DWORD: | |
| __ movl(Address(dest, 0), wide_value); | |
| break; | |
| case USM_QUADWORD: | |
| __ movq(Address(dest, 0), wide_value); | |
| break; | |
| } | |
| __ addq(dest, incr >> 3); | |
| __ decrementq(size); | |
| __ jccb(Assembler::notZero, L_TailLoop); | |
| } | |
| // Generate 'unsafe' set memory stub | |
| // Though just as safe as the other stubs, it takes an unscaled | |
| // size_t (# bytes) argument instead of an element count. | |
| // | |
| // Input: | |
| // c_rarg0 - destination array address | |
| // c_rarg1 - byte count (size_t) | |
| // c_rarg2 - byte value | |
| // | |
| // Examines the alignment of the operands and dispatches | |
| // to an int, short, or byte fill loop. | |
| // | |
| address StubGenerator::generate_unsafe_setmemory(address unsafe_byte_fill) { | |
| __ align(CodeEntryAlignment); | |
| StubGenStubId stub_id = StubGenStubId::unsafe_setmemory_id; | |
| StubCodeMark mark(this, stub_id); | |
| address start = __ pc(); | |
| __ enter(); // required for proper stackwalking of RuntimeStub frame | |
| assert(unsafe_byte_fill != nullptr, "Invalid call"); | |
| // bump this on entry, not on exit: | |
| INC_COUNTER_NP(SharedRuntime::_unsafe_set_memory_ctr, rscratch1); | |
| { | |
| Label L_exit, L_fillQuadwords, L_fillDwords, L_fillBytes; | |
| const Register dest = c_rarg0; | |
| const Register size = c_rarg1; | |
| const Register byteVal = c_rarg2; | |
| const Register wide_value = rax; | |
| const Register rScratch1 = r10; | |
| assert_different_registers(dest, size, byteVal, wide_value, rScratch1); | |
| // fill_to_memory_atomic(unsigned char*, unsigned long, unsigned char) | |
| __ testq(size, size); | |
| __ jcc(Assembler::zero, L_exit); | |
| // Propagate byte to full Register | |
| __ movzbl(rScratch1, byteVal); | |
| __ mov64(wide_value, 0x0101010101010101ULL); | |
| __ imulq(wide_value, rScratch1); | |
| // Check for pointer & size alignment | |
| __ movq(rScratch1, dest); | |
| __ orq(rScratch1, size); | |
| __ testb(rScratch1, 7); | |
| __ jcc(Assembler::equal, L_fillQuadwords); | |
| __ testb(rScratch1, 3); | |
| __ jcc(Assembler::equal, L_fillDwords); | |
| __ testb(rScratch1, 1); | |
| __ jcc(Assembler::notEqual, L_fillBytes); | |
| // Fill words | |
| { | |
| UnsafeMemoryAccessMark umam(this, true, true); | |
| // At this point, we know the lower bit of size is zero and a | |
| // multiple of 2 | |
| do_setmemory_atomic_loop(USM_SHORT, dest, size, wide_value, rScratch1, | |
| L_exit, _masm); | |
| } | |
| __ jmpb(L_exit); | |
| __ BIND(L_fillQuadwords); | |
| // Fill QUADWORDs | |
| { | |
| UnsafeMemoryAccessMark umam(this, true, true); | |
| // At this point, we know the lower 3 bits of size are zero and a | |
| // multiple of 8 | |
| do_setmemory_atomic_loop(USM_QUADWORD, dest, size, wide_value, rScratch1, | |
| L_exit, _masm); | |
| } | |
| __ BIND(L_exit); | |
| __ leave(); // required for proper stackwalking of RuntimeStub frame | |
| __ ret(0); | |
| __ BIND(L_fillDwords); | |
| // Fill DWORDs | |
| { | |
| UnsafeMemoryAccessMark umam(this, true, true); | |
| // At this point, we know the lower 2 bits of size are zero and a | |
| // multiple of 4 | |
| do_setmemory_atomic_loop(USM_DWORD, dest, size, wide_value, rScratch1, | |
| L_exit, _masm); | |
| } | |
| __ jmpb(L_exit); | |
| __ BIND(L_fillBytes); | |
| // Set up for tail call to previously generated byte fill routine | |
| // Parameter order is (ptr, byteVal, size) | |
| __ xchgq(c_rarg1, c_rarg2); | |
| __ leave(); // Clear effect of enter() | |
| __ jump(RuntimeAddress(unsafe_byte_fill)); | |
| } | |
| return start; | |
| } | |
| // Perform range checks on the proposed arraycopy. | |
| // Kills temp, but nothing else. | |
| // Also, clean the sign bits of src_pos and dst_pos. | |
| void StubGenerator::arraycopy_range_checks(Register src, // source array oop (c_rarg0) | |
| Register src_pos, // source position (c_rarg1) | |
| Register dst, // destination array oo (c_rarg2) | |
| Register dst_pos, // destination position (c_rarg3) | |
| Register length, | |
| Register temp, | |
| Label& L_failed) { | |
| BLOCK_COMMENT("arraycopy_range_checks:"); | |
| // if (src_pos + length > arrayOop(src)->length()) FAIL; | |
| __ movl(temp, length); | |
| __ addl(temp, src_pos); // src_pos + length | |
| __ cmpl(temp, Address(src, arrayOopDesc::length_offset_in_bytes())); | |
| __ jcc(Assembler::above, L_failed); | |
| // if (dst_pos + length > arrayOop(dst)->length()) FAIL; | |
| __ movl(temp, length); | |
| __ addl(temp, dst_pos); // dst_pos + length | |
| __ cmpl(temp, Address(dst, arrayOopDesc::length_offset_in_bytes())); | |
| __ jcc(Assembler::above, L_failed); | |
| // Have to clean up high 32-bits of 'src_pos' and 'dst_pos'. | |
| // Move with sign extension can be used since they are positive. | |
| __ movslq(src_pos, src_pos); | |
| __ movslq(dst_pos, dst_pos); | |
| BLOCK_COMMENT("arraycopy_range_checks done"); | |
| } | |
| // Generate generic array copy stubs | |
| // | |
| // Input: | |
| // c_rarg0 - src oop | |
| // c_rarg1 - src_pos (32-bits) | |
| // c_rarg2 - dst oop | |
| // c_rarg3 - dst_pos (32-bits) | |
| // not Win64 | |
| // c_rarg4 - element count (32-bits) | |
| // Win64 | |
| // rsp+40 - element count (32-bits) | |
| // | |
| // Output: | |
| // rax == 0 - success | |
| // rax == -1^K - failure, where K is partial transfer count | |
| // | |
| address StubGenerator::generate_generic_copy(address byte_copy_entry, address short_copy_entry, | |
| address int_copy_entry, address oop_copy_entry, | |
| address long_copy_entry, address checkcast_copy_entry) { | |
| Label L_failed, L_failed_0, L_objArray; | |
| Label L_copy_shorts, L_copy_ints, L_copy_longs; | |
| // Input registers | |
| const Register src = c_rarg0; // source array oop | |
| const Register src_pos = c_rarg1; // source position | |
| const Register dst = c_rarg2; // destination array oop | |
| const Register dst_pos = c_rarg3; // destination position | |
| #ifndef _WIN64 | |
| const Register length = c_rarg4; | |
| const Register rklass_tmp = r9; // load_klass | |
| #else | |
| const Address length(rsp, 7 * wordSize); // elements count is on stack on Win64 | |
| const Register rklass_tmp = rdi; // load_klass | |
| #endif | |
| { int modulus = CodeEntryAlignment; | |
| int target = modulus - 5; // 5 = sizeof jmp(L_failed) | |
| int advance = target - (__ offset() % modulus); | |
| if (advance < 0) advance += modulus; | |
| if (advance > 0) __ nop(advance); | |
| } | |
| StubGenStubId stub_id = StubGenStubId::generic_arraycopy_id; | |
| StubCodeMark mark(this, stub_id); | |
| // Short-hop target to L_failed. Makes for denser prologue code. | |
| __ BIND(L_failed_0); | |
| __ jmp(L_failed); | |
| assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed"); | |
| __ align(CodeEntryAlignment); | |
| address start = __ pc(); | |
| __ enter(); // required for proper stackwalking of RuntimeStub frame | |
| #ifdef _WIN64 | |
| __ push(rklass_tmp); // rdi is callee-save on Windows | |
| #endif | |
| // bump this on entry, not on exit: | |
| INC_COUNTER_NP(SharedRuntime::_generic_array_copy_ctr, rscratch1); | |
| //----------------------------------------------------------------------- | |
| // Assembler stub will be used for this call to arraycopy | |
| // if the following conditions are met: | |
| // | |
| // (1) src and dst must not be null. | |
| // (2) src_pos must not be negative. | |
| // (3) dst_pos must not be negative. | |
| // (4) length must not be negative. | |
| // (5) src klass and dst klass should be the same and not null. | |
| // (6) src and dst should be arrays. | |
| // (7) src_pos + length must not exceed length of src. | |
| // (8) dst_pos + length must not exceed length of dst. | |
| // | |
| // if (src == nullptr) return -1; | |
| __ testptr(src, src); // src oop | |
| size_t j1off = __ offset(); | |
| __ jccb(Assembler::zero, L_failed_0); | |
| // if (src_pos < 0) return -1; | |
| __ testl(src_pos, src_pos); // src_pos (32-bits) | |
| __ jccb(Assembler::negative, L_failed_0); | |
| // if (dst == nullptr) return -1; | |
| __ testptr(dst, dst); // dst oop | |
| __ jccb(Assembler::zero, L_failed_0); | |
| // if (dst_pos < 0) return -1; | |
| __ testl(dst_pos, dst_pos); // dst_pos (32-bits) | |
| size_t j4off = __ offset(); | |
| __ jccb(Assembler::negative, L_failed_0); | |
| // The first four tests are very dense code, | |
| // but not quite dense enough to put four | |
| // jumps in a 16-byte instruction fetch buffer. | |
| // That's good, because some branch predicters | |
| // do not like jumps so close together. | |
| // Make sure of this. | |
| guarantee(((j1off ^ j4off) & ~15) != 0, "I$ line of 1st & 4th jumps"); | |
| // registers used as temp | |
| const Register r11_length = r11; // elements count to copy | |
| const Register r10_src_klass = r10; // array klass | |
| // if (length < 0) return -1; | |
| __ movl(r11_length, length); // length (elements count, 32-bits value) | |
| __ testl(r11_length, r11_length); | |
| __ jccb(Assembler::negative, L_failed_0); | |
| __ load_klass(r10_src_klass, src, rklass_tmp); | |
| #ifdef ASSERT | |
| // assert(src->klass() != nullptr); | |
| { | |
| BLOCK_COMMENT("assert klasses not null {"); | |
| Label L1, L2; | |
| __ testptr(r10_src_klass, r10_src_klass); | |
| __ jcc(Assembler::notZero, L2); // it is broken if klass is null | |
| __ bind(L1); | |
| __ stop("broken null klass"); | |
| __ bind(L2); | |
| __ load_klass(rax, dst, rklass_tmp); | |
| __ cmpq(rax, 0); | |
| __ jcc(Assembler::equal, L1); // this would be broken also | |
| BLOCK_COMMENT("} assert klasses not null done"); | |
| } | |
| #endif | |
| // Load layout helper (32-bits) | |
| // | |
| // |array_tag| | header_size | element_type | |log2_element_size| | |
| // 32 30 24 16 8 2 0 | |
| // | |
| // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0 | |
| // | |
| const int lh_offset = in_bytes(Klass::layout_helper_offset()); | |
| // Handle objArrays completely differently... | |
| const jint objArray_lh = Klass::array_layout_helper(T_OBJECT); | |
| __ cmpl(Address(r10_src_klass, lh_offset), objArray_lh); | |
| __ jcc(Assembler::equal, L_objArray); | |
| // if (src->klass() != dst->klass()) return -1; | |
| __ load_klass(rax, dst, rklass_tmp); | |
| __ cmpq(r10_src_klass, rax); | |
| __ jcc(Assembler::notEqual, L_failed); | |
| const Register rax_lh = rax; // layout helper | |
| __ movl(rax_lh, Address(r10_src_klass, lh_offset)); | |
| // if (!src->is_Array()) return -1; | |
| __ cmpl(rax_lh, Klass::_lh_neutral_value); | |
| __ jcc(Assembler::greaterEqual, L_failed); | |
| // At this point, it is known to be a typeArray (array_tag 0x3). | |
| #ifdef ASSERT | |
| { | |
| BLOCK_COMMENT("assert primitive array {"); | |
| Label L; | |
| __ cmpl(rax_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift)); | |
| __ jcc(Assembler::greaterEqual, L); | |
| __ stop("must be a primitive array"); | |
| __ bind(L); | |
| BLOCK_COMMENT("} assert primitive array done"); | |
| } | |
| #endif | |
| arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length, | |
| r10, L_failed); | |
| // TypeArrayKlass | |
| // | |
| // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize); | |
| // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize); | |
| // | |
| const Register r10_offset = r10; // array offset | |
| const Register rax_elsize = rax_lh; // element size | |
| __ movl(r10_offset, rax_lh); | |
| __ shrl(r10_offset, Klass::_lh_header_size_shift); | |
| __ andptr(r10_offset, Klass::_lh_header_size_mask); // array_offset | |
| __ addptr(src, r10_offset); // src array offset | |
| __ addptr(dst, r10_offset); // dst array offset | |
| BLOCK_COMMENT("choose copy loop based on element size"); | |
| __ andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize | |
| #ifdef _WIN64 | |
| __ pop(rklass_tmp); // Restore callee-save rdi | |
| #endif | |
| // next registers should be set before the jump to corresponding stub | |
| const Register from = c_rarg0; // source array address | |
| const Register to = c_rarg1; // destination array address | |
| const Register count = c_rarg2; // elements count | |
| // 'from', 'to', 'count' registers should be set in such order | |
| // since they are the same as 'src', 'src_pos', 'dst'. | |
| __ cmpl(rax_elsize, 0); | |
| __ jccb(Assembler::notEqual, L_copy_shorts); | |
| __ lea(from, Address(src, src_pos, Address::times_1, 0));// src_addr | |
| __ lea(to, Address(dst, dst_pos, Address::times_1, 0));// dst_addr | |
| __ movl2ptr(count, r11_length); // length | |
| __ jump(RuntimeAddress(byte_copy_entry)); | |
| __ BIND(L_copy_shorts); | |
| __ cmpl(rax_elsize, LogBytesPerShort); | |
| __ jccb(Assembler::notEqual, L_copy_ints); | |
| __ lea(from, Address(src, src_pos, Address::times_2, 0));// src_addr | |
| __ lea(to, Address(dst, dst_pos, Address::times_2, 0));// dst_addr | |
| __ movl2ptr(count, r11_length); // length | |
| __ jump(RuntimeAddress(short_copy_entry)); | |
| __ BIND(L_copy_ints); | |
| __ cmpl(rax_elsize, LogBytesPerInt); | |
| __ jccb(Assembler::notEqual, L_copy_longs); | |
| __ lea(from, Address(src, src_pos, Address::times_4, 0));// src_addr | |
| __ lea(to, Address(dst, dst_pos, Address::times_4, 0));// dst_addr | |
| __ movl2ptr(count, r11_length); // length | |
| __ jump(RuntimeAddress(int_copy_entry)); | |
| __ BIND(L_copy_longs); | |
| #ifdef ASSERT | |
| { | |
| BLOCK_COMMENT("assert long copy {"); | |
| Label L; | |
| __ cmpl(rax_elsize, LogBytesPerLong); | |
| __ jcc(Assembler::equal, L); | |
| __ stop("must be long copy, but elsize is wrong"); | |
| __ bind(L); | |
| BLOCK_COMMENT("} assert long copy done"); | |
| } | |
| #endif | |
| __ lea(from, Address(src, src_pos, Address::times_8, 0));// src_addr | |
| __ lea(to, Address(dst, dst_pos, Address::times_8, 0));// dst_addr | |
| __ movl2ptr(count, r11_length); // length | |
| __ jump(RuntimeAddress(long_copy_entry)); | |
| // ObjArrayKlass | |
| __ BIND(L_objArray); | |
| // live at this point: r10_src_klass, r11_length, src[_pos], dst[_pos] | |
| Label L_plain_copy, L_checkcast_copy; | |
| // test array classes for subtyping | |
| __ load_klass(rax, dst, rklass_tmp); | |
| __ cmpq(r10_src_klass, rax); // usual case is exact equality | |
| __ jcc(Assembler::notEqual, L_checkcast_copy); | |
| // Identically typed arrays can be copied without element-wise checks. | |
| arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length, | |
| r10, L_failed); | |
| __ lea(from, Address(src, src_pos, TIMES_OOP, | |
| arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr | |
| __ lea(to, Address(dst, dst_pos, TIMES_OOP, | |
| arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr | |
| __ movl2ptr(count, r11_length); // length | |
| __ BIND(L_plain_copy); | |
| #ifdef _WIN64 | |
| __ pop(rklass_tmp); // Restore callee-save rdi | |
| #endif | |
| __ jump(RuntimeAddress(oop_copy_entry)); | |
| __ BIND(L_checkcast_copy); | |
| // live at this point: r10_src_klass, r11_length, rax (dst_klass) | |
| { | |
| // Before looking at dst.length, make sure dst is also an objArray. | |
| __ cmpl(Address(rax, lh_offset), objArray_lh); | |
| __ jcc(Assembler::notEqual, L_failed); | |
| // It is safe to examine both src.length and dst.length. | |
| arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length, | |
| rax, L_failed); | |
| const Register r11_dst_klass = r11; | |
| __ load_klass(r11_dst_klass, dst, rklass_tmp); // reload | |
| // Marshal the base address arguments now, freeing registers. | |
| __ lea(from, Address(src, src_pos, TIMES_OOP, | |
| arrayOopDesc::base_offset_in_bytes(T_OBJECT))); | |
| __ lea(to, Address(dst, dst_pos, TIMES_OOP, | |
| arrayOopDesc::base_offset_in_bytes(T_OBJECT))); | |
| __ movl(count, length); // length (reloaded) | |
| Register sco_temp = c_rarg3; // this register is free now | |
| assert_different_registers(from, to, count, sco_temp, | |
| r11_dst_klass, r10_src_klass); | |
| assert_clean_int(count, sco_temp); | |
| // Generate the type check. | |
| const int sco_offset = in_bytes(Klass::super_check_offset_offset()); | |
| __ movl(sco_temp, Address(r11_dst_klass, sco_offset)); | |
| assert_clean_int(sco_temp, rax); | |
| generate_type_check(r10_src_klass, sco_temp, r11_dst_klass, L_plain_copy); | |
| // Fetch destination element klass from the ObjArrayKlass header. | |
| int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset()); | |
| __ movptr(r11_dst_klass, Address(r11_dst_klass, ek_offset)); | |
| __ movl( sco_temp, Address(r11_dst_klass, sco_offset)); | |
| assert_clean_int(sco_temp, rax); | |
| #ifdef _WIN64 | |
| __ pop(rklass_tmp); // Restore callee-save rdi | |
| #endif | |
| // the checkcast_copy loop needs two extra arguments: | |
| assert(c_rarg3 == sco_temp, "#3 already in place"); | |
| // Set up arguments for checkcast_copy_entry. | |
| setup_arg_regs_using_thread(4); | |
| __ movptr(r8, r11_dst_klass); // dst.klass.element_klass, r8 is c_rarg4 on Linux/Solaris | |
| __ jump(RuntimeAddress(checkcast_copy_entry)); | |
| } | |
| __ BIND(L_failed); | |
| #ifdef _WIN64 | |
| __ pop(rklass_tmp); // Restore callee-save rdi | |
| #endif | |
| __ xorptr(rax, rax); | |
| __ notptr(rax); // return -1 | |
| __ leave(); // required for proper stackwalking of RuntimeStub frame | |
| __ ret(0); | |
| return start; | |
| } | |
| #undef __ |