| /* |
| * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved. |
| * Copyright (c) 2014, 2025, Red Hat Inc. All rights reserved. |
| * Copyright (c) 2020, 2025, Huawei Technologies Co., Ltd. All rights reserved. |
| * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| * |
| * This code is free software; you can redistribute it and/or modify it |
| * under the terms of the GNU General Public License version 2 only, as |
| * published by the Free Software Foundation. |
| * |
| * This code is distributed in the hope that it will be useful, but WITHOUT |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| * version 2 for more details (a copy is included in the LICENSE file that |
| * accompanied this code). |
| * |
| * You should have received a copy of the GNU General Public License version |
| * 2 along with this work; if not, write to the Free Software Foundation, |
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| * |
| * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| * or visit www.oracle.com if you need additional information or have any |
| * questions. |
| * |
| */ |
| |
| #include "asm/macroAssembler.hpp" |
| #include "asm/macroAssembler.inline.hpp" |
| #include "compiler/oopMap.hpp" |
| #include "gc/shared/barrierSet.hpp" |
| #include "gc/shared/barrierSetAssembler.hpp" |
| #include "interpreter/interpreter.hpp" |
| #include "memory/universe.hpp" |
| #include "nativeInst_riscv.hpp" |
| #include "oops/instanceOop.hpp" |
| #include "oops/method.hpp" |
| #include "oops/objArrayKlass.hpp" |
| #include "oops/oop.inline.hpp" |
| #include "prims/methodHandles.hpp" |
| #include "prims/upcallLinker.hpp" |
| #include "runtime/continuation.hpp" |
| #include "runtime/continuationEntry.inline.hpp" |
| #include "runtime/frame.inline.hpp" |
| #include "runtime/handles.inline.hpp" |
| #include "runtime/javaThread.hpp" |
| #include "runtime/sharedRuntime.hpp" |
| #include "runtime/stubCodeGenerator.hpp" |
| #include "runtime/stubRoutines.hpp" |
| #include "utilities/align.hpp" |
| #include "utilities/powerOfTwo.hpp" |
| #ifdef COMPILER2 |
| #include "opto/runtime.hpp" |
| #endif |
| |
| // Declaration and definition of StubGenerator (no .hpp file). |
| // For a more detailed description of the stub routine structure |
| // see the comment in stubRoutines.hpp |
| |
| #undef __ |
| #define __ _masm-> |
| |
| #ifdef PRODUCT |
| #define BLOCK_COMMENT(str) /* nothing */ |
| #else |
| #define BLOCK_COMMENT(str) __ block_comment(str) |
| #endif |
| |
| #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") |
| |
| // Stub Code definitions |
| |
| class StubGenerator: public StubCodeGenerator { |
| private: |
| |
| #ifdef PRODUCT |
| #define inc_counter_np(counter) ((void)0) |
| #else |
| void inc_counter_np_(uint& counter) { |
| __ incrementw(ExternalAddress((address)&counter)); |
| } |
| #define inc_counter_np(counter) \ |
| BLOCK_COMMENT("inc_counter " #counter); \ |
| inc_counter_np_(counter); |
| #endif |
| |
| // Call stubs are used to call Java from C |
| // |
| // Arguments: |
| // c_rarg0: call wrapper address address |
| // c_rarg1: result address |
| // c_rarg2: result type BasicType |
| // c_rarg3: method Method* |
| // c_rarg4: (interpreter) entry point address |
| // c_rarg5: parameters intptr_t* |
| // c_rarg6: parameter size (in words) int |
| // c_rarg7: thread Thread* |
| // |
| // There is no return from the stub itself as any Java result |
| // is written to result |
| // |
| // we save x1 (ra) as the return PC at the base of the frame and |
| // link x8 (fp) below it as the frame pointer installing sp (x2) |
| // into fp. |
| // |
| // we save x10-x17, which accounts for all the c arguments. |
| // |
| // TODO: strictly do we need to save them all? they are treated as |
| // volatile by C so could we omit saving the ones we are going to |
| // place in global registers (thread? method?) or those we only use |
| // during setup of the Java call? |
| // |
| // we don't need to save x5 which C uses as an indirect result location |
| // return register. |
| // |
| // we don't need to save x6-x7 and x28-x31 which both C and Java treat as |
| // volatile |
| // |
| // we save x9, x18-x27, f8-f9, and f18-f27 which Java uses as temporary |
| // registers and C expects to be callee-save |
| // |
| // so the stub frame looks like this when we enter Java code |
| // |
| // [ return_from_Java ] <--- sp |
| // [ argument word n ] |
| // ... |
| // -35 [ argument word 1 ] |
| // -34 [ saved FRM in Floating-point Control and Status Register ] <--- sp_after_call |
| // -33 [ saved f27 ] |
| // -32 [ saved f26 ] |
| // -31 [ saved f25 ] |
| // -30 [ saved f24 ] |
| // -29 [ saved f23 ] |
| // -28 [ saved f22 ] |
| // -27 [ saved f21 ] |
| // -26 [ saved f20 ] |
| // -25 [ saved f19 ] |
| // -24 [ saved f18 ] |
| // -23 [ saved f9 ] |
| // -22 [ saved f8 ] |
| // -21 [ saved x27 ] |
| // -20 [ saved x26 ] |
| // -19 [ saved x25 ] |
| // -18 [ saved x24 ] |
| // -17 [ saved x23 ] |
| // -16 [ saved x22 ] |
| // -15 [ saved x21 ] |
| // -14 [ saved x20 ] |
| // -13 [ saved x19 ] |
| // -12 [ saved x18 ] |
| // -11 [ saved x9 ] |
| // -10 [ call wrapper (x10) ] |
| // -9 [ result (x11) ] |
| // -8 [ result type (x12) ] |
| // -7 [ method (x13) ] |
| // -6 [ entry point (x14) ] |
| // -5 [ parameters (x15) ] |
| // -4 [ parameter size (x16) ] |
| // -3 [ thread (x17) ] |
| // -2 [ saved fp (x8) ] |
| // -1 [ saved ra (x1) ] |
| // 0 [ ] <--- fp == saved sp (x2) |
| |
| // Call stub stack layout word offsets from fp |
| enum call_stub_layout { |
| sp_after_call_off = -34, |
| |
| frm_off = sp_after_call_off, |
| f27_off = -33, |
| f26_off = -32, |
| f25_off = -31, |
| f24_off = -30, |
| f23_off = -29, |
| f22_off = -28, |
| f21_off = -27, |
| f20_off = -26, |
| f19_off = -25, |
| f18_off = -24, |
| f9_off = -23, |
| f8_off = -22, |
| |
| x27_off = -21, |
| x26_off = -20, |
| x25_off = -19, |
| x24_off = -18, |
| x23_off = -17, |
| x22_off = -16, |
| x21_off = -15, |
| x20_off = -14, |
| x19_off = -13, |
| x18_off = -12, |
| x9_off = -11, |
| |
| call_wrapper_off = -10, |
| result_off = -9, |
| result_type_off = -8, |
| method_off = -7, |
| entry_point_off = -6, |
| parameters_off = -5, |
| parameter_size_off = -4, |
| thread_off = -3, |
| fp_f = -2, |
| retaddr_off = -1, |
| }; |
| |
| address generate_call_stub(address& return_address) { |
| assert((int)frame::entry_frame_after_call_words == -(int)sp_after_call_off + 1 && |
| (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off, |
| "adjust this code"); |
| |
| StubGenStubId stub_id = StubGenStubId::call_stub_id; |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| |
| const Address sp_after_call (fp, sp_after_call_off * wordSize); |
| |
| const Address frm_save (fp, frm_off * wordSize); |
| const Address call_wrapper (fp, call_wrapper_off * wordSize); |
| const Address result (fp, result_off * wordSize); |
| const Address result_type (fp, result_type_off * wordSize); |
| const Address method (fp, method_off * wordSize); |
| const Address entry_point (fp, entry_point_off * wordSize); |
| const Address parameters (fp, parameters_off * wordSize); |
| const Address parameter_size(fp, parameter_size_off * wordSize); |
| |
| const Address thread (fp, thread_off * wordSize); |
| |
| const Address f27_save (fp, f27_off * wordSize); |
| const Address f26_save (fp, f26_off * wordSize); |
| const Address f25_save (fp, f25_off * wordSize); |
| const Address f24_save (fp, f24_off * wordSize); |
| const Address f23_save (fp, f23_off * wordSize); |
| const Address f22_save (fp, f22_off * wordSize); |
| const Address f21_save (fp, f21_off * wordSize); |
| const Address f20_save (fp, f20_off * wordSize); |
| const Address f19_save (fp, f19_off * wordSize); |
| const Address f18_save (fp, f18_off * wordSize); |
| const Address f9_save (fp, f9_off * wordSize); |
| const Address f8_save (fp, f8_off * wordSize); |
| |
| const Address x27_save (fp, x27_off * wordSize); |
| const Address x26_save (fp, x26_off * wordSize); |
| const Address x25_save (fp, x25_off * wordSize); |
| const Address x24_save (fp, x24_off * wordSize); |
| const Address x23_save (fp, x23_off * wordSize); |
| const Address x22_save (fp, x22_off * wordSize); |
| const Address x21_save (fp, x21_off * wordSize); |
| const Address x20_save (fp, x20_off * wordSize); |
| const Address x19_save (fp, x19_off * wordSize); |
| const Address x18_save (fp, x18_off * wordSize); |
| |
| const Address x9_save (fp, x9_off * wordSize); |
| |
| // stub code |
| |
| address riscv_entry = __ pc(); |
| |
| // set up frame and move sp to end of save area |
| __ enter(); |
| __ addi(sp, fp, sp_after_call_off * wordSize); |
| |
| // save register parameters and Java temporary/global registers |
| // n.b. we save thread even though it gets installed in |
| // xthread because we want to sanity check tp later |
| __ sd(c_rarg7, thread); |
| __ sw(c_rarg6, parameter_size); |
| __ sd(c_rarg5, parameters); |
| __ sd(c_rarg4, entry_point); |
| __ sd(c_rarg3, method); |
| __ sd(c_rarg2, result_type); |
| __ sd(c_rarg1, result); |
| __ sd(c_rarg0, call_wrapper); |
| |
| __ sd(x9, x9_save); |
| |
| __ sd(x18, x18_save); |
| __ sd(x19, x19_save); |
| __ sd(x20, x20_save); |
| __ sd(x21, x21_save); |
| __ sd(x22, x22_save); |
| __ sd(x23, x23_save); |
| __ sd(x24, x24_save); |
| __ sd(x25, x25_save); |
| __ sd(x26, x26_save); |
| __ sd(x27, x27_save); |
| |
| __ fsd(f8, f8_save); |
| __ fsd(f9, f9_save); |
| __ fsd(f18, f18_save); |
| __ fsd(f19, f19_save); |
| __ fsd(f20, f20_save); |
| __ fsd(f21, f21_save); |
| __ fsd(f22, f22_save); |
| __ fsd(f23, f23_save); |
| __ fsd(f24, f24_save); |
| __ fsd(f25, f25_save); |
| __ fsd(f26, f26_save); |
| __ fsd(f27, f27_save); |
| |
| __ frrm(t0); |
| __ sd(t0, frm_save); |
| // Set frm to the state we need. We do want Round to Nearest. We |
| // don't want non-IEEE rounding modes. |
| Label skip_fsrmi; |
| guarantee(__ RoundingMode::rne == 0, "must be"); |
| __ beqz(t0, skip_fsrmi); |
| __ fsrmi(__ RoundingMode::rne); |
| __ bind(skip_fsrmi); |
| |
| // install Java thread in global register now we have saved |
| // whatever value it held |
| __ mv(xthread, c_rarg7); |
| |
| // And method |
| __ mv(xmethod, c_rarg3); |
| |
| // set up the heapbase register |
| __ reinit_heapbase(); |
| |
| #ifdef ASSERT |
| // make sure we have no pending exceptions |
| { |
| Label L; |
| __ ld(t0, Address(xthread, in_bytes(Thread::pending_exception_offset()))); |
| __ beqz(t0, L); |
| __ stop("StubRoutines::call_stub: entered with pending exception"); |
| __ BIND(L); |
| } |
| #endif |
| // pass parameters if any |
| __ mv(esp, sp); |
| __ slli(t0, c_rarg6, LogBytesPerWord); |
| __ sub(t0, sp, t0); // Move SP out of the way |
| __ andi(sp, t0, -2 * wordSize); |
| |
| BLOCK_COMMENT("pass parameters if any"); |
| Label parameters_done; |
| // parameter count is still in c_rarg6 |
| // and parameter pointer identifying param 1 is in c_rarg5 |
| __ beqz(c_rarg6, parameters_done); |
| |
| address loop = __ pc(); |
| __ ld(t0, Address(c_rarg5, 0)); |
| __ addi(c_rarg5, c_rarg5, wordSize); |
| __ subi(c_rarg6, c_rarg6, 1); |
| __ push_reg(t0); |
| __ bgtz(c_rarg6, loop); |
| |
| __ BIND(parameters_done); |
| |
| // call Java entry -- passing methdoOop, and current sp |
| // xmethod: Method* |
| // x19_sender_sp: sender sp |
| BLOCK_COMMENT("call Java function"); |
| __ mv(x19_sender_sp, sp); |
| __ jalr(c_rarg4); |
| |
| // save current address for use by exception handling code |
| |
| return_address = __ pc(); |
| |
| // store result depending on type (everything that is not |
| // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT) |
| // n.b. this assumes Java returns an integral result in x10 |
| // and a floating result in j_farg0 |
| __ ld(j_rarg2, result); |
| Label is_long, is_float, is_double, exit; |
| __ ld(j_rarg1, result_type); |
| __ mv(t0, (u1)T_OBJECT); |
| __ beq(j_rarg1, t0, is_long); |
| __ mv(t0, (u1)T_LONG); |
| __ beq(j_rarg1, t0, is_long); |
| __ mv(t0, (u1)T_FLOAT); |
| __ beq(j_rarg1, t0, is_float); |
| __ mv(t0, (u1)T_DOUBLE); |
| __ beq(j_rarg1, t0, is_double); |
| |
| // handle T_INT case |
| __ sw(x10, Address(j_rarg2)); |
| |
| __ BIND(exit); |
| |
| // pop parameters |
| __ addi(esp, fp, sp_after_call_off * wordSize); |
| |
| #ifdef ASSERT |
| // verify that threads correspond |
| { |
| Label L, S; |
| __ ld(t0, thread); |
| __ bne(xthread, t0, S); |
| __ get_thread(t0); |
| __ beq(xthread, t0, L); |
| __ BIND(S); |
| __ stop("StubRoutines::call_stub: threads must correspond"); |
| __ BIND(L); |
| } |
| #endif |
| |
| __ pop_cont_fastpath(xthread); |
| |
| // restore callee-save registers |
| __ fld(f27, f27_save); |
| __ fld(f26, f26_save); |
| __ fld(f25, f25_save); |
| __ fld(f24, f24_save); |
| __ fld(f23, f23_save); |
| __ fld(f22, f22_save); |
| __ fld(f21, f21_save); |
| __ fld(f20, f20_save); |
| __ fld(f19, f19_save); |
| __ fld(f18, f18_save); |
| __ fld(f9, f9_save); |
| __ fld(f8, f8_save); |
| |
| __ ld(x27, x27_save); |
| __ ld(x26, x26_save); |
| __ ld(x25, x25_save); |
| __ ld(x24, x24_save); |
| __ ld(x23, x23_save); |
| __ ld(x22, x22_save); |
| __ ld(x21, x21_save); |
| __ ld(x20, x20_save); |
| __ ld(x19, x19_save); |
| __ ld(x18, x18_save); |
| |
| __ ld(x9, x9_save); |
| |
| // restore frm |
| Label skip_fsrm; |
| __ ld(t0, frm_save); |
| __ frrm(t1); |
| __ beq(t0, t1, skip_fsrm); |
| __ fsrm(t0); |
| __ bind(skip_fsrm); |
| |
| __ ld(c_rarg0, call_wrapper); |
| __ ld(c_rarg1, result); |
| __ ld(c_rarg2, result_type); |
| __ ld(c_rarg3, method); |
| __ ld(c_rarg4, entry_point); |
| __ ld(c_rarg5, parameters); |
| __ ld(c_rarg6, parameter_size); |
| __ ld(c_rarg7, thread); |
| |
| // leave frame and return to caller |
| __ leave(); |
| __ ret(); |
| |
| // handle return types different from T_INT |
| |
| __ BIND(is_long); |
| __ sd(x10, Address(j_rarg2, 0)); |
| __ j(exit); |
| |
| __ BIND(is_float); |
| __ fsw(j_farg0, Address(j_rarg2, 0), t0); |
| __ j(exit); |
| |
| __ BIND(is_double); |
| __ fsd(j_farg0, Address(j_rarg2, 0), t0); |
| __ j(exit); |
| |
| return start; |
| } |
| |
| // Return point for a Java call if there's an exception thrown in |
| // Java code. The exception is caught and transformed into a |
| // pending exception stored in JavaThread that can be tested from |
| // within the VM. |
| // |
| // Note: Usually the parameters are removed by the callee. In case |
| // of an exception crossing an activation frame boundary, that is |
| // not the case if the callee is compiled code => need to setup the |
| // sp. |
| // |
| // x10: exception oop |
| |
| address generate_catch_exception() { |
| StubGenStubId stub_id = StubGenStubId::catch_exception_id; |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| |
| // same as in generate_call_stub(): |
| const Address thread(fp, thread_off * wordSize); |
| |
| #ifdef ASSERT |
| // verify that threads correspond |
| { |
| Label L, S; |
| __ ld(t0, thread); |
| __ bne(xthread, t0, S); |
| __ get_thread(t0); |
| __ beq(xthread, t0, L); |
| __ bind(S); |
| __ stop("StubRoutines::catch_exception: threads must correspond"); |
| __ bind(L); |
| } |
| #endif |
| |
| // set pending exception |
| __ verify_oop(x10); |
| |
| __ sd(x10, Address(xthread, Thread::pending_exception_offset())); |
| __ mv(t0, (address)__FILE__); |
| __ sd(t0, Address(xthread, Thread::exception_file_offset())); |
| __ mv(t0, (int)__LINE__); |
| __ sw(t0, Address(xthread, Thread::exception_line_offset())); |
| |
| // complete return to VM |
| assert(StubRoutines::_call_stub_return_address != nullptr, |
| "_call_stub_return_address must have been generated before"); |
| __ j(RuntimeAddress(StubRoutines::_call_stub_return_address)); |
| |
| return start; |
| } |
| |
| // Continuation point for runtime calls returning with a pending |
| // exception. The pending exception check happened in the runtime |
| // or native call stub. The pending exception in Thread is |
| // converted into a Java-level exception. |
| // |
| // Contract with Java-level exception handlers: |
| // x10: exception |
| // x13: throwing pc |
| // |
| // NOTE: At entry of this stub, exception-pc must be in RA !! |
| |
| // NOTE: this is always used as a jump target within generated code |
| // so it just needs to be generated code with no x86 prolog |
| |
| address generate_forward_exception() { |
| StubGenStubId stub_id = StubGenStubId::forward_exception_id; |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| |
| // Upon entry, RA points to the return address returning into |
| // Java (interpreted or compiled) code; i.e., the return address |
| // becomes the throwing pc. |
| // |
| // Arguments pushed before the runtime call are still on the stack |
| // but the exception handler will reset the stack pointer -> |
| // ignore them. A potential result in registers can be ignored as |
| // well. |
| |
| #ifdef ASSERT |
| // make sure this code is only executed if there is a pending exception |
| { |
| Label L; |
| __ ld(t0, Address(xthread, Thread::pending_exception_offset())); |
| __ bnez(t0, L); |
| __ stop("StubRoutines::forward exception: no pending exception (1)"); |
| __ bind(L); |
| } |
| #endif |
| |
| // compute exception handler into x9 |
| |
| // call the VM to find the handler address associated with the |
| // caller address. pass thread in x10 and caller pc (ret address) |
| // in x11. n.b. the caller pc is in ra, unlike x86 where it is on |
| // the stack. |
| __ mv(c_rarg1, ra); |
| // ra will be trashed by the VM call so we move it to x9 |
| // (callee-saved) because we also need to pass it to the handler |
| // returned by this call. |
| __ mv(x9, ra); |
| BLOCK_COMMENT("call exception_handler_for_return_address"); |
| __ call_VM_leaf(CAST_FROM_FN_PTR(address, |
| SharedRuntime::exception_handler_for_return_address), |
| xthread, c_rarg1); |
| // we should not really care that ra is no longer the callee |
| // address. we saved the value the handler needs in x9 so we can |
| // just copy it to x13. however, the C2 handler will push its own |
| // frame and then calls into the VM and the VM code asserts that |
| // the PC for the frame above the handler belongs to a compiled |
| // Java method. So, we restore ra here to satisfy that assert. |
| __ mv(ra, x9); |
| // setup x10 & x13 & clear pending exception |
| __ mv(x13, x9); |
| __ mv(x9, x10); |
| __ ld(x10, Address(xthread, Thread::pending_exception_offset())); |
| __ sd(zr, Address(xthread, Thread::pending_exception_offset())); |
| |
| #ifdef ASSERT |
| // make sure exception is set |
| { |
| Label L; |
| __ bnez(x10, L); |
| __ stop("StubRoutines::forward exception: no pending exception (2)"); |
| __ bind(L); |
| } |
| #endif |
| |
| // continue at exception handler |
| // x10: exception |
| // x13: throwing pc |
| // x9: exception handler |
| __ verify_oop(x10); |
| __ jr(x9); |
| |
| return start; |
| } |
| |
| // Non-destructive plausibility checks for oops |
| // |
| // Arguments: |
| // x10: oop to verify |
| // t0: error message |
| // |
| // Stack after saving c_rarg3: |
| // [tos + 0]: saved c_rarg3 |
| // [tos + 1]: saved c_rarg2 |
| // [tos + 2]: saved ra |
| // [tos + 3]: saved t1 |
| // [tos + 4]: saved x10 |
| // [tos + 5]: saved t0 |
| address generate_verify_oop() { |
| |
| StubGenStubId stub_id = StubGenStubId::verify_oop_id; |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| |
| Label exit, error; |
| |
| __ push_reg(RegSet::of(c_rarg2, c_rarg3), sp); // save c_rarg2 and c_rarg3 |
| |
| __ la(c_rarg2, ExternalAddress((address) StubRoutines::verify_oop_count_addr())); |
| __ ld(c_rarg3, Address(c_rarg2)); |
| __ addi(c_rarg3, c_rarg3, 1); |
| __ sd(c_rarg3, Address(c_rarg2)); |
| |
| // object is in x10 |
| // make sure object is 'reasonable' |
| __ beqz(x10, exit); // if obj is null it is OK |
| |
| BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler(); |
| bs_asm->check_oop(_masm, x10, c_rarg2, c_rarg3, error); |
| |
| // return if everything seems ok |
| __ bind(exit); |
| |
| __ pop_reg(RegSet::of(c_rarg2, c_rarg3), sp); // pop c_rarg2 and c_rarg3 |
| __ ret(); |
| |
| // handle errors |
| __ bind(error); |
| __ pop_reg(RegSet::of(c_rarg2, c_rarg3), sp); // pop c_rarg2 and c_rarg3 |
| |
| __ push_reg(RegSet::range(x0, x31), sp); |
| // debug(char* msg, int64_t pc, int64_t regs[]) |
| __ mv(c_rarg0, t0); // pass address of error message |
| __ mv(c_rarg1, ra); // pass return address |
| __ mv(c_rarg2, sp); // pass address of regs on stack |
| #ifndef PRODUCT |
| assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area"); |
| #endif |
| BLOCK_COMMENT("call MacroAssembler::debug"); |
| __ rt_call(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)); |
| __ ebreak(); |
| |
| return start; |
| } |
| |
| // The inner part of zero_words(). |
| // |
| // Inputs: |
| // x28: the HeapWord-aligned base address of an array to zero. |
| // x29: the count in HeapWords, x29 > 0. |
| // |
| // Returns x28 and x29, adjusted for the caller to clear. |
| // x28: the base address of the tail of words left to clear. |
| // x29: the number of words in the tail. |
| // x29 < MacroAssembler::zero_words_block_size. |
| |
| address generate_zero_blocks() { |
| Label done; |
| |
| const Register base = x28, cnt = x29, tmp1 = x30, tmp2 = x31; |
| |
| __ align(CodeEntryAlignment); |
| StubGenStubId stub_id = StubGenStubId::zero_blocks_id; |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| |
| if (UseBlockZeroing) { |
| int zicboz_block_size = VM_Version::zicboz_block_size.value(); |
| // Ensure count >= 2 * zicboz_block_size so that it still deserves |
| // a cbo.zero after alignment. |
| Label small; |
| int low_limit = MAX2(2 * zicboz_block_size, (int)BlockZeroingLowLimit) / wordSize; |
| __ mv(tmp1, low_limit); |
| __ blt(cnt, tmp1, small); |
| __ zero_dcache_blocks(base, cnt, tmp1, tmp2); |
| __ bind(small); |
| } |
| |
| { |
| // Clear the remaining blocks. |
| Label loop; |
| __ mv(tmp1, MacroAssembler::zero_words_block_size); |
| __ blt(cnt, tmp1, done); |
| __ bind(loop); |
| for (int i = 0; i < MacroAssembler::zero_words_block_size; i++) { |
| __ sd(zr, Address(base, i * wordSize)); |
| } |
| __ addi(base, base, MacroAssembler::zero_words_block_size * wordSize); |
| __ subi(cnt, cnt, MacroAssembler::zero_words_block_size); |
| __ bge(cnt, tmp1, loop); |
| __ bind(done); |
| } |
| |
| __ ret(); |
| |
| return start; |
| } |
| |
| typedef enum { |
| copy_forwards = 1, |
| copy_backwards = -1 |
| } copy_direction; |
| |
| // Bulk copy of blocks of 8 words. |
| // |
| // count is a count of words. |
| // |
| // Precondition: count >= 8 |
| // |
| // Postconditions: |
| // |
| // The least significant bit of count contains the remaining count |
| // of words to copy. The rest of count is trash. |
| // |
| // s and d are adjusted to point to the remaining words to copy |
| // |
| void generate_copy_longs(StubGenStubId stub_id, Label &start, |
| Register s, Register d, Register count) { |
| BasicType type; |
| copy_direction direction; |
| switch (stub_id) { |
| case copy_byte_f_id: |
| direction = copy_forwards; |
| type = T_BYTE; |
| break; |
| case copy_byte_b_id: |
| direction = copy_backwards; |
| type = T_BYTE; |
| break; |
| default: |
| ShouldNotReachHere(); |
| } |
| int unit = wordSize * direction; |
| int bias = wordSize; |
| |
| const Register tmp_reg0 = x13, tmp_reg1 = x14, tmp_reg2 = x15, tmp_reg3 = x16, |
| tmp_reg4 = x17, tmp_reg5 = x7, tmp_reg6 = x28, tmp_reg7 = x29; |
| |
| const Register stride = x30; |
| |
| assert_different_registers(t0, tmp_reg0, tmp_reg1, tmp_reg2, tmp_reg3, |
| tmp_reg4, tmp_reg5, tmp_reg6, tmp_reg7); |
| assert_different_registers(s, d, count, t0); |
| |
| Label again, drain; |
| StubCodeMark mark(this, stub_id); |
| __ align(CodeEntryAlignment); |
| __ bind(start); |
| |
| if (direction == copy_forwards) { |
| __ sub(s, s, bias); |
| __ sub(d, d, bias); |
| } |
| |
| #ifdef ASSERT |
| // Make sure we are never given < 8 words |
| { |
| Label L; |
| |
| __ mv(t0, 8); |
| __ bge(count, t0, L); |
| __ stop("genrate_copy_longs called with < 8 words"); |
| __ bind(L); |
| } |
| #endif |
| |
| __ ld(tmp_reg0, Address(s, 1 * unit)); |
| __ ld(tmp_reg1, Address(s, 2 * unit)); |
| __ ld(tmp_reg2, Address(s, 3 * unit)); |
| __ ld(tmp_reg3, Address(s, 4 * unit)); |
| __ ld(tmp_reg4, Address(s, 5 * unit)); |
| __ ld(tmp_reg5, Address(s, 6 * unit)); |
| __ ld(tmp_reg6, Address(s, 7 * unit)); |
| __ ld(tmp_reg7, Address(s, 8 * unit)); |
| __ addi(s, s, 8 * unit); |
| |
| __ subi(count, count, 16); |
| __ bltz(count, drain); |
| |
| __ bind(again); |
| |
| __ sd(tmp_reg0, Address(d, 1 * unit)); |
| __ sd(tmp_reg1, Address(d, 2 * unit)); |
| __ sd(tmp_reg2, Address(d, 3 * unit)); |
| __ sd(tmp_reg3, Address(d, 4 * unit)); |
| __ sd(tmp_reg4, Address(d, 5 * unit)); |
| __ sd(tmp_reg5, Address(d, 6 * unit)); |
| __ sd(tmp_reg6, Address(d, 7 * unit)); |
| __ sd(tmp_reg7, Address(d, 8 * unit)); |
| |
| __ ld(tmp_reg0, Address(s, 1 * unit)); |
| __ ld(tmp_reg1, Address(s, 2 * unit)); |
| __ ld(tmp_reg2, Address(s, 3 * unit)); |
| __ ld(tmp_reg3, Address(s, 4 * unit)); |
| __ ld(tmp_reg4, Address(s, 5 * unit)); |
| __ ld(tmp_reg5, Address(s, 6 * unit)); |
| __ ld(tmp_reg6, Address(s, 7 * unit)); |
| __ ld(tmp_reg7, Address(s, 8 * unit)); |
| |
| __ addi(s, s, 8 * unit); |
| __ addi(d, d, 8 * unit); |
| |
| __ subi(count, count, 8); |
| __ bgez(count, again); |
| |
| // Drain |
| __ bind(drain); |
| |
| __ sd(tmp_reg0, Address(d, 1 * unit)); |
| __ sd(tmp_reg1, Address(d, 2 * unit)); |
| __ sd(tmp_reg2, Address(d, 3 * unit)); |
| __ sd(tmp_reg3, Address(d, 4 * unit)); |
| __ sd(tmp_reg4, Address(d, 5 * unit)); |
| __ sd(tmp_reg5, Address(d, 6 * unit)); |
| __ sd(tmp_reg6, Address(d, 7 * unit)); |
| __ sd(tmp_reg7, Address(d, 8 * unit)); |
| __ addi(d, d, 8 * unit); |
| |
| { |
| Label L1, L2; |
| __ test_bit(t0, count, 2); |
| __ beqz(t0, L1); |
| |
| __ ld(tmp_reg0, Address(s, 1 * unit)); |
| __ ld(tmp_reg1, Address(s, 2 * unit)); |
| __ ld(tmp_reg2, Address(s, 3 * unit)); |
| __ ld(tmp_reg3, Address(s, 4 * unit)); |
| __ addi(s, s, 4 * unit); |
| |
| __ sd(tmp_reg0, Address(d, 1 * unit)); |
| __ sd(tmp_reg1, Address(d, 2 * unit)); |
| __ sd(tmp_reg2, Address(d, 3 * unit)); |
| __ sd(tmp_reg3, Address(d, 4 * unit)); |
| __ addi(d, d, 4 * unit); |
| |
| __ bind(L1); |
| |
| if (direction == copy_forwards) { |
| __ addi(s, s, bias); |
| __ addi(d, d, bias); |
| } |
| |
| __ test_bit(t0, count, 1); |
| __ beqz(t0, L2); |
| if (direction == copy_backwards) { |
| __ addi(s, s, 2 * unit); |
| __ ld(tmp_reg0, Address(s)); |
| __ ld(tmp_reg1, Address(s, wordSize)); |
| __ addi(d, d, 2 * unit); |
| __ sd(tmp_reg0, Address(d)); |
| __ sd(tmp_reg1, Address(d, wordSize)); |
| } else { |
| __ ld(tmp_reg0, Address(s)); |
| __ ld(tmp_reg1, Address(s, wordSize)); |
| __ addi(s, s, 2 * unit); |
| __ sd(tmp_reg0, Address(d)); |
| __ sd(tmp_reg1, Address(d, wordSize)); |
| __ addi(d, d, 2 * unit); |
| } |
| __ bind(L2); |
| } |
| |
| __ ret(); |
| } |
| |
| Label copy_f, copy_b; |
| |
| typedef void (MacroAssembler::*copy_insn)(Register Rd, const Address &adr, Register temp); |
| |
| void copy_memory_v(Register s, Register d, Register count, int step) { |
| bool is_backward = step < 0; |
| int granularity = g_uabs(step); |
| |
| const Register src = x30, dst = x31, vl = x14, cnt = x15, tmp1 = x16, tmp2 = x17; |
| assert_different_registers(s, d, cnt, vl, tmp1, tmp2); |
| Assembler::SEW sew = Assembler::elembytes_to_sew(granularity); |
| Label loop_forward, loop_backward, done; |
| |
| __ mv(dst, d); |
| __ mv(src, s); |
| __ mv(cnt, count); |
| |
| __ bind(loop_forward); |
| __ vsetvli(vl, cnt, sew, Assembler::m8); |
| if (is_backward) { |
| __ bne(vl, cnt, loop_backward); |
| } |
| |
| __ vlex_v(v0, src, sew); |
| __ sub(cnt, cnt, vl); |
| if (sew != Assembler::e8) { |
| // when sew == e8 (e.g., elem size is 1 byte), slli R, R, 0 is a nop and unnecessary |
| __ slli(vl, vl, sew); |
| } |
| __ add(src, src, vl); |
| |
| __ vsex_v(v0, dst, sew); |
| __ add(dst, dst, vl); |
| __ bnez(cnt, loop_forward); |
| |
| if (is_backward) { |
| __ j(done); |
| |
| __ bind(loop_backward); |
| __ sub(t0, cnt, vl); |
| if (sew != Assembler::e8) { |
| // when sew == e8 (e.g., elem size is 1 byte), slli R, R, 0 is a nop and unnecessary |
| __ slli(t0, t0, sew); |
| } |
| __ add(tmp1, s, t0); |
| __ vlex_v(v0, tmp1, sew); |
| __ add(tmp2, d, t0); |
| __ vsex_v(v0, tmp2, sew); |
| __ sub(cnt, cnt, vl); |
| __ bnez(cnt, loop_forward); |
| __ bind(done); |
| } |
| } |
| |
| // All-singing all-dancing memory copy. |
| // |
| // Copy count units of memory from s to d. The size of a unit is |
| // step, which can be positive or negative depending on the direction |
| // of copy. |
| // |
| void copy_memory(DecoratorSet decorators, BasicType type, bool is_aligned, |
| Register s, Register d, Register count, int step) { |
| BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler(); |
| if (UseRVV && (!is_reference_type(type) || bs_asm->supports_rvv_arraycopy())) { |
| return copy_memory_v(s, d, count, step); |
| } |
| |
| bool is_backwards = step < 0; |
| int granularity = g_uabs(step); |
| |
| const Register src = x30, dst = x31, cnt = x15, tmp3 = x16, tmp4 = x17, tmp5 = x14, tmp6 = x13; |
| const Register gct1 = x28, gct2 = x29, gct3 = t2; |
| |
| Label same_aligned; |
| Label copy_big, copy32_loop, copy8_loop, copy_small, done; |
| |
| // The size of copy32_loop body increases significantly with ZGC GC barriers. |
| // Need conditional far branches to reach a point beyond the loop in this case. |
| bool is_far = UseZGC; |
| |
| __ beqz(count, done, is_far); |
| __ slli(cnt, count, exact_log2(granularity)); |
| if (is_backwards) { |
| __ add(src, s, cnt); |
| __ add(dst, d, cnt); |
| } else { |
| __ mv(src, s); |
| __ mv(dst, d); |
| } |
| |
| if (is_aligned) { |
| __ subi(t0, cnt, 32); |
| __ bgez(t0, copy32_loop); |
| __ subi(t0, cnt, 8); |
| __ bgez(t0, copy8_loop, is_far); |
| __ j(copy_small); |
| } else { |
| __ mv(t0, 16); |
| __ blt(cnt, t0, copy_small, is_far); |
| |
| __ xorr(t0, src, dst); |
| __ andi(t0, t0, 0b111); |
| __ bnez(t0, copy_small, is_far); |
| |
| __ bind(same_aligned); |
| __ andi(t0, src, 0b111); |
| __ beqz(t0, copy_big); |
| if (is_backwards) { |
| __ addi(src, src, step); |
| __ addi(dst, dst, step); |
| } |
| bs_asm->copy_load_at(_masm, decorators, type, granularity, tmp3, Address(src), gct1); |
| bs_asm->copy_store_at(_masm, decorators, type, granularity, Address(dst), tmp3, gct1, gct2, gct3); |
| if (!is_backwards) { |
| __ addi(src, src, step); |
| __ addi(dst, dst, step); |
| } |
| __ subi(cnt, cnt, granularity); |
| __ beqz(cnt, done, is_far); |
| __ j(same_aligned); |
| |
| __ bind(copy_big); |
| __ mv(t0, 32); |
| __ blt(cnt, t0, copy8_loop, is_far); |
| } |
| |
| __ bind(copy32_loop); |
| if (is_backwards) { |
| __ subi(src, src, wordSize * 4); |
| __ subi(dst, dst, wordSize * 4); |
| } |
| // we first load 32 bytes, then write it, so the direction here doesn't matter |
| bs_asm->copy_load_at(_masm, decorators, type, 8, tmp3, Address(src), gct1); |
| bs_asm->copy_load_at(_masm, decorators, type, 8, tmp4, Address(src, 8), gct1); |
| bs_asm->copy_load_at(_masm, decorators, type, 8, tmp5, Address(src, 16), gct1); |
| bs_asm->copy_load_at(_masm, decorators, type, 8, tmp6, Address(src, 24), gct1); |
| |
| bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst), tmp3, gct1, gct2, gct3); |
| bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst, 8), tmp4, gct1, gct2, gct3); |
| bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst, 16), tmp5, gct1, gct2, gct3); |
| bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst, 24), tmp6, gct1, gct2, gct3); |
| |
| if (!is_backwards) { |
| __ addi(src, src, wordSize * 4); |
| __ addi(dst, dst, wordSize * 4); |
| } |
| __ subi(t0, cnt, 32 + wordSize * 4); |
| __ subi(cnt, cnt, wordSize * 4); |
| __ bgez(t0, copy32_loop); // cnt >= 32, do next loop |
| |
| __ beqz(cnt, done); // if that's all - done |
| |
| __ subi(t0, cnt, 8); // if not - copy the reminder |
| __ bltz(t0, copy_small); // cnt < 8, go to copy_small, else fall through to copy8_loop |
| |
| __ bind(copy8_loop); |
| if (is_backwards) { |
| __ subi(src, src, wordSize); |
| __ subi(dst, dst, wordSize); |
| } |
| bs_asm->copy_load_at(_masm, decorators, type, 8, tmp3, Address(src), gct1); |
| bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst), tmp3, gct1, gct2, gct3); |
| |
| if (!is_backwards) { |
| __ addi(src, src, wordSize); |
| __ addi(dst, dst, wordSize); |
| } |
| __ subi(t0, cnt, 8 + wordSize); |
| __ subi(cnt, cnt, wordSize); |
| __ bgez(t0, copy8_loop); // cnt >= 8, do next loop |
| |
| __ beqz(cnt, done); // if that's all - done |
| |
| __ bind(copy_small); |
| if (is_backwards) { |
| __ addi(src, src, step); |
| __ addi(dst, dst, step); |
| } |
| |
| bs_asm->copy_load_at(_masm, decorators, type, granularity, tmp3, Address(src), gct1); |
| bs_asm->copy_store_at(_masm, decorators, type, granularity, Address(dst), tmp3, gct1, gct2, gct3); |
| |
| if (!is_backwards) { |
| __ addi(src, src, step); |
| __ addi(dst, dst, step); |
| } |
| __ subi(cnt, cnt, granularity); |
| __ bgtz(cnt, copy_small); |
| |
| __ bind(done); |
| } |
| |
| // Scan over array at a for count oops, verifying each one. |
| // Preserves a and count, clobbers t0 and t1. |
| void verify_oop_array(size_t size, Register a, Register count, Register temp) { |
| Label loop, end; |
| __ mv(t1, zr); |
| __ slli(t0, count, exact_log2(size)); |
| __ bind(loop); |
| __ bgeu(t1, t0, end); |
| |
| __ add(temp, a, t1); |
| if (size == (size_t)wordSize) { |
| __ ld(temp, Address(temp, 0)); |
| __ verify_oop(temp); |
| } else { |
| __ lwu(temp, Address(temp, 0)); |
| __ decode_heap_oop(temp); // calls verify_oop |
| } |
| __ add(t1, t1, size); |
| __ j(loop); |
| __ bind(end); |
| } |
| |
| // Arguments: |
| // stub_id - is used to name the stub and identify all details of |
| // how to perform the copy. |
| // |
| // entry - is assigned to the stub's post push entry point unless |
| // it is null |
| // |
| // Inputs: |
| // c_rarg0 - source array address |
| // c_rarg1 - destination array address |
| // c_rarg2 - element count, treated as ssize_t, can be zero |
| // |
| // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let |
| // the hardware handle it. The two dwords within qwords that span |
| // cache line boundaries will still be loaded and stored atomically. |
| // |
| // Side Effects: entry is set to the (post push) entry point so it |
| // can be used by the corresponding conjoint copy |
| // method |
| // |
| address generate_disjoint_copy(StubGenStubId stub_id, address* entry) { |
| size_t size; |
| bool aligned; |
| bool is_oop; |
| bool dest_uninitialized; |
| switch (stub_id) { |
| case jbyte_disjoint_arraycopy_id: |
| size = sizeof(jbyte); |
| aligned = false; |
| is_oop = false; |
| dest_uninitialized = false; |
| break; |
| case arrayof_jbyte_disjoint_arraycopy_id: |
| size = sizeof(jbyte); |
| aligned = true; |
| is_oop = false; |
| dest_uninitialized = false; |
| break; |
| case jshort_disjoint_arraycopy_id: |
| size = sizeof(jshort); |
| aligned = false; |
| is_oop = false; |
| dest_uninitialized = false; |
| break; |
| case arrayof_jshort_disjoint_arraycopy_id: |
| size = sizeof(jshort); |
| aligned = true; |
| is_oop = false; |
| dest_uninitialized = false; |
| break; |
| case jint_disjoint_arraycopy_id: |
| size = sizeof(jint); |
| aligned = false; |
| is_oop = false; |
| dest_uninitialized = false; |
| break; |
| case arrayof_jint_disjoint_arraycopy_id: |
| size = sizeof(jint); |
| aligned = true; |
| is_oop = false; |
| dest_uninitialized = false; |
| break; |
| case jlong_disjoint_arraycopy_id: |
| // since this is always aligned we can (should!) use the same |
| // stub as for case arrayof_jlong_disjoint_arraycopy |
| ShouldNotReachHere(); |
| break; |
| case arrayof_jlong_disjoint_arraycopy_id: |
| size = sizeof(jlong); |
| aligned = true; |
| is_oop = false; |
| dest_uninitialized = false; |
| break; |
| case oop_disjoint_arraycopy_id: |
| size = UseCompressedOops ? sizeof (jint) : sizeof (jlong); |
| aligned = !UseCompressedOops; |
| is_oop = true; |
| dest_uninitialized = false; |
| break; |
| case arrayof_oop_disjoint_arraycopy_id: |
| size = UseCompressedOops ? sizeof (jint) : sizeof (jlong); |
| aligned = !UseCompressedOops; |
| is_oop = true; |
| dest_uninitialized = false; |
| break; |
| case oop_disjoint_arraycopy_uninit_id: |
| size = UseCompressedOops ? sizeof (jint) : sizeof (jlong); |
| aligned = !UseCompressedOops; |
| is_oop = true; |
| dest_uninitialized = true; |
| break; |
| case arrayof_oop_disjoint_arraycopy_uninit_id: |
| size = UseCompressedOops ? sizeof (jint) : sizeof (jlong); |
| aligned = !UseCompressedOops; |
| is_oop = true; |
| dest_uninitialized = true; |
| break; |
| default: |
| ShouldNotReachHere(); |
| break; |
| } |
| |
| const Register s = c_rarg0, d = c_rarg1, count = c_rarg2; |
| RegSet saved_reg = RegSet::of(s, d, count); |
| __ align(CodeEntryAlignment); |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| __ enter(); |
| |
| if (entry != nullptr) { |
| *entry = __ pc(); |
| // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) |
| BLOCK_COMMENT("Entry:"); |
| } |
| |
| DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT; |
| if (dest_uninitialized) { |
| decorators |= IS_DEST_UNINITIALIZED; |
| } |
| if (aligned) { |
| decorators |= ARRAYCOPY_ALIGNED; |
| } |
| |
| BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
| bs->arraycopy_prologue(_masm, decorators, is_oop, s, d, count, saved_reg); |
| |
| if (is_oop) { |
| // save regs before copy_memory |
| __ push_reg(RegSet::of(d, count), sp); |
| } |
| |
| { |
| // UnsafeMemoryAccess page error: continue after unsafe access |
| bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size); |
| UnsafeMemoryAccessMark umam(this, add_entry, true); |
| copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, size); |
| } |
| |
| if (is_oop) { |
| __ pop_reg(RegSet::of(d, count), sp); |
| if (VerifyOops) { |
| verify_oop_array(size, d, count, t2); |
| } |
| } |
| |
| bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, t0, RegSet()); |
| |
| __ leave(); |
| __ mv(x10, zr); // return 0 |
| __ ret(); |
| return start; |
| } |
| |
| // Arguments: |
| // stub_id - is used to name the stub and identify all details of |
| // how to perform the copy. |
| // |
| // nooverlap_target - identifes the (post push) entry for the |
| // corresponding disjoint copy routine which can be |
| // jumped to if the ranges do not actually overlap |
| // |
| // entry - is assigned to the stub's post push entry point unless |
| // it is null |
| // |
| // Inputs: |
| // c_rarg0 - source array address |
| // c_rarg1 - destination array address |
| // c_rarg2 - element count, treated as ssize_t, can be zero |
| // |
| // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let |
| // the hardware handle it. The two dwords within qwords that span |
| // cache line boundaries will still be loaded and stored atomically. |
| // |
| // Side Effects: |
| // entry is set to the no-overlap entry point so it can be used by |
| // some other conjoint copy method |
| // |
| address generate_conjoint_copy(StubGenStubId stub_id, address nooverlap_target, address *entry) { |
| const Register s = c_rarg0, d = c_rarg1, count = c_rarg2; |
| RegSet saved_regs = RegSet::of(s, d, count); |
| int size; |
| bool aligned; |
| bool is_oop; |
| bool dest_uninitialized; |
| switch (stub_id) { |
| case jbyte_arraycopy_id: |
| size = sizeof(jbyte); |
| aligned = false; |
| is_oop = false; |
| dest_uninitialized = false; |
| break; |
| case arrayof_jbyte_arraycopy_id: |
| size = sizeof(jbyte); |
| aligned = true; |
| is_oop = false; |
| dest_uninitialized = false; |
| break; |
| case jshort_arraycopy_id: |
| size = sizeof(jshort); |
| aligned = false; |
| is_oop = false; |
| dest_uninitialized = false; |
| break; |
| case arrayof_jshort_arraycopy_id: |
| size = sizeof(jshort); |
| aligned = true; |
| is_oop = false; |
| dest_uninitialized = false; |
| break; |
| case jint_arraycopy_id: |
| size = sizeof(jint); |
| aligned = false; |
| is_oop = false; |
| dest_uninitialized = false; |
| break; |
| case arrayof_jint_arraycopy_id: |
| size = sizeof(jint); |
| aligned = true; |
| is_oop = false; |
| dest_uninitialized = false; |
| break; |
| case jlong_arraycopy_id: |
| // since this is always aligned we can (should!) use the same |
| // stub as for case arrayof_jlong_disjoint_arraycopy |
| ShouldNotReachHere(); |
| break; |
| case arrayof_jlong_arraycopy_id: |
| size = sizeof(jlong); |
| aligned = true; |
| is_oop = false; |
| dest_uninitialized = false; |
| break; |
| case oop_arraycopy_id: |
| size = UseCompressedOops ? sizeof (jint) : sizeof (jlong); |
| aligned = !UseCompressedOops; |
| is_oop = true; |
| dest_uninitialized = false; |
| break; |
| case arrayof_oop_arraycopy_id: |
| size = UseCompressedOops ? sizeof (jint) : sizeof (jlong); |
| aligned = !UseCompressedOops; |
| is_oop = true; |
| dest_uninitialized = false; |
| break; |
| case oop_arraycopy_uninit_id: |
| size = UseCompressedOops ? sizeof (jint) : sizeof (jlong); |
| aligned = !UseCompressedOops; |
| is_oop = true; |
| dest_uninitialized = true; |
| break; |
| case arrayof_oop_arraycopy_uninit_id: |
| size = UseCompressedOops ? sizeof (jint) : sizeof (jlong); |
| aligned = !UseCompressedOops; |
| is_oop = true; |
| dest_uninitialized = true; |
| break; |
| default: |
| ShouldNotReachHere(); |
| } |
| |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| __ enter(); |
| |
| if (entry != nullptr) { |
| *entry = __ pc(); |
| // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) |
| BLOCK_COMMENT("Entry:"); |
| } |
| |
| // use fwd copy when (d-s) above_equal (count*size) |
| __ sub(t0, d, s); |
| __ slli(t1, count, exact_log2(size)); |
| Label L_continue; |
| __ bltu(t0, t1, L_continue); |
| __ j(nooverlap_target); |
| __ bind(L_continue); |
| |
| DecoratorSet decorators = IN_HEAP | IS_ARRAY; |
| if (dest_uninitialized) { |
| decorators |= IS_DEST_UNINITIALIZED; |
| } |
| if (aligned) { |
| decorators |= ARRAYCOPY_ALIGNED; |
| } |
| |
| BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
| bs->arraycopy_prologue(_masm, decorators, is_oop, s, d, count, saved_regs); |
| |
| if (is_oop) { |
| // save regs before copy_memory |
| __ push_reg(RegSet::of(d, count), sp); |
| } |
| |
| { |
| // UnsafeMemoryAccess page error: continue after unsafe access |
| bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size); |
| UnsafeMemoryAccessMark umam(this, add_entry, true); |
| copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, -size); |
| } |
| |
| if (is_oop) { |
| __ pop_reg(RegSet::of(d, count), sp); |
| if (VerifyOops) { |
| verify_oop_array(size, d, count, t2); |
| } |
| } |
| bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, t0, RegSet()); |
| __ leave(); |
| __ mv(x10, zr); // return 0 |
| __ ret(); |
| return start; |
| } |
| |
| // Helper for generating a dynamic type check. |
| // Smashes t0, t1. |
| void generate_type_check(Register sub_klass, |
| Register super_check_offset, |
| Register super_klass, |
| Register result, |
| Register tmp1, |
| Register tmp2, |
| Label& L_success) { |
| assert_different_registers(sub_klass, super_check_offset, super_klass); |
| |
| BLOCK_COMMENT("type_check:"); |
| |
| Label L_miss; |
| |
| __ check_klass_subtype_fast_path(sub_klass, super_klass, noreg, &L_success, &L_miss, nullptr, super_check_offset); |
| __ check_klass_subtype_slow_path(sub_klass, super_klass, tmp1, tmp2, &L_success, nullptr); |
| |
| // Fall through on failure! |
| __ BIND(L_miss); |
| } |
| |
| // |
| // Generate checkcasting array copy stub |
| // |
| // Input: |
| // c_rarg0 - source array address |
| // c_rarg1 - destination array address |
| // c_rarg2 - element count, treated as ssize_t, can be zero |
| // c_rarg3 - size_t ckoff (super_check_offset) |
| // c_rarg4 - oop ckval (super_klass) |
| // |
| // Output: |
| // x10 == 0 - success |
| // x10 == -1^K - failure, where K is partial transfer count |
| // |
| address generate_checkcast_copy(StubGenStubId stub_id, address* entry) { |
| bool dest_uninitialized; |
| switch (stub_id) { |
| case checkcast_arraycopy_id: |
| dest_uninitialized = false; |
| break; |
| case checkcast_arraycopy_uninit_id: |
| dest_uninitialized = true; |
| break; |
| default: |
| ShouldNotReachHere(); |
| } |
| |
| Label L_load_element, L_store_element, L_do_card_marks, L_done, L_done_pop; |
| |
| // Input registers (after setup_arg_regs) |
| const Register from = c_rarg0; // source array address |
| const Register to = c_rarg1; // destination array address |
| const Register count = c_rarg2; // elementscount |
| const Register ckoff = c_rarg3; // super_check_offset |
| const Register ckval = c_rarg4; // super_klass |
| |
| RegSet wb_pre_saved_regs = RegSet::range(c_rarg0, c_rarg4); |
| RegSet wb_post_saved_regs = RegSet::of(count); |
| |
| // Registers used as temps (x7, x9, x18 are save-on-entry) |
| const Register count_save = x19; // orig elementscount |
| const Register start_to = x18; // destination array start address |
| const Register copied_oop = x7; // actual oop copied |
| const Register r9_klass = x9; // oop._klass |
| |
| // Registers used as gc temps (x15, x16, x17 are save-on-call) |
| const Register gct1 = x15, gct2 = x16, gct3 = x17; |
| |
| //--------------------------------------------------------------- |
| // Assembler stub will be used for this call to arraycopy |
| // if the two arrays are subtypes of Object[] but the |
| // destination array type is not equal to or a supertype |
| // of the source type. Each element must be separately |
| // checked. |
| |
| assert_different_registers(from, to, count, ckoff, ckval, start_to, |
| copied_oop, r9_klass, count_save); |
| |
| __ align(CodeEntryAlignment); |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| |
| __ enter(); // required for proper stackwalking of RuntimeStub frame |
| |
| // Caller of this entry point must set up the argument registers. |
| if (entry != nullptr) { |
| *entry = __ pc(); |
| BLOCK_COMMENT("Entry:"); |
| } |
| |
| // Empty array: Nothing to do |
| __ beqz(count, L_done); |
| |
| __ push_reg(RegSet::of(x7, x9, x18, x19), sp); |
| |
| #ifdef ASSERT |
| BLOCK_COMMENT("assert consistent ckoff/ckval"); |
| // The ckoff and ckval must be mutually consistent, |
| // even though caller generates both. |
| { Label L; |
| int sco_offset = in_bytes(Klass::super_check_offset_offset()); |
| __ lwu(start_to, Address(ckval, sco_offset)); |
| __ beq(ckoff, start_to, L); |
| __ stop("super_check_offset inconsistent"); |
| __ bind(L); |
| } |
| #endif //ASSERT |
| |
| DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST | ARRAYCOPY_DISJOINT; |
| if (dest_uninitialized) { |
| decorators |= IS_DEST_UNINITIALIZED; |
| } |
| |
| bool is_oop = true; |
| int element_size = UseCompressedOops ? 4 : 8; |
| |
| BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
| bs->arraycopy_prologue(_masm, decorators, is_oop, from, to, count, wb_pre_saved_regs); |
| |
| // save the original count |
| __ mv(count_save, count); |
| |
| // Copy from low to high addresses |
| __ mv(start_to, to); // Save destination array start address |
| __ j(L_load_element); |
| |
| // ======== begin loop ======== |
| // (Loop is rotated; its entry is L_load_element.) |
| // Loop control: |
| // for count to 0 do |
| // copied_oop = load_heap_oop(from++) |
| // ... generate_type_check ... |
| // store_heap_oop(to++, copied_oop) |
| // end |
| |
| __ align(OptoLoopAlignment); |
| |
| __ BIND(L_store_element); |
| bs->copy_store_at(_masm, decorators, T_OBJECT, element_size, |
| Address(to, 0), copied_oop, |
| gct1, gct2, gct3); |
| __ addi(to, to, UseCompressedOops ? 4 : 8); |
| __ subi(count, count, 1); |
| __ beqz(count, L_do_card_marks); |
| |
| // ======== loop entry is here ======== |
| __ BIND(L_load_element); |
| bs->copy_load_at(_masm, decorators, T_OBJECT, element_size, |
| copied_oop, Address(from, 0), |
| gct1); |
| __ addi(from, from, UseCompressedOops ? 4 : 8); |
| __ beqz(copied_oop, L_store_element); |
| |
| __ load_klass(r9_klass, copied_oop);// query the object klass |
| |
| BLOCK_COMMENT("type_check:"); |
| generate_type_check(r9_klass, /*sub_klass*/ |
| ckoff, /*super_check_offset*/ |
| ckval, /*super_klass*/ |
| x10, /*result*/ |
| gct1, /*tmp1*/ |
| gct2, /*tmp2*/ |
| L_store_element); |
| |
| // Fall through on failure! |
| |
| // ======== end loop ======== |
| |
| // It was a real error; we must depend on the caller to finish the job. |
| // Register count = remaining oops, count_orig = total oops. |
| // Emit GC store barriers for the oops we have copied and report |
| // their number to the caller. |
| |
| __ sub(count, count_save, count); // K = partially copied oop count |
| __ xori(count, count, -1); // report (-1^K) to caller |
| __ beqz(count, L_done_pop); |
| |
| __ BIND(L_do_card_marks); |
| bs->arraycopy_epilogue(_masm, decorators, is_oop, start_to, count_save, t0, wb_post_saved_regs); |
| |
| __ bind(L_done_pop); |
| __ pop_reg(RegSet::of(x7, x9, x18, x19), sp); |
| inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr); |
| |
| __ bind(L_done); |
| __ mv(x10, count); |
| __ leave(); |
| __ ret(); |
| |
| return start; |
| } |
| |
| // Perform range checks on the proposed arraycopy. |
| // Kills temp, but nothing else. |
| // Also, clean the sign bits of src_pos and dst_pos. |
| void arraycopy_range_checks(Register src, // source array oop (c_rarg0) |
| Register src_pos, // source position (c_rarg1) |
| Register dst, // destination array oo (c_rarg2) |
| Register dst_pos, // destination position (c_rarg3) |
| Register length, |
| Register temp, |
| Label& L_failed) { |
| BLOCK_COMMENT("arraycopy_range_checks:"); |
| |
| assert_different_registers(t0, temp); |
| |
| // if [src_pos + length > arrayOop(src)->length()] then FAIL |
| __ lwu(t0, Address(src, arrayOopDesc::length_offset_in_bytes())); |
| __ addw(temp, length, src_pos); |
| __ bgtu(temp, t0, L_failed); |
| |
| // if [dst_pos + length > arrayOop(dst)->length()] then FAIL |
| __ lwu(t0, Address(dst, arrayOopDesc::length_offset_in_bytes())); |
| __ addw(temp, length, dst_pos); |
| __ bgtu(temp, t0, L_failed); |
| |
| // Have to clean up high 32 bits of 'src_pos' and 'dst_pos'. |
| __ zext(src_pos, src_pos, 32); |
| __ zext(dst_pos, dst_pos, 32); |
| |
| BLOCK_COMMENT("arraycopy_range_checks done"); |
| } |
| |
| address generate_unsafecopy_common_error_exit() { |
| address start = __ pc(); |
| __ mv(x10, 0); |
| __ leave(); |
| __ ret(); |
| return start; |
| } |
| |
| // |
| // Generate 'unsafe' set memory stub |
| // Though just as safe as the other stubs, it takes an unscaled |
| // size_t (# bytes) argument instead of an element count. |
| // |
| // Input: |
| // c_rarg0 - destination array address |
| // c_rarg1 - byte count (size_t) |
| // c_rarg2 - byte value |
| // |
| address generate_unsafe_setmemory() { |
| __ align(CodeEntryAlignment); |
| StubGenStubId stub_id = StubGenStubId::unsafe_setmemory_id; |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| |
| // bump this on entry, not on exit: |
| // inc_counter_np(SharedRuntime::_unsafe_set_memory_ctr); |
| |
| Label L_fill_elements; |
| |
| const Register dest = c_rarg0; |
| const Register count = c_rarg1; |
| const Register value = c_rarg2; |
| const Register cnt_words = x28; // temp register |
| const Register tmp_reg = x29; // temp register |
| |
| // Mark remaining code as such which performs Unsafe accesses. |
| UnsafeMemoryAccessMark umam(this, true, false); |
| |
| __ enter(); // required for proper stackwalking of RuntimeStub frame |
| |
| // if count < 8, jump to L_fill_elements |
| __ mv(tmp_reg, 8); // 8 bytes fill by element |
| __ bltu(count, tmp_reg, L_fill_elements); |
| |
| // Propagate byte to 64-bit width |
| // 8 bit -> 16 bit |
| __ zext(value, value, 8); |
| __ slli(tmp_reg, value, 8); |
| __ orr(value, value, tmp_reg); |
| // 16 bit -> 32 bit |
| __ slli(tmp_reg, value, 16); |
| __ orr(value, value, tmp_reg); |
| // 32 bit -> 64 bit |
| __ slli(tmp_reg, value, 32); |
| __ orr(value, value, tmp_reg); |
| |
| // Align source address at 8 bytes address boundary. |
| Label L_skip_align1, L_skip_align2, L_skip_align4; |
| // One byte misalignment happens. |
| __ test_bit(tmp_reg, dest, 0); |
| __ beqz(tmp_reg, L_skip_align1); |
| __ sb(value, Address(dest, 0)); |
| __ addi(dest, dest, 1); |
| __ subi(count, count, 1); |
| |
| __ bind(L_skip_align1); |
| // Two bytes misalignment happens. |
| __ test_bit(tmp_reg, dest, 1); |
| __ beqz(tmp_reg, L_skip_align2); |
| __ sh(value, Address(dest, 0)); |
| __ addi(dest, dest, 2); |
| __ subi(count, count, 2); |
| |
| __ bind(L_skip_align2); |
| // Four bytes misalignment happens. |
| __ test_bit(tmp_reg, dest, 2); |
| __ beqz(tmp_reg, L_skip_align4); |
| __ sw(value, Address(dest, 0)); |
| __ addi(dest, dest, 4); |
| __ subi(count, count, 4); |
| __ bind(L_skip_align4); |
| |
| // Fill large chunks |
| __ srli(cnt_words, count, 3); // number of words |
| __ slli(tmp_reg, cnt_words, 3); |
| __ sub(count, count, tmp_reg); |
| { |
| __ fill_words(dest, cnt_words, value); |
| } |
| |
| // Handle copies less than 8 bytes |
| __ bind(L_fill_elements); |
| Label L_fill_2, L_fill_1, L_exit; |
| __ test_bit(tmp_reg, count, 2); |
| __ beqz(tmp_reg, L_fill_2); |
| __ sb(value, Address(dest, 0)); |
| __ sb(value, Address(dest, 1)); |
| __ sb(value, Address(dest, 2)); |
| __ sb(value, Address(dest, 3)); |
| __ addi(dest, dest, 4); |
| |
| __ bind(L_fill_2); |
| __ test_bit(tmp_reg, count, 1); |
| __ beqz(tmp_reg, L_fill_1); |
| __ sb(value, Address(dest, 0)); |
| __ sb(value, Address(dest, 1)); |
| __ addi(dest, dest, 2); |
| |
| __ bind(L_fill_1); |
| __ test_bit(tmp_reg, count, 0); |
| __ beqz(tmp_reg, L_exit); |
| __ sb(value, Address(dest, 0)); |
| |
| __ bind(L_exit); |
| __ leave(); |
| __ ret(); |
| |
| return start; |
| } |
| |
| // |
| // Generate 'unsafe' array copy stub |
| // Though just as safe as the other stubs, it takes an unscaled |
| // size_t argument instead of an element count. |
| // |
| // Input: |
| // c_rarg0 - source array address |
| // c_rarg1 - destination array address |
| // c_rarg2 - byte count, treated as ssize_t, can be zero |
| // |
| // Examines the alignment of the operands and dispatches |
| // to a long, int, short, or byte copy loop. |
| // |
| address generate_unsafe_copy(address byte_copy_entry, |
| address short_copy_entry, |
| address int_copy_entry, |
| address long_copy_entry) { |
| assert_cond(byte_copy_entry != nullptr && short_copy_entry != nullptr && |
| int_copy_entry != nullptr && long_copy_entry != nullptr); |
| Label L_long_aligned, L_int_aligned, L_short_aligned; |
| const Register s = c_rarg0, d = c_rarg1, count = c_rarg2; |
| |
| __ align(CodeEntryAlignment); |
| StubGenStubId stub_id = StubGenStubId::unsafe_arraycopy_id; |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| __ enter(); // required for proper stackwalking of RuntimeStub frame |
| |
| // bump this on entry, not on exit: |
| inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr); |
| |
| __ orr(t0, s, d); |
| __ orr(t0, t0, count); |
| |
| __ andi(t0, t0, BytesPerLong - 1); |
| __ beqz(t0, L_long_aligned); |
| __ andi(t0, t0, BytesPerInt - 1); |
| __ beqz(t0, L_int_aligned); |
| __ test_bit(t0, t0, 0); |
| __ beqz(t0, L_short_aligned); |
| __ j(RuntimeAddress(byte_copy_entry)); |
| |
| __ BIND(L_short_aligned); |
| __ srli(count, count, LogBytesPerShort); // size => short_count |
| __ j(RuntimeAddress(short_copy_entry)); |
| __ BIND(L_int_aligned); |
| __ srli(count, count, LogBytesPerInt); // size => int_count |
| __ j(RuntimeAddress(int_copy_entry)); |
| __ BIND(L_long_aligned); |
| __ srli(count, count, LogBytesPerLong); // size => long_count |
| __ j(RuntimeAddress(long_copy_entry)); |
| |
| return start; |
| } |
| |
| // |
| // Generate generic array copy stubs |
| // |
| // Input: |
| // c_rarg0 - src oop |
| // c_rarg1 - src_pos (32-bits) |
| // c_rarg2 - dst oop |
| // c_rarg3 - dst_pos (32-bits) |
| // c_rarg4 - element count (32-bits) |
| // |
| // Output: |
| // x10 == 0 - success |
| // x10 == -1^K - failure, where K is partial transfer count |
| // |
| address generate_generic_copy(address byte_copy_entry, address short_copy_entry, |
| address int_copy_entry, address oop_copy_entry, |
| address long_copy_entry, address checkcast_copy_entry) { |
| assert_cond(byte_copy_entry != nullptr && short_copy_entry != nullptr && |
| int_copy_entry != nullptr && oop_copy_entry != nullptr && |
| long_copy_entry != nullptr && checkcast_copy_entry != nullptr); |
| Label L_failed, L_failed_0, L_objArray; |
| Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs; |
| |
| // Input registers |
| const Register src = c_rarg0; // source array oop |
| const Register src_pos = c_rarg1; // source position |
| const Register dst = c_rarg2; // destination array oop |
| const Register dst_pos = c_rarg3; // destination position |
| const Register length = c_rarg4; |
| |
| // Registers used as temps |
| const Register dst_klass = c_rarg5; |
| |
| __ align(CodeEntryAlignment); |
| |
| StubGenStubId stub_id = StubGenStubId::generic_arraycopy_id; |
| StubCodeMark mark(this, stub_id); |
| |
| address start = __ pc(); |
| |
| __ enter(); // required for proper stackwalking of RuntimeStub frame |
| |
| // bump this on entry, not on exit: |
| inc_counter_np(SharedRuntime::_generic_array_copy_ctr); |
| |
| //----------------------------------------------------------------------- |
| // Assembler stub will be used for this call to arraycopy |
| // if the following conditions are met: |
| // |
| // (1) src and dst must not be null. |
| // (2) src_pos must not be negative. |
| // (3) dst_pos must not be negative. |
| // (4) length must not be negative. |
| // (5) src klass and dst klass should be the same and not null. |
| // (6) src and dst should be arrays. |
| // (7) src_pos + length must not exceed length of src. |
| // (8) dst_pos + length must not exceed length of dst. |
| // |
| |
| // if src is null then return -1 |
| __ beqz(src, L_failed); |
| |
| // if [src_pos < 0] then return -1 |
| __ sext(t0, src_pos, 32); |
| __ bltz(t0, L_failed); |
| |
| // if dst is null then return -1 |
| __ beqz(dst, L_failed); |
| |
| // if [dst_pos < 0] then return -1 |
| __ sext(t0, dst_pos, 32); |
| __ bltz(t0, L_failed); |
| |
| // registers used as temp |
| const Register scratch_length = x28; // elements count to copy |
| const Register scratch_src_klass = x29; // array klass |
| const Register lh = x30; // layout helper |
| |
| // if [length < 0] then return -1 |
| __ sext(scratch_length, length, 32); // length (elements count, 32-bits value) |
| __ bltz(scratch_length, L_failed); |
| |
| __ load_klass(scratch_src_klass, src); |
| #ifdef ASSERT |
| { |
| BLOCK_COMMENT("assert klasses not null {"); |
| Label L1, L2; |
| __ bnez(scratch_src_klass, L2); // it is broken if klass is null |
| __ bind(L1); |
| __ stop("broken null klass"); |
| __ bind(L2); |
| __ load_klass(t0, dst, t1); |
| __ beqz(t0, L1); // this would be broken also |
| BLOCK_COMMENT("} assert klasses not null done"); |
| } |
| #endif |
| |
| // Load layout helper (32-bits) |
| // |
| // |array_tag| | header_size | element_type | |log2_element_size| |
| // 32 30 24 16 8 2 0 |
| // |
| // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0 |
| // |
| |
| const int lh_offset = in_bytes(Klass::layout_helper_offset()); |
| |
| // Handle objArrays completely differently... |
| const jint objArray_lh = Klass::array_layout_helper(T_OBJECT); |
| __ lw(lh, Address(scratch_src_klass, lh_offset)); |
| __ mv(t0, objArray_lh); |
| __ beq(lh, t0, L_objArray); |
| |
| // if [src->klass() != dst->klass()] then return -1 |
| __ load_klass(t1, dst); |
| __ bne(t1, scratch_src_klass, L_failed); |
| |
| // if src->is_Array() isn't null then return -1 |
| // i.e. (lh >= 0) |
| __ bgez(lh, L_failed); |
| |
| // At this point, it is known to be a typeArray (array_tag 0x3). |
| #ifdef ASSERT |
| { |
| BLOCK_COMMENT("assert primitive array {"); |
| Label L; |
| __ mv(t1, (int32_t)(Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift)); |
| __ bge(lh, t1, L); |
| __ stop("must be a primitive array"); |
| __ bind(L); |
| BLOCK_COMMENT("} assert primitive array done"); |
| } |
| #endif |
| |
| arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length, |
| t1, L_failed); |
| |
| // TypeArrayKlass |
| // |
| // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize) |
| // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize) |
| // |
| |
| const Register t0_offset = t0; // array offset |
| const Register x30_elsize = lh; // element size |
| |
| // Get array_header_in_bytes() |
| int lh_header_size_width = exact_log2(Klass::_lh_header_size_mask + 1); |
| int lh_header_size_msb = Klass::_lh_header_size_shift + lh_header_size_width; |
| __ slli(t0_offset, lh, XLEN - lh_header_size_msb); // left shift to remove 24 ~ 32; |
| __ srli(t0_offset, t0_offset, XLEN - lh_header_size_width); // array_offset |
| |
| __ add(src, src, t0_offset); // src array offset |
| __ add(dst, dst, t0_offset); // dst array offset |
| BLOCK_COMMENT("choose copy loop based on element size"); |
| |
| // next registers should be set before the jump to corresponding stub |
| const Register from = c_rarg0; // source array address |
| const Register to = c_rarg1; // destination array address |
| const Register count = c_rarg2; // elements count |
| |
| // 'from', 'to', 'count' registers should be set in such order |
| // since they are the same as 'src', 'src_pos', 'dst'. |
| |
| assert(Klass::_lh_log2_element_size_shift == 0, "fix this code"); |
| |
| // The possible values of elsize are 0-3, i.e. exact_log2(element |
| // size in bytes). We do a simple bitwise binary search. |
| __ BIND(L_copy_bytes); |
| __ test_bit(t0, x30_elsize, 1); |
| __ bnez(t0, L_copy_ints); |
| __ test_bit(t0, x30_elsize, 0); |
| __ bnez(t0, L_copy_shorts); |
| __ add(from, src, src_pos); // src_addr |
| __ add(to, dst, dst_pos); // dst_addr |
| __ sext(count, scratch_length, 32); // length |
| __ j(RuntimeAddress(byte_copy_entry)); |
| |
| __ BIND(L_copy_shorts); |
| __ shadd(from, src_pos, src, t0, 1); // src_addr |
| __ shadd(to, dst_pos, dst, t0, 1); // dst_addr |
| __ sext(count, scratch_length, 32); // length |
| __ j(RuntimeAddress(short_copy_entry)); |
| |
| __ BIND(L_copy_ints); |
| __ test_bit(t0, x30_elsize, 0); |
| __ bnez(t0, L_copy_longs); |
| __ shadd(from, src_pos, src, t0, 2); // src_addr |
| __ shadd(to, dst_pos, dst, t0, 2); // dst_addr |
| __ sext(count, scratch_length, 32); // length |
| __ j(RuntimeAddress(int_copy_entry)); |
| |
| __ BIND(L_copy_longs); |
| #ifdef ASSERT |
| { |
| BLOCK_COMMENT("assert long copy {"); |
| Label L; |
| __ andi(lh, lh, Klass::_lh_log2_element_size_mask); // lh -> x30_elsize |
| __ sext(lh, lh, 32); |
| __ mv(t0, LogBytesPerLong); |
| __ beq(x30_elsize, t0, L); |
| __ stop("must be long copy, but elsize is wrong"); |
| __ bind(L); |
| BLOCK_COMMENT("} assert long copy done"); |
| } |
| #endif |
| __ shadd(from, src_pos, src, t0, 3); // src_addr |
| __ shadd(to, dst_pos, dst, t0, 3); // dst_addr |
| __ sext(count, scratch_length, 32); // length |
| __ j(RuntimeAddress(long_copy_entry)); |
| |
| // ObjArrayKlass |
| __ BIND(L_objArray); |
| // live at this point: scratch_src_klass, scratch_length, src[_pos], dst[_pos] |
| |
| Label L_plain_copy, L_checkcast_copy; |
| // test array classes for subtyping |
| __ load_klass(t2, dst); |
| __ bne(scratch_src_klass, t2, L_checkcast_copy); // usual case is exact equality |
| |
| // Identically typed arrays can be copied without element-wise checks. |
| arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length, |
| t1, L_failed); |
| |
| __ shadd(from, src_pos, src, t0, LogBytesPerHeapOop); |
| __ addi(from, from, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); |
| __ shadd(to, dst_pos, dst, t0, LogBytesPerHeapOop); |
| __ addi(to, to, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); |
| __ sext(count, scratch_length, 32); // length |
| __ BIND(L_plain_copy); |
| __ j(RuntimeAddress(oop_copy_entry)); |
| |
| __ BIND(L_checkcast_copy); |
| // live at this point: scratch_src_klass, scratch_length, t2 (dst_klass) |
| { |
| // Before looking at dst.length, make sure dst is also an objArray. |
| __ lwu(t0, Address(t2, lh_offset)); |
| __ mv(t1, objArray_lh); |
| __ bne(t0, t1, L_failed); |
| |
| // It is safe to examine both src.length and dst.length. |
| arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length, |
| t2, L_failed); |
| |
| __ load_klass(dst_klass, dst); // reload |
| |
| // Marshal the base address arguments now, freeing registers. |
| __ shadd(from, src_pos, src, t0, LogBytesPerHeapOop); |
| __ addi(from, from, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); |
| __ shadd(to, dst_pos, dst, t0, LogBytesPerHeapOop); |
| __ addi(to, to, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); |
| __ sext(count, length, 32); // length (reloaded) |
| const Register sco_temp = c_rarg3; // this register is free now |
| assert_different_registers(from, to, count, sco_temp, |
| dst_klass, scratch_src_klass); |
| |
| // Generate the type check. |
| const int sco_offset = in_bytes(Klass::super_check_offset_offset()); |
| __ lwu(sco_temp, Address(dst_klass, sco_offset)); |
| |
| // Smashes t0, t1 |
| generate_type_check(scratch_src_klass, sco_temp, dst_klass, noreg, noreg, noreg, L_plain_copy); |
| |
| // Fetch destination element klass from the ObjArrayKlass header. |
| int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset()); |
| __ ld(dst_klass, Address(dst_klass, ek_offset)); |
| __ lwu(sco_temp, Address(dst_klass, sco_offset)); |
| |
| // the checkcast_copy loop needs two extra arguments: |
| assert(c_rarg3 == sco_temp, "#3 already in place"); |
| // Set up arguments for checkcast_copy_entry. |
| __ mv(c_rarg4, dst_klass); // dst.klass.element_klass |
| __ j(RuntimeAddress(checkcast_copy_entry)); |
| } |
| |
| __ BIND(L_failed); |
| __ mv(x10, -1); |
| __ leave(); // required for proper stackwalking of RuntimeStub frame |
| __ ret(); |
| |
| return start; |
| } |
| |
| // |
| // Generate stub for array fill. If "aligned" is true, the |
| // "to" address is assumed to be heapword aligned. |
| // |
| // Arguments for generated stub: |
| // to: c_rarg0 |
| // value: c_rarg1 |
| // count: c_rarg2 treated as signed |
| // |
| address generate_fill(StubGenStubId stub_id) { |
| BasicType t; |
| bool aligned; |
| |
| switch (stub_id) { |
| case jbyte_fill_id: |
| t = T_BYTE; |
| aligned = false; |
| break; |
| case jshort_fill_id: |
| t = T_SHORT; |
| aligned = false; |
| break; |
| case jint_fill_id: |
| t = T_INT; |
| aligned = false; |
| break; |
| case arrayof_jbyte_fill_id: |
| t = T_BYTE; |
| aligned = true; |
| break; |
| case arrayof_jshort_fill_id: |
| t = T_SHORT; |
| aligned = true; |
| break; |
| case arrayof_jint_fill_id: |
| t = T_INT; |
| aligned = true; |
| break; |
| default: |
| ShouldNotReachHere(); |
| }; |
| |
| __ align(CodeEntryAlignment); |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| |
| BLOCK_COMMENT("Entry:"); |
| |
| const Register to = c_rarg0; // source array address |
| const Register value = c_rarg1; // value |
| const Register count = c_rarg2; // elements count |
| |
| const Register bz_base = x28; // base for block_zero routine |
| const Register cnt_words = x29; // temp register |
| const Register tmp_reg = t1; |
| |
| __ enter(); |
| |
| Label L_fill_elements; |
| |
| int shift = -1; |
| switch (t) { |
| case T_BYTE: |
| shift = 0; |
| // Short arrays (< 8 bytes) fill by element |
| __ mv(tmp_reg, 8 >> shift); |
| __ bltu(count, tmp_reg, L_fill_elements); |
| |
| // Zero extend value |
| // 8 bit -> 16 bit |
| __ zext(value, value, 8); |
| __ slli(tmp_reg, value, 8); |
| __ orr(value, value, tmp_reg); |
| |
| // 16 bit -> 32 bit |
| __ slli(tmp_reg, value, 16); |
| __ orr(value, value, tmp_reg); |
| break; |
| case T_SHORT: |
| shift = 1; |
| // Short arrays (< 8 bytes) fill by element |
| __ mv(tmp_reg, 8 >> shift); |
| __ bltu(count, tmp_reg, L_fill_elements); |
| |
| // Zero extend value |
| // 16 bit -> 32 bit |
| __ zext(value, value, 16); |
| __ slli(tmp_reg, value, 16); |
| __ orr(value, value, tmp_reg); |
| break; |
| case T_INT: |
| shift = 2; |
| // Short arrays (< 8 bytes) fill by element |
| __ mv(tmp_reg, 8 >> shift); |
| __ bltu(count, tmp_reg, L_fill_elements); |
| break; |
| default: ShouldNotReachHere(); |
| } |
| |
| // Align source address at 8 bytes address boundary. |
| Label L_skip_align1, L_skip_align2, L_skip_align4; |
| if (!aligned) { |
| switch (t) { |
| case T_BYTE: |
| // One byte misalignment happens only for byte arrays. |
| __ test_bit(tmp_reg, to, 0); |
| __ beqz(tmp_reg, L_skip_align1); |
| __ sb(value, Address(to, 0)); |
| __ addi(to, to, 1); |
| __ subiw(count, count, 1); |
| __ bind(L_skip_align1); |
| // Fallthrough |
| case T_SHORT: |
| // Two bytes misalignment happens only for byte and short (char) arrays. |
| __ test_bit(tmp_reg, to, 1); |
| __ beqz(tmp_reg, L_skip_align2); |
| __ sh(value, Address(to, 0)); |
| __ addi(to, to, 2); |
| __ subiw(count, count, 2 >> shift); |
| __ bind(L_skip_align2); |
| // Fallthrough |
| case T_INT: |
| // Align to 8 bytes, we know we are 4 byte aligned to start. |
| __ test_bit(tmp_reg, to, 2); |
| __ beqz(tmp_reg, L_skip_align4); |
| __ sw(value, Address(to, 0)); |
| __ addi(to, to, 4); |
| __ subiw(count, count, 4 >> shift); |
| __ bind(L_skip_align4); |
| break; |
| default: ShouldNotReachHere(); |
| } |
| } |
| |
| // |
| // Fill large chunks |
| // |
| __ srliw(cnt_words, count, 3 - shift); // number of words |
| |
| // 32 bit -> 64 bit |
| __ zext(value, value, 32); |
| __ slli(tmp_reg, value, 32); |
| __ orr(value, value, tmp_reg); |
| |
| __ slli(tmp_reg, cnt_words, 3 - shift); |
| __ subw(count, count, tmp_reg); |
| { |
| __ fill_words(to, cnt_words, value); |
| } |
| |
| // Handle copies less than 8 bytes. |
| // Address may not be heapword aligned. |
| Label L_fill_1, L_fill_2, L_exit; |
| __ bind(L_fill_elements); |
| switch (t) { |
| case T_BYTE: |
| __ test_bit(tmp_reg, count, 2); |
| __ beqz(tmp_reg, L_fill_2); |
| __ sb(value, Address(to, 0)); |
| __ sb(value, Address(to, 1)); |
| __ sb(value, Address(to, 2)); |
| __ sb(value, Address(to, 3)); |
| __ addi(to, to, 4); |
| |
| __ bind(L_fill_2); |
| __ test_bit(tmp_reg, count, 1); |
| __ beqz(tmp_reg, L_fill_1); |
| __ sb(value, Address(to, 0)); |
| __ sb(value, Address(to, 1)); |
| __ addi(to, to, 2); |
| |
| __ bind(L_fill_1); |
| __ test_bit(tmp_reg, count, 0); |
| __ beqz(tmp_reg, L_exit); |
| __ sb(value, Address(to, 0)); |
| break; |
| case T_SHORT: |
| __ test_bit(tmp_reg, count, 1); |
| __ beqz(tmp_reg, L_fill_2); |
| __ sh(value, Address(to, 0)); |
| __ sh(value, Address(to, 2)); |
| __ addi(to, to, 4); |
| |
| __ bind(L_fill_2); |
| __ test_bit(tmp_reg, count, 0); |
| __ beqz(tmp_reg, L_exit); |
| __ sh(value, Address(to, 0)); |
| break; |
| case T_INT: |
| __ beqz(count, L_exit); |
| __ sw(value, Address(to, 0)); |
| break; |
| default: ShouldNotReachHere(); |
| } |
| __ bind(L_exit); |
| __ leave(); |
| __ ret(); |
| |
| return start; |
| } |
| |
| void generate_arraycopy_stubs() { |
| address entry = nullptr; |
| address entry_jbyte_arraycopy = nullptr; |
| address entry_jshort_arraycopy = nullptr; |
| address entry_jint_arraycopy = nullptr; |
| address entry_oop_arraycopy = nullptr; |
| address entry_jlong_arraycopy = nullptr; |
| address entry_checkcast_arraycopy = nullptr; |
| |
| generate_copy_longs(StubGenStubId::copy_byte_f_id, copy_f, c_rarg0, c_rarg1, t1); |
| generate_copy_longs(StubGenStubId::copy_byte_b_id, copy_b, c_rarg0, c_rarg1, t1); |
| |
| address ucm_common_error_exit = generate_unsafecopy_common_error_exit(); |
| UnsafeMemoryAccess::set_common_exit_stub_pc(ucm_common_error_exit); |
| |
| StubRoutines::riscv::_zero_blocks = generate_zero_blocks(); |
| |
| //*** jbyte |
| // Always need aligned and unaligned versions |
| StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_copy(StubGenStubId::jbyte_disjoint_arraycopy_id, &entry); |
| StubRoutines::_jbyte_arraycopy = generate_conjoint_copy(StubGenStubId::jbyte_arraycopy_id, entry, &entry_jbyte_arraycopy); |
| StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_disjoint_copy(StubGenStubId::arrayof_jbyte_disjoint_arraycopy_id, &entry); |
| StubRoutines::_arrayof_jbyte_arraycopy = generate_conjoint_copy(StubGenStubId::arrayof_jbyte_arraycopy_id, entry, nullptr); |
| |
| //*** jshort |
| // Always need aligned and unaligned versions |
| StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_copy(StubGenStubId::jshort_disjoint_arraycopy_id, &entry); |
| StubRoutines::_jshort_arraycopy = generate_conjoint_copy(StubGenStubId::jshort_arraycopy_id, entry, &entry_jshort_arraycopy); |
| StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_copy(StubGenStubId::arrayof_jshort_disjoint_arraycopy_id, &entry); |
| StubRoutines::_arrayof_jshort_arraycopy = generate_conjoint_copy(StubGenStubId::arrayof_jshort_arraycopy_id, entry, nullptr); |
| |
| //*** jint |
| // Aligned versions |
| StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_disjoint_copy(StubGenStubId::arrayof_jint_disjoint_arraycopy_id, &entry); |
| StubRoutines::_arrayof_jint_arraycopy = generate_conjoint_copy(StubGenStubId::arrayof_jint_arraycopy_id, entry, &entry_jint_arraycopy); |
| // In 64 bit we need both aligned and unaligned versions of jint arraycopy. |
| // entry_jint_arraycopy always points to the unaligned version |
| StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_copy(StubGenStubId::jint_disjoint_arraycopy_id, &entry); |
| StubRoutines::_jint_arraycopy = generate_conjoint_copy(StubGenStubId::jint_arraycopy_id, entry, &entry_jint_arraycopy); |
| |
| //*** jlong |
| // It is always aligned |
| StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_copy(StubGenStubId::arrayof_jlong_disjoint_arraycopy_id, &entry); |
| StubRoutines::_arrayof_jlong_arraycopy = generate_conjoint_copy(StubGenStubId::arrayof_jlong_arraycopy_id, entry, &entry_jlong_arraycopy); |
| StubRoutines::_jlong_disjoint_arraycopy = StubRoutines::_arrayof_jlong_disjoint_arraycopy; |
| StubRoutines::_jlong_arraycopy = StubRoutines::_arrayof_jlong_arraycopy; |
| |
| //*** oops |
| StubRoutines::_arrayof_oop_disjoint_arraycopy |
| = generate_disjoint_copy(StubGenStubId::arrayof_oop_disjoint_arraycopy_id, &entry); |
| StubRoutines::_arrayof_oop_arraycopy |
| = generate_conjoint_copy(StubGenStubId::arrayof_oop_arraycopy_id, entry, &entry_oop_arraycopy); |
| // Aligned versions without pre-barriers |
| StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit |
| = generate_disjoint_copy(StubGenStubId::arrayof_oop_disjoint_arraycopy_uninit_id, &entry); |
| StubRoutines::_arrayof_oop_arraycopy_uninit |
| = generate_conjoint_copy(StubGenStubId::arrayof_oop_arraycopy_uninit_id, entry, nullptr); |
| |
| StubRoutines::_oop_disjoint_arraycopy = StubRoutines::_arrayof_oop_disjoint_arraycopy; |
| StubRoutines::_oop_arraycopy = StubRoutines::_arrayof_oop_arraycopy; |
| StubRoutines::_oop_disjoint_arraycopy_uninit = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit; |
| StubRoutines::_oop_arraycopy_uninit = StubRoutines::_arrayof_oop_arraycopy_uninit; |
| |
| StubRoutines::_checkcast_arraycopy = generate_checkcast_copy(StubGenStubId::checkcast_arraycopy_id, &entry_checkcast_arraycopy); |
| StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy(StubGenStubId::checkcast_arraycopy_uninit_id, nullptr); |
| |
| |
| StubRoutines::_unsafe_arraycopy = generate_unsafe_copy(entry_jbyte_arraycopy, |
| entry_jshort_arraycopy, |
| entry_jint_arraycopy, |
| entry_jlong_arraycopy); |
| |
| StubRoutines::_generic_arraycopy = generate_generic_copy(entry_jbyte_arraycopy, |
| entry_jshort_arraycopy, |
| entry_jint_arraycopy, |
| entry_oop_arraycopy, |
| entry_jlong_arraycopy, |
| entry_checkcast_arraycopy); |
| |
| StubRoutines::_jbyte_fill = generate_fill(StubGenStubId::jbyte_fill_id); |
| StubRoutines::_jshort_fill = generate_fill(StubGenStubId::jshort_fill_id); |
| StubRoutines::_jint_fill = generate_fill(StubGenStubId::jint_fill_id); |
| StubRoutines::_arrayof_jbyte_fill = generate_fill(StubGenStubId::arrayof_jbyte_fill_id); |
| StubRoutines::_arrayof_jshort_fill = generate_fill(StubGenStubId::arrayof_jshort_fill_id); |
| StubRoutines::_arrayof_jint_fill = generate_fill(StubGenStubId::arrayof_jint_fill_id); |
| |
| StubRoutines::_unsafe_setmemory = generate_unsafe_setmemory(); |
| } |
| |
| void generate_aes_loadkeys(const Register &key, VectorRegister *working_vregs, int rounds) { |
| const int step = 16; |
| for (int i = 0; i < rounds; i++) { |
| __ vle32_v(working_vregs[i], key); |
| // The keys are stored in little-endian array, while we need |
| // to operate in big-endian. |
| // So performing an endian-swap here with vrev8.v instruction |
| __ vrev8_v(working_vregs[i], working_vregs[i]); |
| __ addi(key, key, step); |
| } |
| } |
| |
| void generate_aes_encrypt(const VectorRegister &res, VectorRegister *working_vregs, int rounds) { |
| assert(rounds <= 15, "rounds should be less than or equal to working_vregs size"); |
| |
| __ vxor_vv(res, res, working_vregs[0]); |
| for (int i = 1; i < rounds - 1; i++) { |
| __ vaesem_vv(res, working_vregs[i]); |
| } |
| __ vaesef_vv(res, working_vregs[rounds - 1]); |
| } |
| |
| // Arguments: |
| // |
| // Inputs: |
| // c_rarg0 - source byte array address |
| // c_rarg1 - destination byte array address |
| // c_rarg2 - K (key) in little endian int array |
| // |
| address generate_aescrypt_encryptBlock() { |
| assert(UseAESIntrinsics, "need AES instructions (Zvkned extension) support"); |
| |
| __ align(CodeEntryAlignment); |
| StubGenStubId stub_id = StubGenStubId::aescrypt_encryptBlock_id; |
| StubCodeMark mark(this, stub_id); |
| |
| Label L_aes128, L_aes192; |
| |
| const Register from = c_rarg0; // source array address |
| const Register to = c_rarg1; // destination array address |
| const Register key = c_rarg2; // key array address |
| const Register keylen = c_rarg3; |
| |
| VectorRegister working_vregs[] = { |
| v4, v5, v6, v7, v8, v9, v10, v11, |
| v12, v13, v14, v15, v16, v17, v18 |
| }; |
| const VectorRegister res = v19; |
| |
| address start = __ pc(); |
| __ enter(); |
| |
| __ lwu(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT))); |
| |
| __ vsetivli(x0, 4, Assembler::e32, Assembler::m1); |
| __ vle32_v(res, from); |
| |
| __ mv(t2, 52); |
| __ blt(keylen, t2, L_aes128); |
| __ beq(keylen, t2, L_aes192); |
| // Else we fallthrough to the biggest case (256-bit key size) |
| |
| // Note: the following function performs key += 15*16 |
| generate_aes_loadkeys(key, working_vregs, 15); |
| generate_aes_encrypt(res, working_vregs, 15); |
| __ vse32_v(res, to); |
| __ mv(c_rarg0, 0); |
| __ leave(); |
| __ ret(); |
| |
| __ bind(L_aes192); |
| // Note: the following function performs key += 13*16 |
| generate_aes_loadkeys(key, working_vregs, 13); |
| generate_aes_encrypt(res, working_vregs, 13); |
| __ vse32_v(res, to); |
| __ mv(c_rarg0, 0); |
| __ leave(); |
| __ ret(); |
| |
| __ bind(L_aes128); |
| // Note: the following function performs key += 11*16 |
| generate_aes_loadkeys(key, working_vregs, 11); |
| generate_aes_encrypt(res, working_vregs, 11); |
| __ vse32_v(res, to); |
| __ mv(c_rarg0, 0); |
| __ leave(); |
| __ ret(); |
| |
| return start; |
| } |
| |
| void generate_aes_decrypt(const VectorRegister &res, VectorRegister *working_vregs, int rounds) { |
| assert(rounds <= 15, "rounds should be less than or equal to working_vregs size"); |
| |
| __ vxor_vv(res, res, working_vregs[rounds - 1]); |
| for (int i = rounds - 2; i > 0; i--) { |
| __ vaesdm_vv(res, working_vregs[i]); |
| } |
| __ vaesdf_vv(res, working_vregs[0]); |
| } |
| |
| // Arguments: |
| // |
| // Inputs: |
| // c_rarg0 - source byte array address |
| // c_rarg1 - destination byte array address |
| // c_rarg2 - K (key) in little endian int array |
| // |
| address generate_aescrypt_decryptBlock() { |
| assert(UseAESIntrinsics, "need AES instructions (Zvkned extension) support"); |
| |
| __ align(CodeEntryAlignment); |
| StubGenStubId stub_id = StubGenStubId::aescrypt_decryptBlock_id; |
| StubCodeMark mark(this, stub_id); |
| |
| Label L_aes128, L_aes192; |
| |
| const Register from = c_rarg0; // source array address |
| const Register to = c_rarg1; // destination array address |
| const Register key = c_rarg2; // key array address |
| const Register keylen = c_rarg3; |
| |
| VectorRegister working_vregs[] = { |
| v4, v5, v6, v7, v8, v9, v10, v11, |
| v12, v13, v14, v15, v16, v17, v18 |
| }; |
| const VectorRegister res = v19; |
| |
| address start = __ pc(); |
| __ enter(); // required for proper stackwalking of RuntimeStub frame |
| |
| __ lwu(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT))); |
| |
| __ vsetivli(x0, 4, Assembler::e32, Assembler::m1); |
| __ vle32_v(res, from); |
| |
| __ mv(t2, 52); |
| __ blt(keylen, t2, L_aes128); |
| __ beq(keylen, t2, L_aes192); |
| // Else we fallthrough to the biggest case (256-bit key size) |
| |
| // Note: the following function performs key += 15*16 |
| generate_aes_loadkeys(key, working_vregs, 15); |
| generate_aes_decrypt(res, working_vregs, 15); |
| __ vse32_v(res, to); |
| __ mv(c_rarg0, 0); |
| __ leave(); |
| __ ret(); |
| |
| __ bind(L_aes192); |
| // Note: the following function performs key += 13*16 |
| generate_aes_loadkeys(key, working_vregs, 13); |
| generate_aes_decrypt(res, working_vregs, 13); |
| __ vse32_v(res, to); |
| __ mv(c_rarg0, 0); |
| __ leave(); |
| __ ret(); |
| |
| __ bind(L_aes128); |
| // Note: the following function performs key += 11*16 |
| generate_aes_loadkeys(key, working_vregs, 11); |
| generate_aes_decrypt(res, working_vregs, 11); |
| __ vse32_v(res, to); |
| __ mv(c_rarg0, 0); |
| __ leave(); |
| __ ret(); |
| |
| return start; |
| } |
| |
| // code for comparing 8 characters of strings with Latin1 and Utf16 encoding |
| void compare_string_8_x_LU(Register tmpL, Register tmpU, |
| Register strL, Register strU, Label& DIFF) { |
| const Register tmp = x30, tmpLval = x12; |
| |
| int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE); |
| assert((base_offset % (UseCompactObjectHeaders ? 4 : |
| (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be"); |
| |
| #ifdef ASSERT |
| if (AvoidUnalignedAccesses) { |
| Label align_ok; |
| __ andi(t0, strL, 0x7); |
| __ beqz(t0, align_ok); |
| __ stop("bad alignment"); |
| __ bind(align_ok); |
| } |
| #endif |
| __ ld(tmpLval, Address(strL)); |
| __ addi(strL, strL, wordSize); |
| |
| // compare first 4 characters |
| __ load_long_misaligned(tmpU, Address(strU), tmp, (base_offset % 8) != 0 ? 4 : 8); |
| __ addi(strU, strU, wordSize); |
| __ inflate_lo32(tmpL, tmpLval); |
| __ xorr(tmp, tmpU, tmpL); |
| __ bnez(tmp, DIFF); |
| |
| // compare second 4 characters |
| __ load_long_misaligned(tmpU, Address(strU), tmp, (base_offset % 8) != 0 ? 4 : 8); |
| __ addi(strU, strU, wordSize); |
| __ inflate_hi32(tmpL, tmpLval); |
| __ xorr(tmp, tmpU, tmpL); |
| __ bnez(tmp, DIFF); |
| } |
| |
| // x10 = result |
| // x11 = str1 |
| // x12 = cnt1 |
| // x13 = str2 |
| // x14 = cnt2 |
| // x28 = tmp1 |
| // x29 = tmp2 |
| // x30 = tmp3 |
| address generate_compare_long_string_different_encoding(StubGenStubId stub_id) { |
| bool isLU; |
| switch (stub_id) { |
| case compare_long_string_LU_id: |
| isLU = true; |
| break; |
| case compare_long_string_UL_id: |
| isLU = false; |
| break; |
| default: |
| ShouldNotReachHere(); |
| }; |
| __ align(CodeEntryAlignment); |
| StubCodeMark mark(this, stub_id); |
| address entry = __ pc(); |
| Label SMALL_LOOP, TAIL, LOAD_LAST, DONE, CALCULATE_DIFFERENCE; |
| const Register result = x10, str1 = x11, str2 = x13, cnt2 = x14, |
| tmp1 = x28, tmp2 = x29, tmp3 = x30, tmp4 = x12; |
| |
| int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE); |
| assert((base_offset % (UseCompactObjectHeaders ? 4 : |
| (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be"); |
| |
| Register strU = isLU ? str2 : str1, |
| strL = isLU ? str1 : str2, |
| tmpU = isLU ? tmp2 : tmp1, // where to keep U for comparison |
| tmpL = isLU ? tmp1 : tmp2; // where to keep L for comparison |
| |
| if (AvoidUnalignedAccesses && (base_offset % 8) != 0) { |
| // Load 4 bytes from strL to make sure main loop is 8-byte aligned |
| // cnt2 is >= 68 here, no need to check it for >= 0 |
| __ lwu(tmpL, Address(strL)); |
| __ addi(strL, strL, wordSize / 2); |
| __ load_long_misaligned(tmpU, Address(strU), tmp4, (base_offset % 8) != 0 ? 4 : 8); |
| __ addi(strU, strU, wordSize); |
| __ inflate_lo32(tmp3, tmpL); |
| __ mv(tmpL, tmp3); |
| __ xorr(tmp3, tmpU, tmpL); |
| __ bnez(tmp3, CALCULATE_DIFFERENCE); |
| __ subi(cnt2, cnt2, wordSize / 2); |
| } |
| |
| // we are now 8-bytes aligned on strL when AvoidUnalignedAccesses is true |
| __ subi(cnt2, cnt2, wordSize * 2); |
| __ bltz(cnt2, TAIL); |
| __ bind(SMALL_LOOP); // smaller loop |
| __ subi(cnt2, cnt2, wordSize * 2); |
| compare_string_8_x_LU(tmpL, tmpU, strL, strU, CALCULATE_DIFFERENCE); |
| compare_string_8_x_LU(tmpL, tmpU, strL, strU, CALCULATE_DIFFERENCE); |
| __ bgez(cnt2, SMALL_LOOP); |
| __ addi(t0, cnt2, wordSize * 2); |
| __ beqz(t0, DONE); |
| __ bind(TAIL); // 1..15 characters left |
| // Aligned access. Load bytes in portions - 4, 2, 1. |
| |
| __ addi(t0, cnt2, wordSize); |
| __ addi(cnt2, cnt2, wordSize * 2); // amount of characters left to process |
| __ bltz(t0, LOAD_LAST); |
| // remaining characters are greater than or equals to 8, we can do one compare_string_8_x_LU |
| compare_string_8_x_LU(tmpL, tmpU, strL, strU, CALCULATE_DIFFERENCE); |
| __ subi(cnt2, cnt2, wordSize); |
| __ beqz(cnt2, DONE); // no character left |
| __ bind(LOAD_LAST); // cnt2 = 1..7 characters left |
| |
| __ subi(cnt2, cnt2, wordSize); // cnt2 is now an offset in strL which points to last 8 bytes |
| __ slli(t0, cnt2, 1); // t0 is now an offset in strU which points to last 16 bytes |
| __ add(strL, strL, cnt2); // Address of last 8 bytes in Latin1 string |
| __ add(strU, strU, t0); // Address of last 16 bytes in UTF-16 string |
| __ load_int_misaligned(tmpL, Address(strL), t0, false); |
| __ load_long_misaligned(tmpU, Address(strU), t0, 2); |
| __ inflate_lo32(tmp3, tmpL); |
| __ mv(tmpL, tmp3); |
| __ xorr(tmp3, tmpU, tmpL); |
| __ bnez(tmp3, CALCULATE_DIFFERENCE); |
| |
| __ addi(strL, strL, wordSize / 2); // Address of last 4 bytes in Latin1 string |
| __ addi(strU, strU, wordSize); // Address of last 8 bytes in UTF-16 string |
| __ load_int_misaligned(tmpL, Address(strL), t0, false); |
| __ load_long_misaligned(tmpU, Address(strU), t0, 2); |
| __ inflate_lo32(tmp3, tmpL); |
| __ mv(tmpL, tmp3); |
| __ xorr(tmp3, tmpU, tmpL); |
| __ bnez(tmp3, CALCULATE_DIFFERENCE); |
| __ j(DONE); // no character left |
| |
| // Find the first different characters in the longwords and |
| // compute their difference. |
| __ bind(CALCULATE_DIFFERENCE); |
| // count bits of trailing zero chars |
| __ ctzc_bits(tmp4, tmp3); |
| __ srl(tmp1, tmp1, tmp4); |
| __ srl(tmp2, tmp2, tmp4); |
| __ zext(tmp1, tmp1, 16); |
| __ zext(tmp2, tmp2, 16); |
| __ sub(result, tmp1, tmp2); |
| __ bind(DONE); |
| __ ret(); |
| return entry; |
| } |
| |
| address generate_method_entry_barrier() { |
| __ align(CodeEntryAlignment); |
| StubGenStubId stub_id = StubGenStubId::method_entry_barrier_id; |
| StubCodeMark mark(this, stub_id); |
| |
| Label deoptimize_label; |
| |
| address start = __ pc(); |
| |
| BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler(); |
| |
| if (bs_asm->nmethod_patching_type() == NMethodPatchingType::conc_instruction_and_data_patch) { |
| BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); |
| Address thread_epoch_addr(xthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()) + 4); |
| __ la(t1, ExternalAddress(bs_asm->patching_epoch_addr())); |
| __ lwu(t1, t1); |
| __ sw(t1, thread_epoch_addr); |
| // There are two ways this can work: |
| // - The writer did system icache shootdown after the instruction stream update. |
| // Hence do nothing. |
| // - The writer trust us to make sure our icache is in sync before entering. |
| // Hence use cmodx fence (fence.i, may change). |
| if (UseCtxFencei) { |
| __ cmodx_fence(); |
| } |
| __ membar(__ LoadLoad); |
| } |
| |
| __ set_last_Java_frame(sp, fp, ra); |
| |
| __ enter(); |
| __ addi(t1, sp, wordSize); |
| |
| __ subi(sp, sp, 4 * wordSize); |
| |
| __ push_call_clobbered_registers(); |
| |
| __ mv(c_rarg0, t1); |
| __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetNMethod::nmethod_stub_entry_barrier), 1); |
| |
| __ reset_last_Java_frame(true); |
| |
| __ mv(t0, x10); |
| |
| __ pop_call_clobbered_registers(); |
| |
| __ bnez(t0, deoptimize_label); |
| |
| __ leave(); |
| __ ret(); |
| |
| __ BIND(deoptimize_label); |
| |
| __ ld(t0, Address(sp, 0)); |
| __ ld(fp, Address(sp, wordSize)); |
| __ ld(ra, Address(sp, wordSize * 2)); |
| __ ld(t1, Address(sp, wordSize * 3)); |
| |
| __ mv(sp, t0); |
| __ jr(t1); |
| |
| return start; |
| } |
| |
| // x10 = result |
| // x11 = str1 |
| // x12 = cnt1 |
| // x13 = str2 |
| // x14 = cnt2 |
| // x28 = tmp1 |
| // x29 = tmp2 |
| // x30 = tmp3 |
| // x31 = tmp4 |
| address generate_compare_long_string_same_encoding(StubGenStubId stub_id) { |
| bool isLL; |
| switch (stub_id) { |
| case compare_long_string_LL_id: |
| isLL = true; |
| break; |
| case compare_long_string_UU_id: |
| isLL = false; |
| break; |
| default: |
| ShouldNotReachHere(); |
| }; |
| __ align(CodeEntryAlignment); |
| StubCodeMark mark(this, stub_id); |
| address entry = __ pc(); |
| Label SMALL_LOOP, CHECK_LAST, DIFF2, TAIL, |
| LENGTH_DIFF, DIFF, LAST_CHECK_AND_LENGTH_DIFF; |
| const Register result = x10, str1 = x11, cnt1 = x12, str2 = x13, cnt2 = x14, |
| tmp1 = x28, tmp2 = x29, tmp3 = x30, tmp4 = x7, tmp5 = x31; |
| RegSet spilled_regs = RegSet::of(tmp4, tmp5); |
| |
| // cnt1/cnt2 contains amount of characters to compare. cnt1 can be re-used |
| // update cnt2 counter with already loaded 8 bytes |
| __ subi(cnt2, cnt2, wordSize / (isLL ? 1 : 2)); |
| // update pointers, because of previous read |
| __ addi(str1, str1, wordSize); |
| __ addi(str2, str2, wordSize); |
| // less than 16 bytes left? |
| __ subi(cnt2, cnt2, isLL ? 16 : 8); |
| __ push_reg(spilled_regs, sp); |
| __ bltz(cnt2, TAIL); |
| __ bind(SMALL_LOOP); |
| // compare 16 bytes of strings with same encoding |
| __ ld(tmp5, Address(str1)); |
| __ addi(str1, str1, 8); |
| __ xorr(tmp4, tmp1, tmp2); |
| __ ld(cnt1, Address(str2)); |
| __ addi(str2, str2, 8); |
| __ bnez(tmp4, DIFF); |
| __ ld(tmp1, Address(str1)); |
| __ addi(str1, str1, 8); |
| __ xorr(tmp4, tmp5, cnt1); |
| __ ld(tmp2, Address(str2)); |
| __ addi(str2, str2, 8); |
| __ bnez(tmp4, DIFF2); |
| |
| __ subi(cnt2, cnt2, isLL ? 16 : 8); |
| __ bgez(cnt2, SMALL_LOOP); |
| __ bind(TAIL); |
| __ addi(cnt2, cnt2, isLL ? 16 : 8); |
| __ beqz(cnt2, LAST_CHECK_AND_LENGTH_DIFF); |
| __ subi(cnt2, cnt2, isLL ? 8 : 4); |
| __ blez(cnt2, CHECK_LAST); |
| __ xorr(tmp4, tmp1, tmp2); |
| __ bnez(tmp4, DIFF); |
| __ ld(tmp1, Address(str1)); |
| __ addi(str1, str1, 8); |
| __ ld(tmp2, Address(str2)); |
| __ addi(str2, str2, 8); |
| __ subi(cnt2, cnt2, isLL ? 8 : 4); |
| __ bind(CHECK_LAST); |
| if (!isLL) { |
| __ add(cnt2, cnt2, cnt2); // now in bytes |
| } |
| __ xorr(tmp4, tmp1, tmp2); |
| __ bnez(tmp4, DIFF); |
| __ add(str1, str1, cnt2); |
| __ load_long_misaligned(tmp5, Address(str1), tmp3, isLL ? 1 : 2); |
| __ add(str2, str2, cnt2); |
| __ load_long_misaligned(cnt1, Address(str2), tmp3, isLL ? 1 : 2); |
| __ xorr(tmp4, tmp5, cnt1); |
| __ beqz(tmp4, LENGTH_DIFF); |
| // Find the first different characters in the longwords and |
| // compute their difference. |
| __ bind(DIFF2); |
| // count bits of trailing zero chars |
| __ ctzc_bits(tmp3, tmp4, isLL); |
| __ srl(tmp5, tmp5, tmp3); |
| __ srl(cnt1, cnt1, tmp3); |
| if (isLL) { |
| __ zext(tmp5, tmp5, 8); |
| __ zext(cnt1, cnt1, 8); |
| } else { |
| __ zext(tmp5, tmp5, 16); |
| __ zext(cnt1, cnt1, 16); |
| } |
| __ sub(result, tmp5, cnt1); |
| __ j(LENGTH_DIFF); |
| __ bind(DIFF); |
| // count bits of trailing zero chars |
| __ ctzc_bits(tmp3, tmp4, isLL); |
| __ srl(tmp1, tmp1, tmp3); |
| __ srl(tmp2, tmp2, tmp3); |
| if (isLL) { |
| __ zext(tmp1, tmp1, 8); |
| __ zext(tmp2, tmp2, 8); |
| } else { |
| __ zext(tmp1, tmp1, 16); |
| __ zext(tmp2, tmp2, 16); |
| } |
| __ sub(result, tmp1, tmp2); |
| __ j(LENGTH_DIFF); |
| __ bind(LAST_CHECK_AND_LENGTH_DIFF); |
| __ xorr(tmp4, tmp1, tmp2); |
| __ bnez(tmp4, DIFF); |
| __ bind(LENGTH_DIFF); |
| __ pop_reg(spilled_regs, sp); |
| __ ret(); |
| return entry; |
| } |
| |
| void generate_compare_long_strings() { |
| StubRoutines::riscv::_compare_long_string_LL = generate_compare_long_string_same_encoding(StubGenStubId::compare_long_string_LL_id); |
| StubRoutines::riscv::_compare_long_string_UU = generate_compare_long_string_same_encoding(StubGenStubId::compare_long_string_UU_id); |
| StubRoutines::riscv::_compare_long_string_LU = generate_compare_long_string_different_encoding(StubGenStubId::compare_long_string_LU_id); |
| StubRoutines::riscv::_compare_long_string_UL = generate_compare_long_string_different_encoding(StubGenStubId::compare_long_string_UL_id); |
| } |
| |
| // x10 result |
| // x11 src |
| // x12 src count |
| // x13 pattern |
| // x14 pattern count |
| address generate_string_indexof_linear(StubGenStubId stub_id) |
| { |
| bool needle_isL; |
| bool haystack_isL; |
| switch (stub_id) { |
| case string_indexof_linear_ll_id: |
| needle_isL = true; |
| haystack_isL = true; |
| break; |
| case string_indexof_linear_ul_id: |
| needle_isL = true; |
| haystack_isL = false; |
| break; |
| case string_indexof_linear_uu_id: |
| needle_isL = false; |
| haystack_isL = false; |
| break; |
| default: |
| ShouldNotReachHere(); |
| }; |
| |
| __ align(CodeEntryAlignment); |
| StubCodeMark mark(this, stub_id); |
| address entry = __ pc(); |
| |
| int needle_chr_size = needle_isL ? 1 : 2; |
| int haystack_chr_size = haystack_isL ? 1 : 2; |
| int needle_chr_shift = needle_isL ? 0 : 1; |
| int haystack_chr_shift = haystack_isL ? 0 : 1; |
| bool isL = needle_isL && haystack_isL; |
| // parameters |
| Register result = x10, haystack = x11, haystack_len = x12, needle = x13, needle_len = x14; |
| // temporary registers |
| Register mask1 = x20, match_mask = x21, first = x22, trailing_zeros = x23, mask2 = x24, tmp = x25; |
| // redefinitions |
| Register ch1 = x28, ch2 = x29; |
| RegSet spilled_regs = RegSet::range(x20, x25) + RegSet::range(x28, x29); |
| |
| __ push_reg(spilled_regs, sp); |
| |
| Label L_LOOP, L_LOOP_PROCEED, L_SMALL, L_HAS_ZERO, |
| L_HAS_ZERO_LOOP, L_CMP_LOOP, L_CMP_LOOP_NOMATCH, L_SMALL_PROCEED, |
| L_SMALL_HAS_ZERO_LOOP, L_SMALL_CMP_LOOP_NOMATCH, L_SMALL_CMP_LOOP, |
| L_POST_LOOP, L_CMP_LOOP_LAST_CMP, L_HAS_ZERO_LOOP_NOMATCH, |
| L_SMALL_CMP_LOOP_LAST_CMP, L_SMALL_CMP_LOOP_LAST_CMP2, |
| L_CMP_LOOP_LAST_CMP2, DONE, NOMATCH; |
| |
| __ ld(ch1, Address(needle)); |
| __ ld(ch2, Address(haystack)); |
| // src.length - pattern.length |
| __ sub(haystack_len, haystack_len, needle_len); |
| |
| // first is needle[0] |
| __ zext(first, ch1, needle_isL ? 8 : 16); |
| |
| uint64_t mask0101 = UCONST64(0x0101010101010101); |
| uint64_t mask0001 = UCONST64(0x0001000100010001); |
| __ mv(mask1, haystack_isL ? mask0101 : mask0001); |
| __ mul(first, first, mask1); |
| uint64_t mask7f7f = UCONST64(0x7f7f7f7f7f7f7f7f); |
| uint64_t mask7fff = UCONST64(0x7fff7fff7fff7fff); |
| __ mv(mask2, haystack_isL ? mask7f7f : mask7fff); |
| if (needle_isL != haystack_isL) { |
| __ mv(tmp, ch1); |
| } |
| __ subi(haystack_len, haystack_len, wordSize / haystack_chr_size - 1); |
| __ blez(haystack_len, L_SMALL); |
| |
| if (needle_isL != haystack_isL) { |
| __ inflate_lo32(ch1, tmp, match_mask, trailing_zeros); |
| } |
| // xorr, sub, orr, notr, andr |
| // compare and set match_mask[i] with 0x80/0x8000 (Latin1/UTF16) if ch2[i] == first[i] |
| // eg: |
| // first: aa aa aa aa aa aa aa aa |
| // ch2: aa aa li nx jd ka aa aa |
| // match_mask: 80 80 00 00 00 00 80 80 |
| __ compute_match_mask(ch2, first, match_mask, mask1, mask2); |
| |
| // search first char of needle, if success, goto L_HAS_ZERO; |
| __ bnez(match_mask, L_HAS_ZERO); |
| __ subi(haystack_len, haystack_len, wordSize / haystack_chr_size); |
| __ addi(result, result, wordSize / haystack_chr_size); |
| __ addi(haystack, haystack, wordSize); |
| __ bltz(haystack_len, L_POST_LOOP); |
| |
| __ bind(L_LOOP); |
| __ ld(ch2, Address(haystack)); |
| __ compute_match_mask(ch2, first, match_mask, mask1, mask2); |
| __ bnez(match_mask, L_HAS_ZERO); |
| |
| __ bind(L_LOOP_PROCEED); |
| __ subi(haystack_len, haystack_len, wordSize / haystack_chr_size); |
| __ addi(haystack, haystack, wordSize); |
| __ addi(result, result, wordSize / haystack_chr_size); |
| __ bgez(haystack_len, L_LOOP); |
| |
| __ bind(L_POST_LOOP); |
| __ mv(ch2, -wordSize / haystack_chr_size); |
| __ ble(haystack_len, ch2, NOMATCH); // no extra characters to check |
| __ ld(ch2, Address(haystack)); |
| __ slli(haystack_len, haystack_len, LogBitsPerByte + haystack_chr_shift); |
| __ neg(haystack_len, haystack_len); |
| __ xorr(ch2, first, ch2); |
| __ sub(match_mask, ch2, mask1); |
| __ orr(ch2, ch2, mask2); |
| __ mv(trailing_zeros, -1); // all bits set |
| __ j(L_SMALL_PROCEED); |
| |
| __ align(OptoLoopAlignment); |
| __ bind(L_SMALL); |
| __ slli(haystack_len, haystack_len, LogBitsPerByte + haystack_chr_shift); |
| __ neg(haystack_len, haystack_len); |
| if (needle_isL != haystack_isL) { |
| __ inflate_lo32(ch1, tmp, match_mask, trailing_zeros); |
| } |
| __ xorr(ch2, first, ch2); |
| __ sub(match_mask, ch2, mask1); |
| __ orr(ch2, ch2, mask2); |
| __ mv(trailing_zeros, -1); // all bits set |
| |
| __ bind(L_SMALL_PROCEED); |
| __ srl(trailing_zeros, trailing_zeros, haystack_len); // mask. zeroes on useless bits. |
| __ notr(ch2, ch2); |
| __ andr(match_mask, match_mask, ch2); |
| __ andr(match_mask, match_mask, trailing_zeros); // clear useless bits and check |
| __ beqz(match_mask, NOMATCH); |
| |
| __ bind(L_SMALL_HAS_ZERO_LOOP); |
| // count bits of trailing zero chars |
| __ ctzc_bits(trailing_zeros, match_mask, haystack_isL, ch2, tmp); |
| __ addi(trailing_zeros, trailing_zeros, haystack_isL ? 7 : 15); |
| __ mv(ch2, wordSize / haystack_chr_size); |
| __ ble(needle_len, ch2, L_SMALL_CMP_LOOP_LAST_CMP2); |
| __ compute_index(haystack, trailing_zeros, match_mask, result, ch2, tmp, haystack_isL); |
| __ mv(trailing_zeros, wordSize / haystack_chr_size); |
| __ bne(ch1, ch2, L_SMALL_CMP_LOOP_NOMATCH); |
| |
| __ bind(L_SMALL_CMP_LOOP); |
| __ shadd(first, trailing_zeros, needle, first, needle_chr_shift); |
| __ shadd(ch2, trailing_zeros, haystack, ch2, haystack_chr_shift); |
| needle_isL ? __ lbu(first, Address(first)) : __ lhu(first, Address(first)); |
| haystack_isL ? __ lbu(ch2, Address(ch2)) : __ lhu(ch2, Address(ch2)); |
| __ addi(trailing_zeros, trailing_zeros, 1); |
| __ bge(trailing_zeros, needle_len, L_SMALL_CMP_LOOP_LAST_CMP); |
| __ beq(first, ch2, L_SMALL_CMP_LOOP); |
| |
| __ bind(L_SMALL_CMP_LOOP_NOMATCH); |
| __ beqz(match_mask, NOMATCH); |
| // count bits of trailing zero chars |
| __ ctzc_bits(trailing_zeros, match_mask, haystack_isL, tmp, ch2); |
| __ addi(trailing_zeros, trailing_zeros, haystack_isL ? 7 : 15); |
| __ addi(result, result, 1); |
| __ addi(haystack, haystack, haystack_chr_size); |
| __ j(L_SMALL_HAS_ZERO_LOOP); |
| |
| __ align(OptoLoopAlignment); |
| __ bind(L_SMALL_CMP_LOOP_LAST_CMP); |
| __ bne(first, ch2, L_SMALL_CMP_LOOP_NOMATCH); |
| __ j(DONE); |
| |
| __ align(OptoLoopAlignment); |
| __ bind(L_SMALL_CMP_LOOP_LAST_CMP2); |
| __ compute_index(haystack, trailing_zeros, match_mask, result, ch2, tmp, haystack_isL); |
| __ bne(ch1, ch2, L_SMALL_CMP_LOOP_NOMATCH); |
| __ j(DONE); |
| |
| __ align(OptoLoopAlignment); |
| __ bind(L_HAS_ZERO); |
| // count bits of trailing zero chars |
| __ ctzc_bits(trailing_zeros, match_mask, haystack_isL, tmp, ch2); |
| __ addi(trailing_zeros, trailing_zeros, haystack_isL ? 7 : 15); |
| __ slli(needle_len, needle_len, BitsPerByte * wordSize / 2); |
| __ orr(haystack_len, haystack_len, needle_len); // restore needle_len(32bits) |
| __ subi(result, result, 1); // array index from 0, so result -= 1 |
| |
| __ bind(L_HAS_ZERO_LOOP); |
| __ mv(needle_len, wordSize / haystack_chr_size); |
| __ srli(ch2, haystack_len, BitsPerByte * wordSize / 2); |
| __ bge(needle_len, ch2, L_CMP_LOOP_LAST_CMP2); |
| // load next 8 bytes from haystack, and increase result index |
| __ compute_index(haystack, trailing_zeros, match_mask, result, ch2, tmp, haystack_isL); |
| __ addi(result, result, 1); |
| __ mv(trailing_zeros, wordSize / haystack_chr_size); |
| __ bne(ch1, ch2, L_CMP_LOOP_NOMATCH); |
| |
| // compare one char |
| __ bind(L_CMP_LOOP); |
| __ shadd(needle_len, trailing_zeros, needle, needle_len, needle_chr_shift); |
| needle_isL ? __ lbu(needle_len, Address(needle_len)) : __ lhu(needle_len, Address(needle_len)); |
| __ shadd(ch2, trailing_zeros, haystack, ch2, haystack_chr_shift); |
| haystack_isL ? __ lbu(ch2, Address(ch2)) : __ lhu(ch2, Address(ch2)); |
| __ addi(trailing_zeros, trailing_zeros, 1); // next char index |
| __ srli(tmp, haystack_len, BitsPerByte * wordSize / 2); |
| __ bge(trailing_zeros, tmp, L_CMP_LOOP_LAST_CMP); |
| __ beq(needle_len, ch2, L_CMP_LOOP); |
| |
| __ bind(L_CMP_LOOP_NOMATCH); |
| __ beqz(match_mask, L_HAS_ZERO_LOOP_NOMATCH); |
| // count bits of trailing zero chars |
| __ ctzc_bits(trailing_zeros, match_mask, haystack_isL, needle_len, ch2); |
| __ addi(trailing_zeros, trailing_zeros, haystack_isL ? 7 : 15); |
| __ addi(haystack, haystack, haystack_chr_size); |
| __ j(L_HAS_ZERO_LOOP); |
| |
| __ align(OptoLoopAlignment); |
| __ bind(L_CMP_LOOP_LAST_CMP); |
| __ bne(needle_len, ch2, L_CMP_LOOP_NOMATCH); |
| __ j(DONE); |
| |
| __ align(OptoLoopAlignment); |
| __ bind(L_CMP_LOOP_LAST_CMP2); |
| __ compute_index(haystack, trailing_zeros, match_mask, result, ch2, tmp, haystack_isL); |
| __ addi(result, result, 1); |
| __ bne(ch1, ch2, L_CMP_LOOP_NOMATCH); |
| __ j(DONE); |
| |
| __ align(OptoLoopAlignment); |
| __ bind(L_HAS_ZERO_LOOP_NOMATCH); |
| // 1) Restore "result" index. Index was wordSize/str2_chr_size * N until |
| // L_HAS_ZERO block. Byte octet was analyzed in L_HAS_ZERO_LOOP, |
| // so, result was increased at max by wordSize/str2_chr_size - 1, so, |
| // respective high bit wasn't changed. L_LOOP_PROCEED will increase |
| // result by analyzed characters value, so, we can just reset lower bits |
| // in result here. Clear 2 lower bits for UU/UL and 3 bits for LL |
| // 2) restore needle_len and haystack_len values from "compressed" haystack_len |
| // 3) advance haystack value to represent next haystack octet. result & 7/3 is |
| // index of last analyzed substring inside current octet. So, haystack in at |
| // respective start address. We need to advance it to next octet |
| __ andi(match_mask, result, wordSize / haystack_chr_size - 1); |
| __ srli(needle_len, haystack_len, BitsPerByte * wordSize / 2); |
| __ andi(result, result, haystack_isL ? -8 : -4); |
| __ slli(tmp, match_mask, haystack_chr_shift); |
| __ sub(haystack, haystack, tmp); |
| __ sext(haystack_len, haystack_len, 32); |
| __ j(L_LOOP_PROCEED); |
| |
| __ align(OptoLoopAlignment); |
| __ bind(NOMATCH); |
| __ mv(result, -1); |
| |
| __ bind(DONE); |
| __ pop_reg(spilled_regs, sp); |
| __ ret(); |
| return entry; |
| } |
| |
| void generate_string_indexof_stubs() |
| { |
| StubRoutines::riscv::_string_indexof_linear_ll = generate_string_indexof_linear(StubGenStubId::string_indexof_linear_ll_id); |
| StubRoutines::riscv::_string_indexof_linear_uu = generate_string_indexof_linear(StubGenStubId::string_indexof_linear_uu_id); |
| StubRoutines::riscv::_string_indexof_linear_ul = generate_string_indexof_linear(StubGenStubId::string_indexof_linear_ul_id); |
| } |
| |
| #ifdef COMPILER2 |
| void generate_lookup_secondary_supers_table_stub() { |
| StubGenStubId stub_id = StubGenStubId::lookup_secondary_supers_table_id; |
| StubCodeMark mark(this, stub_id); |
| |
| const Register |
| r_super_klass = x10, |
| r_array_base = x11, |
| r_array_length = x12, |
| r_array_index = x13, |
| r_sub_klass = x14, |
| result = x15, |
| r_bitmap = x16; |
| |
| for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) { |
| StubRoutines::_lookup_secondary_supers_table_stubs[slot] = __ pc(); |
| Label L_success; |
| __ enter(); |
| __ lookup_secondary_supers_table_const(r_sub_klass, r_super_klass, result, |
| r_array_base, r_array_length, r_array_index, |
| r_bitmap, slot, /*stub_is_near*/true); |
| __ leave(); |
| __ ret(); |
| } |
| } |
| |
| // Slow path implementation for UseSecondarySupersTable. |
| address generate_lookup_secondary_supers_table_slow_path_stub() { |
| StubGenStubId stub_id = StubGenStubId::lookup_secondary_supers_table_slow_path_id; |
| StubCodeMark mark(this, stub_id); |
| |
| address start = __ pc(); |
| const Register |
| r_super_klass = x10, // argument |
| r_array_base = x11, // argument |
| temp1 = x12, // tmp |
| r_array_index = x13, // argument |
| result = x15, // argument |
| r_bitmap = x16; // argument |
| |
| |
| __ lookup_secondary_supers_table_slow_path(r_super_klass, r_array_base, r_array_index, r_bitmap, result, temp1); |
| __ ret(); |
| |
| return start; |
| } |
| |
| address generate_mulAdd() |
| { |
| __ align(CodeEntryAlignment); |
| StubGenStubId stub_id = StubGenStubId::mulAdd_id; |
| StubCodeMark mark(this, stub_id); |
| |
| address entry = __ pc(); |
| |
| const Register out = x10; |
| const Register in = x11; |
| const Register offset = x12; |
| const Register len = x13; |
| const Register k = x14; |
| const Register tmp = x28; |
| |
| BLOCK_COMMENT("Entry:"); |
| __ enter(); |
| __ mul_add(out, in, offset, len, k, tmp); |
| __ leave(); |
| __ ret(); |
| |
| return entry; |
| } |
| |
| /** |
| * Arguments: |
| * |
| * Input: |
| * c_rarg0 - x address |
| * c_rarg1 - x length |
| * c_rarg2 - y address |
| * c_rarg3 - y length |
| * c_rarg4 - z address |
| */ |
| address generate_multiplyToLen() |
| { |
| __ align(CodeEntryAlignment); |
| StubGenStubId stub_id = StubGenStubId::multiplyToLen_id; |
| StubCodeMark mark(this, stub_id); |
| address entry = __ pc(); |
| |
| const Register x = x10; |
| const Register xlen = x11; |
| const Register y = x12; |
| const Register ylen = x13; |
| const Register z = x14; |
| |
| const Register tmp0 = x15; |
| const Register tmp1 = x16; |
| const Register tmp2 = x17; |
| const Register tmp3 = x7; |
| const Register tmp4 = x28; |
| const Register tmp5 = x29; |
| const Register tmp6 = x30; |
| const Register tmp7 = x31; |
| |
| BLOCK_COMMENT("Entry:"); |
| __ enter(); // required for proper stackwalking of RuntimeStub frame |
| __ multiply_to_len(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); |
| __ leave(); // required for proper stackwalking of RuntimeStub frame |
| __ ret(); |
| |
| return entry; |
| } |
| |
| address generate_squareToLen() |
| { |
| __ align(CodeEntryAlignment); |
| StubGenStubId stub_id = StubGenStubId::squareToLen_id; |
| StubCodeMark mark(this, stub_id); |
| address entry = __ pc(); |
| |
| const Register x = x10; |
| const Register xlen = x11; |
| const Register z = x12; |
| const Register y = x14; // == x |
| const Register ylen = x15; // == xlen |
| |
| const Register tmp0 = x13; // zlen, unused |
| const Register tmp1 = x16; |
| const Register tmp2 = x17; |
| const Register tmp3 = x7; |
| const Register tmp4 = x28; |
| const Register tmp5 = x29; |
| const Register tmp6 = x30; |
| const Register tmp7 = x31; |
| |
| BLOCK_COMMENT("Entry:"); |
| __ enter(); |
| __ mv(y, x); |
| __ mv(ylen, xlen); |
| __ multiply_to_len(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); |
| __ leave(); |
| __ ret(); |
| |
| return entry; |
| } |
| |
| // Arguments: |
| // |
| // Input: |
| // c_rarg0 - newArr address |
| // c_rarg1 - oldArr address |
| // c_rarg2 - newIdx |
| // c_rarg3 - shiftCount |
| // c_rarg4 - numIter |
| // |
| address generate_bigIntegerLeftShift() { |
| __ align(CodeEntryAlignment); |
| StubGenStubId stub_id = StubGenStubId::bigIntegerLeftShiftWorker_id; |
| StubCodeMark mark(this, stub_id); |
| address entry = __ pc(); |
| |
| Label loop, exit; |
| |
| Register newArr = c_rarg0; |
| Register oldArr = c_rarg1; |
| Register newIdx = c_rarg2; |
| Register shiftCount = c_rarg3; |
| Register numIter = c_rarg4; |
| |
| Register shiftRevCount = c_rarg5; |
| Register oldArrNext = t1; |
| |
| __ beqz(numIter, exit); |
| __ shadd(newArr, newIdx, newArr, t0, 2); |
| |
| __ mv(shiftRevCount, 32); |
| __ sub(shiftRevCount, shiftRevCount, shiftCount); |
| |
| __ bind(loop); |
| __ addi(oldArrNext, oldArr, 4); |
| __ vsetvli(t0, numIter, Assembler::e32, Assembler::m4); |
| __ vle32_v(v0, oldArr); |
| __ vle32_v(v4, oldArrNext); |
| __ vsll_vx(v0, v0, shiftCount); |
| __ vsrl_vx(v4, v4, shiftRevCount); |
| __ vor_vv(v0, v0, v4); |
| __ vse32_v(v0, newArr); |
| __ sub(numIter, numIter, t0); |
| __ shadd(oldArr, t0, oldArr, t1, 2); |
| __ shadd(newArr, t0, newArr, t1, 2); |
| __ bnez(numIter, loop); |
| |
| __ bind(exit); |
| __ ret(); |
| |
| return entry; |
| } |
| |
| // Arguments: |
| // |
| // Input: |
| // c_rarg0 - newArr address |
| // c_rarg1 - oldArr address |
| // c_rarg2 - newIdx |
| // c_rarg3 - shiftCount |
| // c_rarg4 - numIter |
| // |
| address generate_bigIntegerRightShift() { |
| __ align(CodeEntryAlignment); |
| StubGenStubId stub_id = StubGenStubId::bigIntegerRightShiftWorker_id; |
| StubCodeMark mark(this, stub_id); |
| address entry = __ pc(); |
| |
| Label loop, exit; |
| |
| Register newArr = c_rarg0; |
| Register oldArr = c_rarg1; |
| Register newIdx = c_rarg2; |
| Register shiftCount = c_rarg3; |
| Register numIter = c_rarg4; |
| Register idx = numIter; |
| |
| Register shiftRevCount = c_rarg5; |
| Register oldArrNext = c_rarg6; |
| Register newArrCur = t0; |
| Register oldArrCur = t1; |
| |
| __ beqz(idx, exit); |
| __ shadd(newArr, newIdx, newArr, t0, 2); |
| |
| __ mv(shiftRevCount, 32); |
| __ sub(shiftRevCount, shiftRevCount, shiftCount); |
| |
| __ bind(loop); |
| __ vsetvli(t0, idx, Assembler::e32, Assembler::m4); |
| __ sub(idx, idx, t0); |
| __ shadd(oldArrNext, idx, oldArr, t1, 2); |
| __ shadd(newArrCur, idx, newArr, t1, 2); |
| __ addi(oldArrCur, oldArrNext, 4); |
| __ vle32_v(v0, oldArrCur); |
| __ vle32_v(v4, oldArrNext); |
| __ vsrl_vx(v0, v0, shiftCount); |
| __ vsll_vx(v4, v4, shiftRevCount); |
| __ vor_vv(v0, v0, v4); |
| __ vse32_v(v0, newArrCur); |
| __ bnez(idx, loop); |
| |
| __ bind(exit); |
| __ ret(); |
| |
| return entry; |
| } |
| #endif |
| |
| #ifdef COMPILER2 |
| class MontgomeryMultiplyGenerator : public MacroAssembler { |
| |
| Register Pa_base, Pb_base, Pn_base, Pm_base, inv, Rlen, Ra, Rb, Rm, Rn, |
| Pa, Pb, Pn, Pm, Rhi_ab, Rlo_ab, Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2, Ri, Rj; |
| |
| RegSet _toSave; |
| bool _squaring; |
| |
| public: |
| MontgomeryMultiplyGenerator (Assembler *as, bool squaring) |
| : MacroAssembler(as->code()), _squaring(squaring) { |
| |
| // Register allocation |
| |
| RegSetIterator<Register> regs = RegSet::range(x10, x26).begin(); |
| Pa_base = *regs; // Argument registers |
| if (squaring) { |
| Pb_base = Pa_base; |
| } else { |
| Pb_base = *++regs; |
| } |
| Pn_base = *++regs; |
| Rlen= *++regs; |
| inv = *++regs; |
| Pm_base = *++regs; |
| |
| // Working registers: |
| Ra = *++regs; // The current digit of a, b, n, and m. |
| Rb = *++regs; |
| Rm = *++regs; |
| Rn = *++regs; |
| |
| Pa = *++regs; // Pointers to the current/next digit of a, b, n, and m. |
| Pb = *++regs; |
| Pm = *++regs; |
| Pn = *++regs; |
| |
| tmp0 = *++regs; // Three registers which form a |
| tmp1 = *++regs; // triple-precision accumuator. |
| tmp2 = *++regs; |
| |
| Ri = x6; // Inner and outer loop indexes. |
| Rj = x7; |
| |
| Rhi_ab = x28; // Product registers: low and high parts |
| Rlo_ab = x29; // of a*b and m*n. |
| Rhi_mn = x30; |
| Rlo_mn = x31; |
| |
| // x18 and up are callee-saved. |
| _toSave = RegSet::range(x18, *regs) + Pm_base; |
| } |
| |
| private: |
| void save_regs() { |
| push_reg(_toSave, sp); |
| } |
| |
| void restore_regs() { |
| pop_reg(_toSave, sp); |
| } |
| |
| template <typename T> |
| void unroll_2(Register count, T block) { |
| Label loop, end, odd; |
| beqz(count, end); |
| test_bit(t0, count, 0); |
| bnez(t0, odd); |
| align(16); |
| bind(loop); |
| (this->*block)(); |
| bind(odd); |
| (this->*block)(); |
| subi(count, count, 2); |
| bgtz(count, loop); |
| bind(end); |
| } |
| |
| template <typename T> |
| void unroll_2(Register count, T block, Register d, Register s, Register tmp) { |
| Label loop, end, odd; |
| beqz(count, end); |
| test_bit(tmp, count, 0); |
| bnez(tmp, odd); |
| align(16); |
| bind(loop); |
| (this->*block)(d, s, tmp); |
| bind(odd); |
| (this->*block)(d, s, tmp); |
| subi(count, count, 2); |
| bgtz(count, loop); |
| bind(end); |
| } |
| |
| void pre1(RegisterOrConstant i) { |
| block_comment("pre1"); |
| // Pa = Pa_base; |
| // Pb = Pb_base + i; |
| // Pm = Pm_base; |
| // Pn = Pn_base + i; |
| // Ra = *Pa; |
| // Rb = *Pb; |
| // Rm = *Pm; |
| // Rn = *Pn; |
| if (i.is_register()) { |
| slli(t0, i.as_register(), LogBytesPerWord); |
| } else { |
| mv(t0, i.as_constant()); |
| slli(t0, t0, LogBytesPerWord); |
| } |
| |
| mv(Pa, Pa_base); |
| add(Pb, Pb_base, t0); |
| mv(Pm, Pm_base); |
| add(Pn, Pn_base, t0); |
| |
| ld(Ra, Address(Pa)); |
| ld(Rb, Address(Pb)); |
| ld(Rm, Address(Pm)); |
| ld(Rn, Address(Pn)); |
| |
| // Zero the m*n result. |
| mv(Rhi_mn, zr); |
| mv(Rlo_mn, zr); |
| } |
| |
| // The core multiply-accumulate step of a Montgomery |
| // multiplication. The idea is to schedule operations as a |
| // pipeline so that instructions with long latencies (loads and |
| // multiplies) have time to complete before their results are |
| // used. This most benefits in-order implementations of the |
| // architecture but out-of-order ones also benefit. |
| void step() { |
| block_comment("step"); |
| // MACC(Ra, Rb, tmp0, tmp1, tmp2); |
| // Ra = *++Pa; |
| // Rb = *--Pb; |
| mulhu(Rhi_ab, Ra, Rb); |
| mul(Rlo_ab, Ra, Rb); |
| addi(Pa, Pa, wordSize); |
| ld(Ra, Address(Pa)); |
| subi(Pb, Pb, wordSize); |
| ld(Rb, Address(Pb)); |
| acc(Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2); // The pending m*n from the |
| // previous iteration. |
| // MACC(Rm, Rn, tmp0, tmp1, tmp2); |
| // Rm = *++Pm; |
| // Rn = *--Pn; |
| mulhu(Rhi_mn, Rm, Rn); |
| mul(Rlo_mn, Rm, Rn); |
| addi(Pm, Pm, wordSize); |
| ld(Rm, Address(Pm)); |
| subi(Pn, Pn, wordSize); |
| ld(Rn, Address(Pn)); |
| acc(Rhi_ab, Rlo_ab, tmp0, tmp1, tmp2); |
| } |
| |
| void post1() { |
| block_comment("post1"); |
| |
| // MACC(Ra, Rb, tmp0, tmp1, tmp2); |
| // Ra = *++Pa; |
| // Rb = *--Pb; |
| mulhu(Rhi_ab, Ra, Rb); |
| mul(Rlo_ab, Ra, Rb); |
| acc(Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2); // The pending m*n |
| acc(Rhi_ab, Rlo_ab, tmp0, tmp1, tmp2); |
| |
| // *Pm = Rm = tmp0 * inv; |
| mul(Rm, tmp0, inv); |
| sd(Rm, Address(Pm)); |
| |
| // MACC(Rm, Rn, tmp0, tmp1, tmp2); |
| // tmp0 = tmp1; tmp1 = tmp2; tmp2 = 0; |
| mulhu(Rhi_mn, Rm, Rn); |
| |
| #ifndef PRODUCT |
| // assert(m[i] * n[0] + tmp0 == 0, "broken Montgomery multiply"); |
| { |
| mul(Rlo_mn, Rm, Rn); |
| add(Rlo_mn, tmp0, Rlo_mn); |
| Label ok; |
| beqz(Rlo_mn, ok); |
| stop("broken Montgomery multiply"); |
| bind(ok); |
| } |
| #endif |
| // We have very carefully set things up so that |
| // m[i]*n[0] + tmp0 == 0 (mod b), so we don't have to calculate |
| // the lower half of Rm * Rn because we know the result already: |
| // it must be -tmp0. tmp0 + (-tmp0) must generate a carry iff |
| // tmp0 != 0. So, rather than do a mul and an cad we just set |
| // the carry flag iff tmp0 is nonzero. |
| // |
| // mul(Rlo_mn, Rm, Rn); |
| // cad(zr, tmp0, Rlo_mn); |
| subi(t0, tmp0, 1); |
| sltu(t0, t0, tmp0); // Set carry iff tmp0 is nonzero |
| cadc(tmp0, tmp1, Rhi_mn, t0); |
| adc(tmp1, tmp2, zr, t0); |
| mv(tmp2, zr); |
| } |
| |
| void pre2(Register i, Register len) { |
| block_comment("pre2"); |
| // Pa = Pa_base + i-len; |
| // Pb = Pb_base + len; |
| // Pm = Pm_base + i-len; |
| // Pn = Pn_base + len; |
| |
| sub(Rj, i, len); |
| // Rj == i-len |
| |
| // Ra as temp register |
| slli(Ra, Rj, LogBytesPerWord); |
| add(Pa, Pa_base, Ra); |
| add(Pm, Pm_base, Ra); |
| slli(Ra, len, LogBytesPerWord); |
| add(Pb, Pb_base, Ra); |
| add(Pn, Pn_base, Ra); |
| |
| // Ra = *++Pa; |
| // Rb = *--Pb; |
| // Rm = *++Pm; |
| // Rn = *--Pn; |
| addi(Pa, Pa, wordSize); |
| ld(Ra, Address(Pa)); |
| subi(Pb, Pb, wordSize); |
| ld(Rb, Address(Pb)); |
| addi(Pm, Pm, wordSize); |
| ld(Rm, Address(Pm)); |
| subi(Pn, Pn, wordSize); |
| ld(Rn, Address(Pn)); |
| |
| mv(Rhi_mn, zr); |
| mv(Rlo_mn, zr); |
| } |
| |
| void post2(Register i, Register len) { |
| block_comment("post2"); |
| sub(Rj, i, len); |
| |
| cad(tmp0, tmp0, Rlo_mn, t0); // The pending m*n, low part |
| |
| // As soon as we know the least significant digit of our result, |
| // store it. |
| // Pm_base[i-len] = tmp0; |
| // Rj as temp register |
| slli(Rj, Rj, LogBytesPerWord); |
| add(Rj, Pm_base, Rj); |
| sd(tmp0, Address(Rj)); |
| |
| // tmp0 = tmp1; tmp1 = tmp2; tmp2 = 0; |
| cadc(tmp0, tmp1, Rhi_mn, t0); // The pending m*n, high part |
| adc(tmp1, tmp2, zr, t0); |
| mv(tmp2, zr); |
| } |
| |
| // A carry in tmp0 after Montgomery multiplication means that we |
| // should subtract multiples of n from our result in m. We'll |
| // keep doing that until there is no carry. |
| void normalize(Register len) { |
| block_comment("normalize"); |
| // while (tmp0) |
| // tmp0 = sub(Pm_base, Pn_base, tmp0, len); |
| Label loop, post, again; |
| Register cnt = tmp1, i = tmp2; // Re-use registers; we're done with them now |
| beqz(tmp0, post); { |
| bind(again); { |
| mv(i, zr); |
| mv(cnt, len); |
| slli(Rn, i, LogBytesPerWord); |
| add(Rm, Pm_base, Rn); |
| ld(Rm, Address(Rm)); |
| add(Rn, Pn_base, Rn); |
| ld(Rn, Address(Rn)); |
| mv(t0, 1); // set carry flag, i.e. no borrow |
| align(16); |
| bind(loop); { |
| notr(Rn, Rn); |
| add(Rm, Rm, t0); |
| add(Rm, Rm, Rn); |
| sltu(t0, Rm, Rn); |
| slli(Rn, i, LogBytesPerWord); // Rn as temp register |
| add(Rn, Pm_base, Rn); |
| sd(Rm, Address(Rn)); |
| addi(i, i, 1); |
| slli(Rn, i, LogBytesPerWord); |
| add(Rm, Pm_base, Rn); |
| ld(Rm, Address(Rm)); |
| add(Rn, Pn_base, Rn); |
| ld(Rn, Address(Rn)); |
| subi(cnt, cnt, 1); |
| } bnez(cnt, loop); |
| subi(tmp0, tmp0, 1); |
| add(tmp0, tmp0, t0); |
| } bnez(tmp0, again); |
| } bind(post); |
| } |
| |
| // Move memory at s to d, reversing words. |
| // Increments d to end of copied memory |
| // Destroys tmp1, tmp2 |
| // Preserves len |
| // Leaves s pointing to the address which was in d at start |
| void reverse(Register d, Register s, Register len, Register tmp1, Register tmp2) { |
| assert(tmp1->encoding() < x28->encoding(), "register corruption"); |
| assert(tmp2->encoding() < x28->encoding(), "register corruption"); |
| |
| shadd(s, len, s, tmp1, LogBytesPerWord); |
| mv(tmp1, len); |
| unroll_2(tmp1, &MontgomeryMultiplyGenerator::reverse1, d, s, tmp2); |
| slli(tmp1, len, LogBytesPerWord); |
| sub(s, d, tmp1); |
| } |
| // [63...0] -> [31...0][63...32] |
| void reverse1(Register d, Register s, Register tmp) { |
| subi(s, s, wordSize); |
| ld(tmp, Address(s)); |
| ror(tmp, tmp, 32, t0); |
| sd(tmp, Address(d)); |
| addi(d, d, wordSize); |
| } |
| |
| void step_squaring() { |
| // An extra ACC |
| step(); |
| acc(Rhi_ab, Rlo_ab, tmp0, tmp1, tmp2); |
| } |
| |
| void last_squaring(Register i) { |
| Label dont; |
| // if ((i & 1) == 0) { |
| test_bit(t0, i, 0); |
| bnez(t0, dont); { |
| // MACC(Ra, Rb, tmp0, tmp1, tmp2); |
| // Ra = *++Pa; |
| // Rb = *--Pb; |
| mulhu(Rhi_ab, Ra, Rb); |
| mul(Rlo_ab, Ra, Rb); |
| acc(Rhi_ab, Rlo_ab, tmp0, tmp1, tmp2); |
| } bind(dont); |
| } |
| |
| void extra_step_squaring() { |
| acc(Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2); // The pending m*n |
| |
| // MACC(Rm, Rn, tmp0, tmp1, tmp2); |
| // Rm = *++Pm; |
| // Rn = *--Pn; |
| mulhu(Rhi_mn, Rm, Rn); |
| mul(Rlo_mn, Rm, Rn); |
| addi(Pm, Pm, wordSize); |
| ld(Rm, Address(Pm)); |
| subi(Pn, Pn, wordSize); |
| ld(Rn, Address(Pn)); |
| } |
| |
| void post1_squaring() { |
| acc(Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2); // The pending m*n |
| |
| // *Pm = Rm = tmp0 * inv; |
| mul(Rm, tmp0, inv); |
| sd(Rm, Address(Pm)); |
| |
| // MACC(Rm, Rn, tmp0, tmp1, tmp2); |
| // tmp0 = tmp1; tmp1 = tmp2; tmp2 = 0; |
| mulhu(Rhi_mn, Rm, Rn); |
| |
| #ifndef PRODUCT |
| // assert(m[i] * n[0] + tmp0 == 0, "broken Montgomery multiply"); |
| { |
| mul(Rlo_mn, Rm, Rn); |
| add(Rlo_mn, tmp0, Rlo_mn); |
| Label ok; |
| beqz(Rlo_mn, ok); { |
| stop("broken Montgomery multiply"); |
| } bind(ok); |
| } |
| #endif |
| // We have very carefully set things up so that |
| // m[i]*n[0] + tmp0 == 0 (mod b), so we don't have to calculate |
| // the lower half of Rm * Rn because we know the result already: |
| // it must be -tmp0. tmp0 + (-tmp0) must generate a carry iff |
| // tmp0 != 0. So, rather than do a mul and a cad we just set |
| // the carry flag iff tmp0 is nonzero. |
| // |
| // mul(Rlo_mn, Rm, Rn); |
| // cad(zr, tmp, Rlo_mn); |
| subi(t0, tmp0, 1); |
| sltu(t0, t0, tmp0); // Set carry iff tmp0 is nonzero |
| cadc(tmp0, tmp1, Rhi_mn, t0); |
| adc(tmp1, tmp2, zr, t0); |
| mv(tmp2, zr); |
| } |
| |
| // use t0 as carry |
| void acc(Register Rhi, Register Rlo, |
| Register tmp0, Register tmp1, Register tmp2) { |
| cad(tmp0, tmp0, Rlo, t0); |
| cadc(tmp1, tmp1, Rhi, t0); |
| adc(tmp2, tmp2, zr, t0); |
| } |
| |
| public: |
| /** |
| * Fast Montgomery multiplication. The derivation of the |
| * algorithm is in A Cryptographic Library for the Motorola |
| * DSP56000, Dusse and Kaliski, Proc. EUROCRYPT 90, pp. 230-237. |
| * |
| * Arguments: |
| * |
| * Inputs for multiplication: |
| * c_rarg0 - int array elements a |
| * c_rarg1 - int array elements b |
| * c_rarg2 - int array elements n (the modulus) |
| * c_rarg3 - int length |
| * c_rarg4 - int inv |
| * c_rarg5 - int array elements m (the result) |
| * |
| * Inputs for squaring: |
| * c_rarg0 - int array elements a |
| * c_rarg1 - int array elements n (the modulus) |
| * c_rarg2 - int length |
| * c_rarg3 - int inv |
| * c_rarg4 - int array elements m (the result) |
| * |
| */ |
| address generate_multiply() { |
| Label argh, nothing; |
| bind(argh); |
| stop("MontgomeryMultiply total_allocation must be <= 8192"); |
| |
| align(CodeEntryAlignment); |
| address entry = pc(); |
| |
| beqz(Rlen, nothing); |
| |
| enter(); |
| |
| // Make room. |
| mv(Ra, 512); |
| bgt(Rlen, Ra, argh); |
| slli(Ra, Rlen, exact_log2(4 * sizeof(jint))); |
| sub(Ra, sp, Ra); |
| andi(sp, Ra, -2 * wordSize); |
| |
| srliw(Rlen, Rlen, 1); // length in longwords = len/2 |
| |
| { |
| // Copy input args, reversing as we go. We use Ra as a |
| // temporary variable. |
| reverse(Ra, Pa_base, Rlen, Ri, Rj); |
| if (!_squaring) |
| reverse(Ra, Pb_base, Rlen, Ri, Rj); |
| reverse(Ra, Pn_base, Rlen, Ri, Rj); |
| } |
| |
| // Push all call-saved registers and also Pm_base which we'll need |
| // at the end. |
| save_regs(); |
| |
| #ifndef PRODUCT |
| // assert(inv * n[0] == -1UL, "broken inverse in Montgomery multiply"); |
| { |
| ld(Rn, Address(Pn_base)); |
| mul(Rlo_mn, Rn, inv); |
| mv(t0, -1); |
| Label ok; |
| beq(Rlo_mn, t0, ok); |
| stop("broken inverse in Montgomery multiply"); |
| bind(ok); |
| } |
| #endif |
| |
| mv(Pm_base, Ra); |
| |
| mv(tmp0, zr); |
| mv(tmp1, zr); |
| mv(tmp2, zr); |
| |
| block_comment("for (int i = 0; i < len; i++) {"); |
| mv(Ri, zr); { |
| Label loop, end; |
| bge(Ri, Rlen, end); |
| |
| bind(loop); |
| pre1(Ri); |
| |
| block_comment(" for (j = i; j; j--) {"); { |
| mv(Rj, Ri); |
| unroll_2(Rj, &MontgomeryMultiplyGenerator::step); |
| } block_comment(" } // j"); |
| |
| post1(); |
| addiw(Ri, Ri, 1); |
| blt(Ri, Rlen, loop); |
| bind(end); |
| block_comment("} // i"); |
| } |
| |
| block_comment("for (int i = len; i < 2*len; i++) {"); |
| mv(Ri, Rlen); { |
| Label loop, end; |
| slli(t0, Rlen, 1); |
| bge(Ri, t0, end); |
| |
| bind(loop); |
| pre2(Ri, Rlen); |
| |
| block_comment(" for (j = len*2-i-1; j; j--) {"); { |
| slliw(Rj, Rlen, 1); |
| subw(Rj, Rj, Ri); |
| subiw(Rj, Rj, 1); |
| unroll_2(Rj, &MontgomeryMultiplyGenerator::step); |
| } block_comment(" } // j"); |
| |
| post2(Ri, Rlen); |
| addiw(Ri, Ri, 1); |
| slli(t0, Rlen, 1); |
| blt(Ri, t0, loop); |
| bind(end); |
| } |
| block_comment("} // i"); |
| |
| normalize(Rlen); |
| |
| mv(Ra, Pm_base); // Save Pm_base in Ra |
| restore_regs(); // Restore caller's Pm_base |
| |
| // Copy our result into caller's Pm_base |
| reverse(Pm_base, Ra, Rlen, Ri, Rj); |
| |
| leave(); |
| bind(nothing); |
| ret(); |
| |
| return entry; |
| } |
| |
| /** |
| * |
| * Arguments: |
| * |
| * Inputs: |
| * c_rarg0 - int array elements a |
| * c_rarg1 - int array elements n (the modulus) |
| * c_rarg2 - int length |
| * c_rarg3 - int inv |
| * c_rarg4 - int array elements m (the result) |
| * |
| */ |
| address generate_square() { |
| Label argh; |
| bind(argh); |
| stop("MontgomeryMultiply total_allocation must be <= 8192"); |
| |
| align(CodeEntryAlignment); |
| address entry = pc(); |
| |
| enter(); |
| |
| // Make room. |
| mv(Ra, 512); |
| bgt(Rlen, Ra, argh); |
| slli(Ra, Rlen, exact_log2(4 * sizeof(jint))); |
| sub(Ra, sp, Ra); |
| andi(sp, Ra, -2 * wordSize); |
| |
| srliw(Rlen, Rlen, 1); // length in longwords = len/2 |
| |
| { |
| // Copy input args, reversing as we go. We use Ra as a |
| // temporary variable. |
| reverse(Ra, Pa_base, Rlen, Ri, Rj); |
| reverse(Ra, Pn_base, Rlen, Ri, Rj); |
| } |
| |
| // Push all call-saved registers and also Pm_base which we'll need |
| // at the end. |
| save_regs(); |
| |
| mv(Pm_base, Ra); |
| |
| mv(tmp0, zr); |
| mv(tmp1, zr); |
| mv(tmp2, zr); |
| |
| block_comment("for (int i = 0; i < len; i++) {"); |
| mv(Ri, zr); { |
| Label loop, end; |
| bind(loop); |
| bge(Ri, Rlen, end); |
| |
| pre1(Ri); |
| |
| block_comment("for (j = (i+1)/2; j; j--) {"); { |
| addi(Rj, Ri, 1); |
| srliw(Rj, Rj, 1); |
| unroll_2(Rj, &MontgomeryMultiplyGenerator::step_squaring); |
| } block_comment(" } // j"); |
| |
| last_squaring(Ri); |
| |
| block_comment(" for (j = i/2; j; j--) {"); { |
| srliw(Rj, Ri, 1); |
| unroll_2(Rj, &MontgomeryMultiplyGenerator::extra_step_squaring); |
| } block_comment(" } // j"); |
| |
| post1_squaring(); |
| addi(Ri, Ri, 1); |
| blt(Ri, Rlen, loop); |
| |
| bind(end); |
| block_comment("} // i"); |
| } |
| |
| block_comment("for (int i = len; i < 2*len; i++) {"); |
| mv(Ri, Rlen); { |
| Label loop, end; |
| bind(loop); |
| slli(t0, Rlen, 1); |
| bge(Ri, t0, end); |
| |
| pre2(Ri, Rlen); |
| |
| block_comment(" for (j = (2*len-i-1)/2; j; j--) {"); { |
| slli(Rj, Rlen, 1); |
| sub(Rj, Rj, Ri); |
| subi(Rj, Rj, 1); |
| srliw(Rj, Rj, 1); |
| unroll_2(Rj, &MontgomeryMultiplyGenerator::step_squaring); |
| } block_comment(" } // j"); |
| |
| last_squaring(Ri); |
| |
| block_comment(" for (j = (2*len-i)/2; j; j--) {"); { |
| slli(Rj, Rlen, 1); |
| sub(Rj, Rj, Ri); |
| srliw(Rj, Rj, 1); |
| unroll_2(Rj, &MontgomeryMultiplyGenerator::extra_step_squaring); |
| } block_comment(" } // j"); |
| |
| post2(Ri, Rlen); |
| addi(Ri, Ri, 1); |
| slli(t0, Rlen, 1); |
| blt(Ri, t0, loop); |
| |
| bind(end); |
| block_comment("} // i"); |
| } |
| |
| normalize(Rlen); |
| |
| mv(Ra, Pm_base); // Save Pm_base in Ra |
| restore_regs(); // Restore caller's Pm_base |
| |
| // Copy our result into caller's Pm_base |
| reverse(Pm_base, Ra, Rlen, Ri, Rj); |
| |
| leave(); |
| ret(); |
| |
| return entry; |
| } |
| }; |
| |
| #endif // COMPILER2 |
| |
| address generate_cont_thaw(Continuation::thaw_kind kind) { |
| bool return_barrier = Continuation::is_thaw_return_barrier(kind); |
| bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind); |
| |
| address start = __ pc(); |
| |
| if (return_barrier) { |
| __ ld(sp, Address(xthread, JavaThread::cont_entry_offset())); |
| } |
| |
| #ifndef PRODUCT |
| { |
| Label OK; |
| __ ld(t0, Address(xthread, JavaThread::cont_entry_offset())); |
| __ beq(sp, t0, OK); |
| __ stop("incorrect sp"); |
| __ bind(OK); |
| } |
| #endif |
| |
| if (return_barrier) { |
| // preserve possible return value from a method returning to the return barrier |
| __ subi(sp, sp, 2 * wordSize); |
| __ fsd(f10, Address(sp, 0 * wordSize)); |
| __ sd(x10, Address(sp, 1 * wordSize)); |
| } |
| |
| __ mv(c_rarg1, (return_barrier ? 1 : 0)); |
| __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), xthread, c_rarg1); |
| __ mv(t1, x10); // x10 contains the size of the frames to thaw, 0 if overflow or no more frames |
| |
| if (return_barrier) { |
| // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK) |
| __ ld(x10, Address(sp, 1 * wordSize)); |
| __ fld(f10, Address(sp, 0 * wordSize)); |
| __ addi(sp, sp, 2 * wordSize); |
| } |
| |
| #ifndef PRODUCT |
| { |
| Label OK; |
| __ ld(t0, Address(xthread, JavaThread::cont_entry_offset())); |
| __ beq(sp, t0, OK); |
| __ stop("incorrect sp"); |
| __ bind(OK); |
| } |
| #endif |
| |
| Label thaw_success; |
| // t1 contains the size of the frames to thaw, 0 if overflow or no more frames |
| __ bnez(t1, thaw_success); |
| __ j(RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry())); |
| __ bind(thaw_success); |
| |
| // make room for the thawed frames |
| __ sub(t0, sp, t1); |
| __ andi(sp, t0, -16); // align |
| |
| if (return_barrier) { |
| // save original return value -- again |
| __ subi(sp, sp, 2 * wordSize); |
| __ fsd(f10, Address(sp, 0 * wordSize)); |
| __ sd(x10, Address(sp, 1 * wordSize)); |
| } |
| |
| // If we want, we can templatize thaw by kind, and have three different entries |
| __ mv(c_rarg1, kind); |
| |
| __ call_VM_leaf(Continuation::thaw_entry(), xthread, c_rarg1); |
| __ mv(t1, x10); // x10 is the sp of the yielding frame |
| |
| if (return_barrier) { |
| // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK) |
| __ ld(x10, Address(sp, 1 * wordSize)); |
| __ fld(f10, Address(sp, 0 * wordSize)); |
| __ addi(sp, sp, 2 * wordSize); |
| } else { |
| __ mv(x10, zr); // return 0 (success) from doYield |
| } |
| |
| // we're now on the yield frame (which is in an address above us b/c sp has been pushed down) |
| __ mv(fp, t1); |
| __ subi(sp, t1, 2 * wordSize); // now pointing to fp spill |
| |
| if (return_barrier_exception) { |
| __ ld(c_rarg1, Address(fp, -1 * wordSize)); // return address |
| __ verify_oop(x10); |
| __ mv(x9, x10); // save return value contaning the exception oop in callee-saved x9 |
| |
| __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), xthread, c_rarg1); |
| |
| // see OptoRuntime::generate_exception_blob: x10 -- exception oop, x13 -- exception pc |
| |
| __ mv(x11, x10); // the exception handler |
| __ mv(x10, x9); // restore return value contaning the exception oop |
| __ verify_oop(x10); |
| |
| __ leave(); |
| __ mv(x13, ra); |
| __ jr(x11); // the exception handler |
| } else { |
| // We're "returning" into the topmost thawed frame; see Thaw::push_return_frame |
| __ leave(); |
| __ ret(); |
| } |
| |
| return start; |
| } |
| |
| address generate_cont_thaw() { |
| if (!Continuations::enabled()) return nullptr; |
| |
| StubGenStubId stub_id = StubGenStubId::cont_thaw_id; |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| generate_cont_thaw(Continuation::thaw_top); |
| return start; |
| } |
| |
| address generate_cont_returnBarrier() { |
| if (!Continuations::enabled()) return nullptr; |
| |
| // TODO: will probably need multiple return barriers depending on return type |
| StubGenStubId stub_id = StubGenStubId::cont_returnBarrier_id; |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| |
| generate_cont_thaw(Continuation::thaw_return_barrier); |
| |
| return start; |
| } |
| |
| address generate_cont_returnBarrier_exception() { |
| if (!Continuations::enabled()) return nullptr; |
| |
| StubGenStubId stub_id = StubGenStubId::cont_returnBarrierExc_id; |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| |
| generate_cont_thaw(Continuation::thaw_return_barrier_exception); |
| |
| return start; |
| } |
| |
| address generate_cont_preempt_stub() { |
| if (!Continuations::enabled()) return nullptr; |
| StubGenStubId stub_id = StubGenStubId::cont_preempt_id; |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| |
| __ reset_last_Java_frame(true); |
| |
| // Set sp to enterSpecial frame, i.e. remove all frames copied into the heap. |
| __ ld(sp, Address(xthread, JavaThread::cont_entry_offset())); |
| |
| Label preemption_cancelled; |
| __ lbu(t0, Address(xthread, JavaThread::preemption_cancelled_offset())); |
| __ bnez(t0, preemption_cancelled); |
| |
| // Remove enterSpecial frame from the stack and return to Continuation.run() to unmount. |
| SharedRuntime::continuation_enter_cleanup(_masm); |
| __ leave(); |
| __ ret(); |
| |
| // We acquired the monitor after freezing the frames so call thaw to continue execution. |
| __ bind(preemption_cancelled); |
| __ sb(zr, Address(xthread, JavaThread::preemption_cancelled_offset())); |
| __ la(fp, Address(sp, checked_cast<int32_t>(ContinuationEntry::size() + 2 * wordSize))); |
| __ la(t1, ExternalAddress(ContinuationEntry::thaw_call_pc_address())); |
| __ ld(t1, Address(t1)); |
| __ jr(t1); |
| |
| return start; |
| } |
| |
| #if COMPILER2_OR_JVMCI |
| |
| #undef __ |
| #define __ this-> |
| |
| class Sha2Generator : public MacroAssembler { |
| StubCodeGenerator* _cgen; |
| public: |
| Sha2Generator(MacroAssembler* masm, StubCodeGenerator* cgen) : MacroAssembler(masm->code()), _cgen(cgen) {} |
| address generate_sha256_implCompress(StubGenStubId stub_id) { |
| return generate_sha2_implCompress(Assembler::e32, stub_id); |
| } |
| address generate_sha512_implCompress(StubGenStubId stub_id) { |
| return generate_sha2_implCompress(Assembler::e64, stub_id); |
| } |
| private: |
| |
| void vleXX_v(Assembler::SEW vset_sew, VectorRegister vr, Register sr) { |
| if (vset_sew == Assembler::e32) __ vle32_v(vr, sr); |
| else __ vle64_v(vr, sr); |
| } |
| |
| void vseXX_v(Assembler::SEW vset_sew, VectorRegister vr, Register sr) { |
| if (vset_sew == Assembler::e32) __ vse32_v(vr, sr); |
| else __ vse64_v(vr, sr); |
| } |
| |
| // Overview of the logic in each "quad round". |
| // |
| // The code below repeats 16/20 times the logic implementing four rounds |
| // of the SHA-256/512 core loop as documented by NIST. 16/20 "quad rounds" |
| // to implementing the 64/80 single rounds. |
| // |
| // // Load four word (u32/64) constants (K[t+3], K[t+2], K[t+1], K[t+0]) |
| // // Output: |
| // // vTmp1 = {K[t+3], K[t+2], K[t+1], K[t+0]} |
| // vl1reXX.v vTmp1, ofs |
| // |
| // // Increment word constant address by stride (16/32 bytes, 4*4B/8B, 128b/256b) |
| // addi ofs, ofs, 16/32 |
| // |
| // // Add constants to message schedule words: |
| // // Input |
| // // vTmp1 = {K[t+3], K[t+2], K[t+1], K[t+0]} |
| // // vW0 = {W[t+3], W[t+2], W[t+1], W[t+0]}; // Vt0 = W[3:0]; |
| // // Output |
| // // vTmp0 = {W[t+3]+K[t+3], W[t+2]+K[t+2], W[t+1]+K[t+1], W[t+0]+K[t+0]} |
| // vadd.vv vTmp0, vTmp1, vW0 |
| // |
| // // 2 rounds of working variables updates. |
| // // vState1[t+4] <- vState1[t], vState0[t], vTmp0[t] |
| // // Input: |
| // // vState1 = {c[t],d[t],g[t],h[t]} " = vState1[t] " |
| // // vState0 = {a[t],b[t],e[t],f[t]} |
| // // vTmp0 = {W[t+3]+K[t+3], W[t+2]+K[t+2], W[t+1]+K[t+1], W[t+0]+K[t+0]} |
| // // Output: |
| // // vState1 = {f[t+2],e[t+2],b[t+2],a[t+2]} " = vState0[t+2] " |
| // // = {h[t+4],g[t+4],d[t+4],c[t+4]} " = vState1[t+4] " |
| // vsha2cl.vv vState1, vState0, vTmp0 |
| // |
| // // 2 rounds of working variables updates. |
| // // vState0[t+4] <- vState0[t], vState0[t+2], vTmp0[t] |
| // // Input |
| // // vState0 = {a[t],b[t],e[t],f[t]} " = vState0[t] " |
| // // = {h[t+2],g[t+2],d[t+2],c[t+2]} " = vState1[t+2] " |
| // // vState1 = {f[t+2],e[t+2],b[t+2],a[t+2]} " = vState0[t+2] " |
| // // vTmp0 = {W[t+3]+K[t+3], W[t+2]+K[t+2], W[t+1]+K[t+1], W[t+0]+K[t+0]} |
| // // Output: |
| // // vState0 = {f[t+4],e[t+4],b[t+4],a[t+4]} " = vState0[t+4] " |
| // vsha2ch.vv vState0, vState1, vTmp0 |
| // |
| // // Combine 2QW into 1QW |
| // // |
| // // To generate the next 4 words, "new_vW0"/"vTmp0" from vW0-vW3, vsha2ms needs |
| // // vW0[0..3], vW1[0], vW2[1..3], vW3[0, 2..3] |
| // // and it can only take 3 vectors as inputs. Hence we need to combine |
| // // vW1[0] and vW2[1..3] in a single vector. |
| // // |
| // // vmerge Vt4, Vt1, Vt2, V0 |
| // // Input |
| // // V0 = mask // first word from vW2, 1..3 words from vW1 |
| // // vW2 = {Wt-8, Wt-7, Wt-6, Wt-5} |
| // // vW1 = {Wt-12, Wt-11, Wt-10, Wt-9} |
| // // Output |
| // // Vt4 = {Wt-12, Wt-7, Wt-6, Wt-5} |
| // vmerge.vvm vTmp0, vW2, vW1, v0 |
| // |
| // // Generate next Four Message Schedule Words (hence allowing for 4 more rounds) |
| // // Input |
| // // vW0 = {W[t+ 3], W[t+ 2], W[t+ 1], W[t+ 0]} W[ 3: 0] |
| // // vW3 = {W[t+15], W[t+14], W[t+13], W[t+12]} W[15:12] |
| // // vTmp0 = {W[t+11], W[t+10], W[t+ 9], W[t+ 4]} W[11: 9,4] |
| // // Output (next four message schedule words) |
| // // vW0 = {W[t+19], W[t+18], W[t+17], W[t+16]} W[19:16] |
| // vsha2ms.vv vW0, vTmp0, vW3 |
| // |
| // BEFORE |
| // vW0 - vW3 hold the message schedule words (initially the block words) |
| // vW0 = W[ 3: 0] "oldest" |
| // vW1 = W[ 7: 4] |
| // vW2 = W[11: 8] |
| // vW3 = W[15:12] "newest" |
| // |
| // vt6 - vt7 hold the working state variables |
| // vState0 = {a[t],b[t],e[t],f[t]} // initially {H5,H4,H1,H0} |
| // vState1 = {c[t],d[t],g[t],h[t]} // initially {H7,H6,H3,H2} |
| // |
| // AFTER |
| // vW0 - vW3 hold the message schedule words (initially the block words) |
| // vW1 = W[ 7: 4] "oldest" |
| // vW2 = W[11: 8] |
| // vW3 = W[15:12] |
| // vW0 = W[19:16] "newest" |
| // |
| // vState0 and vState1 hold the working state variables |
| // vState0 = {a[t+4],b[t+4],e[t+4],f[t+4]} |
| // vState1 = {c[t+4],d[t+4],g[t+4],h[t+4]} |
| // |
| // The group of vectors vW0,vW1,vW2,vW3 is "rotated" by one in each quad-round, |
| // hence the uses of those vectors rotate in each round, and we get back to the |
| // initial configuration every 4 quad-rounds. We could avoid those changes at |
| // the cost of moving those vectors at the end of each quad-rounds. |
| void sha2_quad_round(Assembler::SEW vset_sew, VectorRegister rot1, VectorRegister rot2, VectorRegister rot3, VectorRegister rot4, |
| Register scalarconst, VectorRegister vtemp, VectorRegister vtemp2, VectorRegister v_abef, VectorRegister v_cdgh, |
| bool gen_words = true, bool step_const = true) { |
| __ vleXX_v(vset_sew, vtemp, scalarconst); |
| if (step_const) { |
| __ addi(scalarconst, scalarconst, vset_sew == Assembler::e32 ? 16 : 32); |
| } |
| __ vadd_vv(vtemp2, vtemp, rot1); |
| __ vsha2cl_vv(v_cdgh, v_abef, vtemp2); |
| __ vsha2ch_vv(v_abef, v_cdgh, vtemp2); |
| if (gen_words) { |
| __ vmerge_vvm(vtemp2, rot3, rot2); |
| __ vsha2ms_vv(rot1, vtemp2, rot4); |
| } |
| } |
| |
| // Arguments: |
| // |
| // Inputs: |
| // c_rarg0 - byte[] source+offset |
| // c_rarg1 - int[] SHA.state |
| // c_rarg2 - int offset |
| // c_rarg3 - int limit |
| // |
| address generate_sha2_implCompress(Assembler::SEW vset_sew, StubGenStubId stub_id) { |
| alignas(64) static const uint32_t round_consts_256[64] = { |
| 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, |
| 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, |
| 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, |
| 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, |
| 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, |
| 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, |
| 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, |
| 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, |
| 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, |
| 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, |
| 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, |
| 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, |
| 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, |
| 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, |
| 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, |
| 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, |
| }; |
| alignas(64) static const uint64_t round_consts_512[80] = { |
| 0x428a2f98d728ae22l, 0x7137449123ef65cdl, 0xb5c0fbcfec4d3b2fl, |
| 0xe9b5dba58189dbbcl, 0x3956c25bf348b538l, 0x59f111f1b605d019l, |
| 0x923f82a4af194f9bl, 0xab1c5ed5da6d8118l, 0xd807aa98a3030242l, |
| 0x12835b0145706fbel, 0x243185be4ee4b28cl, 0x550c7dc3d5ffb4e2l, |
| 0x72be5d74f27b896fl, 0x80deb1fe3b1696b1l, 0x9bdc06a725c71235l, |
| 0xc19bf174cf692694l, 0xe49b69c19ef14ad2l, 0xefbe4786384f25e3l, |
| 0x0fc19dc68b8cd5b5l, 0x240ca1cc77ac9c65l, 0x2de92c6f592b0275l, |
| 0x4a7484aa6ea6e483l, 0x5cb0a9dcbd41fbd4l, 0x76f988da831153b5l, |
| 0x983e5152ee66dfabl, 0xa831c66d2db43210l, 0xb00327c898fb213fl, |
| 0xbf597fc7beef0ee4l, 0xc6e00bf33da88fc2l, 0xd5a79147930aa725l, |
| 0x06ca6351e003826fl, 0x142929670a0e6e70l, 0x27b70a8546d22ffcl, |
| 0x2e1b21385c26c926l, 0x4d2c6dfc5ac42aedl, 0x53380d139d95b3dfl, |
| 0x650a73548baf63del, 0x766a0abb3c77b2a8l, 0x81c2c92e47edaee6l, |
| 0x92722c851482353bl, 0xa2bfe8a14cf10364l, 0xa81a664bbc423001l, |
| 0xc24b8b70d0f89791l, 0xc76c51a30654be30l, 0xd192e819d6ef5218l, |
| 0xd69906245565a910l, 0xf40e35855771202al, 0x106aa07032bbd1b8l, |
| 0x19a4c116b8d2d0c8l, 0x1e376c085141ab53l, 0x2748774cdf8eeb99l, |
| 0x34b0bcb5e19b48a8l, 0x391c0cb3c5c95a63l, 0x4ed8aa4ae3418acbl, |
| 0x5b9cca4f7763e373l, 0x682e6ff3d6b2b8a3l, 0x748f82ee5defb2fcl, |
| 0x78a5636f43172f60l, 0x84c87814a1f0ab72l, 0x8cc702081a6439ecl, |
| 0x90befffa23631e28l, 0xa4506cebde82bde9l, 0xbef9a3f7b2c67915l, |
| 0xc67178f2e372532bl, 0xca273eceea26619cl, 0xd186b8c721c0c207l, |
| 0xeada7dd6cde0eb1el, 0xf57d4f7fee6ed178l, 0x06f067aa72176fbal, |
| 0x0a637dc5a2c898a6l, 0x113f9804bef90dael, 0x1b710b35131c471bl, |
| 0x28db77f523047d84l, 0x32caab7b40c72493l, 0x3c9ebe0a15c9bebcl, |
| 0x431d67c49c100d4cl, 0x4cc5d4becb3e42b6l, 0x597f299cfc657e2al, |
| 0x5fcb6fab3ad6faecl, 0x6c44198c4a475817l |
| }; |
| const int const_add = vset_sew == Assembler::e32 ? 16 : 32; |
| |
| bool multi_block; |
| switch (stub_id) { |
| case sha256_implCompress_id: |
| assert (vset_sew == Assembler::e32, "wrong macroassembler for stub"); |
| multi_block = false; |
| break; |
| case sha256_implCompressMB_id: |
| assert (vset_sew == Assembler::e32, "wrong macroassembler for stub"); |
| multi_block = true; |
| break; |
| case sha512_implCompress_id: |
| assert (vset_sew == Assembler::e64, "wrong macroassembler for stub"); |
| multi_block = false; |
| break; |
| case sha512_implCompressMB_id: |
| assert (vset_sew == Assembler::e64, "wrong macroassembler for stub"); |
| multi_block = true; |
| break; |
| default: |
| ShouldNotReachHere(); |
| }; |
| __ align(CodeEntryAlignment); |
| StubCodeMark mark(_cgen, stub_id); |
| address start = __ pc(); |
| |
| Register buf = c_rarg0; |
| Register state = c_rarg1; |
| Register ofs = c_rarg2; |
| Register limit = c_rarg3; |
| Register consts = t2; // caller saved |
| Register state_c = x28; // caller saved |
| VectorRegister vindex = v2; |
| VectorRegister vW0 = v4; |
| VectorRegister vW1 = v6; |
| VectorRegister vW2 = v8; |
| VectorRegister vW3 = v10; |
| VectorRegister vState0 = v12; |
| VectorRegister vState1 = v14; |
| VectorRegister vHash0 = v16; |
| VectorRegister vHash1 = v18; |
| VectorRegister vTmp0 = v20; |
| VectorRegister vTmp1 = v22; |
| |
| Label multi_block_loop; |
| |
| __ enter(); |
| |
| address constant_table = vset_sew == Assembler::e32 ? (address)round_consts_256 : (address)round_consts_512; |
| la(consts, ExternalAddress(constant_table)); |
| |
| // Register use in this function: |
| // |
| // VECTORS |
| // vW0 - vW3 (512/1024-bits / 4*128/256 bits / 4*4*32/65 bits), hold the message |
| // schedule words (Wt). They start with the message block |
| // content (W0 to W15), then further words in the message |
| // schedule generated via vsha2ms from previous Wt. |
| // Initially: |
| // vW0 = W[ 3:0] = { W3, W2, W1, W0} |
| // vW1 = W[ 7:4] = { W7, W6, W5, W4} |
| // vW2 = W[ 11:8] = {W11, W10, W9, W8} |
| // vW3 = W[15:12] = {W15, W14, W13, W12} |
| // |
| // vState0 - vState1 hold the working state variables (a, b, ..., h) |
| // vState0 = {f[t],e[t],b[t],a[t]} |
| // vState1 = {h[t],g[t],d[t],c[t]} |
| // Initially: |
| // vState0 = {H5i-1, H4i-1, H1i-1 , H0i-1} |
| // vState1 = {H7i-i, H6i-1, H3i-1 , H2i-1} |
| // |
| // v0 = masks for vrgather/vmerge. Single value during the 16 rounds. |
| // |
| // vTmp0 = temporary, Wt+Kt |
| // vTmp1 = temporary, Kt |
| // |
| // vHash0/vHash1 = hold the initial values of the hash, byte-swapped. |
| // |
| // During most of the function the vector state is configured so that each |
| // vector is interpreted as containing four 32/64 bits (e32/e64) elements (128/256 bits). |
| |
| // vsha2ch/vsha2cl uses EGW of 4*SEW. |
| // SHA256 SEW = e32, EGW = 128-bits |
| // SHA512 SEW = e64, EGW = 256-bits |
| // |
| // VLEN is required to be at least 128. |
| // For the case of VLEN=128 and SHA512 we need LMUL=2 to work with 4*e64 (EGW = 256) |
| // |
| // m1: LMUL=1/2 |
| // ta: tail agnostic (don't care about those lanes) |
| // ma: mask agnostic (don't care about those lanes) |
| // x0 is not written, we known the number of vector elements. |
| |
| if (vset_sew == Assembler::e64 && MaxVectorSize == 16) { // SHA512 and VLEN = 128 |
| __ vsetivli(x0, 4, vset_sew, Assembler::m2, Assembler::ma, Assembler::ta); |
| } else { |
| __ vsetivli(x0, 4, vset_sew, Assembler::m1, Assembler::ma, Assembler::ta); |
| } |
| |
| int64_t indexes = vset_sew == Assembler::e32 ? 0x00041014ul : 0x00082028ul; |
| __ li(t0, indexes); |
| __ vmv_v_x(vindex, t0); |
| |
| // Step-over a,b, so we are pointing to c. |
| // const_add is equal to 4x state variable, div by 2 is thus 2, a,b |
| __ addi(state_c, state, const_add/2); |
| |
| // Use index-load to get {f,e,b,a},{h,g,d,c} |
| __ vluxei8_v(vState0, state, vindex); |
| __ vluxei8_v(vState1, state_c, vindex); |
| |
| __ bind(multi_block_loop); |
| |
| // Capture the initial H values in vHash0 and vHash1 to allow for computing |
| // the resulting H', since H' = H+{a',b',c',...,h'}. |
| __ vmv_v_v(vHash0, vState0); |
| __ vmv_v_v(vHash1, vState1); |
| |
| // Load the 512/1024-bits of the message block in vW0-vW3 and perform |
| // an endian swap on each 4/8 bytes element. |
| // |
| // If Zvkb is not implemented one can use vrgather |
| // with an index sequence to byte-swap. |
| // sequence = [3 2 1 0 7 6 5 4 11 10 9 8 15 14 13 12] |
| // <https://oeis.org/A004444> gives us "N ^ 3" as a nice formula to generate |
| // this sequence. 'vid' gives us the N. |
| __ vleXX_v(vset_sew, vW0, buf); |
| __ vrev8_v(vW0, vW0); |
| __ addi(buf, buf, const_add); |
| __ vleXX_v(vset_sew, vW1, buf); |
| __ vrev8_v(vW1, vW1); |
| __ addi(buf, buf, const_add); |
| __ vleXX_v(vset_sew, vW2, buf); |
| __ vrev8_v(vW2, vW2); |
| __ addi(buf, buf, const_add); |
| __ vleXX_v(vset_sew, vW3, buf); |
| __ vrev8_v(vW3, vW3); |
| __ addi(buf, buf, const_add); |
| |
| // Set v0 up for the vmerge that replaces the first word (idx==0) |
| __ vid_v(v0); |
| __ vmseq_vi(v0, v0, 0x0); // v0.mask[i] = (i == 0 ? 1 : 0) |
| |
| VectorRegister rotation_regs[] = {vW0, vW1, vW2, vW3}; |
| int rot_pos = 0; |
| // Quad-round #0 (+0, vW0->vW1->vW2->vW3) ... #11 (+3, vW3->vW0->vW1->vW2) |
| const int qr_end = vset_sew == Assembler::e32 ? 12 : 16; |
| for (int i = 0; i < qr_end; i++) { |
| sha2_quad_round(vset_sew, |
| rotation_regs[(rot_pos + 0) & 0x3], |
| rotation_regs[(rot_pos + 1) & 0x3], |
| rotation_regs[(rot_pos + 2) & 0x3], |
| rotation_regs[(rot_pos + 3) & 0x3], |
| consts, |
| vTmp1, vTmp0, vState0, vState1); |
| ++rot_pos; |
| } |
| // Quad-round #12 (+0, vW0->vW1->vW2->vW3) ... #15 (+3, vW3->vW0->vW1->vW2) |
| // Note that we stop generating new message schedule words (Wt, vW0-13) |
| // as we already generated all the words we end up consuming (i.e., W[63:60]). |
| const int qr_c_end = qr_end + 4; |
| for (int i = qr_end; i < qr_c_end; i++) { |
| sha2_quad_round(vset_sew, |
| rotation_regs[(rot_pos + 0) & 0x3], |
| rotation_regs[(rot_pos + 1) & 0x3], |
| rotation_regs[(rot_pos + 2) & 0x3], |
| rotation_regs[(rot_pos + 3) & 0x3], |
| consts, |
| vTmp1, vTmp0, vState0, vState1, false, i < (qr_c_end-1)); |
| ++rot_pos; |
| } |
| |
| //-------------------------------------------------------------------------------- |
| // Compute the updated hash value H' |
| // H' = H + {h',g',...,b',a'} |
| // = {h,g,...,b,a} + {h',g',...,b',a'} |
| // = {h+h',g+g',...,b+b',a+a'} |
| |
| // H' = H+{a',b',c',...,h'} |
| __ vadd_vv(vState0, vHash0, vState0); |
| __ vadd_vv(vState1, vHash1, vState1); |
| |
| if (multi_block) { |
| int total_adds = vset_sew == Assembler::e32 ? 240 : 608; |
| __ subi(consts, consts, total_adds); |
| __ addi(ofs, ofs, vset_sew == Assembler::e32 ? 64 : 128); |
| __ ble(ofs, limit, multi_block_loop); |
| __ mv(c_rarg0, ofs); // return ofs |
| } |
| |
| // Store H[0..8] = {a,b,c,d,e,f,g,h} from |
| // vState0 = {f,e,b,a} |
| // vState1 = {h,g,d,c} |
| __ vsuxei8_v(vState0, state, vindex); |
| __ vsuxei8_v(vState1, state_c, vindex); |
| |
| __ leave(); |
| __ ret(); |
| |
| return start; |
| } |
| }; |
| |
| #undef __ |
| #define __ _masm-> |
| |
| // Set of L registers that correspond to a contiguous memory area. |
| // Each 64-bit register typically corresponds to 2 32-bit integers. |
| template <uint L> |
| class RegCache { |
| private: |
| MacroAssembler *_masm; |
| Register _regs[L]; |
| |
| public: |
| RegCache(MacroAssembler *masm, RegSet rs): _masm(masm) { |
| assert(rs.size() == L, "%u registers are used to cache %u 4-byte data", rs.size(), 2 * L); |
| auto it = rs.begin(); |
| for (auto &r: _regs) { |
| r = *it; |
| ++it; |
| } |
| } |
| |
| // generate load for the i'th register |
| void gen_load(uint i, Register base) { |
| assert(i < L, "invalid i: %u", i); |
| __ ld(_regs[i], Address(base, 8 * i)); |
| } |
| |
| // add i'th 32-bit integer to dest |
| void add_u32(const Register dest, uint i, const Register rtmp = t0) { |
| assert(i < 2 * L, "invalid i: %u", i); |
| |
| if (is_even(i)) { |
| // Use the bottom 32 bits. No need to mask off the top 32 bits |
| // as addw will do the right thing. |
| __ addw(dest, dest, _regs[i / 2]); |
| } else { |
| // Use the top 32 bits by right-shifting them. |
| __ srli(rtmp, _regs[i / 2], 32); |
| __ addw(dest, dest, rtmp); |
| } |
| } |
| }; |
| |
| typedef RegCache<8> BufRegCache; |
| |
| // a += value + x + ac; |
| // a = Integer.rotateLeft(a, s) + b; |
| void m5_FF_GG_HH_II_epilogue(BufRegCache& reg_cache, |
| Register a, Register b, Register c, Register d, |
| int k, int s, int t, |
| Register value) { |
| // a += ac |
| __ addw(a, a, t, t1); |
| |
| // a += x; |
| reg_cache.add_u32(a, k); |
| // a += value; |
| __ addw(a, a, value); |
| |
| // a = Integer.rotateLeft(a, s) + b; |
| __ rolw(a, a, s); |
| __ addw(a, a, b); |
| } |
| |
| // a += ((b & c) | ((~b) & d)) + x + ac; |
| // a = Integer.rotateLeft(a, s) + b; |
| void md5_FF(BufRegCache& reg_cache, |
| Register a, Register b, Register c, Register d, |
| int k, int s, int t, |
| Register rtmp1, Register rtmp2) { |
| // rtmp1 = b & c |
| __ andr(rtmp1, b, c); |
| |
| // rtmp2 = (~b) & d |
| __ andn(rtmp2, d, b); |
| |
| // rtmp1 = (b & c) | ((~b) & d) |
| __ orr(rtmp1, rtmp1, rtmp2); |
| |
| m5_FF_GG_HH_II_epilogue(reg_cache, a, b, c, d, k, s, t, rtmp1); |
| } |
| |
| // a += ((b & d) | (c & (~d))) + x + ac; |
| // a = Integer.rotateLeft(a, s) + b; |
| void md5_GG(BufRegCache& reg_cache, |
| Register a, Register b, Register c, Register d, |
| int k, int s, int t, |
| Register rtmp1, Register rtmp2) { |
| // rtmp1 = b & d |
| __ andr(rtmp1, b, d); |
| |
| // rtmp2 = c & (~d) |
| __ andn(rtmp2, c, d); |
| |
| // rtmp1 = (b & d) | (c & (~d)) |
| __ orr(rtmp1, rtmp1, rtmp2); |
| |
| m5_FF_GG_HH_II_epilogue(reg_cache, a, b, c, d, k, s, t, rtmp1); |
| } |
| |
| // a += ((b ^ c) ^ d) + x + ac; |
| // a = Integer.rotateLeft(a, s) + b; |
| void md5_HH(BufRegCache& reg_cache, |
| Register a, Register b, Register c, Register d, |
| int k, int s, int t, |
| Register rtmp1, Register rtmp2) { |
| // rtmp1 = (b ^ c) ^ d |
| __ xorr(rtmp2, b, c); |
| __ xorr(rtmp1, rtmp2, d); |
| |
| m5_FF_GG_HH_II_epilogue(reg_cache, a, b, c, d, k, s, t, rtmp1); |
| } |
| |
| // a += (c ^ (b | (~d))) + x + ac; |
| // a = Integer.rotateLeft(a, s) + b; |
| void md5_II(BufRegCache& reg_cache, |
| Register a, Register b, Register c, Register d, |
| int k, int s, int t, |
| Register rtmp1, Register rtmp2) { |
| // rtmp1 = c ^ (b | (~d)) |
| __ orn(rtmp2, b, d); |
| __ xorr(rtmp1, c, rtmp2); |
| |
| m5_FF_GG_HH_II_epilogue(reg_cache, a, b, c, d, k, s, t, rtmp1); |
| } |
| |
| // Arguments: |
| // |
| // Inputs: |
| // c_rarg0 - byte[] source+offset |
| // c_rarg1 - int[] SHA.state |
| // c_rarg2 - int offset (multi_block == True) |
| // c_rarg3 - int limit (multi_block == True) |
| // |
| // Registers: |
| // x0 zero (zero) |
| // x1 ra (return address) |
| // x2 sp (stack pointer) |
| // x3 gp (global pointer) |
| // x4 tp (thread pointer) |
| // x5 t0 (tmp register) |
| // x6 t1 (tmp register) |
| // x7 t2 state0 |
| // x8 f0/s0 (frame pointer) |
| // x9 s1 |
| // x10 a0 rtmp1 / c_rarg0 |
| // x11 a1 rtmp2 / c_rarg1 |
| // x12 a2 a / c_rarg2 |
| // x13 a3 b / c_rarg3 |
| // x14 a4 c |
| // x15 a5 d |
| // x16 a6 buf |
| // x17 a7 state |
| // x18 s2 ofs [saved-reg] (multi_block == True) |
| // x19 s3 limit [saved-reg] (multi_block == True) |
| // x20 s4 state1 [saved-reg] |
| // x21 s5 state2 [saved-reg] |
| // x22 s6 state3 [saved-reg] |
| // x23 s7 |
| // x24 s8 buf0 [saved-reg] |
| // x25 s9 buf1 [saved-reg] |
| // x26 s10 buf2 [saved-reg] |
| // x27 s11 buf3 [saved-reg] |
| // x28 t3 buf4 |
| // x29 t4 buf5 |
| // x30 t5 buf6 |
| // x31 t6 buf7 |
| address generate_md5_implCompress(StubGenStubId stub_id) { |
| __ align(CodeEntryAlignment); |
| bool multi_block; |
| switch (stub_id) { |
| case md5_implCompress_id: |
| multi_block = false; |
| break; |
| case md5_implCompressMB_id: |
| multi_block = true; |
| break; |
| default: |
| ShouldNotReachHere(); |
| }; |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| |
| // rotation constants |
| const int S11 = 7; |
| const int S12 = 12; |
| const int S13 = 17; |
| const int S14 = 22; |
| const int S21 = 5; |
| const int S22 = 9; |
| const int S23 = 14; |
| const int S24 = 20; |
| const int S31 = 4; |
| const int S32 = 11; |
| const int S33 = 16; |
| const int S34 = 23; |
| const int S41 = 6; |
| const int S42 = 10; |
| const int S43 = 15; |
| const int S44 = 21; |
| |
| const int64_t mask32 = 0xffffffff; |
| |
| Register buf_arg = c_rarg0; // a0 |
| Register state_arg = c_rarg1; // a1 |
| Register ofs_arg = c_rarg2; // a2 |
| Register limit_arg = c_rarg3; // a3 |
| |
| // we'll copy the args to these registers to free up a0-a3 |
| // to use for other values manipulated by instructions |
| // that can be compressed |
| Register buf = x16; // a6 |
| Register state = x17; // a7 |
| Register ofs = x18; // s2 |
| Register limit = x19; // s3 |
| |
| // using x12->15 to allow compressed instructions |
| Register a = x12; // a2 |
| Register b = x13; // a3 |
| Register c = x14; // a4 |
| Register d = x15; // a5 |
| |
| Register state0 = x7; // t2 |
| Register state1 = x20; // s4 |
| Register state2 = x21; // s5 |
| Register state3 = x22; // s6 |
| |
| // using x10->x11 to allow compressed instructions |
| Register rtmp1 = x10; // a0 |
| Register rtmp2 = x11; // a1 |
| |
| RegSet reg_cache_saved_regs = RegSet::of(x24, x25, x26, x27); // s8, s9, s10, s11 |
| RegSet reg_cache_regs; |
| reg_cache_regs += reg_cache_saved_regs; |
| reg_cache_regs += RegSet::of(t3, t4, t5, t6); |
| BufRegCache reg_cache(_masm, reg_cache_regs); |
| |
| RegSet saved_regs; |
| if (multi_block) { |
| saved_regs += RegSet::of(ofs, limit); |
| } |
| saved_regs += RegSet::of(state1, state2, state3); |
| saved_regs += reg_cache_saved_regs; |
| |
| __ push_reg(saved_regs, sp); |
| |
| __ mv(buf, buf_arg); |
| __ mv(state, state_arg); |
| if (multi_block) { |
| __ mv(ofs, ofs_arg); |
| __ mv(limit, limit_arg); |
| } |
| |
| // to minimize the number of memory operations: |
| // read the 4 state 4-byte values in pairs, with a single ld, |
| // and split them into 2 registers. |
| // |
| // And, as the core algorithm of md5 works on 32-bits words, so |
| // in the following code, it does not care about the content of |
| // higher 32-bits in state[x]. Based on this observation, |
| // we can apply further optimization, which is to just ignore the |
| // higher 32-bits in state0/state2, rather than set the higher |
| // 32-bits of state0/state2 to zero explicitly with extra instructions. |
| __ ld(state0, Address(state)); |
| __ srli(state1, state0, 32); |
| __ ld(state2, Address(state, 8)); |
| __ srli(state3, state2, 32); |
| |
| Label md5_loop; |
| __ BIND(md5_loop); |
| |
| __ mv(a, state0); |
| __ mv(b, state1); |
| __ mv(c, state2); |
| __ mv(d, state3); |
| |
| // Round 1 |
| reg_cache.gen_load(0, buf); |
| md5_FF(reg_cache, a, b, c, d, 0, S11, 0xd76aa478, rtmp1, rtmp2); |
| md5_FF(reg_cache, d, a, b, c, 1, S12, 0xe8c7b756, rtmp1, rtmp2); |
| reg_cache.gen_load(1, buf); |
| md5_FF(reg_cache, c, d, a, b, 2, S13, 0x242070db, rtmp1, rtmp2); |
| md5_FF(reg_cache, b, c, d, a, 3, S14, 0xc1bdceee, rtmp1, rtmp2); |
| reg_cache.gen_load(2, buf); |
| md5_FF(reg_cache, a, b, c, d, 4, S11, 0xf57c0faf, rtmp1, rtmp2); |
| md5_FF(reg_cache, d, a, b, c, 5, S12, 0x4787c62a, rtmp1, rtmp2); |
| reg_cache.gen_load(3, buf); |
| md5_FF(reg_cache, c, d, a, b, 6, S13, 0xa8304613, rtmp1, rtmp2); |
| md5_FF(reg_cache, b, c, d, a, 7, S14, 0xfd469501, rtmp1, rtmp2); |
| reg_cache.gen_load(4, buf); |
| md5_FF(reg_cache, a, b, c, d, 8, S11, 0x698098d8, rtmp1, rtmp2); |
| md5_FF(reg_cache, d, a, b, c, 9, S12, 0x8b44f7af, rtmp1, rtmp2); |
| reg_cache.gen_load(5, buf); |
| md5_FF(reg_cache, c, d, a, b, 10, S13, 0xffff5bb1, rtmp1, rtmp2); |
| md5_FF(reg_cache, b, c, d, a, 11, S14, 0x895cd7be, rtmp1, rtmp2); |
| reg_cache.gen_load(6, buf); |
| md5_FF(reg_cache, a, b, c, d, 12, S11, 0x6b901122, rtmp1, rtmp2); |
| md5_FF(reg_cache, d, a, b, c, 13, S12, 0xfd987193, rtmp1, rtmp2); |
| reg_cache.gen_load(7, buf); |
| md5_FF(reg_cache, c, d, a, b, 14, S13, 0xa679438e, rtmp1, rtmp2); |
| md5_FF(reg_cache, b, c, d, a, 15, S14, 0x49b40821, rtmp1, rtmp2); |
| |
| // Round 2 |
| md5_GG(reg_cache, a, b, c, d, 1, S21, 0xf61e2562, rtmp1, rtmp2); |
| md5_GG(reg_cache, d, a, b, c, 6, S22, 0xc040b340, rtmp1, rtmp2); |
| md5_GG(reg_cache, c, d, a, b, 11, S23, 0x265e5a51, rtmp1, rtmp2); |
| md5_GG(reg_cache, b, c, d, a, 0, S24, 0xe9b6c7aa, rtmp1, rtmp2); |
| md5_GG(reg_cache, a, b, c, d, 5, S21, 0xd62f105d, rtmp1, rtmp2); |
| md5_GG(reg_cache, d, a, b, c, 10, S22, 0x02441453, rtmp1, rtmp2); |
| md5_GG(reg_cache, c, d, a, b, 15, S23, 0xd8a1e681, rtmp1, rtmp2); |
| md5_GG(reg_cache, b, c, d, a, 4, S24, 0xe7d3fbc8, rtmp1, rtmp2); |
| md5_GG(reg_cache, a, b, c, d, 9, S21, 0x21e1cde6, rtmp1, rtmp2); |
| md5_GG(reg_cache, d, a, b, c, 14, S22, 0xc33707d6, rtmp1, rtmp2); |
| md5_GG(reg_cache, c, d, a, b, 3, S23, 0xf4d50d87, rtmp1, rtmp2); |
| md5_GG(reg_cache, b, c, d, a, 8, S24, 0x455a14ed, rtmp1, rtmp2); |
| md5_GG(reg_cache, a, b, c, d, 13, S21, 0xa9e3e905, rtmp1, rtmp2); |
| md5_GG(reg_cache, d, a, b, c, 2, S22, 0xfcefa3f8, rtmp1, rtmp2); |
| md5_GG(reg_cache, c, d, a, b, 7, S23, 0x676f02d9, rtmp1, rtmp2); |
| md5_GG(reg_cache, b, c, d, a, 12, S24, 0x8d2a4c8a, rtmp1, rtmp2); |
| |
| // Round 3 |
| md5_HH(reg_cache, a, b, c, d, 5, S31, 0xfffa3942, rtmp1, rtmp2); |
| md5_HH(reg_cache, d, a, b, c, 8, S32, 0x8771f681, rtmp1, rtmp2); |
| md5_HH(reg_cache, c, d, a, b, 11, S33, 0x6d9d6122, rtmp1, rtmp2); |
| md5_HH(reg_cache, b, c, d, a, 14, S34, 0xfde5380c, rtmp1, rtmp2); |
| md5_HH(reg_cache, a, b, c, d, 1, S31, 0xa4beea44, rtmp1, rtmp2); |
| md5_HH(reg_cache, d, a, b, c, 4, S32, 0x4bdecfa9, rtmp1, rtmp2); |
| md5_HH(reg_cache, c, d, a, b, 7, S33, 0xf6bb4b60, rtmp1, rtmp2); |
| md5_HH(reg_cache, b, c, d, a, 10, S34, 0xbebfbc70, rtmp1, rtmp2); |
| md5_HH(reg_cache, a, b, c, d, 13, S31, 0x289b7ec6, rtmp1, rtmp2); |
| md5_HH(reg_cache, d, a, b, c, 0, S32, 0xeaa127fa, rtmp1, rtmp2); |
| md5_HH(reg_cache, c, d, a, b, 3, S33, 0xd4ef3085, rtmp1, rtmp2); |
| md5_HH(reg_cache, b, c, d, a, 6, S34, 0x04881d05, rtmp1, rtmp2); |
| md5_HH(reg_cache, a, b, c, d, 9, S31, 0xd9d4d039, rtmp1, rtmp2); |
| md5_HH(reg_cache, d, a, b, c, 12, S32, 0xe6db99e5, rtmp1, rtmp2); |
| md5_HH(reg_cache, c, d, a, b, 15, S33, 0x1fa27cf8, rtmp1, rtmp2); |
| md5_HH(reg_cache, b, c, d, a, 2, S34, 0xc4ac5665, rtmp1, rtmp2); |
| |
| // Round 4 |
| md5_II(reg_cache, a, b, c, d, 0, S41, 0xf4292244, rtmp1, rtmp2); |
| md5_II(reg_cache, d, a, b, c, 7, S42, 0x432aff97, rtmp1, rtmp2); |
| md5_II(reg_cache, c, d, a, b, 14, S43, 0xab9423a7, rtmp1, rtmp2); |
| md5_II(reg_cache, b, c, d, a, 5, S44, 0xfc93a039, rtmp1, rtmp2); |
| md5_II(reg_cache, a, b, c, d, 12, S41, 0x655b59c3, rtmp1, rtmp2); |
| md5_II(reg_cache, d, a, b, c, 3, S42, 0x8f0ccc92, rtmp1, rtmp2); |
| md5_II(reg_cache, c, d, a, b, 10, S43, 0xffeff47d, rtmp1, rtmp2); |
| md5_II(reg_cache, b, c, d, a, 1, S44, 0x85845dd1, rtmp1, rtmp2); |
| md5_II(reg_cache, a, b, c, d, 8, S41, 0x6fa87e4f, rtmp1, rtmp2); |
| md5_II(reg_cache, d, a, b, c, 15, S42, 0xfe2ce6e0, rtmp1, rtmp2); |
| md5_II(reg_cache, c, d, a, b, 6, S43, 0xa3014314, rtmp1, rtmp2); |
| md5_II(reg_cache, b, c, d, a, 13, S44, 0x4e0811a1, rtmp1, rtmp2); |
| md5_II(reg_cache, a, b, c, d, 4, S41, 0xf7537e82, rtmp1, rtmp2); |
| md5_II(reg_cache, d, a, b, c, 11, S42, 0xbd3af235, rtmp1, rtmp2); |
| md5_II(reg_cache, c, d, a, b, 2, S43, 0x2ad7d2bb, rtmp1, rtmp2); |
| md5_II(reg_cache, b, c, d, a, 9, S44, 0xeb86d391, rtmp1, rtmp2); |
| |
| __ addw(state0, state0, a); |
| __ addw(state1, state1, b); |
| __ addw(state2, state2, c); |
| __ addw(state3, state3, d); |
| |
| if (multi_block) { |
| __ addi(buf, buf, 64); |
| __ addi(ofs, ofs, 64); |
| // if (ofs <= limit) goto m5_loop |
| __ bge(limit, ofs, md5_loop); |
| __ mv(c_rarg0, ofs); // return ofs |
| } |
| |
| // to minimize the number of memory operations: |
| // write back the 4 state 4-byte values in pairs, with a single sd |
| __ mv(t0, mask32); |
| __ andr(state0, state0, t0); |
| __ slli(state1, state1, 32); |
| __ orr(state0, state0, state1); |
| __ sd(state0, Address(state)); |
| __ andr(state2, state2, t0); |
| __ slli(state3, state3, 32); |
| __ orr(state2, state2, state3); |
| __ sd(state2, Address(state, 8)); |
| |
| __ pop_reg(saved_regs, sp); |
| __ ret(); |
| |
| return (address) start; |
| } |
| |
| /** |
| * Perform the quarter round calculations on values contained within four vector registers. |
| * |
| * @param aVec the SIMD register containing only the "a" values |
| * @param bVec the SIMD register containing only the "b" values |
| * @param cVec the SIMD register containing only the "c" values |
| * @param dVec the SIMD register containing only the "d" values |
| * @param tmp_vr temporary vector register holds intermedia values. |
| */ |
| void chacha20_quarter_round(VectorRegister aVec, VectorRegister bVec, |
| VectorRegister cVec, VectorRegister dVec, VectorRegister tmp_vr) { |
| // a += b, d ^= a, d <<<= 16 |
| __ vadd_vv(aVec, aVec, bVec); |
| __ vxor_vv(dVec, dVec, aVec); |
| __ vrole32_vi(dVec, 16, tmp_vr); |
| |
| // c += d, b ^= c, b <<<= 12 |
| __ vadd_vv(cVec, cVec, dVec); |
| __ vxor_vv(bVec, bVec, cVec); |
| __ vrole32_vi(bVec, 12, tmp_vr); |
| |
| // a += b, d ^= a, d <<<= 8 |
| __ vadd_vv(aVec, aVec, bVec); |
| __ vxor_vv(dVec, dVec, aVec); |
| __ vrole32_vi(dVec, 8, tmp_vr); |
| |
| // c += d, b ^= c, b <<<= 7 |
| __ vadd_vv(cVec, cVec, dVec); |
| __ vxor_vv(bVec, bVec, cVec); |
| __ vrole32_vi(bVec, 7, tmp_vr); |
| } |
| |
| /** |
| * int com.sun.crypto.provider.ChaCha20Cipher.implChaCha20Block(int[] initState, byte[] result) |
| * |
| * Input arguments: |
| * c_rarg0 - state, the starting state |
| * c_rarg1 - key_stream, the array that will hold the result of the ChaCha20 block function |
| * |
| * Implementation Note: |
| * Parallelization is achieved by loading individual state elements into vectors for N blocks. |
| * N depends on single vector register length. |
| */ |
| address generate_chacha20Block() { |
| Label L_Rounds; |
| |
| __ align(CodeEntryAlignment); |
| StubGenStubId stub_id = StubGenStubId::chacha20Block_id; |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| __ enter(); |
| |
| const int states_len = 16; |
| const int step = 4; |
| const Register state = c_rarg0; |
| const Register key_stream = c_rarg1; |
| const Register tmp_addr = t0; |
| const Register length = t1; |
| |
| // Organize vector registers in an array that facilitates |
| // putting repetitive opcodes into loop structures below. |
| const VectorRegister work_vrs[16] = { |
| v0, v1, v2, v3, v4, v5, v6, v7, |
| v8, v9, v10, v11, v12, v13, v14, v15 |
| }; |
| const VectorRegister tmp_vr = v16; |
| const VectorRegister counter_vr = v17; |
| |
| { |
| // Put 16 here, as com.sun.crypto.providerChaCha20Cipher.KS_MAX_LEN is 1024 |
| // in java level. |
| __ vsetivli(length, 16, Assembler::e32, Assembler::m1); |
| } |
| |
| // Load from source state. |
| // Every element in source state is duplicated to all elements in the corresponding vector. |
| __ mv(tmp_addr, state); |
| for (int i = 0; i < states_len; i += 1) { |
| __ vlse32_v(work_vrs[i], tmp_addr, zr); |
| __ addi(tmp_addr, tmp_addr, step); |
| } |
| // Adjust counter for every individual block. |
| __ vid_v(counter_vr); |
| __ vadd_vv(work_vrs[12], work_vrs[12], counter_vr); |
| |
| // Perform 10 iterations of the 8 quarter round set |
| { |
| const Register loop = t2; // share t2 with other non-overlapping usages. |
| __ mv(loop, 10); |
| __ BIND(L_Rounds); |
| |
| chacha20_quarter_round(work_vrs[0], work_vrs[4], work_vrs[8], work_vrs[12], tmp_vr); |
| chacha20_quarter_round(work_vrs[1], work_vrs[5], work_vrs[9], work_vrs[13], tmp_vr); |
| chacha20_quarter_round(work_vrs[2], work_vrs[6], work_vrs[10], work_vrs[14], tmp_vr); |
| chacha20_quarter_round(work_vrs[3], work_vrs[7], work_vrs[11], work_vrs[15], tmp_vr); |
| |
| chacha20_quarter_round(work_vrs[0], work_vrs[5], work_vrs[10], work_vrs[15], tmp_vr); |
| chacha20_quarter_round(work_vrs[1], work_vrs[6], work_vrs[11], work_vrs[12], tmp_vr); |
| chacha20_quarter_round(work_vrs[2], work_vrs[7], work_vrs[8], work_vrs[13], tmp_vr); |
| chacha20_quarter_round(work_vrs[3], work_vrs[4], work_vrs[9], work_vrs[14], tmp_vr); |
| |
| __ subi(loop, loop, 1); |
| __ bnez(loop, L_Rounds); |
| } |
| |
| // Add the original state into the end working state. |
| // We do this by first duplicating every element in source state array to the corresponding |
| // vector, then adding it to the post-loop working state. |
| __ mv(tmp_addr, state); |
| for (int i = 0; i < states_len; i += 1) { |
| __ vlse32_v(tmp_vr, tmp_addr, zr); |
| __ addi(tmp_addr, tmp_addr, step); |
| __ vadd_vv(work_vrs[i], work_vrs[i], tmp_vr); |
| } |
| // Add the counter overlay onto work_vrs[12] at the end. |
| __ vadd_vv(work_vrs[12], work_vrs[12], counter_vr); |
| |
| // Store result to key stream. |
| { |
| const Register stride = t2; // share t2 with other non-overlapping usages. |
| // Every block occupies 64 bytes, so we use 64 as stride of the vector store. |
| __ mv(stride, 64); |
| for (int i = 0; i < states_len; i += 1) { |
| __ vsse32_v(work_vrs[i], key_stream, stride); |
| __ addi(key_stream, key_stream, step); |
| } |
| } |
| |
| // Return length of output key_stream |
| __ slli(c_rarg0, length, 6); |
| |
| __ leave(); |
| __ ret(); |
| |
| return (address) start; |
| } |
| |
| |
| // ------------------------ SHA-1 intrinsic ------------------------ |
| |
| // K't = |
| // 5a827999, 0 <= t <= 19 |
| // 6ed9eba1, 20 <= t <= 39 |
| // 8f1bbcdc, 40 <= t <= 59 |
| // ca62c1d6, 60 <= t <= 79 |
| void sha1_prepare_k(Register cur_k, int round) { |
| assert(round >= 0 && round < 80, "must be"); |
| |
| static const int64_t ks[] = {0x5a827999, 0x6ed9eba1, 0x8f1bbcdc, 0xca62c1d6}; |
| if ((round % 20) == 0) { |
| __ mv(cur_k, ks[round/20]); |
| } |
| } |
| |
| // W't = |
| // M't, 0 <= t <= 15 |
| // ROTL'1(W't-3 ^ W't-8 ^ W't-14 ^ W't-16), 16 <= t <= 79 |
| void sha1_prepare_w(Register cur_w, Register ws[], Register buf, int round) { |
| assert(round >= 0 && round < 80, "must be"); |
| |
| if (round < 16) { |
| // in the first 16 rounds, in ws[], every register contains 2 W't, e.g. |
| // in ws[0], high part contains W't-0, low part contains W't-1, |
| // in ws[1], high part contains W't-2, low part contains W't-3, |
| // ... |
| // in ws[7], high part contains W't-14, low part contains W't-15. |
| |
| if ((round % 2) == 0) { |
| __ ld(ws[round/2], Address(buf, (round/2) * 8)); |
| // reverse bytes, as SHA-1 is defined in big-endian. |
| __ revb(ws[round/2], ws[round/2]); |
| __ srli(cur_w, ws[round/2], 32); |
| } else { |
| __ mv(cur_w, ws[round/2]); |
| } |
| |
| return; |
| } |
| |
| if ((round % 2) == 0) { |
| int idx = 16; |
| // W't = ROTL'1(W't-3 ^ W't-8 ^ W't-14 ^ W't-16), 16 <= t <= 79 |
| __ srli(t1, ws[(idx-8)/2], 32); |
| __ xorr(t0, ws[(idx-3)/2], t1); |
| |
| __ srli(t1, ws[(idx-14)/2], 32); |
| __ srli(cur_w, ws[(idx-16)/2], 32); |
| __ xorr(cur_w, cur_w, t1); |
| |
| __ xorr(cur_w, cur_w, t0); |
| __ rolw(cur_w, cur_w, 1, t0); |
| |
| // copy the cur_w value to ws[8]. |
| // now, valid w't values are at: |
| // w0: ws[0]'s lower 32 bits |
| // w1 ~ w14: ws[1] ~ ws[7] |
| // w15: ws[8]'s higher 32 bits |
| __ slli(ws[idx/2], cur_w, 32); |
| |
| return; |
| } |
| |
| int idx = 17; |
| // W't = ROTL'1(W't-3 ^ W't-8 ^ W't-14 ^ W't-16), 16 <= t <= 79 |
| __ srli(t1, ws[(idx-3)/2], 32); |
| __ xorr(t0, t1, ws[(idx-8)/2]); |
| |
| __ xorr(cur_w, ws[(idx-16)/2], ws[(idx-14)/2]); |
| |
| __ xorr(cur_w, cur_w, t0); |
| __ rolw(cur_w, cur_w, 1, t0); |
| |
| // copy the cur_w value to ws[8] |
| __ zext(cur_w, cur_w, 32); |
| __ orr(ws[idx/2], ws[idx/2], cur_w); |
| |
| // shift the w't registers, so they start from ws[0] again. |
| // now, valid w't values are at: |
| // w0 ~ w15: ws[0] ~ ws[7] |
| Register ws_0 = ws[0]; |
| for (int i = 0; i < 16/2; i++) { |
| ws[i] = ws[i+1]; |
| } |
| ws[8] = ws_0; |
| } |
| |
| // f't(x, y, z) = |
| // Ch(x, y, z) = (x & y) ^ (~x & z) , 0 <= t <= 19 |
| // Parity(x, y, z) = x ^ y ^ z , 20 <= t <= 39 |
| // Maj(x, y, z) = (x & y) ^ (x & z) ^ (y & z) , 40 <= t <= 59 |
| // Parity(x, y, z) = x ^ y ^ z , 60 <= t <= 79 |
| void sha1_f(Register dst, Register x, Register y, Register z, int round) { |
| assert(round >= 0 && round < 80, "must be"); |
| assert_different_registers(dst, x, y, z, t0, t1); |
| |
| if (round < 20) { |
| // (x & y) ^ (~x & z) |
| __ andr(t0, x, y); |
| __ andn(dst, z, x); |
| __ xorr(dst, dst, t0); |
| } else if (round >= 40 && round < 60) { |
| // (x & y) ^ (x & z) ^ (y & z) |
| __ andr(t0, x, y); |
| __ andr(t1, x, z); |
| __ andr(dst, y, z); |
| __ xorr(dst, dst, t0); |
| __ xorr(dst, dst, t1); |
| } else { |
| // x ^ y ^ z |
| __ xorr(dst, x, y); |
| __ xorr(dst, dst, z); |
| } |
| } |
| |
| // T = ROTL'5(a) + f't(b, c, d) + e + K't + W't |
| // e = d |
| // d = c |
| // c = ROTL'30(b) |
| // b = a |
| // a = T |
| void sha1_process_round(Register a, Register b, Register c, Register d, Register e, |
| Register cur_k, Register cur_w, Register tmp, int round) { |
| assert(round >= 0 && round < 80, "must be"); |
| assert_different_registers(a, b, c, d, e, cur_w, cur_k, tmp, t0); |
| |
| // T = ROTL'5(a) + f't(b, c, d) + e + K't + W't |
| |
| // cur_w will be recalculated at the beginning of each round, |
| // so, we can reuse it as a temp register here. |
| Register tmp2 = cur_w; |
| |
| // reuse e as a temporary register, as we will mv new value into it later |
| Register tmp3 = e; |
| __ add(tmp2, cur_k, tmp2); |
| __ add(tmp3, tmp3, tmp2); |
| __ rolw(tmp2, a, 5, t0); |
| |
| sha1_f(tmp, b, c, d, round); |
| |
| __ add(tmp2, tmp2, tmp); |
| __ add(tmp2, tmp2, tmp3); |
| |
| // e = d |
| // d = c |
| // c = ROTL'30(b) |
| // b = a |
| // a = T |
| __ mv(e, d); |
| __ mv(d, c); |
| |
| __ rolw(c, b, 30); |
| __ mv(b, a); |
| __ mv(a, tmp2); |
| } |
| |
| // H(i)0 = a + H(i-1)0 |
| // H(i)1 = b + H(i-1)1 |
| // H(i)2 = c + H(i-1)2 |
| // H(i)3 = d + H(i-1)3 |
| // H(i)4 = e + H(i-1)4 |
| void sha1_calculate_im_hash(Register a, Register b, Register c, Register d, Register e, |
| Register prev_ab, Register prev_cd, Register prev_e) { |
| assert_different_registers(a, b, c, d, e, prev_ab, prev_cd, prev_e); |
| |
| __ add(a, a, prev_ab); |
| __ srli(prev_ab, prev_ab, 32); |
| __ add(b, b, prev_ab); |
| |
| __ add(c, c, prev_cd); |
| __ srli(prev_cd, prev_cd, 32); |
| __ add(d, d, prev_cd); |
| |
| __ add(e, e, prev_e); |
| } |
| |
| void sha1_preserve_prev_abcde(Register a, Register b, Register c, Register d, Register e, |
| Register prev_ab, Register prev_cd, Register prev_e) { |
| assert_different_registers(a, b, c, d, e, prev_ab, prev_cd, prev_e, t0); |
| |
| __ slli(t0, b, 32); |
| __ zext(prev_ab, a, 32); |
| __ orr(prev_ab, prev_ab, t0); |
| |
| __ slli(t0, d, 32); |
| __ zext(prev_cd, c, 32); |
| __ orr(prev_cd, prev_cd, t0); |
| |
| __ mv(prev_e, e); |
| } |
| |
| // Intrinsic for: |
| // void sun.security.provider.SHA.implCompress0(byte[] buf, int ofs) |
| // void sun.security.provider.DigestBase.implCompressMultiBlock0(byte[] b, int ofs, int limit) |
| // |
| // Arguments: |
| // |
| // Inputs: |
| // c_rarg0: byte[] src array + offset |
| // c_rarg1: int[] SHA.state |
| // - - - - - - below are only for implCompressMultiBlock0 - - - - - - |
| // c_rarg2: int offset |
| // c_rarg3: int limit |
| // |
| // Outputs: |
| // - - - - - - below are only for implCompressMultiBlock0 - - - - - - |
| // c_rarg0: int offset, when (multi_block == true) |
| // |
| address generate_sha1_implCompress(StubGenStubId stub_id) { |
| bool multi_block; |
| switch (stub_id) { |
| case sha1_implCompress_id: |
| multi_block = false; |
| break; |
| case sha1_implCompressMB_id: |
| multi_block = true; |
| break; |
| default: |
| ShouldNotReachHere(); |
| }; |
| __ align(CodeEntryAlignment); |
| StubCodeMark mark(this, stub_id); |
| |
| address start = __ pc(); |
| __ enter(); |
| |
| RegSet saved_regs = RegSet::range(x18, x27); |
| if (multi_block) { |
| // use x9 as src below. |
| saved_regs += RegSet::of(x9); |
| } |
| __ push_reg(saved_regs, sp); |
| |
| // c_rarg0 - c_rarg3: x10 - x13 |
| Register buf = c_rarg0; |
| Register state = c_rarg1; |
| Register offset = c_rarg2; |
| Register limit = c_rarg3; |
| // use src to contain the original start point of the array. |
| Register src = x9; |
| |
| if (multi_block) { |
| __ sub(limit, limit, offset); |
| __ add(limit, limit, buf); |
| __ sub(src, buf, offset); |
| } |
| |
| // [args-reg]: x14 - x17 |
| // [temp-reg]: x28 - x31 |
| // [saved-reg]: x18 - x27 |
| |
| // h0/1/2/3/4 |
| const Register a = x14, b = x15, c = x16, d = x17, e = x28; |
| // w0, w1, ... w15 |
| // put two adjecent w's in one register: |
| // one at high word part, another at low word part |
| // at different round (even or odd), w't value reside in different items in ws[]. |
| // w0 ~ w15, either reside in |
| // ws[0] ~ ws[7], where |
| // w0 at higher 32 bits of ws[0], |
| // w1 at lower 32 bits of ws[0], |
| // ... |
| // w14 at higher 32 bits of ws[7], |
| // w15 at lower 32 bits of ws[7]. |
| // or, reside in |
| // w0: ws[0]'s lower 32 bits |
| // w1 ~ w14: ws[1] ~ ws[7] |
| // w15: ws[8]'s higher 32 bits |
| Register ws[9] = {x29, x30, x31, x18, |
| x19, x20, x21, x22, |
| x23}; // auxiliary register for calculating w's value |
| // current k't's value |
| const Register cur_k = x24; |
| // current w't's value |
| const Register cur_w = x25; |
| // values of a, b, c, d, e in the previous round |
| const Register prev_ab = x26, prev_cd = x27; |
| const Register prev_e = offset; // reuse offset/c_rarg2 |
| |
| // load 5 words state into a, b, c, d, e. |
| // |
| // To minimize the number of memory operations, we apply following |
| // optimization: read the states (a/b/c/d) of 4-byte values in pairs, |
| // with a single ld, and split them into 2 registers. |
| // |
| // And, as the core algorithm of SHA-1 works on 32-bits words, so |
| // in the following code, it does not care about the content of |
| // higher 32-bits in a/b/c/d/e. Based on this observation, |
| // we can apply further optimization, which is to just ignore the |
| // higher 32-bits in a/c/e, rather than set the higher |
| // 32-bits of a/c/e to zero explicitly with extra instructions. |
| __ ld(a, Address(state, 0)); |
| __ srli(b, a, 32); |
| __ ld(c, Address(state, 8)); |
| __ srli(d, c, 32); |
| __ lw(e, Address(state, 16)); |
| |
| Label L_sha1_loop; |
| if (multi_block) { |
| __ BIND(L_sha1_loop); |
| } |
| |
| sha1_preserve_prev_abcde(a, b, c, d, e, prev_ab, prev_cd, prev_e); |
| |
| for (int round = 0; round < 80; round++) { |
| // prepare K't value |
| sha1_prepare_k(cur_k, round); |
| |
| // prepare W't value |
| sha1_prepare_w(cur_w, ws, buf, round); |
| |
| // one round process |
| sha1_process_round(a, b, c, d, e, cur_k, cur_w, t2, round); |
| } |
| |
| // compute the intermediate hash value |
| sha1_calculate_im_hash(a, b, c, d, e, prev_ab, prev_cd, prev_e); |
| |
| if (multi_block) { |
| int64_t block_bytes = 16 * 4; |
| __ addi(buf, buf, block_bytes); |
| |
| __ bge(limit, buf, L_sha1_loop, true); |
| } |
| |
| // store back the state. |
| __ zext(a, a, 32); |
| __ slli(b, b, 32); |
| __ orr(a, a, b); |
| __ sd(a, Address(state, 0)); |
| __ zext(c, c, 32); |
| __ slli(d, d, 32); |
| __ orr(c, c, d); |
| __ sd(c, Address(state, 8)); |
| __ sw(e, Address(state, 16)); |
| |
| // return offset |
| if (multi_block) { |
| __ sub(c_rarg0, buf, src); |
| } |
| |
| __ pop_reg(saved_regs, sp); |
| |
| __ leave(); |
| __ ret(); |
| |
| return (address) start; |
| } |
| |
| /** |
| * vector registers: |
| * input VectorRegister's: intputV1-V3, for m2 they could be v2, v4, v6, for m1 they could be v1, v2, v3 |
| * index VectorRegister's: idxV1-V4, for m2 they could be v8, v10, v12, v14, for m1 they could be v4, v5, v6, v7 |
| * output VectorRegister's: outputV1-V4, for m2 they could be v16, v18, v20, v22, for m1 they could be v8, v9, v10, v11 |
| * |
| * NOTE: each field will occupy a vector register group |
| */ |
| void base64_vector_encode_round(Register src, Register dst, Register codec, |
| Register size, Register stepSrc, Register stepDst, |
| VectorRegister inputV1, VectorRegister inputV2, VectorRegister inputV3, |
| VectorRegister idxV1, VectorRegister idxV2, VectorRegister idxV3, VectorRegister idxV4, |
| VectorRegister outputV1, VectorRegister outputV2, VectorRegister outputV3, VectorRegister outputV4, |
| Assembler::LMUL lmul) { |
| // set vector register type/len |
| __ vsetvli(x0, size, Assembler::e8, lmul); |
| |
| // segmented load src into v registers: mem(src) => vr(3) |
| __ vlseg3e8_v(inputV1, src); |
| |
| // src = src + register_group_len_bytes * 3 |
| __ add(src, src, stepSrc); |
| |
| // encoding |
| // 1. compute index into lookup table: vr(3) => vr(4) |
| __ vsrl_vi(idxV1, inputV1, 2); |
| |
| __ vsrl_vi(idxV2, inputV2, 2); |
| __ vsll_vi(inputV1, inputV1, 6); |
| __ vor_vv(idxV2, idxV2, inputV1); |
| __ vsrl_vi(idxV2, idxV2, 2); |
| |
| __ vsrl_vi(idxV3, inputV3, 4); |
| __ vsll_vi(inputV2, inputV2, 4); |
| __ vor_vv(idxV3, inputV2, idxV3); |
| __ vsrl_vi(idxV3, idxV3, 2); |
| |
| __ vsll_vi(idxV4, inputV3, 2); |
| __ vsrl_vi(idxV4, idxV4, 2); |
| |
| // 2. indexed load: vr(4) => vr(4) |
| __ vluxei8_v(outputV1, codec, idxV1); |
| __ vluxei8_v(outputV2, codec, idxV2); |
| __ vluxei8_v(outputV3, codec, idxV3); |
| __ vluxei8_v(outputV4, codec, idxV4); |
| |
| // segmented store encoded data in v registers back to dst: vr(4) => mem(dst) |
| __ vsseg4e8_v(outputV1, dst); |
| |
| // dst = dst + register_group_len_bytes * 4 |
| __ add(dst, dst, stepDst); |
| } |
| |
| /** |
| * void j.u.Base64.Encoder.encodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, boolean isURL) |
| * |
| * Input arguments: |
| * c_rarg0 - src, source array |
| * c_rarg1 - sp, src start offset |
| * c_rarg2 - sl, src end offset |
| * c_rarg3 - dst, dest array |
| * c_rarg4 - dp, dst start offset |
| * c_rarg5 - isURL, Base64 or URL character set |
| */ |
| address generate_base64_encodeBlock() { |
| alignas(64) static const char toBase64[64] = { |
| 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', |
| 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', |
| 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', |
| 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', |
| '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/' |
| }; |
| |
| alignas(64) static const char toBase64URL[64] = { |
| 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', |
| 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', |
| 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', |
| 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', |
| '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_' |
| }; |
| |
| __ align(CodeEntryAlignment); |
| StubGenStubId stub_id = StubGenStubId::base64_encodeBlock_id; |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| __ enter(); |
| |
| Register src = c_rarg0; |
| Register soff = c_rarg1; |
| Register send = c_rarg2; |
| Register dst = c_rarg3; |
| Register doff = c_rarg4; |
| Register isURL = c_rarg5; |
| |
| Register codec = c_rarg6; |
| Register length = c_rarg7; // total length of src data in bytes |
| |
| Label ProcessData, Exit; |
| |
| // length should be multiple of 3 |
| __ sub(length, send, soff); |
| // real src/dst to process data |
| __ add(src, src, soff); |
| __ add(dst, dst, doff); |
| |
| // load the codec base address |
| __ la(codec, ExternalAddress((address) toBase64)); |
| __ beqz(isURL, ProcessData); |
| __ la(codec, ExternalAddress((address) toBase64URL)); |
| __ BIND(ProcessData); |
| |
| // vector version |
| if (UseRVV) { |
| Label ProcessM2, ProcessM1, ProcessScalar; |
| |
| Register size = soff; |
| Register stepSrcM1 = send; |
| Register stepSrcM2 = doff; |
| Register stepDst = isURL; |
| |
| __ mv(size, MaxVectorSize * 2); |
| __ mv(stepSrcM1, MaxVectorSize * 3); |
| __ slli(stepSrcM2, stepSrcM1, 1); |
| __ mv(stepDst, MaxVectorSize * 2 * 4); |
| |
| __ blt(length, stepSrcM2, ProcessM1); |
| |
| __ BIND(ProcessM2); |
| base64_vector_encode_round(src, dst, codec, |
| size, stepSrcM2, stepDst, |
| v2, v4, v6, // inputs |
| v8, v10, v12, v14, // indexes |
| v16, v18, v20, v22, // outputs |
| Assembler::m2); |
| |
| __ sub(length, length, stepSrcM2); |
| __ bge(length, stepSrcM2, ProcessM2); |
| |
| __ BIND(ProcessM1); |
| __ blt(length, stepSrcM1, ProcessScalar); |
| |
| __ srli(size, size, 1); |
| __ srli(stepDst, stepDst, 1); |
| base64_vector_encode_round(src, dst, codec, |
| size, stepSrcM1, stepDst, |
| v1, v2, v3, // inputs |
| v4, v5, v6, v7, // indexes |
| v8, v9, v10, v11, // outputs |
| Assembler::m1); |
| __ sub(length, length, stepSrcM1); |
| |
| __ BIND(ProcessScalar); |
| } |
| |
| // scalar version |
| { |
| Register byte1 = soff, byte0 = send, byte2 = doff; |
| Register combined24Bits = isURL; |
| |
| __ beqz(length, Exit); |
| |
| Label ScalarLoop; |
| __ BIND(ScalarLoop); |
| { |
| // plain: [byte0[7:0] : byte1[7:0] : byte2[7:0]] => |
| // encoded: [byte0[7:2] : byte0[1:0]+byte1[7:4] : byte1[3:0]+byte2[7:6] : byte2[5:0]] |
| |
| // load 3 bytes src data |
| __ lbu(byte0, Address(src, 0)); |
| __ lbu(byte1, Address(src, 1)); |
| __ lbu(byte2, Address(src, 2)); |
| __ addi(src, src, 3); |
| |
| // construct 24 bits from 3 bytes |
| __ slliw(byte0, byte0, 16); |
| __ slliw(byte1, byte1, 8); |
| __ orr(combined24Bits, byte0, byte1); |
| __ orr(combined24Bits, combined24Bits, byte2); |
| |
| // get codec index and encode(ie. load from codec by index) |
| __ slliw(byte0, combined24Bits, 8); |
| __ srliw(byte0, byte0, 26); |
| __ add(byte0, codec, byte0); |
| __ lbu(byte0, byte0); |
| |
| __ slliw(byte1, combined24Bits, 14); |
| __ srliw(byte1, byte1, 26); |
| __ add(byte1, codec, byte1); |
| __ lbu(byte1, byte1); |
| |
| __ slliw(byte2, combined24Bits, 20); |
| __ srliw(byte2, byte2, 26); |
| __ add(byte2, codec, byte2); |
| __ lbu(byte2, byte2); |
| |
| __ andi(combined24Bits, combined24Bits, 0x3f); |
| __ add(combined24Bits, codec, combined24Bits); |
| __ lbu(combined24Bits, combined24Bits); |
| |
| // store 4 bytes encoded data |
| __ sb(byte0, Address(dst, 0)); |
| __ sb(byte1, Address(dst, 1)); |
| __ sb(byte2, Address(dst, 2)); |
| __ sb(combined24Bits, Address(dst, 3)); |
| |
| __ subi(length, length, 3); |
| __ addi(dst, dst, 4); |
| // loop back |
| __ bnez(length, ScalarLoop); |
| } |
| } |
| |
| __ BIND(Exit); |
| |
| __ leave(); |
| __ ret(); |
| |
| return (address) start; |
| } |
| |
| /** |
| * vector registers: |
| * input VectorRegister's: intputV1-V4, for m2 they could be v2, v4, v6, for m1 they could be v2, v4, v6, v8 |
| * index VectorRegister's: idxV1-V3, for m2 they could be v8, v10, v12, v14, for m1 they could be v10, v12, v14, v16 |
| * output VectorRegister's: outputV1-V4, for m2 they could be v16, v18, v20, v22, for m1 they could be v18, v20, v22 |
| * |
| * NOTE: each field will occupy a single vector register group |
| */ |
| void base64_vector_decode_round(Register src, Register dst, Register codec, |
| Register size, Register stepSrc, Register stepDst, Register failedIdx, |
| VectorRegister inputV1, VectorRegister inputV2, VectorRegister inputV3, VectorRegister inputV4, |
| VectorRegister idxV1, VectorRegister idxV2, VectorRegister idxV3, VectorRegister idxV4, |
| VectorRegister outputV1, VectorRegister outputV2, VectorRegister outputV3, |
| Assembler::LMUL lmul) { |
| // set vector register type/len |
| __ vsetvli(x0, size, Assembler::e8, lmul, Assembler::ma, Assembler::ta); |
| |
| // segmented load src into v registers: mem(src) => vr(4) |
| __ vlseg4e8_v(inputV1, src); |
| |
| // src = src + register_group_len_bytes * 4 |
| __ add(src, src, stepSrc); |
| |
| // decoding |
| // 1. indexed load: vr(4) => vr(4) |
| __ vluxei8_v(idxV1, codec, inputV1); |
| __ vluxei8_v(idxV2, codec, inputV2); |
| __ vluxei8_v(idxV3, codec, inputV3); |
| __ vluxei8_v(idxV4, codec, inputV4); |
| |
| // 2. check wrong data |
| __ vor_vv(outputV1, idxV1, idxV2); |
| __ vor_vv(outputV2, idxV3, idxV4); |
| __ vor_vv(outputV1, outputV1, outputV2); |
| __ vmseq_vi(v0, outputV1, -1); |
| __ vfirst_m(failedIdx, v0); |
| Label NoFailure, FailureAtIdx0; |
| // valid value can only be -1 when < 0 |
| __ bltz(failedIdx, NoFailure); |
| // when the first data (at index 0) fails, no need to process data anymore |
| __ beqz(failedIdx, FailureAtIdx0); |
| __ vsetvli(x0, failedIdx, Assembler::e8, lmul, Assembler::mu, Assembler::tu); |
| __ slli(stepDst, failedIdx, 1); |
| __ add(stepDst, failedIdx, stepDst); |
| __ BIND(NoFailure); |
| |
| // 3. compute the decoded data: vr(4) => vr(3) |
| __ vsll_vi(idxV1, idxV1, 2); |
| __ vsrl_vi(outputV1, idxV2, 4); |
| __ vor_vv(outputV1, outputV1, idxV1); |
| |
| __ vsll_vi(idxV2, idxV2, 4); |
| __ vsrl_vi(outputV2, idxV3, 2); |
| __ vor_vv(outputV2, outputV2, idxV2); |
| |
| __ vsll_vi(idxV3, idxV3, 6); |
| __ vor_vv(outputV3, idxV4, idxV3); |
| |
| // segmented store encoded data in v registers back to dst: vr(3) => mem(dst) |
| __ vsseg3e8_v(outputV1, dst); |
| |
| // dst = dst + register_group_len_bytes * 3 |
| __ add(dst, dst, stepDst); |
| __ BIND(FailureAtIdx0); |
| } |
| |
| /** |
| * int j.u.Base64.Decoder.decodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, boolean isURL, boolean isMIME) |
| * |
| * Input arguments: |
| * c_rarg0 - src, source array |
| * c_rarg1 - sp, src start offset |
| * c_rarg2 - sl, src end offset |
| * c_rarg3 - dst, dest array |
| * c_rarg4 - dp, dst start offset |
| * c_rarg5 - isURL, Base64 or URL character set |
| * c_rarg6 - isMIME, Decoding MIME block |
| */ |
| address generate_base64_decodeBlock() { |
| |
| static const uint8_t fromBase64[256] = { |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u, |
| 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u, |
| 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u, |
| 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| }; |
| |
| static const uint8_t fromBase64URL[256] = { |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, |
| 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u, |
| 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 63u, |
| 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u, |
| 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, |
| }; |
| |
| __ align(CodeEntryAlignment); |
| StubGenStubId stub_id = StubGenStubId::base64_decodeBlock_id; |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| __ enter(); |
| |
| Register src = c_rarg0; |
| Register soff = c_rarg1; |
| Register send = c_rarg2; |
| Register dst = c_rarg3; |
| Register doff = c_rarg4; |
| Register isURL = c_rarg5; |
| Register isMIME = c_rarg6; |
| |
| Register codec = c_rarg7; |
| Register dstBackup = t6; |
| Register length = t3; // total length of src data in bytes |
| |
| Label ProcessData, Exit; |
| Label ProcessScalar, ScalarLoop; |
| |
| // passed in length (send - soff) is guaranteed to be > 4, |
| // and in this intrinsic we only process data of length in multiple of 4, |
| // it's not guaranteed to be multiple of 4 by java level, so do it explicitly |
| __ sub(length, send, soff); |
| __ andi(length, length, -4); |
| // real src/dst to process data |
| __ add(src, src, soff); |
| __ add(dst, dst, doff); |
| // backup of dst, used to calculate the return value at exit |
| __ mv(dstBackup, dst); |
| |
| // load the codec base address |
| __ la(codec, ExternalAddress((address) fromBase64)); |
| __ beqz(isURL, ProcessData); |
| __ la(codec, ExternalAddress((address) fromBase64URL)); |
| __ BIND(ProcessData); |
| |
| // vector version |
| if (UseRVV) { |
| // for MIME case, it has a default length limit of 76 which could be |
| // different(smaller) from (send - soff), so in MIME case, we go through |
| // the scalar code path directly. |
| __ bnez(isMIME, ScalarLoop); |
| |
| Label ProcessM1, ProcessM2; |
| |
| Register failedIdx = soff; |
| Register stepSrcM1 = send; |
| Register stepSrcM2 = doff; |
| Register stepDst = isURL; |
| Register size = t4; |
| |
| __ mv(size, MaxVectorSize * 2); |
| __ mv(stepSrcM1, MaxVectorSize * 4); |
| __ slli(stepSrcM2, stepSrcM1, 1); |
| __ mv(stepDst, MaxVectorSize * 2 * 3); |
| |
| __ blt(length, stepSrcM2, ProcessM1); |
| |
| |
| // Assembler::m2 |
| __ BIND(ProcessM2); |
| base64_vector_decode_round(src, dst, codec, |
| size, stepSrcM2, stepDst, failedIdx, |
| v2, v4, v6, v8, // inputs |
| v10, v12, v14, v16, // indexes |
| v18, v20, v22, // outputs |
| Assembler::m2); |
| __ sub(length, length, stepSrcM2); |
| |
| // error check |
| // valid value of failedIdx can only be -1 when < 0 |
| __ bgez(failedIdx, Exit); |
| |
| __ bge(length, stepSrcM2, ProcessM2); |
| |
| |
| // Assembler::m1 |
| __ BIND(ProcessM1); |
| __ blt(length, stepSrcM1, ProcessScalar); |
| |
| __ srli(size, size, 1); |
| __ srli(stepDst, stepDst, 1); |
| base64_vector_decode_round(src, dst, codec, |
| size, stepSrcM1, stepDst, failedIdx, |
| v1, v2, v3, v4, // inputs |
| v5, v6, v7, v8, // indexes |
| v9, v10, v11, // outputs |
| Assembler::m1); |
| __ sub(length, length, stepSrcM1); |
| |
| // error check |
| // valid value of failedIdx can only be -1 when < 0 |
| __ bgez(failedIdx, Exit); |
| |
| __ BIND(ProcessScalar); |
| __ beqz(length, Exit); |
| } |
| |
| // scalar version |
| { |
| Register byte0 = soff, byte1 = send, byte2 = doff, byte3 = isURL; |
| Register combined32Bits = t4; |
| |
| // encoded: [byte0[5:0] : byte1[5:0] : byte2[5:0]] : byte3[5:0]] => |
| // plain: [byte0[5:0]+byte1[5:4] : byte1[3:0]+byte2[5:2] : byte2[1:0]+byte3[5:0]] |
| __ BIND(ScalarLoop); |
| |
| // load 4 bytes encoded src data |
| __ lbu(byte0, Address(src, 0)); |
| __ lbu(byte1, Address(src, 1)); |
| __ lbu(byte2, Address(src, 2)); |
| __ lbu(byte3, Address(src, 3)); |
| __ addi(src, src, 4); |
| |
| // get codec index and decode (ie. load from codec by index) |
| __ add(byte0, codec, byte0); |
| __ add(byte1, codec, byte1); |
| __ lb(byte0, Address(byte0, 0)); |
| __ lb(byte1, Address(byte1, 0)); |
| __ add(byte2, codec, byte2); |
| __ add(byte3, codec, byte3); |
| __ lb(byte2, Address(byte2, 0)); |
| __ lb(byte3, Address(byte3, 0)); |
| __ slliw(byte0, byte0, 18); |
| __ slliw(byte1, byte1, 12); |
| __ orr(byte0, byte0, byte1); |
| __ orr(byte0, byte0, byte3); |
| __ slliw(byte2, byte2, 6); |
| // For performance consideration, `combined32Bits` is constructed for 2 purposes at the same time, |
| // 1. error check below |
| // 2. decode below |
| __ orr(combined32Bits, byte0, byte2); |
| |
| // error check |
| __ bltz(combined32Bits, Exit); |
| |
| // store 3 bytes decoded data |
| __ sraiw(byte0, combined32Bits, 16); |
| __ sraiw(byte1, combined32Bits, 8); |
| __ sb(byte0, Address(dst, 0)); |
| __ sb(byte1, Address(dst, 1)); |
| __ sb(combined32Bits, Address(dst, 2)); |
| |
| __ subi(length, length, 4); |
| __ addi(dst, dst, 3); |
| // loop back |
| __ bnez(length, ScalarLoop); |
| } |
| |
| __ BIND(Exit); |
| __ sub(c_rarg0, dst, dstBackup); |
| |
| __ leave(); |
| __ ret(); |
| |
| return (address) start; |
| } |
| |
| void adler32_process_bytes(Register buff, Register s1, Register s2, VectorRegister vtable, |
| VectorRegister vzero, VectorRegister vbytes, VectorRegister vs1acc, VectorRegister vs2acc, |
| Register temp0, Register temp1, Register temp2, Register temp3, |
| VectorRegister vtemp1, VectorRegister vtemp2, int step, Assembler::LMUL lmul) { |
| |
| assert((lmul == Assembler::m4 && step == 64) || |
| (lmul == Assembler::m2 && step == 32) || |
| (lmul == Assembler::m1 && step == 16), |
| "LMUL should be aligned with step: m4 and 64, m2 and 32 or m1 and 16"); |
| // Below is function for calculating Adler32 checksum with 64-, 32- or 16-byte step. LMUL=m4, m2 or m1 is used. |
| // The results are in v12, v13, ..., v22, v23. Example below is for 64-byte step case. |
| // We use b1, b2, ..., b64 to denote the 64 bytes loaded in each iteration. |
| // In non-vectorized code, we update s1 and s2 as: |
| // s1 <- s1 + b1 |
| // s2 <- s2 + s1 |
| // s1 <- s1 + b2 |
| // s2 <- s2 + b1 |
| // ... |
| // s1 <- s1 + b64 |
| // s2 <- s2 + s1 |
| // Putting above assignments together, we have: |
| // s1_new = s1 + b1 + b2 + ... + b64 |
| // s2_new = s2 + (s1 + b1) + (s1 + b1 + b2) + ... + (s1 + b1 + b2 + ... + b64) = |
| // = s2 + s1 * 64 + (b1 * 64 + b2 * 63 + ... + b64 * 1) = |
| // = s2 + s1 * 64 + (b1, b2, ... b64) dot (64, 63, ... 1) |
| |
| __ mv(temp3, step); |
| // Load data |
| __ vsetvli(temp0, temp3, Assembler::e8, lmul); |
| __ vle8_v(vbytes, buff); |
| __ addi(buff, buff, step); |
| |
| // Upper bound reduction sum for s1_new: |
| // 0xFF * 64 = 0x3FC0, so: |
| // 1. Need to do vector-widening reduction sum |
| // 2. It is safe to perform sign-extension during vmv.x.s with 16-bits elements |
| __ vwredsumu_vs(vs1acc, vbytes, vzero); |
| // Multiplication for s2_new |
| __ vwmulu_vv(vs2acc, vtable, vbytes); |
| |
| // s2 = s2 + s1 * log2(step) |
| __ slli(temp1, s1, exact_log2(step)); |
| __ add(s2, s2, temp1); |
| |
| // Summing up calculated results for s2_new |
| if (MaxVectorSize > 16) { |
| __ vsetvli(temp0, temp3, Assembler::e16, lmul); |
| } else { |
| // Half of vector-widening multiplication result is in successor of vs2acc |
| // group for vlen == 16, in which case we need to double vector register |
| // group width in order to reduction sum all of them |
| Assembler::LMUL lmulx2 = (lmul == Assembler::m1) ? Assembler::m2 : |
| (lmul == Assembler::m2) ? Assembler::m4 : Assembler::m8; |
| __ vsetvli(temp0, temp3, Assembler::e16, lmulx2); |
| } |
| // Upper bound for reduction sum: |
| // 0xFF * (64 + 63 + ... + 2 + 1) = 0x817E0 max for whole register group, so: |
| // 1. Need to do vector-widening reduction sum |
| // 2. It is safe to perform sign-extension during vmv.x.s with 32-bits elements |
| __ vwredsumu_vs(vtemp1, vs2acc, vzero); |
| |
| // Extracting results for: |
| // s1_new |
| __ vmv_x_s(temp0, vs1acc); |
| __ add(s1, s1, temp0); |
| // s2_new |
| __ vsetvli(temp0, temp3, Assembler::e32, Assembler::m1); |
| __ vmv_x_s(temp1, vtemp1); |
| __ add(s2, s2, temp1); |
| } |
| |
| /*** |
| * int java.util.zip.Adler32.updateBytes(int adler, byte[] b, int off, int len) |
| * |
| * Arguments: |
| * |
| * Inputs: |
| * c_rarg0 - int adler |
| * c_rarg1 - byte* buff (b + off) |
| * c_rarg2 - int len |
| * |
| * Output: |
| * c_rarg0 - int adler result |
| */ |
| address generate_updateBytesAdler32() { |
| __ align(CodeEntryAlignment); |
| StubGenStubId stub_id = StubGenStubId::updateBytesAdler32_id; |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| |
| Label L_nmax, L_nmax_loop, L_nmax_loop_entry, L_by16, L_by16_loop, |
| L_by16_loop_unroll, L_by1_loop, L_do_mod, L_combine, L_by1; |
| |
| // Aliases |
| Register adler = c_rarg0; |
| Register s1 = c_rarg0; |
| Register s2 = c_rarg3; |
| Register buff = c_rarg1; |
| Register len = c_rarg2; |
| Register nmax = c_rarg4; |
| Register base = c_rarg5; |
| Register count = c_rarg6; |
| Register temp0 = t3; |
| Register temp1 = t4; |
| Register temp2 = t5; |
| Register temp3 = t6; |
| |
| VectorRegister vzero = v31; |
| VectorRegister vbytes = v8; // group: v8, v9, v10, v11 |
| VectorRegister vs1acc = v12; // group: v12, v13, v14, v15 |
| VectorRegister vs2acc = v16; // group: v16, v17, v18, v19, v20, v21, v22, v23 |
| VectorRegister vtable_64 = v24; // group: v24, v25, v26, v27 |
| VectorRegister vtable_32 = v4; // group: v4, v5 |
| VectorRegister vtable_16 = v30; |
| VectorRegister vtemp1 = v28; |
| VectorRegister vtemp2 = v29; |
| |
| // Max number of bytes we can process before having to take the mod |
| // 0x15B0 is 5552 in decimal, the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 |
| const uint64_t BASE = 0xfff1; |
| const uint64_t NMAX = 0x15B0; |
| |
| // Loops steps |
| int step_64 = 64; |
| int step_32 = 32; |
| int step_16 = 16; |
| int step_1 = 1; |
| |
| __ enter(); // Required for proper stackwalking of RuntimeStub frame |
| __ mv(temp1, 64); |
| __ vsetvli(temp0, temp1, Assembler::e8, Assembler::m4); |
| |
| // Generating accumulation coefficients for further calculations |
| // vtable_64: |
| __ vid_v(vtemp1); |
| __ vrsub_vx(vtable_64, vtemp1, temp1); |
| // vtable_64 group now contains { 0x40, 0x3f, 0x3e, ..., 0x3, 0x2, 0x1 } |
| |
| // vtable_32: |
| __ mv(temp1, 32); |
| __ vsetvli(temp0, temp1, Assembler::e8, Assembler::m2); |
| __ vid_v(vtemp1); |
| __ vrsub_vx(vtable_32, vtemp1, temp1); |
| // vtable_32 group now contains { 0x20, 0x1f, 0x1e, ..., 0x3, 0x2, 0x1 } |
| |
| __ vsetivli(temp0, 16, Assembler::e8, Assembler::m1); |
| // vtable_16: |
| __ mv(temp1, 16); |
| __ vid_v(vtemp1); |
| __ vrsub_vx(vtable_16, vtemp1, temp1); |
| // vtable_16 now contains { 0x10, 0xf, 0xe, ..., 0x3, 0x2, 0x1 } |
| |
| __ vmv_v_i(vzero, 0); |
| |
| __ mv(base, BASE); |
| __ mv(nmax, NMAX); |
| |
| // s1 is initialized to the lower 16 bits of adler |
| // s2 is initialized to the upper 16 bits of adler |
| __ srliw(s2, adler, 16); // s2 = ((adler >> 16) & 0xffff) |
| __ zext(s1, adler, 16); // s1 = (adler & 0xffff) |
| |
| // The pipelined loop needs at least 16 elements for 1 iteration |
| // It does check this, but it is more effective to skip to the cleanup loop |
| __ mv(temp0, step_16); |
| __ bgeu(len, temp0, L_nmax); |
| __ beqz(len, L_combine); |
| |
| // Jumping to L_by1_loop |
| __ subi(len, len, step_1); |
| __ j(L_by1_loop); |
| |
| __ bind(L_nmax); |
| __ sub(len, len, nmax); |
| __ subi(count, nmax, 16); |
| __ bltz(len, L_by16); |
| |
| // Align L_nmax loop by 64 |
| __ bind(L_nmax_loop_entry); |
| __ subi(count, count, 32); |
| |
| __ bind(L_nmax_loop); |
| adler32_process_bytes(buff, s1, s2, vtable_64, vzero, |
| vbytes, vs1acc, vs2acc, temp0, temp1, temp2, temp3, |
| vtemp1, vtemp2, step_64, Assembler::m4); |
| __ subi(count, count, step_64); |
| __ bgtz(count, L_nmax_loop); |
| |
| // There are three iterations left to do |
| adler32_process_bytes(buff, s1, s2, vtable_32, vzero, |
| vbytes, vs1acc, vs2acc, temp0, temp1, temp2, temp3, |
| vtemp1, vtemp2, step_32, Assembler::m2); |
| adler32_process_bytes(buff, s1, s2, vtable_16, vzero, |
| vbytes, vs1acc, vs2acc, temp0, temp1, temp2, temp3, |
| vtemp1, vtemp2, step_16, Assembler::m1); |
| |
| // s1 = s1 % BASE |
| __ remuw(s1, s1, base); |
| // s2 = s2 % BASE |
| __ remuw(s2, s2, base); |
| |
| __ sub(len, len, nmax); |
| __ subi(count, nmax, 16); |
| __ bgez(len, L_nmax_loop_entry); |
| |
| __ bind(L_by16); |
| __ add(len, len, count); |
| __ bltz(len, L_by1); |
| // Trying to unroll |
| __ mv(temp3, step_64); |
| __ blt(len, temp3, L_by16_loop); |
| |
| __ bind(L_by16_loop_unroll); |
| adler32_process_bytes(buff, s1, s2, vtable_64, vzero, |
| vbytes, vs1acc, vs2acc, temp0, temp1, temp2, temp3, |
| vtemp1, vtemp2, step_64, Assembler::m4); |
| __ subi(len, len, step_64); |
| // By now the temp3 should still be 64 |
| __ bge(len, temp3, L_by16_loop_unroll); |
| |
| __ bind(L_by16_loop); |
| adler32_process_bytes(buff, s1, s2, vtable_16, vzero, |
| vbytes, vs1acc, vs2acc, temp0, temp1, temp2, temp3, |
| vtemp1, vtemp2, step_16, Assembler::m1); |
| __ subi(len, len, step_16); |
| __ bgez(len, L_by16_loop); |
| |
| __ bind(L_by1); |
| __ addi(len, len, 15); |
| __ bltz(len, L_do_mod); |
| |
| __ bind(L_by1_loop); |
| __ lbu(temp0, Address(buff, 0)); |
| __ addi(buff, buff, step_1); |
| __ add(s1, temp0, s1); |
| __ add(s2, s2, s1); |
| __ subi(len, len, step_1); |
| __ bgez(len, L_by1_loop); |
| |
| __ bind(L_do_mod); |
| // s1 = s1 % BASE |
| __ remuw(s1, s1, base); |
| // s2 = s2 % BASE |
| __ remuw(s2, s2, base); |
| |
| // Combine lower bits and higher bits |
| // adler = s1 | (s2 << 16) |
| __ bind(L_combine); |
| __ slli(s2, s2, 16); |
| __ orr(s1, s1, s2); |
| |
| __ leave(); // Required for proper stackwalking of RuntimeStub frame |
| __ ret(); |
| |
| return start; |
| } |
| |
| #endif // COMPILER2_OR_JVMCI |
| |
| // x10 = input (float16) |
| // f10 = result (float) |
| // t1 = temporary register |
| address generate_float16ToFloat() { |
| __ align(CodeEntryAlignment); |
| StubGenStubId stub_id = StubGenStubId::hf2f_id; |
| StubCodeMark mark(this, stub_id); |
| address entry = __ pc(); |
| BLOCK_COMMENT("float16ToFloat:"); |
| |
| FloatRegister dst = f10; |
| Register src = x10; |
| Label NaN_SLOW; |
| |
| assert(VM_Version::supports_float16_float_conversion(), "must"); |
| |
| // On riscv, NaN needs a special process as fcvt does not work in that case. |
| // On riscv, Inf does not need a special process as fcvt can handle it correctly. |
| // but we consider to get the slow path to process NaN and Inf at the same time, |
| // as both of them are rare cases, and if we try to get the slow path to handle |
| // only NaN case it would sacrifise the performance for normal cases, |
| // i.e. non-NaN and non-Inf cases. |
| |
| // check whether it's a NaN or +/- Inf. |
| __ mv(t0, 0x7c00); |
| __ andr(t1, src, t0); |
| // jump to stub processing NaN and Inf cases. |
| __ beq(t0, t1, NaN_SLOW); |
| |
| // non-NaN or non-Inf cases, just use built-in instructions. |
| __ fmv_h_x(dst, src); |
| __ fcvt_s_h(dst, dst); |
| __ ret(); |
| |
| __ bind(NaN_SLOW); |
| // following instructions mainly focus on NaN, as riscv does not handle |
| // NaN well with fcvt, but the code also works for Inf at the same time. |
| |
| // construct a NaN in 32 bits from the NaN in 16 bits, |
| // we need the payloads of non-canonical NaNs to be preserved. |
| __ mv(t1, 0x7f800000); |
| // sign-bit was already set via sign-extension if necessary. |
| __ slli(t0, src, 13); |
| __ orr(t1, t0, t1); |
| __ fmv_w_x(dst, t1); |
| |
| __ ret(); |
| return entry; |
| } |
| |
| // f10 = input (float) |
| // x10 = result (float16) |
| // f11 = temporary float register |
| // t1 = temporary register |
| address generate_floatToFloat16() { |
| __ align(CodeEntryAlignment); |
| StubGenStubId stub_id = StubGenStubId::f2hf_id; |
| StubCodeMark mark(this, stub_id); |
| address entry = __ pc(); |
| BLOCK_COMMENT("floatToFloat16:"); |
| |
| Register dst = x10; |
| FloatRegister src = f10, ftmp = f11; |
| Label NaN_SLOW; |
| |
| assert(VM_Version::supports_float16_float_conversion(), "must"); |
| |
| // On riscv, NaN needs a special process as fcvt does not work in that case. |
| |
| // check whether it's a NaN. |
| // replace fclass with feq as performance optimization. |
| __ feq_s(t0, src, src); |
| // jump to stub processing NaN cases. |
| __ beqz(t0, NaN_SLOW); |
| |
| // non-NaN cases, just use built-in instructions. |
| __ fcvt_h_s(ftmp, src); |
| __ fmv_x_h(dst, ftmp); |
| __ ret(); |
| |
| __ bind(NaN_SLOW); |
| |
| __ float_to_float16_NaN(dst, src, t0, t1); |
| |
| __ ret(); |
| return entry; |
| } |
| |
| #ifdef COMPILER2 |
| |
| static const int64_t right_2_bits = right_n_bits(2); |
| static const int64_t right_3_bits = right_n_bits(3); |
| |
| // In sun.security.util.math.intpoly.IntegerPolynomial1305, integers |
| // are represented as long[5], with BITS_PER_LIMB = 26. |
| // Pack five 26-bit limbs into three 64-bit registers. |
| void poly1305_pack_26(Register dest0, Register dest1, Register dest2, Register src, Register tmp1, Register tmp2) { |
| assert_different_registers(dest0, dest1, dest2, src, tmp1, tmp2); |
| |
| // The goal is to have 128-bit value in dest2:dest1:dest0 |
| __ ld(dest0, Address(src, 0)); // 26 bits in dest0 |
| |
| __ ld(tmp1, Address(src, sizeof(jlong))); |
| __ slli(tmp1, tmp1, 26); |
| __ add(dest0, dest0, tmp1); // 52 bits in dest0 |
| |
| __ ld(tmp2, Address(src, 2 * sizeof(jlong))); |
| __ slli(tmp1, tmp2, 52); |
| __ add(dest0, dest0, tmp1); // dest0 is full |
| |
| __ srli(dest1, tmp2, 12); // 14-bit in dest1 |
| |
| __ ld(tmp1, Address(src, 3 * sizeof(jlong))); |
| __ slli(tmp1, tmp1, 14); |
| __ add(dest1, dest1, tmp1); // 40-bit in dest1 |
| |
| __ ld(tmp1, Address(src, 4 * sizeof(jlong))); |
| __ slli(tmp2, tmp1, 40); |
| __ add(dest1, dest1, tmp2); // dest1 is full |
| |
| if (dest2->is_valid()) { |
| __ srli(tmp1, tmp1, 24); |
| __ mv(dest2, tmp1); // 2 bits in dest2 |
| } else { |
| #ifdef ASSERT |
| Label OK; |
| __ srli(tmp1, tmp1, 24); |
| __ beq(zr, tmp1, OK); // 2 bits |
| __ stop("high bits of Poly1305 integer should be zero"); |
| __ should_not_reach_here(); |
| __ bind(OK); |
| #endif |
| } |
| } |
| |
| // As above, but return only a 128-bit integer, packed into two |
| // 64-bit registers. |
| void poly1305_pack_26(Register dest0, Register dest1, Register src, Register tmp1, Register tmp2) { |
| poly1305_pack_26(dest0, dest1, noreg, src, tmp1, tmp2); |
| } |
| |
| // U_2:U_1:U_0: += (U_2 >> 2) * 5 |
| void poly1305_reduce(Register U_2, Register U_1, Register U_0, Register tmp1, Register tmp2) { |
| assert_different_registers(U_2, U_1, U_0, tmp1, tmp2); |
| |
| // First, U_2:U_1:U_0 += (U_2 >> 2) |
| __ srli(tmp1, U_2, 2); |
| __ cad(U_0, U_0, tmp1, tmp2); // Add tmp1 to U_0 with carry output to tmp2 |
| __ andi(U_2, U_2, right_2_bits); // Clear U_2 except for the lowest two bits |
| __ cad(U_1, U_1, tmp2, tmp2); // Add carry to U_1 with carry output to tmp2 |
| __ add(U_2, U_2, tmp2); |
| |
| // Second, U_2:U_1:U_0 += (U_2 >> 2) << 2 |
| __ slli(tmp1, tmp1, 2); |
| __ cad(U_0, U_0, tmp1, tmp2); // Add tmp1 to U_0 with carry output to tmp2 |
| __ cad(U_1, U_1, tmp2, tmp2); // Add carry to U_1 with carry output to tmp2 |
| __ add(U_2, U_2, tmp2); |
| } |
| |
| // Poly1305, RFC 7539 |
| // void com.sun.crypto.provider.Poly1305.processMultipleBlocks(byte[] input, int offset, int length, long[] aLimbs, long[] rLimbs) |
| |
| // Arguments: |
| // c_rarg0: input_start -- where the input is stored |
| // c_rarg1: length |
| // c_rarg2: acc_start -- where the output will be stored |
| // c_rarg3: r_start -- where the randomly generated 128-bit key is stored |
| |
| // See https://loup-vaillant.fr/tutorials/poly1305-design for a |
| // description of the tricks used to simplify and accelerate this |
| // computation. |
| |
| address generate_poly1305_processBlocks() { |
| __ align(CodeEntryAlignment); |
| StubGenStubId stub_id = StubGenStubId::poly1305_processBlocks_id; |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| __ enter(); |
| Label here; |
| |
| RegSet saved_regs = RegSet::range(x18, x21); |
| RegSetIterator<Register> regs = (RegSet::range(x14, x31) - RegSet::range(x22, x27)).begin(); |
| __ push_reg(saved_regs, sp); |
| |
| // Arguments |
| const Register input_start = c_rarg0, length = c_rarg1, acc_start = c_rarg2, r_start = c_rarg3; |
| |
| // R_n is the 128-bit randomly-generated key, packed into two |
| // registers. The caller passes this key to us as long[5], with |
| // BITS_PER_LIMB = 26. |
| const Register R_0 = *regs, R_1 = *++regs; |
| poly1305_pack_26(R_0, R_1, r_start, t1, t2); |
| |
| // RR_n is (R_n >> 2) * 5 |
| const Register RR_0 = *++regs, RR_1 = *++regs; |
| __ srli(t1, R_0, 2); |
| __ shadd(RR_0, t1, t1, t2, 2); |
| __ srli(t1, R_1, 2); |
| __ shadd(RR_1, t1, t1, t2, 2); |
| |
| // U_n is the current checksum |
| const Register U_0 = *++regs, U_1 = *++regs, U_2 = *++regs; |
| poly1305_pack_26(U_0, U_1, U_2, acc_start, t1, t2); |
| |
| static constexpr int BLOCK_LENGTH = 16; |
| Label DONE, LOOP; |
| |
| __ mv(t1, BLOCK_LENGTH); |
| __ blt(length, t1, DONE); { |
| __ bind(LOOP); |
| |
| // S_n is to be the sum of U_n and the next block of data |
| const Register S_0 = *++regs, S_1 = *++regs, S_2 = *++regs; |
| __ ld(S_0, Address(input_start, 0)); |
| __ ld(S_1, Address(input_start, wordSize)); |
| |
| __ cad(S_0, S_0, U_0, t1); // Add U_0 to S_0 with carry output to t1 |
| __ cadc(S_1, S_1, U_1, t1); // Add U_1 with carry to S_1 with carry output to t1 |
| __ add(S_2, U_2, t1); |
| |
| __ addi(S_2, S_2, 1); |
| |
| const Register U_0HI = *++regs, U_1HI = *++regs; |
| |
| // NB: this logic depends on some of the special properties of |
| // Poly1305 keys. In particular, because we know that the top |
| // four bits of R_0 and R_1 are zero, we can add together |
| // partial products without any risk of needing to propagate a |
| // carry out. |
| __ wide_mul(U_0, U_0HI, S_0, R_0); |
| __ wide_madd(U_0, U_0HI, S_1, RR_1, t1, t2); |
| __ wide_madd(U_0, U_0HI, S_2, RR_0, t1, t2); |
| |
| __ wide_mul(U_1, U_1HI, S_0, R_1); |
| __ wide_madd(U_1, U_1HI, S_1, R_0, t1, t2); |
| __ wide_madd(U_1, U_1HI, S_2, RR_1, t1, t2); |
| |
| __ andi(U_2, R_0, right_2_bits); |
| __ mul(U_2, S_2, U_2); |
| |
| // Partial reduction mod 2**130 - 5 |
| __ cad(U_1, U_1, U_0HI, t1); // Add U_0HI to U_1 with carry output to t1 |
| __ adc(U_2, U_2, U_1HI, t1); |
| // Sum is now in U_2:U_1:U_0. |
| |
| // U_2:U_1:U_0: += (U_2 >> 2) * 5 |
| poly1305_reduce(U_2, U_1, U_0, t1, t2); |
| |
| __ subi(length, length, BLOCK_LENGTH); |
| __ addi(input_start, input_start, BLOCK_LENGTH); |
| __ mv(t1, BLOCK_LENGTH); |
| __ bge(length, t1, LOOP); |
| } |
| |
| // Further reduce modulo 2^130 - 5 |
| poly1305_reduce(U_2, U_1, U_0, t1, t2); |
| |
| // Unpack the sum into five 26-bit limbs and write to memory. |
| // First 26 bits is the first limb |
| __ slli(t1, U_0, 38); // Take lowest 26 bits |
| __ srli(t1, t1, 38); |
| __ sd(t1, Address(acc_start)); // First 26-bit limb |
| |
| // 27-52 bits of U_0 is the second limb |
| __ slli(t1, U_0, 12); // Take next 27-52 bits |
| __ srli(t1, t1, 38); |
| __ sd(t1, Address(acc_start, sizeof (jlong))); // Second 26-bit limb |
| |
| // Getting 53-64 bits of U_0 and 1-14 bits of U_1 in one register |
| __ srli(t1, U_0, 52); |
| __ slli(t2, U_1, 50); |
| __ srli(t2, t2, 38); |
| __ add(t1, t1, t2); |
| __ sd(t1, Address(acc_start, 2 * sizeof (jlong))); // Third 26-bit limb |
| |
| // Storing 15-40 bits of U_1 |
| __ slli(t1, U_1, 24); // Already used up 14 bits |
| __ srli(t1, t1, 38); // Clear all other bits from t1 |
| __ sd(t1, Address(acc_start, 3 * sizeof (jlong))); // Fourth 26-bit limb |
| |
| // Storing 41-64 bits of U_1 and first three bits from U_2 in one register |
| __ srli(t1, U_1, 40); |
| __ andi(t2, U_2, right_3_bits); |
| __ slli(t2, t2, 24); |
| __ add(t1, t1, t2); |
| __ sd(t1, Address(acc_start, 4 * sizeof (jlong))); // Fifth 26-bit limb |
| |
| __ bind(DONE); |
| __ pop_reg(saved_regs, sp); |
| __ leave(); // Required for proper stackwalking |
| __ ret(); |
| |
| return start; |
| } |
| |
| #endif // COMPILER2 |
| |
| /** |
| * Arguments: |
| * |
| * Inputs: |
| * c_rarg0 - int crc |
| * c_rarg1 - byte* buf |
| * c_rarg2 - int length |
| * |
| * Output: |
| * c_rarg0 - int crc result |
| */ |
| address generate_updateBytesCRC32() { |
| assert(UseCRC32Intrinsics, "what are we doing here?"); |
| |
| __ align(CodeEntryAlignment); |
| StubGenStubId stub_id = StubGenStubId::updateBytesCRC32_id; |
| StubCodeMark mark(this, stub_id); |
| |
| address start = __ pc(); |
| |
| // input parameters |
| const Register crc = c_rarg0; // crc |
| const Register buf = c_rarg1; // source java byte array address |
| const Register len = c_rarg2; // length |
| |
| BLOCK_COMMENT("Entry:"); |
| __ enter(); // required for proper stackwalking of RuntimeStub frame |
| |
| __ kernel_crc32(crc, buf, len, |
| c_rarg3, c_rarg4, c_rarg5, c_rarg6, // tmp's for tables |
| c_rarg7, t2, t3, t4, t5, t6); // misc tmps |
| |
| __ leave(); // required for proper stackwalking of RuntimeStub frame |
| __ ret(); |
| |
| return start; |
| } |
| |
| // exception handler for upcall stubs |
| address generate_upcall_stub_exception_handler() { |
| StubGenStubId stub_id = StubGenStubId::upcall_stub_exception_handler_id; |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| |
| // Native caller has no idea how to handle exceptions, |
| // so we just crash here. Up to callee to catch exceptions. |
| __ verify_oop(x10); // return a exception oop in a0 |
| __ rt_call(CAST_FROM_FN_PTR(address, UpcallLinker::handle_uncaught_exception)); |
| __ should_not_reach_here(); |
| |
| return start; |
| } |
| |
| // load Method* target of MethodHandle |
| // j_rarg0 = jobject receiver |
| // xmethod = Method* result |
| address generate_upcall_stub_load_target() { |
| |
| StubGenStubId stub_id = StubGenStubId::upcall_stub_load_target_id; |
| StubCodeMark mark(this, stub_id); |
| address start = __ pc(); |
| |
| __ resolve_global_jobject(j_rarg0, t0, t1); |
| // Load target method from receiver |
| __ load_heap_oop(xmethod, Address(j_rarg0, java_lang_invoke_MethodHandle::form_offset()), t0, t1); |
| __ load_heap_oop(xmethod, Address(xmethod, java_lang_invoke_LambdaForm::vmentry_offset()), t0, t1); |
| __ load_heap_oop(xmethod, Address(xmethod, java_lang_invoke_MemberName::method_offset()), t0, t1); |
| __ access_load_at(T_ADDRESS, IN_HEAP, xmethod, |
| Address(xmethod, java_lang_invoke_ResolvedMethodName::vmtarget_offset()), |
| noreg, noreg); |
| __ sd(xmethod, Address(xthread, JavaThread::callee_target_offset())); // just in case callee is deoptimized |
| |
| __ ret(); |
| |
| return start; |
| } |
| |
| #undef __ |
| |
| // Initialization |
| void generate_initial_stubs() { |
| // Generate initial stubs and initializes the entry points |
| |
| // entry points that exist in all platforms Note: This is code |
| // that could be shared among different platforms - however the |
| // benefit seems to be smaller than the disadvantage of having a |
| // much more complicated generator structure. See also comment in |
| // stubRoutines.hpp. |
| |
| StubRoutines::_forward_exception_entry = generate_forward_exception(); |
| |
| if (UnsafeMemoryAccess::_table == nullptr) { |
| UnsafeMemoryAccess::create_table(8 + 4); // 8 for copyMemory; 4 for setMemory |
| } |
| |
| StubRoutines::_call_stub_entry = |
| generate_call_stub(StubRoutines::_call_stub_return_address); |
| |
| // is referenced by megamorphic call |
| StubRoutines::_catch_exception_entry = generate_catch_exception(); |
| |
| if (UseCRC32Intrinsics) { |
| // set table address before stub generation which use it |
| StubRoutines::_crc_table_adr = (address)StubRoutines::riscv::_crc_table; |
| StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32(); |
| } |
| |
| if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) && |
| vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) { |
| StubRoutines::_hf2f = generate_float16ToFloat(); |
| StubRoutines::_f2hf = generate_floatToFloat16(); |
| } |
| } |
| |
| void generate_continuation_stubs() { |
| // Continuation stubs: |
| StubRoutines::_cont_thaw = generate_cont_thaw(); |
| StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); |
| StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception(); |
| StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub(); |
| } |
| |
| void generate_final_stubs() { |
| // support for verify_oop (must happen after universe_init) |
| if (VerifyOops) { |
| StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop(); |
| } |
| |
| // arraycopy stubs used by compilers |
| generate_arraycopy_stubs(); |
| |
| StubRoutines::_method_entry_barrier = generate_method_entry_barrier(); |
| |
| #ifdef COMPILER2 |
| if (UseSecondarySupersTable) { |
| StubRoutines::_lookup_secondary_supers_table_slow_path_stub = generate_lookup_secondary_supers_table_slow_path_stub(); |
| if (!InlineSecondarySupersTest) { |
| generate_lookup_secondary_supers_table_stub(); |
| } |
| } |
| #endif // COMPILER2 |
| |
| StubRoutines::_upcall_stub_exception_handler = generate_upcall_stub_exception_handler(); |
| StubRoutines::_upcall_stub_load_target = generate_upcall_stub_load_target(); |
| |
| StubRoutines::riscv::set_completed(); |
| } |
| |
| void generate_compiler_stubs() { |
| #ifdef COMPILER2 |
| if (UseMulAddIntrinsic) { |
| StubRoutines::_mulAdd = generate_mulAdd(); |
| } |
| |
| if (UseMultiplyToLenIntrinsic) { |
| StubRoutines::_multiplyToLen = generate_multiplyToLen(); |
| } |
| |
| if (UseSquareToLenIntrinsic) { |
| StubRoutines::_squareToLen = generate_squareToLen(); |
| } |
| |
| if (UseMontgomeryMultiplyIntrinsic) { |
| StubGenStubId stub_id = StubGenStubId::montgomeryMultiply_id; |
| StubCodeMark mark(this, stub_id); |
| MontgomeryMultiplyGenerator g(_masm, /*squaring*/false); |
| StubRoutines::_montgomeryMultiply = g.generate_multiply(); |
| } |
| |
| if (UseMontgomerySquareIntrinsic) { |
| StubGenStubId stub_id = StubGenStubId::montgomerySquare_id; |
| StubCodeMark mark(this, stub_id); |
| MontgomeryMultiplyGenerator g(_masm, /*squaring*/true); |
| StubRoutines::_montgomerySquare = g.generate_square(); |
| } |
| |
| if (UseAESIntrinsics) { |
| StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock(); |
| StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock(); |
| } |
| |
| if (UsePoly1305Intrinsics) { |
| StubRoutines::_poly1305_processBlocks = generate_poly1305_processBlocks(); |
| } |
| |
| if (UseRVV) { |
| StubRoutines::_bigIntegerLeftShiftWorker = generate_bigIntegerLeftShift(); |
| StubRoutines::_bigIntegerRightShiftWorker = generate_bigIntegerRightShift(); |
| } |
| |
| if (UseSHA256Intrinsics) { |
| Sha2Generator sha2(_masm, this); |
| StubRoutines::_sha256_implCompress = sha2.generate_sha256_implCompress(StubGenStubId::sha256_implCompress_id); |
| StubRoutines::_sha256_implCompressMB = sha2.generate_sha256_implCompress(StubGenStubId::sha256_implCompressMB_id); |
| } |
| |
| if (UseSHA512Intrinsics) { |
| Sha2Generator sha2(_masm, this); |
| StubRoutines::_sha512_implCompress = sha2.generate_sha512_implCompress(StubGenStubId::sha512_implCompress_id); |
| StubRoutines::_sha512_implCompressMB = sha2.generate_sha512_implCompress(StubGenStubId::sha512_implCompressMB_id); |
| } |
| |
| if (UseMD5Intrinsics) { |
| StubRoutines::_md5_implCompress = generate_md5_implCompress(StubGenStubId::md5_implCompress_id); |
| StubRoutines::_md5_implCompressMB = generate_md5_implCompress(StubGenStubId::md5_implCompressMB_id); |
| } |
| |
| if (UseChaCha20Intrinsics) { |
| StubRoutines::_chacha20Block = generate_chacha20Block(); |
| } |
| |
| if (UseSHA1Intrinsics) { |
| StubRoutines::_sha1_implCompress = generate_sha1_implCompress(StubGenStubId::sha1_implCompress_id); |
| StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(StubGenStubId::sha1_implCompressMB_id); |
| } |
| |
| if (UseBASE64Intrinsics) { |
| StubRoutines::_base64_encodeBlock = generate_base64_encodeBlock(); |
| StubRoutines::_base64_decodeBlock = generate_base64_decodeBlock(); |
| } |
| |
| if (UseAdler32Intrinsics) { |
| StubRoutines::_updateBytesAdler32 = generate_updateBytesAdler32(); |
| } |
| |
| generate_compare_long_strings(); |
| |
| generate_string_indexof_stubs(); |
| |
| #endif // COMPILER2 |
| } |
| |
| public: |
| StubGenerator(CodeBuffer* code, StubGenBlobId blob_id) : StubCodeGenerator(code, blob_id) { |
| switch(blob_id) { |
| case initial_id: |
| generate_initial_stubs(); |
| break; |
| case continuation_id: |
| generate_continuation_stubs(); |
| break; |
| case compiler_id: |
| generate_compiler_stubs(); |
| break; |
| case final_id: |
| generate_final_stubs(); |
| break; |
| default: |
| fatal("unexpected blob id: %d", blob_id); |
| break; |
| }; |
| } |
| }; // end class declaration |
| |
| void StubGenerator_generate(CodeBuffer* code, StubGenBlobId blob_id) { |
| StubGenerator g(code, blob_id); |
| } |
