/*
 * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
 * Copyright (c) 2014, 2025, Red Hat Inc. All rights reserved.
 * Copyright (c) 2020, 2025, Huawei Technologies Co., Ltd. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 *
 */

#include "asm/macroAssembler.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "compiler/oopMap.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
#include "interpreter/interpreter.hpp"
#include "memory/universe.hpp"
#include "nativeInst_riscv.hpp"
#include "oops/instanceOop.hpp"
#include "oops/method.hpp"
#include "oops/objArrayKlass.hpp"
#include "oops/oop.inline.hpp"
#include "prims/methodHandles.hpp"
#include "prims/upcallLinker.hpp"
#include "runtime/continuation.hpp"
#include "runtime/continuationEntry.inline.hpp"
#include "runtime/frame.inline.hpp"
#include "runtime/handles.inline.hpp"
#include "runtime/javaThread.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubCodeGenerator.hpp"
#include "runtime/stubRoutines.hpp"
#include "utilities/align.hpp"
#include "utilities/powerOfTwo.hpp"
#ifdef COMPILER2
#include "opto/runtime.hpp"
#endif

// Declaration and definition of StubGenerator (no .hpp file).
// For a more detailed description of the stub routine structure
// see the comment in stubRoutines.hpp

#undef __
#define __ _masm->

#ifdef PRODUCT
#define BLOCK_COMMENT(str) /* nothing */
#else
#define BLOCK_COMMENT(str) __ block_comment(str)
#endif

#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")

// Stub Code definitions

class StubGenerator: public StubCodeGenerator {
 private:

#ifdef PRODUCT
#define inc_counter_np(counter) ((void)0)
#else
  void inc_counter_np_(uint& counter) {
    __ incrementw(ExternalAddress((address)&counter));
  }
#define inc_counter_np(counter) \
  BLOCK_COMMENT("inc_counter " #counter); \
  inc_counter_np_(counter);
#endif

  // Call stubs are used to call Java from C
  //
  // Arguments:
  //    c_rarg0:   call wrapper address                   address
  //    c_rarg1:   result                                 address
  //    c_rarg2:   result type                            BasicType
  //    c_rarg3:   method                                 Method*
  //    c_rarg4:   (interpreter) entry point              address
  //    c_rarg5:   parameters                             intptr_t*
  //    c_rarg6:   parameter size (in words)              int
  //    c_rarg7:   thread                                 Thread*
  //
  // There is no return from the stub itself as any Java result
  // is written to result
  //
  // we save x1 (ra) as the return PC at the base of the frame and
  // link x8 (fp) below it as the frame pointer installing sp (x2)
  // into fp.
  //
  // we save x10-x17, which accounts for all the c arguments.
  //
  // TODO: strictly do we need to save them all? they are treated as
  // volatile by C so could we omit saving the ones we are going to
  // place in global registers (thread? method?) or those we only use
  // during setup of the Java call?
  //
  // we don't need to save x5 which C uses as an indirect result location
  // return register.
  //
  // we don't need to save x6-x7 and x28-x31 which both C and Java treat as
  // volatile
  //
  // we save x9, x18-x27, f8-f9, and f18-f27 which Java uses as temporary
  // registers and C expects to be callee-save
  //
  // so the stub frame looks like this when we enter Java code
  //
  //     [ return_from_Java     ] <--- sp
  //     [ argument word n      ]
  //      ...
  // -35 [ argument word 1      ]
  // -34 [ saved FRM in Floating-point Control and Status Register ] <--- sp_after_call
  // -33 [ saved f27            ]
  // -32 [ saved f26            ]
  // -31 [ saved f25            ]
  // -30 [ saved f24            ]
  // -29 [ saved f23            ]
  // -28 [ saved f22            ]
  // -27 [ saved f21            ]
  // -26 [ saved f20            ]
  // -25 [ saved f19            ]
  // -24 [ saved f18            ]
  // -23 [ saved f9             ]
  // -22 [ saved f8             ]
  // -21 [ saved x27            ]
  // -20 [ saved x26            ]
  // -19 [ saved x25            ]
  // -18 [ saved x24            ]
  // -17 [ saved x23            ]
  // -16 [ saved x22            ]
  // -15 [ saved x21            ]
  // -14 [ saved x20            ]
  // -13 [ saved x19            ]
  // -12 [ saved x18            ]
  // -11 [ saved x9             ]
  // -10 [ call wrapper   (x10) ]
  //  -9 [ result         (x11) ]
  //  -8 [ result type    (x12) ]
  //  -7 [ method         (x13) ]
  //  -6 [ entry point    (x14) ]
  //  -5 [ parameters     (x15) ]
  //  -4 [ parameter size (x16) ]
  //  -3 [ thread         (x17) ]
  //  -2 [ saved fp       (x8)  ]
  //  -1 [ saved ra       (x1)  ]
  //   0 [                      ] <--- fp == saved sp (x2)

  // Call stub stack layout word offsets from fp
  enum call_stub_layout {
    sp_after_call_off  = -34,

    frm_off            = sp_after_call_off,
    f27_off            = -33,
    f26_off            = -32,
    f25_off            = -31,
    f24_off            = -30,
    f23_off            = -29,
    f22_off            = -28,
    f21_off            = -27,
    f20_off            = -26,
    f19_off            = -25,
    f18_off            = -24,
    f9_off             = -23,
    f8_off             = -22,

    x27_off            = -21,
    x26_off            = -20,
    x25_off            = -19,
    x24_off            = -18,
    x23_off            = -17,
    x22_off            = -16,
    x21_off            = -15,
    x20_off            = -14,
    x19_off            = -13,
    x18_off            = -12,
    x9_off             = -11,

    call_wrapper_off   = -10,
    result_off         = -9,
    result_type_off    = -8,
    method_off         = -7,
    entry_point_off    = -6,
    parameters_off     = -5,
    parameter_size_off = -4,
    thread_off         = -3,
    fp_f               = -2,
    retaddr_off        = -1,
  };

  address generate_call_stub(address& return_address) {
    assert((int)frame::entry_frame_after_call_words == -(int)sp_after_call_off + 1 &&
           (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,
           "adjust this code");

    StubGenStubId stub_id = StubGenStubId::call_stub_id;
    StubCodeMark mark(this, stub_id);
    address start = __ pc();

    const Address sp_after_call (fp, sp_after_call_off  * wordSize);

    const Address frm_save      (fp, frm_off           * wordSize);
    const Address call_wrapper  (fp, call_wrapper_off   * wordSize);
    const Address result        (fp, result_off         * wordSize);
    const Address result_type   (fp, result_type_off    * wordSize);
    const Address method        (fp, method_off         * wordSize);
    const Address entry_point   (fp, entry_point_off    * wordSize);
    const Address parameters    (fp, parameters_off     * wordSize);
    const Address parameter_size(fp, parameter_size_off * wordSize);

    const Address thread        (fp, thread_off         * wordSize);

    const Address f27_save      (fp, f27_off            * wordSize);
    const Address f26_save      (fp, f26_off            * wordSize);
    const Address f25_save      (fp, f25_off            * wordSize);
    const Address f24_save      (fp, f24_off            * wordSize);
    const Address f23_save      (fp, f23_off            * wordSize);
    const Address f22_save      (fp, f22_off            * wordSize);
    const Address f21_save      (fp, f21_off            * wordSize);
    const Address f20_save      (fp, f20_off            * wordSize);
    const Address f19_save      (fp, f19_off            * wordSize);
    const Address f18_save      (fp, f18_off            * wordSize);
    const Address f9_save       (fp, f9_off             * wordSize);
    const Address f8_save       (fp, f8_off             * wordSize);

    const Address x27_save      (fp, x27_off            * wordSize);
    const Address x26_save      (fp, x26_off            * wordSize);
    const Address x25_save      (fp, x25_off            * wordSize);
    const Address x24_save      (fp, x24_off            * wordSize);
    const Address x23_save      (fp, x23_off            * wordSize);
    const Address x22_save      (fp, x22_off            * wordSize);
    const Address x21_save      (fp, x21_off            * wordSize);
    const Address x20_save      (fp, x20_off            * wordSize);
    const Address x19_save      (fp, x19_off            * wordSize);
    const Address x18_save      (fp, x18_off            * wordSize);

    const Address x9_save       (fp, x9_off             * wordSize);

    // stub code

    address riscv_entry = __ pc();

    // set up frame and move sp to end of save area
    __ enter();
    __ addi(sp, fp, sp_after_call_off * wordSize);

    // save register parameters and Java temporary/global registers
    // n.b. we save thread even though it gets installed in
    // xthread because we want to sanity check tp later
    __ sd(c_rarg7, thread);
    __ sw(c_rarg6, parameter_size);
    __ sd(c_rarg5, parameters);
    __ sd(c_rarg4, entry_point);
    __ sd(c_rarg3, method);
    __ sd(c_rarg2, result_type);
    __ sd(c_rarg1, result);
    __ sd(c_rarg0, call_wrapper);

    __ sd(x9, x9_save);

    __ sd(x18, x18_save);
    __ sd(x19, x19_save);
    __ sd(x20, x20_save);
    __ sd(x21, x21_save);
    __ sd(x22, x22_save);
    __ sd(x23, x23_save);
    __ sd(x24, x24_save);
    __ sd(x25, x25_save);
    __ sd(x26, x26_save);
    __ sd(x27, x27_save);

    __ fsd(f8,  f8_save);
    __ fsd(f9,  f9_save);
    __ fsd(f18, f18_save);
    __ fsd(f19, f19_save);
    __ fsd(f20, f20_save);
    __ fsd(f21, f21_save);
    __ fsd(f22, f22_save);
    __ fsd(f23, f23_save);
    __ fsd(f24, f24_save);
    __ fsd(f25, f25_save);
    __ fsd(f26, f26_save);
    __ fsd(f27, f27_save);

    __ frrm(t0);
    __ sd(t0, frm_save);
    // Set frm to the state we need. We do want Round to Nearest. We
    // don't want non-IEEE rounding modes.
    Label skip_fsrmi;
    guarantee(__ RoundingMode::rne == 0, "must be");
    __ beqz(t0, skip_fsrmi);
    __ fsrmi(__ RoundingMode::rne);
    __ bind(skip_fsrmi);

    // install Java thread in global register now we have saved
    // whatever value it held
    __ mv(xthread, c_rarg7);

    // And method
    __ mv(xmethod, c_rarg3);

    // set up the heapbase register
    __ reinit_heapbase();

#ifdef ASSERT
    // make sure we have no pending exceptions
    {
      Label L;
      __ ld(t0, Address(xthread, in_bytes(Thread::pending_exception_offset())));
      __ beqz(t0, L);
      __ stop("StubRoutines::call_stub: entered with pending exception");
      __ BIND(L);
    }
#endif
    // pass parameters if any
    __ mv(esp, sp);
    __ slli(t0, c_rarg6, LogBytesPerWord);
    __ sub(t0, sp, t0); // Move SP out of the way
    __ andi(sp, t0, -2 * wordSize);

    BLOCK_COMMENT("pass parameters if any");
    Label parameters_done;
    // parameter count is still in c_rarg6
    // and parameter pointer identifying param 1 is in c_rarg5
    __ beqz(c_rarg6, parameters_done);

    address loop = __ pc();
    __ ld(t0, Address(c_rarg5, 0));
    __ addi(c_rarg5, c_rarg5, wordSize);
    __ subi(c_rarg6, c_rarg6, 1);
    __ push_reg(t0);
    __ bgtz(c_rarg6, loop);

    __ BIND(parameters_done);

    // call Java entry -- passing methdoOop, and current sp
    //      xmethod: Method*
    //      x19_sender_sp: sender sp
    BLOCK_COMMENT("call Java function");
    __ mv(x19_sender_sp, sp);
    __ jalr(c_rarg4);

    // save current address for use by exception handling code

    return_address = __ pc();

    // store result depending on type (everything that is not
    // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
    // n.b. this assumes Java returns an integral result in x10
    // and a floating result in j_farg0
    __ ld(j_rarg2, result);
    Label is_long, is_float, is_double, exit;
    __ ld(j_rarg1, result_type);
    __ mv(t0, (u1)T_OBJECT);
    __ beq(j_rarg1, t0, is_long);
    __ mv(t0, (u1)T_LONG);
    __ beq(j_rarg1, t0, is_long);
    __ mv(t0, (u1)T_FLOAT);
    __ beq(j_rarg1, t0, is_float);
    __ mv(t0, (u1)T_DOUBLE);
    __ beq(j_rarg1, t0, is_double);

    // handle T_INT case
    __ sw(x10, Address(j_rarg2));

    __ BIND(exit);

    // pop parameters
    __ addi(esp, fp, sp_after_call_off * wordSize);

#ifdef ASSERT
    // verify that threads correspond
    {
      Label L, S;
      __ ld(t0, thread);
      __ bne(xthread, t0, S);
      __ get_thread(t0);
      __ beq(xthread, t0, L);
      __ BIND(S);
      __ stop("StubRoutines::call_stub: threads must correspond");
      __ BIND(L);
    }
#endif

    __ pop_cont_fastpath(xthread);

    // restore callee-save registers
    __ fld(f27, f27_save);
    __ fld(f26, f26_save);
    __ fld(f25, f25_save);
    __ fld(f24, f24_save);
    __ fld(f23, f23_save);
    __ fld(f22, f22_save);
    __ fld(f21, f21_save);
    __ fld(f20, f20_save);
    __ fld(f19, f19_save);
    __ fld(f18, f18_save);
    __ fld(f9,  f9_save);
    __ fld(f8,  f8_save);

    __ ld(x27, x27_save);
    __ ld(x26, x26_save);
    __ ld(x25, x25_save);
    __ ld(x24, x24_save);
    __ ld(x23, x23_save);
    __ ld(x22, x22_save);
    __ ld(x21, x21_save);
    __ ld(x20, x20_save);
    __ ld(x19, x19_save);
    __ ld(x18, x18_save);

    __ ld(x9, x9_save);

    // restore frm
    Label skip_fsrm;
    __ ld(t0, frm_save);
    __ frrm(t1);
    __ beq(t0, t1, skip_fsrm);
    __ fsrm(t0);
    __ bind(skip_fsrm);

    __ ld(c_rarg0, call_wrapper);
    __ ld(c_rarg1, result);
    __ ld(c_rarg2, result_type);
    __ ld(c_rarg3, method);
    __ ld(c_rarg4, entry_point);
    __ ld(c_rarg5, parameters);
    __ ld(c_rarg6, parameter_size);
    __ ld(c_rarg7, thread);

    // leave frame and return to caller
    __ leave();
    __ ret();

    // handle return types different from T_INT

    __ BIND(is_long);
    __ sd(x10, Address(j_rarg2, 0));
    __ j(exit);

    __ BIND(is_float);
    __ fsw(j_farg0, Address(j_rarg2, 0), t0);
    __ j(exit);

    __ BIND(is_double);
    __ fsd(j_farg0, Address(j_rarg2, 0), t0);
    __ j(exit);

    return start;
  }

  // Return point for a Java call if there's an exception thrown in
  // Java code.  The exception is caught and transformed into a
  // pending exception stored in JavaThread that can be tested from
  // within the VM.
  //
  // Note: Usually the parameters are removed by the callee. In case
  // of an exception crossing an activation frame boundary, that is
  // not the case if the callee is compiled code => need to setup the
  // sp.
  //
  // x10: exception oop

  address generate_catch_exception() {
    StubGenStubId stub_id = StubGenStubId::catch_exception_id;
    StubCodeMark mark(this, stub_id);
    address start = __ pc();

    // same as in generate_call_stub():
    const Address thread(fp, thread_off * wordSize);

#ifdef ASSERT
    // verify that threads correspond
    {
      Label L, S;
      __ ld(t0, thread);
      __ bne(xthread, t0, S);
      __ get_thread(t0);
      __ beq(xthread, t0, L);
      __ bind(S);
      __ stop("StubRoutines::catch_exception: threads must correspond");
      __ bind(L);
    }
#endif

    // set pending exception
    __ verify_oop(x10);

    __ sd(x10, Address(xthread, Thread::pending_exception_offset()));
    __ mv(t0, (address)__FILE__);
    __ sd(t0, Address(xthread, Thread::exception_file_offset()));
    __ mv(t0, (int)__LINE__);
    __ sw(t0, Address(xthread, Thread::exception_line_offset()));

    // complete return to VM
    assert(StubRoutines::_call_stub_return_address != nullptr,
           "_call_stub_return_address must have been generated before");
    __ j(RuntimeAddress(StubRoutines::_call_stub_return_address));

    return start;
  }

  // Continuation point for runtime calls returning with a pending
  // exception.  The pending exception check happened in the runtime
  // or native call stub.  The pending exception in Thread is
  // converted into a Java-level exception.
  //
  // Contract with Java-level exception handlers:
  // x10: exception
  // x13: throwing pc
  //
  // NOTE: At entry of this stub, exception-pc must be in RA !!

  // NOTE: this is always used as a jump target within generated code
  // so it just needs to be generated code with no x86 prolog

  address generate_forward_exception() {
    StubGenStubId stub_id = StubGenStubId::forward_exception_id;
    StubCodeMark mark(this, stub_id);
    address start = __ pc();

    // Upon entry, RA points to the return address returning into
    // Java (interpreted or compiled) code; i.e., the return address
    // becomes the throwing pc.
    //
    // Arguments pushed before the runtime call are still on the stack
    // but the exception handler will reset the stack pointer ->
    // ignore them.  A potential result in registers can be ignored as
    // well.

#ifdef ASSERT
    // make sure this code is only executed if there is a pending exception
    {
      Label L;
      __ ld(t0, Address(xthread, Thread::pending_exception_offset()));
      __ bnez(t0, L);
      __ stop("StubRoutines::forward exception: no pending exception (1)");
      __ bind(L);
    }
#endif

    // compute exception handler into x9

    // call the VM to find the handler address associated with the
    // caller address. pass thread in x10 and caller pc (ret address)
    // in x11. n.b. the caller pc is in ra, unlike x86 where it is on
    // the stack.
    __ mv(c_rarg1, ra);
    // ra will be trashed by the VM call so we move it to x9
    // (callee-saved) because we also need to pass it to the handler
    // returned by this call.
    __ mv(x9, ra);
    BLOCK_COMMENT("call exception_handler_for_return_address");
    __ call_VM_leaf(CAST_FROM_FN_PTR(address,
                         SharedRuntime::exception_handler_for_return_address),
                    xthread, c_rarg1);
    // we should not really care that ra is no longer the callee
    // address. we saved the value the handler needs in x9 so we can
    // just copy it to x13. however, the C2 handler will push its own
    // frame and then calls into the VM and the VM code asserts that
    // the PC for the frame above the handler belongs to a compiled
    // Java method. So, we restore ra here to satisfy that assert.
    __ mv(ra, x9);
    // setup x10 & x13 & clear pending exception
    __ mv(x13, x9);
    __ mv(x9, x10);
    __ ld(x10, Address(xthread, Thread::pending_exception_offset()));
    __ sd(zr, Address(xthread, Thread::pending_exception_offset()));

#ifdef ASSERT
    // make sure exception is set
    {
      Label L;
      __ bnez(x10, L);
      __ stop("StubRoutines::forward exception: no pending exception (2)");
      __ bind(L);
    }
#endif

    // continue at exception handler
    // x10: exception
    // x13: throwing pc
    // x9: exception handler
    __ verify_oop(x10);
    __ jr(x9);

    return start;
  }

  // Non-destructive plausibility checks for oops
  //
  // Arguments:
  //    x10: oop to verify
  //    t0: error message
  //
  // Stack after saving c_rarg3:
  //    [tos + 0]: saved c_rarg3
  //    [tos + 1]: saved c_rarg2
  //    [tos + 2]: saved ra
  //    [tos + 3]: saved t1
  //    [tos + 4]: saved x10
  //    [tos + 5]: saved t0
  address generate_verify_oop() {

    StubGenStubId stub_id = StubGenStubId::verify_oop_id;
    StubCodeMark mark(this, stub_id);
    address start = __ pc();

    Label exit, error;

    __ push_reg(RegSet::of(c_rarg2, c_rarg3), sp); // save c_rarg2 and c_rarg3

    __ la(c_rarg2, ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
    __ ld(c_rarg3, Address(c_rarg2));
    __ addi(c_rarg3, c_rarg3, 1);
    __ sd(c_rarg3, Address(c_rarg2));

    // object is in x10
    // make sure object is 'reasonable'
    __ beqz(x10, exit); // if obj is null it is OK

    BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
    bs_asm->check_oop(_masm, x10, c_rarg2, c_rarg3, error);

    // return if everything seems ok
    __ bind(exit);

    __ pop_reg(RegSet::of(c_rarg2, c_rarg3), sp);  // pop c_rarg2 and c_rarg3
    __ ret();

    // handle errors
    __ bind(error);
    __ pop_reg(RegSet::of(c_rarg2, c_rarg3), sp); // pop c_rarg2 and c_rarg3

    __ push_reg(RegSet::range(x0, x31), sp);
    // debug(char* msg, int64_t pc, int64_t regs[])
    __ mv(c_rarg0, t0);             // pass address of error message
    __ mv(c_rarg1, ra);             // pass return address
    __ mv(c_rarg2, sp);             // pass address of regs on stack
#ifndef PRODUCT
    assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
#endif
    BLOCK_COMMENT("call MacroAssembler::debug");
    __ rt_call(CAST_FROM_FN_PTR(address, MacroAssembler::debug64));
    __ ebreak();

    return start;
  }

  // The inner part of zero_words().
  //
  // Inputs:
  // x28: the HeapWord-aligned base address of an array to zero.
  // x29: the count in HeapWords, x29 > 0.
  //
  // Returns x28 and x29, adjusted for the caller to clear.
  // x28: the base address of the tail of words left to clear.
  // x29: the number of words in the tail.
  //      x29 < MacroAssembler::zero_words_block_size.

  address generate_zero_blocks() {
    Label done;

    const Register base = x28, cnt = x29, tmp1 = x30, tmp2 = x31;

    __ align(CodeEntryAlignment);
    StubGenStubId stub_id = StubGenStubId::zero_blocks_id;
    StubCodeMark mark(this, stub_id);
    address start = __ pc();

    if (UseBlockZeroing) {
      int zicboz_block_size = VM_Version::zicboz_block_size.value();
      // Ensure count >= 2 * zicboz_block_size so that it still deserves
      // a cbo.zero after alignment.
      Label small;
      int low_limit = MAX2(2 * zicboz_block_size, (int)BlockZeroingLowLimit) / wordSize;
      __ mv(tmp1, low_limit);
      __ blt(cnt, tmp1, small);
      __ zero_dcache_blocks(base, cnt, tmp1, tmp2);
      __ bind(small);
    }

    {
      // Clear the remaining blocks.
      Label loop;
      __ mv(tmp1, MacroAssembler::zero_words_block_size);
      __ blt(cnt, tmp1, done);
      __ bind(loop);
      for (int i = 0; i < MacroAssembler::zero_words_block_size; i++) {
        __ sd(zr, Address(base, i * wordSize));
      }
      __ addi(base, base, MacroAssembler::zero_words_block_size * wordSize);
      __ subi(cnt, cnt, MacroAssembler::zero_words_block_size);
      __ bge(cnt, tmp1, loop);
      __ bind(done);
    }

    __ ret();

    return start;
  }

  typedef enum {
    copy_forwards = 1,
    copy_backwards = -1
  } copy_direction;

  // Bulk copy of blocks of 8 words.
  //
  // count is a count of words.
  //
  // Precondition: count >= 8
  //
  // Postconditions:
  //
  // The least significant bit of count contains the remaining count
  // of words to copy.  The rest of count is trash.
  //
  // s and d are adjusted to point to the remaining words to copy
  //
  void generate_copy_longs(StubGenStubId stub_id, Label &start,
                           Register s, Register d, Register count) {
    BasicType type;
    copy_direction direction;
    switch (stub_id) {
    case copy_byte_f_id:
      direction = copy_forwards;
      type = T_BYTE;
      break;
    case copy_byte_b_id:
      direction = copy_backwards;
      type = T_BYTE;
      break;
    default:
      ShouldNotReachHere();
    }
    int unit = wordSize * direction;
    int bias = wordSize;

    const Register tmp_reg0 = x13, tmp_reg1 = x14, tmp_reg2 = x15, tmp_reg3 = x16,
      tmp_reg4 = x17, tmp_reg5 = x7, tmp_reg6 = x28, tmp_reg7 = x29;

    const Register stride = x30;

    assert_different_registers(t0, tmp_reg0, tmp_reg1, tmp_reg2, tmp_reg3,
      tmp_reg4, tmp_reg5, tmp_reg6, tmp_reg7);
    assert_different_registers(s, d, count, t0);

    Label again, drain;
    StubCodeMark mark(this, stub_id);
    __ align(CodeEntryAlignment);
    __ bind(start);

    if (direction == copy_forwards) {
      __ sub(s, s, bias);
      __ sub(d, d, bias);
    }

#ifdef ASSERT
    // Make sure we are never given < 8 words
    {
      Label L;

      __ mv(t0, 8);
      __ bge(count, t0, L);
      __ stop("genrate_copy_longs called with < 8 words");
      __ bind(L);
    }
#endif

    __ ld(tmp_reg0, Address(s, 1 * unit));
    __ ld(tmp_reg1, Address(s, 2 * unit));
    __ ld(tmp_reg2, Address(s, 3 * unit));
    __ ld(tmp_reg3, Address(s, 4 * unit));
    __ ld(tmp_reg4, Address(s, 5 * unit));
    __ ld(tmp_reg5, Address(s, 6 * unit));
    __ ld(tmp_reg6, Address(s, 7 * unit));
    __ ld(tmp_reg7, Address(s, 8 * unit));
    __ addi(s, s, 8 * unit);

    __ subi(count, count, 16);
    __ bltz(count, drain);

    __ bind(again);

    __ sd(tmp_reg0, Address(d, 1 * unit));
    __ sd(tmp_reg1, Address(d, 2 * unit));
    __ sd(tmp_reg2, Address(d, 3 * unit));
    __ sd(tmp_reg3, Address(d, 4 * unit));
    __ sd(tmp_reg4, Address(d, 5 * unit));
    __ sd(tmp_reg5, Address(d, 6 * unit));
    __ sd(tmp_reg6, Address(d, 7 * unit));
    __ sd(tmp_reg7, Address(d, 8 * unit));

    __ ld(tmp_reg0, Address(s, 1 * unit));
    __ ld(tmp_reg1, Address(s, 2 * unit));
    __ ld(tmp_reg2, Address(s, 3 * unit));
    __ ld(tmp_reg3, Address(s, 4 * unit));
    __ ld(tmp_reg4, Address(s, 5 * unit));
    __ ld(tmp_reg5, Address(s, 6 * unit));
    __ ld(tmp_reg6, Address(s, 7 * unit));
    __ ld(tmp_reg7, Address(s, 8 * unit));

    __ addi(s, s, 8 * unit);
    __ addi(d, d, 8 * unit);

    __ subi(count, count, 8);
    __ bgez(count, again);

    // Drain
    __ bind(drain);

    __ sd(tmp_reg0, Address(d, 1 * unit));
    __ sd(tmp_reg1, Address(d, 2 * unit));
    __ sd(tmp_reg2, Address(d, 3 * unit));
    __ sd(tmp_reg3, Address(d, 4 * unit));
    __ sd(tmp_reg4, Address(d, 5 * unit));
    __ sd(tmp_reg5, Address(d, 6 * unit));
    __ sd(tmp_reg6, Address(d, 7 * unit));
    __ sd(tmp_reg7, Address(d, 8 * unit));
    __ addi(d, d, 8 * unit);

    {
      Label L1, L2;
      __ test_bit(t0, count, 2);
      __ beqz(t0, L1);

      __ ld(tmp_reg0, Address(s, 1 * unit));
      __ ld(tmp_reg1, Address(s, 2 * unit));
      __ ld(tmp_reg2, Address(s, 3 * unit));
      __ ld(tmp_reg3, Address(s, 4 * unit));
      __ addi(s, s, 4 * unit);

      __ sd(tmp_reg0, Address(d, 1 * unit));
      __ sd(tmp_reg1, Address(d, 2 * unit));
      __ sd(tmp_reg2, Address(d, 3 * unit));
      __ sd(tmp_reg3, Address(d, 4 * unit));
      __ addi(d, d, 4 * unit);

      __ bind(L1);

      if (direction == copy_forwards) {
        __ addi(s, s, bias);
        __ addi(d, d, bias);
      }

      __ test_bit(t0, count, 1);
      __ beqz(t0, L2);
      if (direction == copy_backwards) {
        __ addi(s, s, 2 * unit);
        __ ld(tmp_reg0, Address(s));
        __ ld(tmp_reg1, Address(s, wordSize));
        __ addi(d, d, 2 * unit);
        __ sd(tmp_reg0, Address(d));
        __ sd(tmp_reg1, Address(d, wordSize));
      } else {
        __ ld(tmp_reg0, Address(s));
        __ ld(tmp_reg1, Address(s, wordSize));
        __ addi(s, s, 2 * unit);
        __ sd(tmp_reg0, Address(d));
        __ sd(tmp_reg1, Address(d, wordSize));
        __ addi(d, d, 2 * unit);
      }
      __ bind(L2);
    }

    __ ret();
  }

  Label copy_f, copy_b;

  typedef void (MacroAssembler::*copy_insn)(Register Rd, const Address &adr, Register temp);

  void copy_memory_v(Register s, Register d, Register count, int step) {
    bool is_backward = step < 0;
    int granularity = g_uabs(step);

    const Register src = x30, dst = x31, vl = x14, cnt = x15, tmp1 = x16, tmp2 = x17;
    assert_different_registers(s, d, cnt, vl, tmp1, tmp2);
    Assembler::SEW sew = Assembler::elembytes_to_sew(granularity);
    Label loop_forward, loop_backward, done;

    __ mv(dst, d);
    __ mv(src, s);
    __ mv(cnt, count);

    __ bind(loop_forward);
    __ vsetvli(vl, cnt, sew, Assembler::m8);
    if (is_backward) {
      __ bne(vl, cnt, loop_backward);
    }

    __ vlex_v(v0, src, sew);
    __ sub(cnt, cnt, vl);
    if (sew != Assembler::e8) {
      // when sew == e8 (e.g., elem size is 1 byte), slli R, R, 0 is a nop and unnecessary
      __ slli(vl, vl, sew);
    }
    __ add(src, src, vl);

    __ vsex_v(v0, dst, sew);
    __ add(dst, dst, vl);
    __ bnez(cnt, loop_forward);

    if (is_backward) {
      __ j(done);

      __ bind(loop_backward);
      __ sub(t0, cnt, vl);
      if (sew != Assembler::e8) {
        // when sew == e8 (e.g., elem size is 1 byte), slli R, R, 0 is a nop and unnecessary
        __ slli(t0, t0, sew);
      }
      __ add(tmp1, s, t0);
      __ vlex_v(v0, tmp1, sew);
      __ add(tmp2, d, t0);
      __ vsex_v(v0, tmp2, sew);
      __ sub(cnt, cnt, vl);
      __ bnez(cnt, loop_forward);
      __ bind(done);
    }
  }

  // All-singing all-dancing memory copy.
  //
  // Copy count units of memory from s to d.  The size of a unit is
  // step, which can be positive or negative depending on the direction
  // of copy.
  //
  void copy_memory(DecoratorSet decorators, BasicType type, bool is_aligned,
                   Register s, Register d, Register count, int step) {
    BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
    if (UseRVV && (!is_reference_type(type) || bs_asm->supports_rvv_arraycopy())) {
      return copy_memory_v(s, d, count, step);
    }

    bool is_backwards = step < 0;
    int granularity = g_uabs(step);

    const Register src = x30, dst = x31, cnt = x15, tmp3 = x16, tmp4 = x17, tmp5 = x14, tmp6 = x13;
    const Register gct1 = x28, gct2 = x29, gct3 = t2;

    Label same_aligned;
    Label copy_big, copy32_loop, copy8_loop, copy_small, done;

    // The size of copy32_loop body increases significantly with ZGC GC barriers.
    // Need conditional far branches to reach a point beyond the loop in this case.
    bool is_far = UseZGC;

    __ beqz(count, done, is_far);
    __ slli(cnt, count, exact_log2(granularity));
    if (is_backwards) {
      __ add(src, s, cnt);
      __ add(dst, d, cnt);
    } else {
      __ mv(src, s);
      __ mv(dst, d);
    }

    if (is_aligned) {
      __ subi(t0, cnt, 32);
      __ bgez(t0, copy32_loop);
      __ subi(t0, cnt, 8);
      __ bgez(t0, copy8_loop, is_far);
      __ j(copy_small);
    } else {
      __ mv(t0, 16);
      __ blt(cnt, t0, copy_small, is_far);

      __ xorr(t0, src, dst);
      __ andi(t0, t0, 0b111);
      __ bnez(t0, copy_small, is_far);

      __ bind(same_aligned);
      __ andi(t0, src, 0b111);
      __ beqz(t0, copy_big);
      if (is_backwards) {
        __ addi(src, src, step);
        __ addi(dst, dst, step);
      }
      bs_asm->copy_load_at(_masm, decorators, type, granularity, tmp3, Address(src), gct1);
      bs_asm->copy_store_at(_masm, decorators, type, granularity, Address(dst), tmp3, gct1, gct2, gct3);
      if (!is_backwards) {
        __ addi(src, src, step);
        __ addi(dst, dst, step);
      }
      __ subi(cnt, cnt, granularity);
      __ beqz(cnt, done, is_far);
      __ j(same_aligned);

      __ bind(copy_big);
      __ mv(t0, 32);
      __ blt(cnt, t0, copy8_loop, is_far);
    }

    __ bind(copy32_loop);
    if (is_backwards) {
      __ subi(src, src, wordSize * 4);
      __ subi(dst, dst, wordSize * 4);
    }
    // we first load 32 bytes, then write it, so the direction here doesn't matter
    bs_asm->copy_load_at(_masm, decorators, type, 8, tmp3, Address(src),     gct1);
    bs_asm->copy_load_at(_masm, decorators, type, 8, tmp4, Address(src, 8),  gct1);
    bs_asm->copy_load_at(_masm, decorators, type, 8, tmp5, Address(src, 16), gct1);
    bs_asm->copy_load_at(_masm, decorators, type, 8, tmp6, Address(src, 24), gct1);

    bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst),     tmp3, gct1, gct2, gct3);
    bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst, 8),  tmp4, gct1, gct2, gct3);
    bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst, 16), tmp5, gct1, gct2, gct3);
    bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst, 24), tmp6, gct1, gct2, gct3);

    if (!is_backwards) {
      __ addi(src, src, wordSize * 4);
      __ addi(dst, dst, wordSize * 4);
    }
    __ subi(t0, cnt, 32 + wordSize * 4);
    __ subi(cnt, cnt, wordSize * 4);
    __ bgez(t0, copy32_loop); // cnt >= 32, do next loop

    __ beqz(cnt, done); // if that's all - done

    __ subi(t0, cnt, 8); // if not - copy the reminder
    __ bltz(t0, copy_small); // cnt < 8, go to copy_small, else fall through to copy8_loop

    __ bind(copy8_loop);
    if (is_backwards) {
      __ subi(src, src, wordSize);
      __ subi(dst, dst, wordSize);
    }
    bs_asm->copy_load_at(_masm, decorators, type, 8, tmp3, Address(src), gct1);
    bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst), tmp3, gct1, gct2, gct3);

    if (!is_backwards) {
      __ addi(src, src, wordSize);
      __ addi(dst, dst, wordSize);
    }
    __ subi(t0, cnt, 8 + wordSize);
    __ subi(cnt, cnt, wordSize);
    __ bgez(t0, copy8_loop); // cnt >= 8, do next loop

    __ beqz(cnt, done); // if that's all - done

    __ bind(copy_small);
    if (is_backwards) {
      __ addi(src, src, step);
      __ addi(dst, dst, step);
    }

    bs_asm->copy_load_at(_masm, decorators, type, granularity, tmp3, Address(src), gct1);
    bs_asm->copy_store_at(_masm, decorators, type, granularity, Address(dst), tmp3, gct1, gct2, gct3);

    if (!is_backwards) {
      __ addi(src, src, step);
      __ addi(dst, dst, step);
    }
    __ subi(cnt, cnt, granularity);
    __ bgtz(cnt, copy_small);

    __ bind(done);
  }

  // Scan over array at a for count oops, verifying each one.
  // Preserves a and count, clobbers t0 and t1.
  void verify_oop_array(size_t size, Register a, Register count, Register temp) {
    Label loop, end;
    __ mv(t1, zr);
    __ slli(t0, count, exact_log2(size));
    __ bind(loop);
    __ bgeu(t1, t0, end);

    __ add(temp, a, t1);
    if (size == (size_t)wordSize) {
      __ ld(temp, Address(temp, 0));
      __ verify_oop(temp);
    } else {
      __ lwu(temp, Address(temp, 0));
      __ decode_heap_oop(temp); // calls verify_oop
    }
    __ add(t1, t1, size);
    __ j(loop);
    __ bind(end);
  }

  // Arguments:
  //   stub_id - is used to name the stub and identify all details of
  //             how to perform the copy.
  //
  //   entry - is assigned to the stub's post push entry point unless
  //           it is null
  //
  // Inputs:
  //   c_rarg0   - source array address
  //   c_rarg1   - destination array address
  //   c_rarg2   - element count, treated as ssize_t, can be zero
  //
  // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
  // the hardware handle it.  The two dwords within qwords that span
  // cache line boundaries will still be loaded and stored atomically.
  //
  // Side Effects: entry is set to the (post push) entry point so it
  //               can be used by the corresponding conjoint copy
  //               method
  //
  address generate_disjoint_copy(StubGenStubId stub_id, address* entry) {
    size_t size;
    bool aligned;
    bool is_oop;
    bool dest_uninitialized;
    switch (stub_id) {
    case jbyte_disjoint_arraycopy_id:
      size = sizeof(jbyte);
      aligned = false;
      is_oop = false;
      dest_uninitialized = false;
      break;
    case arrayof_jbyte_disjoint_arraycopy_id:
      size = sizeof(jbyte);
      aligned = true;
      is_oop = false;
      dest_uninitialized = false;
      break;
    case jshort_disjoint_arraycopy_id:
      size = sizeof(jshort);
      aligned = false;
      is_oop = false;
      dest_uninitialized = false;
      break;
    case arrayof_jshort_disjoint_arraycopy_id:
      size = sizeof(jshort);
      aligned = true;
      is_oop = false;
      dest_uninitialized = false;
      break;
    case jint_disjoint_arraycopy_id:
      size = sizeof(jint);
      aligned = false;
      is_oop = false;
      dest_uninitialized = false;
      break;
    case arrayof_jint_disjoint_arraycopy_id:
      size = sizeof(jint);
      aligned = true;
      is_oop = false;
      dest_uninitialized = false;
      break;
    case jlong_disjoint_arraycopy_id:
      // since this is always aligned we can (should!) use the same
      // stub as for case arrayof_jlong_disjoint_arraycopy
      ShouldNotReachHere();
      break;
    case arrayof_jlong_disjoint_arraycopy_id:
      size = sizeof(jlong);
      aligned = true;
      is_oop = false;
      dest_uninitialized = false;
      break;
    case oop_disjoint_arraycopy_id:
      size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
      aligned = !UseCompressedOops;
      is_oop = true;
      dest_uninitialized = false;
      break;
    case arrayof_oop_disjoint_arraycopy_id:
      size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
      aligned = !UseCompressedOops;
      is_oop = true;
      dest_uninitialized = false;
      break;
    case oop_disjoint_arraycopy_uninit_id:
      size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
      aligned = !UseCompressedOops;
      is_oop = true;
      dest_uninitialized = true;
      break;
    case arrayof_oop_disjoint_arraycopy_uninit_id:
      size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
      aligned = !UseCompressedOops;
      is_oop = true;
      dest_uninitialized = true;
      break;
    default:
      ShouldNotReachHere();
      break;
    }

    const Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
    RegSet saved_reg = RegSet::of(s, d, count);
    __ align(CodeEntryAlignment);
    StubCodeMark mark(this, stub_id);
    address start = __ pc();
    __ enter();

    if (entry != nullptr) {
      *entry = __ pc();
      // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
      BLOCK_COMMENT("Entry:");
    }

    DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
    if (dest_uninitialized) {
      decorators |= IS_DEST_UNINITIALIZED;
    }
    if (aligned) {
      decorators |= ARRAYCOPY_ALIGNED;
    }

    BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
    bs->arraycopy_prologue(_masm, decorators, is_oop, s, d, count, saved_reg);

    if (is_oop) {
      // save regs before copy_memory
      __ push_reg(RegSet::of(d, count), sp);
    }

    {
      // UnsafeMemoryAccess page error: continue after unsafe access
      bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size);
      UnsafeMemoryAccessMark umam(this, add_entry, true);
      copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, size);
    }

    if (is_oop) {
      __ pop_reg(RegSet::of(d, count), sp);
      if (VerifyOops) {
        verify_oop_array(size, d, count, t2);
      }
    }

    bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, t0, RegSet());

    __ leave();
    __ mv(x10, zr); // return 0
    __ ret();
    return start;
  }

  // Arguments:
  //   stub_id - is used to name the stub and identify all details of
  //             how to perform the copy.
  //
  //   nooverlap_target - identifes the (post push) entry for the
  //             corresponding disjoint copy routine which can be
  //             jumped to if the ranges do not actually overlap
  //
  //   entry - is assigned to the stub's post push entry point unless
  //           it is null
  //
  // Inputs:
  //   c_rarg0   - source array address
  //   c_rarg1   - destination array address
  //   c_rarg2   - element count, treated as ssize_t, can be zero
  //
  // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
  // the hardware handle it.  The two dwords within qwords that span
  // cache line boundaries will still be loaded and stored atomically.
  //
  // Side Effects:
  //   entry is set to the no-overlap entry point so it can be used by
  //   some other conjoint copy method
  //
  address generate_conjoint_copy(StubGenStubId stub_id, address nooverlap_target, address *entry) {
    const Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
    RegSet saved_regs = RegSet::of(s, d, count);
    int size;
    bool aligned;
    bool is_oop;
    bool dest_uninitialized;
    switch (stub_id) {
    case jbyte_arraycopy_id:
      size = sizeof(jbyte);
      aligned = false;
      is_oop = false;
      dest_uninitialized = false;
      break;
    case arrayof_jbyte_arraycopy_id:
      size = sizeof(jbyte);
      aligned = true;
      is_oop = false;
      dest_uninitialized = false;
      break;
    case jshort_arraycopy_id:
      size = sizeof(jshort);
      aligned = false;
      is_oop = false;
      dest_uninitialized = false;
      break;
    case arrayof_jshort_arraycopy_id:
      size = sizeof(jshort);
      aligned = true;
      is_oop = false;
      dest_uninitialized = false;
      break;
    case jint_arraycopy_id:
      size = sizeof(jint);
      aligned = false;
      is_oop = false;
      dest_uninitialized = false;
      break;
    case arrayof_jint_arraycopy_id:
      size = sizeof(jint);
      aligned = true;
      is_oop = false;
      dest_uninitialized = false;
      break;
    case jlong_arraycopy_id:
      // since this is always aligned we can (should!) use the same
      // stub as for case arrayof_jlong_disjoint_arraycopy
      ShouldNotReachHere();
      break;
    case arrayof_jlong_arraycopy_id:
      size = sizeof(jlong);
      aligned = true;
      is_oop = false;
      dest_uninitialized = false;
      break;
    case oop_arraycopy_id:
      size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
      aligned = !UseCompressedOops;
      is_oop = true;
      dest_uninitialized = false;
      break;
    case arrayof_oop_arraycopy_id:
      size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
      aligned = !UseCompressedOops;
      is_oop = true;
      dest_uninitialized = false;
      break;
    case oop_arraycopy_uninit_id:
      size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
      aligned = !UseCompressedOops;
      is_oop = true;
      dest_uninitialized = true;
      break;
    case arrayof_oop_arraycopy_uninit_id:
      size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
      aligned = !UseCompressedOops;
      is_oop = true;
      dest_uninitialized = true;
      break;
    default:
      ShouldNotReachHere();
    }

    StubCodeMark mark(this, stub_id);
    address start = __ pc();
    __ enter();

    if (entry != nullptr) {
      *entry = __ pc();
      // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
      BLOCK_COMMENT("Entry:");
    }

    // use fwd copy when (d-s) above_equal (count*size)
    __ sub(t0, d, s);
    __ slli(t1, count, exact_log2(size));
    Label L_continue;
    __ bltu(t0, t1, L_continue);
    __ j(nooverlap_target);
    __ bind(L_continue);

    DecoratorSet decorators = IN_HEAP | IS_ARRAY;
    if (dest_uninitialized) {
      decorators |= IS_DEST_UNINITIALIZED;
    }
    if (aligned) {
      decorators |= ARRAYCOPY_ALIGNED;
    }

    BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
    bs->arraycopy_prologue(_masm, decorators, is_oop, s, d, count, saved_regs);

    if (is_oop) {
      // save regs before copy_memory
      __ push_reg(RegSet::of(d, count), sp);
    }

    {
      // UnsafeMemoryAccess page error: continue after unsafe access
      bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size);
      UnsafeMemoryAccessMark umam(this, add_entry, true);
      copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, -size);
    }

    if (is_oop) {
      __ pop_reg(RegSet::of(d, count), sp);
      if (VerifyOops) {
        verify_oop_array(size, d, count, t2);
      }
    }
    bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, t0, RegSet());
    __ leave();
    __ mv(x10, zr); // return 0
    __ ret();
    return start;
  }

  // Helper for generating a dynamic type check.
  // Smashes t0, t1.
  void generate_type_check(Register sub_klass,
                           Register super_check_offset,
                           Register super_klass,
                           Register result,
                           Register tmp1,
                           Register tmp2,
                           Label& L_success) {
    assert_different_registers(sub_klass, super_check_offset, super_klass);

    BLOCK_COMMENT("type_check:");

    Label L_miss;

    __ check_klass_subtype_fast_path(sub_klass, super_klass, noreg, &L_success, &L_miss, nullptr, super_check_offset);
    __ check_klass_subtype_slow_path(sub_klass, super_klass, tmp1, tmp2, &L_success, nullptr);

    // Fall through on failure!
    __ BIND(L_miss);
  }

  //
  //  Generate checkcasting array copy stub
  //
  //  Input:
  //    c_rarg0   - source array address
  //    c_rarg1   - destination array address
  //    c_rarg2   - element count, treated as ssize_t, can be zero
  //    c_rarg3   - size_t ckoff (super_check_offset)
  //    c_rarg4   - oop ckval (super_klass)
  //
  //  Output:
  //    x10 ==  0  -  success
  //    x10 == -1^K - failure, where K is partial transfer count
  //
  address generate_checkcast_copy(StubGenStubId stub_id, address* entry) {
    bool dest_uninitialized;
    switch (stub_id) {
    case checkcast_arraycopy_id:
      dest_uninitialized = false;
      break;
    case checkcast_arraycopy_uninit_id:
      dest_uninitialized = true;
      break;
    default:
      ShouldNotReachHere();
    }

    Label L_load_element, L_store_element, L_do_card_marks, L_done, L_done_pop;

    // Input registers (after setup_arg_regs)
    const Register from        = c_rarg0;   // source array address
    const Register to          = c_rarg1;   // destination array address
    const Register count       = c_rarg2;   // elementscount
    const Register ckoff       = c_rarg3;   // super_check_offset
    const Register ckval       = c_rarg4;   // super_klass

    RegSet wb_pre_saved_regs   = RegSet::range(c_rarg0, c_rarg4);
    RegSet wb_post_saved_regs  = RegSet::of(count);

    // Registers used as temps (x7, x9, x18 are save-on-entry)
    const Register count_save  = x19;       // orig elementscount
    const Register start_to    = x18;       // destination array start address
    const Register copied_oop  = x7;        // actual oop copied
    const Register r9_klass    = x9;        // oop._klass

    // Registers used as gc temps (x15, x16, x17 are save-on-call)
    const Register gct1 = x15, gct2 = x16, gct3 = x17;

    //---------------------------------------------------------------
    // Assembler stub will be used for this call to arraycopy
    // if the two arrays are subtypes of Object[] but the
    // destination array type is not equal to or a supertype
    // of the source type.  Each element must be separately
    // checked.

    assert_different_registers(from, to, count, ckoff, ckval, start_to,
                               copied_oop, r9_klass, count_save);

    __ align(CodeEntryAlignment);
    StubCodeMark mark(this, stub_id);
    address start = __ pc();

    __ enter(); // required for proper stackwalking of RuntimeStub frame

    // Caller of this entry point must set up the argument registers.
    if (entry != nullptr) {
      *entry = __ pc();
      BLOCK_COMMENT("Entry:");
    }

    // Empty array:  Nothing to do
    __ beqz(count, L_done);

    __ push_reg(RegSet::of(x7, x9, x18, x19), sp);

#ifdef ASSERT
    BLOCK_COMMENT("assert consistent ckoff/ckval");
    // The ckoff and ckval must be mutually consistent,
    // even though caller generates both.
    { Label L;
      int sco_offset = in_bytes(Klass::super_check_offset_offset());
      __ lwu(start_to, Address(ckval, sco_offset));
      __ beq(ckoff, start_to, L);
      __ stop("super_check_offset inconsistent");
      __ bind(L);
    }
#endif //ASSERT

    DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST | ARRAYCOPY_DISJOINT;
    if (dest_uninitialized) {
      decorators |= IS_DEST_UNINITIALIZED;
    }

    bool is_oop = true;
    int element_size = UseCompressedOops ? 4 : 8;

    BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
    bs->arraycopy_prologue(_masm, decorators, is_oop, from, to, count, wb_pre_saved_regs);

    // save the original count
    __ mv(count_save, count);

    // Copy from low to high addresses
    __ mv(start_to, to);              // Save destination array start address
    __ j(L_load_element);

    // ======== begin loop ========
    // (Loop is rotated; its entry is L_load_element.)
    // Loop control:
    //   for count to 0 do
    //     copied_oop = load_heap_oop(from++)
    //     ... generate_type_check ...
    //     store_heap_oop(to++, copied_oop)
    //   end

    __ align(OptoLoopAlignment);

    __ BIND(L_store_element);
    bs->copy_store_at(_masm, decorators, T_OBJECT, element_size,
                      Address(to, 0), copied_oop,
                      gct1, gct2, gct3);
    __ addi(to, to, UseCompressedOops ? 4 : 8);
    __ subi(count, count, 1);
    __ beqz(count, L_do_card_marks);

    // ======== loop entry is here ========
    __ BIND(L_load_element);
    bs->copy_load_at(_masm, decorators, T_OBJECT, element_size,
                     copied_oop, Address(from, 0),
                     gct1);
    __ addi(from, from, UseCompressedOops ? 4 : 8);
    __ beqz(copied_oop, L_store_element);

    __ load_klass(r9_klass, copied_oop);// query the object klass

    BLOCK_COMMENT("type_check:");
    generate_type_check(r9_klass, /*sub_klass*/
                        ckoff,    /*super_check_offset*/
                        ckval,    /*super_klass*/
                        x10,      /*result*/
                        gct1,     /*tmp1*/
                        gct2,     /*tmp2*/
                        L_store_element);

    // Fall through on failure!

    // ======== end loop ========

    // It was a real error; we must depend on the caller to finish the job.
    // Register count = remaining oops, count_orig = total oops.
    // Emit GC store barriers for the oops we have copied and report
    // their number to the caller.

    __ sub(count, count_save, count);     // K = partially copied oop count
    __ xori(count, count, -1);            // report (-1^K) to caller
    __ beqz(count, L_done_pop);

    __ BIND(L_do_card_marks);
    bs->arraycopy_epilogue(_masm, decorators, is_oop, start_to, count_save, t0, wb_post_saved_regs);

    __ bind(L_done_pop);
    __ pop_reg(RegSet::of(x7, x9, x18, x19), sp);
    inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);

    __ bind(L_done);
    __ mv(x10, count);
    __ leave();
    __ ret();

    return start;
  }

  // Perform range checks on the proposed arraycopy.
  // Kills temp, but nothing else.
  // Also, clean the sign bits of src_pos and dst_pos.
  void arraycopy_range_checks(Register src,     // source array oop (c_rarg0)
                              Register src_pos, // source position (c_rarg1)
                              Register dst,     // destination array oo (c_rarg2)
                              Register dst_pos, // destination position (c_rarg3)
                              Register length,
                              Register temp,
                              Label& L_failed) {
    BLOCK_COMMENT("arraycopy_range_checks:");

    assert_different_registers(t0, temp);

    // if [src_pos + length > arrayOop(src)->length()] then FAIL
    __ lwu(t0, Address(src, arrayOopDesc::length_offset_in_bytes()));
    __ addw(temp, length, src_pos);
    __ bgtu(temp, t0, L_failed);

    // if [dst_pos + length > arrayOop(dst)->length()] then FAIL
    __ lwu(t0, Address(dst, arrayOopDesc::length_offset_in_bytes()));
    __ addw(temp, length, dst_pos);
    __ bgtu(temp, t0, L_failed);

    // Have to clean up high 32 bits of 'src_pos' and 'dst_pos'.
    __ zext(src_pos, src_pos, 32);
    __ zext(dst_pos, dst_pos, 32);

    BLOCK_COMMENT("arraycopy_range_checks done");
  }

  address generate_unsafecopy_common_error_exit() {
    address start = __ pc();
    __ mv(x10, 0);
    __ leave();
    __ ret();
    return start;
  }

  //
  //  Generate 'unsafe' set memory stub
  //  Though just as safe as the other stubs, it takes an unscaled
  //  size_t (# bytes) argument instead of an element count.
  //
  //  Input:
  //    c_rarg0   - destination array address
  //    c_rarg1   - byte count (size_t)
  //    c_rarg2   - byte value
  //
  address generate_unsafe_setmemory() {
    __ align(CodeEntryAlignment);
    StubGenStubId stub_id = StubGenStubId::unsafe_setmemory_id;
    StubCodeMark mark(this, stub_id);
    address start = __ pc();

    // bump this on entry, not on exit:
    // inc_counter_np(SharedRuntime::_unsafe_set_memory_ctr);

    Label L_fill_elements;

    const Register dest = c_rarg0;
    const Register count = c_rarg1;
    const Register value = c_rarg2;
    const Register cnt_words = x28; // temp register
    const Register tmp_reg   = x29; // temp register

    // Mark remaining code as such which performs Unsafe accesses.
    UnsafeMemoryAccessMark umam(this, true, false);

    __ enter(); // required for proper stackwalking of RuntimeStub frame

    // if count < 8, jump to L_fill_elements
    __ mv(tmp_reg, 8); // 8 bytes fill by element
    __ bltu(count, tmp_reg, L_fill_elements);

    // Propagate byte to 64-bit width
    // 8 bit -> 16 bit
    __ zext(value, value, 8);
    __ slli(tmp_reg, value, 8);
    __ orr(value, value, tmp_reg);
    // 16 bit -> 32 bit
    __ slli(tmp_reg, value, 16);
    __ orr(value, value, tmp_reg);
    // 32 bit -> 64 bit
    __ slli(tmp_reg, value, 32);
    __ orr(value, value, tmp_reg);

    // Align source address at 8 bytes address boundary.
    Label L_skip_align1, L_skip_align2, L_skip_align4;
    // One byte misalignment happens.
    __ test_bit(tmp_reg, dest, 0);
    __ beqz(tmp_reg, L_skip_align1);
    __ sb(value, Address(dest, 0));
    __ addi(dest, dest, 1);
    __ subi(count, count, 1);

    __ bind(L_skip_align1);
    // Two bytes misalignment happens.
    __ test_bit(tmp_reg, dest, 1);
    __ beqz(tmp_reg, L_skip_align2);
    __ sh(value, Address(dest, 0));
    __ addi(dest, dest, 2);
    __ subi(count, count, 2);

    __ bind(L_skip_align2);
    // Four bytes misalignment happens.
    __ test_bit(tmp_reg, dest, 2);
    __ beqz(tmp_reg, L_skip_align4);
    __ sw(value, Address(dest, 0));
    __ addi(dest, dest, 4);
    __ subi(count, count, 4);
    __ bind(L_skip_align4);

    //  Fill large chunks
    __ srli(cnt_words, count, 3); // number of words
    __ slli(tmp_reg, cnt_words, 3);
    __ sub(count, count, tmp_reg);
    {
      __ fill_words(dest, cnt_words, value);
    }

    // Handle copies less than 8 bytes
    __ bind(L_fill_elements);
    Label L_fill_2, L_fill_1, L_exit;
    __ test_bit(tmp_reg, count, 2);
    __ beqz(tmp_reg, L_fill_2);
    __ sb(value, Address(dest, 0));
    __ sb(value, Address(dest, 1));
    __ sb(value, Address(dest, 2));
    __ sb(value, Address(dest, 3));
    __ addi(dest, dest, 4);

    __ bind(L_fill_2);
    __ test_bit(tmp_reg, count, 1);
    __ beqz(tmp_reg, L_fill_1);
    __ sb(value, Address(dest, 0));
    __ sb(value, Address(dest, 1));
    __ addi(dest, dest, 2);

    __ bind(L_fill_1);
    __ test_bit(tmp_reg, count, 0);
    __ beqz(tmp_reg, L_exit);
    __ sb(value, Address(dest, 0));

    __ bind(L_exit);
    __ leave();
    __ ret();

    return start;
  }

  //
  //  Generate 'unsafe' array copy stub
  //  Though just as safe as the other stubs, it takes an unscaled
  //  size_t argument instead of an element count.
  //
  //  Input:
  //    c_rarg0   - source array address
  //    c_rarg1   - destination array address
  //    c_rarg2   - byte count, treated as ssize_t, can be zero
  //
  // Examines the alignment of the operands and dispatches
  // to a long, int, short, or byte copy loop.
  //
  address generate_unsafe_copy(address byte_copy_entry,
                               address short_copy_entry,
                               address int_copy_entry,
                               address long_copy_entry) {
    assert_cond(byte_copy_entry != nullptr && short_copy_entry != nullptr &&
                int_copy_entry != nullptr && long_copy_entry != nullptr);
    Label L_long_aligned, L_int_aligned, L_short_aligned;
    const Register s = c_rarg0, d = c_rarg1, count = c_rarg2;

    __ align(CodeEntryAlignment);
    StubGenStubId stub_id = StubGenStubId::unsafe_arraycopy_id;
    StubCodeMark mark(this, stub_id);
    address start = __ pc();
    __ enter(); // required for proper stackwalking of RuntimeStub frame

    // bump this on entry, not on exit:
    inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);

    __ orr(t0, s, d);
    __ orr(t0, t0, count);

    __ andi(t0, t0, BytesPerLong - 1);
    __ beqz(t0, L_long_aligned);
    __ andi(t0, t0, BytesPerInt - 1);
    __ beqz(t0, L_int_aligned);
    __ test_bit(t0, t0, 0);
    __ beqz(t0, L_short_aligned);
    __ j(RuntimeAddress(byte_copy_entry));

    __ BIND(L_short_aligned);
    __ srli(count, count, LogBytesPerShort);  // size => short_count
    __ j(RuntimeAddress(short_copy_entry));
    __ BIND(L_int_aligned);
    __ srli(count, count, LogBytesPerInt);    // size => int_count
    __ j(RuntimeAddress(int_copy_entry));
    __ BIND(L_long_aligned);
    __ srli(count, count, LogBytesPerLong);   // size => long_count
    __ j(RuntimeAddress(long_copy_entry));

    return start;
  }

  //
  //  Generate generic array copy stubs
  //
  //  Input:
  //    c_rarg0    -  src oop
  //    c_rarg1    -  src_pos (32-bits)
  //    c_rarg2    -  dst oop
  //    c_rarg3    -  dst_pos (32-bits)
  //    c_rarg4    -  element count (32-bits)
  //
  //  Output:
  //    x10 ==  0  -  success
  //    x10 == -1^K - failure, where K is partial transfer count
  //
  address generate_generic_copy(address byte_copy_entry, address short_copy_entry,
                                address int_copy_entry, address oop_copy_entry,
                                address long_copy_entry, address checkcast_copy_entry) {
    assert_cond(byte_copy_entry != nullptr && short_copy_entry != nullptr &&
                int_copy_entry != nullptr && oop_copy_entry != nullptr &&
                long_copy_entry != nullptr && checkcast_copy_entry != nullptr);
    Label L_failed, L_failed_0, L_objArray;
    Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs;

    // Input registers
    const Register src        = c_rarg0;  // source array oop
    const Register src_pos    = c_rarg1;  // source position
    const Register dst        = c_rarg2;  // destination array oop
    const Register dst_pos    = c_rarg3;  // destination position
    const Register length     = c_rarg4;

    // Registers used as temps
    const Register dst_klass = c_rarg5;

    __ align(CodeEntryAlignment);

    StubGenStubId stub_id = StubGenStubId::generic_arraycopy_id;
    StubCodeMark mark(this, stub_id);

    address start = __ pc();

    __ enter(); // required for proper stackwalking of RuntimeStub frame

    // bump this on entry, not on exit:
    inc_counter_np(SharedRuntime::_generic_array_copy_ctr);

    //-----------------------------------------------------------------------
    // Assembler stub will be used for this call to arraycopy
    // if the following conditions are met:
    //
    // (1) src and dst must not be null.
    // (2) src_pos must not be negative.
    // (3) dst_pos must not be negative.
    // (4) length  must not be negative.
    // (5) src klass and dst klass should be the same and not null.
    // (6) src and dst should be arrays.
    // (7) src_pos + length must not exceed length of src.
    // (8) dst_pos + length must not exceed length of dst.
    //

    // if src is null then return -1
    __ beqz(src, L_failed);

    // if [src_pos < 0] then return -1
    __ sext(t0, src_pos, 32);
    __ bltz(t0, L_failed);

    // if dst is null then return -1
    __ beqz(dst, L_failed);

    // if [dst_pos < 0] then return -1
    __ sext(t0, dst_pos, 32);
    __ bltz(t0, L_failed);

    // registers used as temp
    const Register scratch_length    = x28; // elements count to copy
    const Register scratch_src_klass = x29; // array klass
    const Register lh                = x30; // layout helper

    // if [length < 0] then return -1
    __ sext(scratch_length, length, 32); // length (elements count, 32-bits value)
    __ bltz(scratch_length, L_failed);

    __ load_klass(scratch_src_klass, src);
#ifdef ASSERT
    {
      BLOCK_COMMENT("assert klasses not null {");
      Label L1, L2;
      __ bnez(scratch_src_klass, L2);   // it is broken if klass is null
      __ bind(L1);
      __ stop("broken null klass");
      __ bind(L2);
      __ load_klass(t0, dst, t1);
      __ beqz(t0, L1);     // this would be broken also
      BLOCK_COMMENT("} assert klasses not null done");
    }
#endif

    // Load layout helper (32-bits)
    //
    //  |array_tag|     | header_size | element_type |     |log2_element_size|
    // 32        30    24            16              8     2                 0
    //
    //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
    //

    const int lh_offset = in_bytes(Klass::layout_helper_offset());

    // Handle objArrays completely differently...
    const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
    __ lw(lh, Address(scratch_src_klass, lh_offset));
    __ mv(t0, objArray_lh);
    __ beq(lh, t0, L_objArray);

    // if [src->klass() != dst->klass()] then return -1
    __ load_klass(t1, dst);
    __ bne(t1, scratch_src_klass, L_failed);

    // if src->is_Array() isn't null then return -1
    // i.e. (lh >= 0)
    __ bgez(lh, L_failed);

    // At this point, it is known to be a typeArray (array_tag 0x3).
#ifdef ASSERT
    {
      BLOCK_COMMENT("assert primitive array {");
      Label L;
      __ mv(t1, (int32_t)(Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
      __ bge(lh, t1, L);
      __ stop("must be a primitive array");
      __ bind(L);
      BLOCK_COMMENT("} assert primitive array done");
    }
#endif

    arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
                           t1, L_failed);

    // TypeArrayKlass
    //
    // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize)
    // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize)
    //

    const Register t0_offset = t0;    // array offset
    const Register x30_elsize = lh;   // element size

    // Get array_header_in_bytes()
    int lh_header_size_width = exact_log2(Klass::_lh_header_size_mask + 1);
    int lh_header_size_msb = Klass::_lh_header_size_shift + lh_header_size_width;
    __ slli(t0_offset, lh, XLEN - lh_header_size_msb);          // left shift to remove 24 ~ 32;
    __ srli(t0_offset, t0_offset, XLEN - lh_header_size_width); // array_offset

    __ add(src, src, t0_offset);           // src array offset
    __ add(dst, dst, t0_offset);           // dst array offset
    BLOCK_COMMENT("choose copy loop based on element size");

    // next registers should be set before the jump to corresponding stub
    const Register from     = c_rarg0;  // source array address
    const Register to       = c_rarg1;  // destination array address
    const Register count    = c_rarg2;  // elements count

    // 'from', 'to', 'count' registers should be set in such order
    // since they are the same as 'src', 'src_pos', 'dst'.

    assert(Klass::_lh_log2_element_size_shift == 0, "fix this code");

    // The possible values of elsize are 0-3, i.e. exact_log2(element
    // size in bytes).  We do a simple bitwise binary search.
  __ BIND(L_copy_bytes);
    __ test_bit(t0, x30_elsize, 1);
    __ bnez(t0, L_copy_ints);
    __ test_bit(t0, x30_elsize, 0);
    __ bnez(t0, L_copy_shorts);
    __ add(from, src, src_pos); // src_addr
    __ add(to, dst, dst_pos); // dst_addr
    __ sext(count, scratch_length, 32); // length
    __ j(RuntimeAddress(byte_copy_entry));

  __ BIND(L_copy_shorts);
    __ shadd(from, src_pos, src, t0, 1); // src_addr
    __ shadd(to, dst_pos, dst, t0, 1); // dst_addr
    __ sext(count, scratch_length, 32); // length
    __ j(RuntimeAddress(short_copy_entry));

  __ BIND(L_copy_ints);
    __ test_bit(t0, x30_elsize, 0);
    __ bnez(t0, L_copy_longs);
    __ shadd(from, src_pos, src, t0, 2); // src_addr
    __ shadd(to, dst_pos, dst, t0, 2); // dst_addr
    __ sext(count, scratch_length, 32); // length
    __ j(RuntimeAddress(int_copy_entry));

  __ BIND(L_copy_longs);
#ifdef ASSERT
    {
      BLOCK_COMMENT("assert long copy {");
      Label L;
      __ andi(lh, lh, Klass::_lh_log2_element_size_mask); // lh -> x30_elsize
      __ sext(lh, lh, 32);
      __ mv(t0, LogBytesPerLong);
      __ beq(x30_elsize, t0, L);
      __ stop("must be long copy, but elsize is wrong");
      __ bind(L);
      BLOCK_COMMENT("} assert long copy done");
    }
#endif
    __ shadd(from, src_pos, src, t0, 3); // src_addr
    __ shadd(to, dst_pos, dst, t0, 3); // dst_addr
    __ sext(count, scratch_length, 32); // length
    __ j(RuntimeAddress(long_copy_entry));

    // ObjArrayKlass
  __ BIND(L_objArray);
    // live at this point:  scratch_src_klass, scratch_length, src[_pos], dst[_pos]

    Label L_plain_copy, L_checkcast_copy;
    // test array classes for subtyping
    __ load_klass(t2, dst);
    __ bne(scratch_src_klass, t2, L_checkcast_copy); // usual case is exact equality

    // Identically typed arrays can be copied without element-wise checks.
    arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
                           t1, L_failed);

    __ shadd(from, src_pos, src, t0, LogBytesPerHeapOop);
    __ addi(from, from, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
    __ shadd(to, dst_pos, dst, t0, LogBytesPerHeapOop);
    __ addi(to, to, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
    __ sext(count, scratch_length, 32); // length
  __ BIND(L_plain_copy);
    __ j(RuntimeAddress(oop_copy_entry));

  __ BIND(L_checkcast_copy);
    // live at this point:  scratch_src_klass, scratch_length, t2 (dst_klass)
    {
      // Before looking at dst.length, make sure dst is also an objArray.
      __ lwu(t0, Address(t2, lh_offset));
      __ mv(t1, objArray_lh);
      __ bne(t0, t1, L_failed);

      // It is safe to examine both src.length and dst.length.
      arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
                             t2, L_failed);

      __ load_klass(dst_klass, dst); // reload

      // Marshal the base address arguments now, freeing registers.
      __ shadd(from, src_pos, src, t0, LogBytesPerHeapOop);
      __ addi(from, from, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
      __ shadd(to, dst_pos, dst, t0, LogBytesPerHeapOop);
      __ addi(to, to, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
      __ sext(count, length, 32); // length (reloaded)
      const Register sco_temp = c_rarg3; // this register is free now
      assert_different_registers(from, to, count, sco_temp,
                                 dst_klass, scratch_src_klass);

      // Generate the type check.
      const int sco_offset = in_bytes(Klass::super_check_offset_offset());
      __ lwu(sco_temp, Address(dst_klass, sco_offset));

      // Smashes t0, t1
      generate_type_check(scratch_src_klass, sco_temp, dst_klass, noreg, noreg, noreg, L_plain_copy);

      // Fetch destination element klass from the ObjArrayKlass header.
      int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
      __ ld(dst_klass, Address(dst_klass, ek_offset));
      __ lwu(sco_temp, Address(dst_klass, sco_offset));

      // the checkcast_copy loop needs two extra arguments:
      assert(c_rarg3 == sco_temp, "#3 already in place");
      // Set up arguments for checkcast_copy_entry.
      __ mv(c_rarg4, dst_klass);  // dst.klass.element_klass
      __ j(RuntimeAddress(checkcast_copy_entry));
    }

  __ BIND(L_failed);
    __ mv(x10, -1);
    __ leave();   // required for proper stackwalking of RuntimeStub frame
    __ ret();

    return start;
  }

  //
  // Generate stub for array fill. If "aligned" is true, the
  // "to" address is assumed to be heapword aligned.
  //
  // Arguments for generated stub:
  //   to:    c_rarg0
  //   value: c_rarg1
  //   count: c_rarg2 treated as signed
  //
  address generate_fill(StubGenStubId stub_id) {
    BasicType t;
    bool aligned;

    switch (stub_id) {
    case jbyte_fill_id:
      t = T_BYTE;
      aligned = false;
      break;
    case jshort_fill_id:
      t = T_SHORT;
      aligned = false;
      break;
    case jint_fill_id:
      t = T_INT;
      aligned = false;
      break;
    case arrayof_jbyte_fill_id:
      t = T_BYTE;
      aligned = true;
      break;
    case arrayof_jshort_fill_id:
      t = T_SHORT;
      aligned = true;
      break;
    case arrayof_jint_fill_id:
      t = T_INT;
      aligned = true;
      break;
    default:
      ShouldNotReachHere();
    };

    __ align(CodeEntryAlignment);
    StubCodeMark mark(this, stub_id);
    address start = __ pc();

    BLOCK_COMMENT("Entry:");

    const Register to        = c_rarg0;  // source array address
    const Register value     = c_rarg1;  // value
    const Register count     = c_rarg2;  // elements count

    const Register bz_base   = x28;      // base for block_zero routine
    const Register cnt_words = x29;      // temp register
    const Register tmp_reg   = t1;

    __ enter();

    Label L_fill_elements;

    int shift = -1;
    switch (t) {
      case T_BYTE:
        shift = 0;
        // Short arrays (< 8 bytes) fill by element
        __ mv(tmp_reg, 8 >> shift);
        __ bltu(count, tmp_reg, L_fill_elements);

        // Zero extend value
        // 8 bit -> 16 bit
        __ zext(value, value, 8);
        __ slli(tmp_reg, value, 8);
        __ orr(value, value, tmp_reg);

        // 16 bit -> 32 bit
        __ slli(tmp_reg, value, 16);
        __ orr(value, value, tmp_reg);
        break;
      case T_SHORT:
        shift = 1;
        // Short arrays (< 8 bytes) fill by element
        __ mv(tmp_reg, 8 >> shift);
        __ bltu(count, tmp_reg, L_fill_elements);

        // Zero extend value
        // 16 bit -> 32 bit
        __ zext(value, value, 16);
        __ slli(tmp_reg, value, 16);
        __ orr(value, value, tmp_reg);
        break;
      case T_INT:
        shift = 2;
        // Short arrays (< 8 bytes) fill by element
        __ mv(tmp_reg, 8 >> shift);
        __ bltu(count, tmp_reg, L_fill_elements);
        break;
      default: ShouldNotReachHere();
    }

    // Align source address at 8 bytes address boundary.
    Label L_skip_align1, L_skip_align2, L_skip_align4;
    if (!aligned) {
      switch (t) {
        case T_BYTE:
          // One byte misalignment happens only for byte arrays.
          __ test_bit(tmp_reg, to, 0);
          __ beqz(tmp_reg, L_skip_align1);
          __ sb(value, Address(to, 0));
          __ addi(to, to, 1);
          __ subiw(count, count, 1);
          __ bind(L_skip_align1);
          // Fallthrough
        case T_SHORT:
          // Two bytes misalignment happens only for byte and short (char) arrays.
          __ test_bit(tmp_reg, to, 1);
          __ beqz(tmp_reg, L_skip_align2);
          __ sh(value, Address(to, 0));
          __ addi(to, to, 2);
          __ subiw(count, count, 2 >> shift);
          __ bind(L_skip_align2);
          // Fallthrough
        case T_INT:
          // Align to 8 bytes, we know we are 4 byte aligned to start.
          __ test_bit(tmp_reg, to, 2);
          __ beqz(tmp_reg, L_skip_align4);
          __ sw(value, Address(to, 0));
          __ addi(to, to, 4);
          __ subiw(count, count, 4 >> shift);
          __ bind(L_skip_align4);
          break;
        default: ShouldNotReachHere();
      }
    }

    //
    //  Fill large chunks
    //
    __ srliw(cnt_words, count, 3 - shift); // number of words

    // 32 bit -> 64 bit
    __ zext(value, value, 32);
    __ slli(tmp_reg, value, 32);
    __ orr(value, value, tmp_reg);

    __ slli(tmp_reg, cnt_words, 3 - shift);
    __ subw(count, count, tmp_reg);
    {
      __ fill_words(to, cnt_words, value);
    }

    // Handle copies less than 8 bytes.
    // Address may not be heapword aligned.
    Label L_fill_1, L_fill_2, L_exit;
    __ bind(L_fill_elements);
    switch (t) {
      case T_BYTE:
        __ test_bit(tmp_reg, count, 2);
        __ beqz(tmp_reg, L_fill_2);
        __ sb(value, Address(to, 0));
        __ sb(value, Address(to, 1));
        __ sb(value, Address(to, 2));
        __ sb(value, Address(to, 3));
        __ addi(to, to, 4);

        __ bind(L_fill_2);
        __ test_bit(tmp_reg, count, 1);
        __ beqz(tmp_reg, L_fill_1);
        __ sb(value, Address(to, 0));
        __ sb(value, Address(to, 1));
        __ addi(to, to, 2);

        __ bind(L_fill_1);
        __ test_bit(tmp_reg, count, 0);
        __ beqz(tmp_reg, L_exit);
        __ sb(value, Address(to, 0));
        break;
      case T_SHORT:
        __ test_bit(tmp_reg, count, 1);
        __ beqz(tmp_reg, L_fill_2);
        __ sh(value, Address(to, 0));
        __ sh(value, Address(to, 2));
        __ addi(to, to, 4);

        __ bind(L_fill_2);
        __ test_bit(tmp_reg, count, 0);
        __ beqz(tmp_reg, L_exit);
        __ sh(value, Address(to, 0));
        break;
      case T_INT:
        __ beqz(count, L_exit);
        __ sw(value, Address(to, 0));
        break;
      default: ShouldNotReachHere();
    }
    __ bind(L_exit);
    __ leave();
    __ ret();

    return start;
  }

  void generate_arraycopy_stubs() {
    address entry                     = nullptr;
    address entry_jbyte_arraycopy     = nullptr;
    address entry_jshort_arraycopy    = nullptr;
    address entry_jint_arraycopy      = nullptr;
    address entry_oop_arraycopy       = nullptr;
    address entry_jlong_arraycopy     = nullptr;
    address entry_checkcast_arraycopy = nullptr;

    generate_copy_longs(StubGenStubId::copy_byte_f_id, copy_f, c_rarg0, c_rarg1, t1);
    generate_copy_longs(StubGenStubId::copy_byte_b_id, copy_b, c_rarg0, c_rarg1, t1);

    address ucm_common_error_exit     = generate_unsafecopy_common_error_exit();
    UnsafeMemoryAccess::set_common_exit_stub_pc(ucm_common_error_exit);

    StubRoutines::riscv::_zero_blocks = generate_zero_blocks();

    //*** jbyte
    // Always need aligned and unaligned versions
    StubRoutines::_jbyte_disjoint_arraycopy          = generate_disjoint_copy(StubGenStubId::jbyte_disjoint_arraycopy_id, &entry);
    StubRoutines::_jbyte_arraycopy                   = generate_conjoint_copy(StubGenStubId::jbyte_arraycopy_id, entry, &entry_jbyte_arraycopy);
    StubRoutines::_arrayof_jbyte_disjoint_arraycopy  = generate_disjoint_copy(StubGenStubId::arrayof_jbyte_disjoint_arraycopy_id, &entry);
    StubRoutines::_arrayof_jbyte_arraycopy           = generate_conjoint_copy(StubGenStubId::arrayof_jbyte_arraycopy_id, entry, nullptr);

    //*** jshort
    // Always need aligned and unaligned versions
    StubRoutines::_jshort_disjoint_arraycopy         = generate_disjoint_copy(StubGenStubId::jshort_disjoint_arraycopy_id, &entry);
    StubRoutines::_jshort_arraycopy                  = generate_conjoint_copy(StubGenStubId::jshort_arraycopy_id, entry, &entry_jshort_arraycopy);
    StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_copy(StubGenStubId::arrayof_jshort_disjoint_arraycopy_id, &entry);
    StubRoutines::_arrayof_jshort_arraycopy          = generate_conjoint_copy(StubGenStubId::arrayof_jshort_arraycopy_id, entry, nullptr);

    //*** jint
    // Aligned versions
    StubRoutines::_arrayof_jint_disjoint_arraycopy   = generate_disjoint_copy(StubGenStubId::arrayof_jint_disjoint_arraycopy_id, &entry);
    StubRoutines::_arrayof_jint_arraycopy            = generate_conjoint_copy(StubGenStubId::arrayof_jint_arraycopy_id, entry, &entry_jint_arraycopy);
    // In 64 bit we need both aligned and unaligned versions of jint arraycopy.
    // entry_jint_arraycopy always points to the unaligned version
    StubRoutines::_jint_disjoint_arraycopy           = generate_disjoint_copy(StubGenStubId::jint_disjoint_arraycopy_id, &entry);
    StubRoutines::_jint_arraycopy                  = generate_conjoint_copy(StubGenStubId::jint_arraycopy_id, entry, &entry_jint_arraycopy);

    //*** jlong
    // It is always aligned
    StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_copy(StubGenStubId::arrayof_jlong_disjoint_arraycopy_id, &entry);
    StubRoutines::_arrayof_jlong_arraycopy          = generate_conjoint_copy(StubGenStubId::arrayof_jlong_arraycopy_id, entry, &entry_jlong_arraycopy);
    StubRoutines::_jlong_disjoint_arraycopy         = StubRoutines::_arrayof_jlong_disjoint_arraycopy;
    StubRoutines::_jlong_arraycopy                  = StubRoutines::_arrayof_jlong_arraycopy;

    //*** oops
    StubRoutines::_arrayof_oop_disjoint_arraycopy
      = generate_disjoint_copy(StubGenStubId::arrayof_oop_disjoint_arraycopy_id, &entry);
    StubRoutines::_arrayof_oop_arraycopy
      = generate_conjoint_copy(StubGenStubId::arrayof_oop_arraycopy_id, entry, &entry_oop_arraycopy);
    // Aligned versions without pre-barriers
    StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit
      = generate_disjoint_copy(StubGenStubId::arrayof_oop_disjoint_arraycopy_uninit_id, &entry);
    StubRoutines::_arrayof_oop_arraycopy_uninit
      = generate_conjoint_copy(StubGenStubId::arrayof_oop_arraycopy_uninit_id, entry, nullptr);

    StubRoutines::_oop_disjoint_arraycopy            = StubRoutines::_arrayof_oop_disjoint_arraycopy;
    StubRoutines::_oop_arraycopy                     = StubRoutines::_arrayof_oop_arraycopy;
    StubRoutines::_oop_disjoint_arraycopy_uninit     = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit;
    StubRoutines::_oop_arraycopy_uninit              = StubRoutines::_arrayof_oop_arraycopy_uninit;

    StubRoutines::_checkcast_arraycopy        = generate_checkcast_copy(StubGenStubId::checkcast_arraycopy_id, &entry_checkcast_arraycopy);
    StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy(StubGenStubId::checkcast_arraycopy_uninit_id, nullptr);


    StubRoutines::_unsafe_arraycopy    = generate_unsafe_copy(entry_jbyte_arraycopy,
                                                              entry_jshort_arraycopy,
                                                              entry_jint_arraycopy,
                                                              entry_jlong_arraycopy);

    StubRoutines::_generic_arraycopy   = generate_generic_copy(entry_jbyte_arraycopy,
                                                               entry_jshort_arraycopy,
                                                               entry_jint_arraycopy,
                                                               entry_oop_arraycopy,
                                                               entry_jlong_arraycopy,
                                                               entry_checkcast_arraycopy);

    StubRoutines::_jbyte_fill = generate_fill(StubGenStubId::jbyte_fill_id);
    StubRoutines::_jshort_fill = generate_fill(StubGenStubId::jshort_fill_id);
    StubRoutines::_jint_fill = generate_fill(StubGenStubId::jint_fill_id);
    StubRoutines::_arrayof_jbyte_fill = generate_fill(StubGenStubId::arrayof_jbyte_fill_id);
    StubRoutines::_arrayof_jshort_fill = generate_fill(StubGenStubId::arrayof_jshort_fill_id);
    StubRoutines::_arrayof_jint_fill = generate_fill(StubGenStubId::arrayof_jint_fill_id);

    StubRoutines::_unsafe_setmemory    = generate_unsafe_setmemory();
  }

  void generate_aes_loadkeys(const Register &key, VectorRegister *working_vregs, int rounds) {
    const int step = 16;
    for (int i = 0; i < rounds; i++) {
      __ vle32_v(working_vregs[i], key);
      // The keys are stored in little-endian array, while we need
      // to operate in big-endian.
      // So performing an endian-swap here with vrev8.v instruction
      __ vrev8_v(working_vregs[i], working_vregs[i]);
      __ addi(key, key, step);
    }
  }

  void generate_aes_encrypt(const VectorRegister &res, VectorRegister *working_vregs, int rounds) {
    assert(rounds <= 15, "rounds should be less than or equal to working_vregs size");

    __ vxor_vv(res, res, working_vregs[0]);
    for (int i = 1; i < rounds - 1; i++) {
      __ vaesem_vv(res, working_vregs[i]);
    }
    __ vaesef_vv(res, working_vregs[rounds - 1]);
  }

  // Arguments:
  //
  // Inputs:
  //   c_rarg0   - source byte array address
  //   c_rarg1   - destination byte array address
  //   c_rarg2   - K (key) in little endian int array
  //
  address generate_aescrypt_encryptBlock() {
    assert(UseAESIntrinsics, "need AES instructions (Zvkned extension) support");

    __ align(CodeEntryAlignment);
    StubGenStubId stub_id = StubGenStubId::aescrypt_encryptBlock_id;
    StubCodeMark mark(this, stub_id);

    Label L_aes128, L_aes192;

    const Register from        = c_rarg0;  // source array address
    const Register to          = c_rarg1;  // destination array address
    const Register key         = c_rarg2;  // key array address
    const Register keylen      = c_rarg3;

    VectorRegister working_vregs[] = {
      v4, v5, v6, v7, v8, v9, v10, v11,
      v12, v13, v14, v15, v16, v17, v18
    };
    const VectorRegister res   = v19;

    address start = __ pc();
    __ enter();

    __ lwu(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));

    __ vsetivli(x0, 4, Assembler::e32, Assembler::m1);
    __ vle32_v(res, from);

    __ mv(t2, 52);
    __ blt(keylen, t2, L_aes128);
    __ beq(keylen, t2, L_aes192);
    // Else we fallthrough to the biggest case (256-bit key size)

    // Note: the following function performs key += 15*16
    generate_aes_loadkeys(key, working_vregs, 15);
    generate_aes_encrypt(res, working_vregs, 15);
    __ vse32_v(res, to);
    __ mv(c_rarg0, 0);
    __ leave();
    __ ret();

  __ bind(L_aes192);
    // Note: the following function performs key += 13*16
    generate_aes_loadkeys(key, working_vregs, 13);
    generate_aes_encrypt(res, working_vregs, 13);
    __ vse32_v(res, to);
    __ mv(c_rarg0, 0);
    __ leave();
    __ ret();

  __ bind(L_aes128);
    // Note: the following function performs key += 11*16
    generate_aes_loadkeys(key, working_vregs, 11);
    generate_aes_encrypt(res, working_vregs, 11);
    __ vse32_v(res, to);
    __ mv(c_rarg0, 0);
    __ leave();
    __ ret();

    return start;
  }

  void generate_aes_decrypt(const VectorRegister &res, VectorRegister *working_vregs, int rounds) {
    assert(rounds <= 15, "rounds should be less than or equal to working_vregs size");

    __ vxor_vv(res, res, working_vregs[rounds - 1]);
    for (int i = rounds - 2; i > 0; i--) {
      __ vaesdm_vv(res, working_vregs[i]);
    }
    __ vaesdf_vv(res, working_vregs[0]);
  }

  // Arguments:
  //
  // Inputs:
  //   c_rarg0   - source byte array address
  //   c_rarg1   - destination byte array address
  //   c_rarg2   - K (key) in little endian int array
  //
  address generate_aescrypt_decryptBlock() {
    assert(UseAESIntrinsics, "need AES instructions (Zvkned extension) support");

    __ align(CodeEntryAlignment);
    StubGenStubId stub_id = StubGenStubId::aescrypt_decryptBlock_id;
    StubCodeMark mark(this, stub_id);

    Label L_aes128, L_aes192;

    const Register from        = c_rarg0;  // source array address
    const Register to          = c_rarg1;  // destination array address
    const Register key         = c_rarg2;  // key array address
    const Register keylen      = c_rarg3;

    VectorRegister working_vregs[] = {
      v4, v5, v6, v7, v8, v9, v10, v11,
      v12, v13, v14, v15, v16, v17, v18
    };
    const VectorRegister res   = v19;

    address start = __ pc();
    __ enter(); // required for proper stackwalking of RuntimeStub frame

    __ lwu(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));

    __ vsetivli(x0, 4, Assembler::e32, Assembler::m1);
    __ vle32_v(res, from);

    __ mv(t2, 52);
    __ blt(keylen, t2, L_aes128);
    __ beq(keylen, t2, L_aes192);
    // Else we fallthrough to the biggest case (256-bit key size)

    // Note: the following function performs key += 15*16
    generate_aes_loadkeys(key, working_vregs, 15);
    generate_aes_decrypt(res, working_vregs, 15);
    __ vse32_v(res, to);
    __ mv(c_rarg0, 0);
    __ leave();
    __ ret();

  __ bind(L_aes192);
    // Note: the following function performs key += 13*16
    generate_aes_loadkeys(key, working_vregs, 13);
    generate_aes_decrypt(res, working_vregs, 13);
    __ vse32_v(res, to);
    __ mv(c_rarg0, 0);
    __ leave();
    __ ret();

  __ bind(L_aes128);
    // Note: the following function performs key += 11*16
    generate_aes_loadkeys(key, working_vregs, 11);
    generate_aes_decrypt(res, working_vregs, 11);
    __ vse32_v(res, to);
    __ mv(c_rarg0, 0);
    __ leave();
    __ ret();

    return start;
  }

  // code for comparing 8 characters of strings with Latin1 and Utf16 encoding
  void compare_string_8_x_LU(Register tmpL, Register tmpU,
                             Register strL, Register strU, Label& DIFF) {
    const Register tmp = x30, tmpLval = x12;

    int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
    assert((base_offset % (UseCompactObjectHeaders ? 4 :
                           (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");

#ifdef ASSERT
    if (AvoidUnalignedAccesses) {
      Label align_ok;
      __ andi(t0, strL, 0x7);
      __ beqz(t0, align_ok);
      __ stop("bad alignment");
      __ bind(align_ok);
    }
#endif
    __ ld(tmpLval, Address(strL));
    __ addi(strL, strL, wordSize);

    // compare first 4 characters
    __ load_long_misaligned(tmpU, Address(strU), tmp, (base_offset % 8) != 0 ? 4 : 8);
    __ addi(strU, strU, wordSize);
    __ inflate_lo32(tmpL, tmpLval);
    __ xorr(tmp, tmpU, tmpL);
    __ bnez(tmp, DIFF);

    // compare second 4 characters
    __ load_long_misaligned(tmpU, Address(strU), tmp, (base_offset % 8) != 0 ? 4 : 8);
    __ addi(strU, strU, wordSize);
    __ inflate_hi32(tmpL, tmpLval);
    __ xorr(tmp, tmpU, tmpL);
    __ bnez(tmp, DIFF);
  }

  // x10  = result
  // x11  = str1
  // x12  = cnt1
  // x13  = str2
  // x14  = cnt2
  // x28  = tmp1
  // x29  = tmp2
  // x30  = tmp3
  address generate_compare_long_string_different_encoding(StubGenStubId stub_id) {
    bool isLU;
    switch (stub_id) {
    case compare_long_string_LU_id:
      isLU = true;
      break;
    case compare_long_string_UL_id:
      isLU = false;
      break;
    default:
      ShouldNotReachHere();
    };
    __ align(CodeEntryAlignment);
    StubCodeMark mark(this, stub_id);
    address entry = __ pc();
    Label SMALL_LOOP, TAIL, LOAD_LAST, DONE, CALCULATE_DIFFERENCE;
    const Register result = x10, str1 = x11, str2 = x13, cnt2 = x14,
                   tmp1 = x28, tmp2 = x29, tmp3 = x30, tmp4 = x12;

    int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
    assert((base_offset % (UseCompactObjectHeaders ? 4 :
                           (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");

    Register strU = isLU ? str2 : str1,
             strL = isLU ? str1 : str2,
             tmpU = isLU ? tmp2 : tmp1, // where to keep U for comparison
             tmpL = isLU ? tmp1 : tmp2; // where to keep L for comparison

    if (AvoidUnalignedAccesses && (base_offset % 8) != 0) {
      // Load 4 bytes from strL to make sure main loop is 8-byte aligned
      // cnt2 is >= 68 here, no need to check it for >= 0
      __ lwu(tmpL, Address(strL));
      __ addi(strL, strL, wordSize / 2);
      __ load_long_misaligned(tmpU, Address(strU), tmp4, (base_offset % 8) != 0 ? 4 : 8);
      __ addi(strU, strU, wordSize);
      __ inflate_lo32(tmp3, tmpL);
      __ mv(tmpL, tmp3);
      __ xorr(tmp3, tmpU, tmpL);
      __ bnez(tmp3, CALCULATE_DIFFERENCE);
      __ subi(cnt2, cnt2, wordSize / 2);
    }

    // we are now 8-bytes aligned on strL when AvoidUnalignedAccesses is true
    __ subi(cnt2, cnt2, wordSize * 2);
    __ bltz(cnt2, TAIL);
    __ bind(SMALL_LOOP); // smaller loop
      __ subi(cnt2, cnt2, wordSize * 2);
      compare_string_8_x_LU(tmpL, tmpU, strL, strU, CALCULATE_DIFFERENCE);
      compare_string_8_x_LU(tmpL, tmpU, strL, strU, CALCULATE_DIFFERENCE);
      __ bgez(cnt2, SMALL_LOOP);
      __ addi(t0, cnt2, wordSize * 2);
      __ beqz(t0, DONE);
    __ bind(TAIL);  // 1..15 characters left
      // Aligned access. Load bytes in portions - 4, 2, 1.

      __ addi(t0, cnt2, wordSize);
      __ addi(cnt2, cnt2, wordSize * 2); // amount of characters left to process
      __ bltz(t0, LOAD_LAST);
      // remaining characters are greater than or equals to 8, we can do one compare_string_8_x_LU
      compare_string_8_x_LU(tmpL, tmpU, strL, strU, CALCULATE_DIFFERENCE);
      __ subi(cnt2, cnt2, wordSize);
      __ beqz(cnt2, DONE);  // no character left
      __ bind(LOAD_LAST);   // cnt2 = 1..7 characters left

      __ subi(cnt2, cnt2, wordSize); // cnt2 is now an offset in strL which points to last 8 bytes
      __ slli(t0, cnt2, 1);     // t0 is now an offset in strU which points to last 16 bytes
      __ add(strL, strL, cnt2); // Address of last 8 bytes in Latin1 string
      __ add(strU, strU, t0);   // Address of last 16 bytes in UTF-16 string
      __ load_int_misaligned(tmpL, Address(strL), t0, false);
      __ load_long_misaligned(tmpU, Address(strU), t0, 2);
      __ inflate_lo32(tmp3, tmpL);
      __ mv(tmpL, tmp3);
      __ xorr(tmp3, tmpU, tmpL);
      __ bnez(tmp3, CALCULATE_DIFFERENCE);

      __ addi(strL, strL, wordSize / 2); // Address of last 4 bytes in Latin1 string
      __ addi(strU, strU, wordSize);   // Address of last 8 bytes in UTF-16 string
      __ load_int_misaligned(tmpL, Address(strL), t0, false);
      __ load_long_misaligned(tmpU, Address(strU), t0, 2);
      __ inflate_lo32(tmp3, tmpL);
      __ mv(tmpL, tmp3);
      __ xorr(tmp3, tmpU, tmpL);
      __ bnez(tmp3, CALCULATE_DIFFERENCE);
      __ j(DONE); // no character left

      // Find the first different characters in the longwords and
      // compute their difference.
    __ bind(CALCULATE_DIFFERENCE);
      // count bits of trailing zero chars
      __ ctzc_bits(tmp4, tmp3);
      __ srl(tmp1, tmp1, tmp4);
      __ srl(tmp2, tmp2, tmp4);
      __ zext(tmp1, tmp1, 16);
      __ zext(tmp2, tmp2, 16);
      __ sub(result, tmp1, tmp2);
    __ bind(DONE);
      __ ret();
    return entry;
  }

  address generate_method_entry_barrier() {
    __ align(CodeEntryAlignment);
    StubGenStubId stub_id = StubGenStubId::method_entry_barrier_id;
    StubCodeMark mark(this, stub_id);

    Label deoptimize_label;

    address start = __ pc();

    BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();

    if (bs_asm->nmethod_patching_type() == NMethodPatchingType::conc_instruction_and_data_patch) {
      BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
      Address thread_epoch_addr(xthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()) + 4);
      __ la(t1, ExternalAddress(bs_asm->patching_epoch_addr()));
      __ lwu(t1, t1);
      __ sw(t1, thread_epoch_addr);
      // There are two ways this can work:
      // - The writer did system icache shootdown after the instruction stream update.
      //   Hence do nothing.
      // - The writer trust us to make sure our icache is in sync before entering.
      //   Hence use cmodx fence (fence.i, may change).
      if (UseCtxFencei) {
        __ cmodx_fence();
      }
      __ membar(__ LoadLoad);
    }

    __ set_last_Java_frame(sp, fp, ra);

    __ enter();
    __ addi(t1, sp, wordSize);

    __ subi(sp, sp, 4 * wordSize);

    __ push_call_clobbered_registers();

    __ mv(c_rarg0, t1);
    __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetNMethod::nmethod_stub_entry_barrier), 1);

    __ reset_last_Java_frame(true);

    __ mv(t0, x10);

    __ pop_call_clobbered_registers();

    __ bnez(t0, deoptimize_label);

    __ leave();
    __ ret();

    __ BIND(deoptimize_label);

    __ ld(t0, Address(sp, 0));
    __ ld(fp, Address(sp, wordSize));
    __ ld(ra, Address(sp, wordSize * 2));
    __ ld(t1, Address(sp, wordSize * 3));

    __ mv(sp, t0);
    __ jr(t1);

    return start;
  }

  // x10  = result
  // x11  = str1
  // x12  = cnt1
  // x13  = str2
  // x14  = cnt2
  // x28  = tmp1
  // x29  = tmp2
  // x30  = tmp3
  // x31  = tmp4
  address generate_compare_long_string_same_encoding(StubGenStubId stub_id) {
    bool isLL;
    switch (stub_id) {
    case compare_long_string_LL_id:
      isLL = true;
      break;
    case compare_long_string_UU_id:
      isLL = false;
      break;
    default:
      ShouldNotReachHere();
    };
    __ align(CodeEntryAlignment);
    StubCodeMark mark(this, stub_id);
    address entry = __ pc();
    Label SMALL_LOOP, CHECK_LAST, DIFF2, TAIL,
          LENGTH_DIFF, DIFF, LAST_CHECK_AND_LENGTH_DIFF;
    const Register result = x10, str1 = x11, cnt1 = x12, str2 = x13, cnt2 = x14,
                   tmp1 = x28, tmp2 = x29, tmp3 = x30, tmp4 = x7, tmp5 = x31;
    RegSet spilled_regs = RegSet::of(tmp4, tmp5);

    // cnt1/cnt2 contains amount of characters to compare. cnt1 can be re-used
    // update cnt2 counter with already loaded 8 bytes
    __ subi(cnt2, cnt2, wordSize / (isLL ? 1 : 2));
    // update pointers, because of previous read
    __ addi(str1, str1, wordSize);
    __ addi(str2, str2, wordSize);
    // less than 16 bytes left?
    __ subi(cnt2, cnt2, isLL ? 16 : 8);
    __ push_reg(spilled_regs, sp);
    __ bltz(cnt2, TAIL);
    __ bind(SMALL_LOOP);
      // compare 16 bytes of strings with same encoding
      __ ld(tmp5, Address(str1));
      __ addi(str1, str1, 8);
      __ xorr(tmp4, tmp1, tmp2);
      __ ld(cnt1, Address(str2));
      __ addi(str2, str2, 8);
      __ bnez(tmp4, DIFF);
      __ ld(tmp1, Address(str1));
      __ addi(str1, str1, 8);
      __ xorr(tmp4, tmp5, cnt1);
      __ ld(tmp2, Address(str2));
      __ addi(str2, str2, 8);
      __ bnez(tmp4, DIFF2);

      __ subi(cnt2, cnt2, isLL ? 16 : 8);
      __ bgez(cnt2, SMALL_LOOP);
    __ bind(TAIL);
      __ addi(cnt2, cnt2, isLL ? 16 : 8);
      __ beqz(cnt2, LAST_CHECK_AND_LENGTH_DIFF);
      __ subi(cnt2, cnt2, isLL ? 8 : 4);
      __ blez(cnt2, CHECK_LAST);
      __ xorr(tmp4, tmp1, tmp2);
      __ bnez(tmp4, DIFF);
      __ ld(tmp1, Address(str1));
      __ addi(str1, str1, 8);
      __ ld(tmp2, Address(str2));
      __ addi(str2, str2, 8);
      __ subi(cnt2, cnt2, isLL ? 8 : 4);
    __ bind(CHECK_LAST);
      if (!isLL) {
        __ add(cnt2, cnt2, cnt2); // now in bytes
      }
      __ xorr(tmp4, tmp1, tmp2);
      __ bnez(tmp4, DIFF);
      __ add(str1, str1, cnt2);
      __ load_long_misaligned(tmp5, Address(str1), tmp3, isLL ? 1 : 2);
      __ add(str2, str2, cnt2);
      __ load_long_misaligned(cnt1, Address(str2), tmp3, isLL ? 1 : 2);
      __ xorr(tmp4, tmp5, cnt1);
      __ beqz(tmp4, LENGTH_DIFF);
      // Find the first different characters in the longwords and
      // compute their difference.
    __ bind(DIFF2);
      // count bits of trailing zero chars
      __ ctzc_bits(tmp3, tmp4, isLL);
      __ srl(tmp5, tmp5, tmp3);
      __ srl(cnt1, cnt1, tmp3);
      if (isLL) {
        __ zext(tmp5, tmp5, 8);
        __ zext(cnt1, cnt1, 8);
      } else {
        __ zext(tmp5, tmp5, 16);
        __ zext(cnt1, cnt1, 16);
      }
      __ sub(result, tmp5, cnt1);
      __ j(LENGTH_DIFF);
    __ bind(DIFF);
      // count bits of trailing zero chars
      __ ctzc_bits(tmp3, tmp4, isLL);
      __ srl(tmp1, tmp1, tmp3);
      __ srl(tmp2, tmp2, tmp3);
      if (isLL) {
        __ zext(tmp1, tmp1, 8);
        __ zext(tmp2, tmp2, 8);
      } else {
        __ zext(tmp1, tmp1, 16);
        __ zext(tmp2, tmp2, 16);
      }
      __ sub(result, tmp1, tmp2);
      __ j(LENGTH_DIFF);
    __ bind(LAST_CHECK_AND_LENGTH_DIFF);
      __ xorr(tmp4, tmp1, tmp2);
      __ bnez(tmp4, DIFF);
    __ bind(LENGTH_DIFF);
      __ pop_reg(spilled_regs, sp);
      __ ret();
    return entry;
  }

  void generate_compare_long_strings() {
    StubRoutines::riscv::_compare_long_string_LL = generate_compare_long_string_same_encoding(StubGenStubId::compare_long_string_LL_id);
    StubRoutines::riscv::_compare_long_string_UU = generate_compare_long_string_same_encoding(StubGenStubId::compare_long_string_UU_id);
    StubRoutines::riscv::_compare_long_string_LU = generate_compare_long_string_different_encoding(StubGenStubId::compare_long_string_LU_id);
    StubRoutines::riscv::_compare_long_string_UL = generate_compare_long_string_different_encoding(StubGenStubId::compare_long_string_UL_id);
  }

  // x10 result
  // x11 src
  // x12 src count
  // x13 pattern
  // x14 pattern count
  address generate_string_indexof_linear(StubGenStubId stub_id)
  {
    bool needle_isL;
    bool haystack_isL;
    switch (stub_id) {
    case string_indexof_linear_ll_id:
      needle_isL = true;
      haystack_isL = true;
      break;
    case string_indexof_linear_ul_id:
      needle_isL = true;
      haystack_isL = false;
      break;
    case string_indexof_linear_uu_id:
      needle_isL = false;
      haystack_isL = false;
      break;
    default:
      ShouldNotReachHere();
    };

    __ align(CodeEntryAlignment);
    StubCodeMark mark(this, stub_id);
    address entry = __ pc();

    int needle_chr_size = needle_isL ? 1 : 2;
    int haystack_chr_size = haystack_isL ? 1 : 2;
    int needle_chr_shift = needle_isL ? 0 : 1;
    int haystack_chr_shift = haystack_isL ? 0 : 1;
    bool isL = needle_isL && haystack_isL;
    // parameters
    Register result = x10, haystack = x11, haystack_len = x12, needle = x13, needle_len = x14;
    // temporary registers
    Register mask1 = x20, match_mask = x21, first = x22, trailing_zeros = x23, mask2 = x24, tmp = x25;
    // redefinitions
    Register ch1 = x28, ch2 = x29;
    RegSet spilled_regs = RegSet::range(x20, x25) + RegSet::range(x28, x29);

    __ push_reg(spilled_regs, sp);

    Label L_LOOP, L_LOOP_PROCEED, L_SMALL, L_HAS_ZERO,
          L_HAS_ZERO_LOOP, L_CMP_LOOP, L_CMP_LOOP_NOMATCH, L_SMALL_PROCEED,
          L_SMALL_HAS_ZERO_LOOP, L_SMALL_CMP_LOOP_NOMATCH, L_SMALL_CMP_LOOP,
          L_POST_LOOP, L_CMP_LOOP_LAST_CMP, L_HAS_ZERO_LOOP_NOMATCH,
          L_SMALL_CMP_LOOP_LAST_CMP, L_SMALL_CMP_LOOP_LAST_CMP2,
          L_CMP_LOOP_LAST_CMP2, DONE, NOMATCH;

    __ ld(ch1, Address(needle));
    __ ld(ch2, Address(haystack));
    // src.length - pattern.length
    __ sub(haystack_len, haystack_len, needle_len);

    // first is needle[0]
    __ zext(first, ch1, needle_isL ? 8 : 16);

    uint64_t mask0101 = UCONST64(0x0101010101010101);
    uint64_t mask0001 = UCONST64(0x0001000100010001);
    __ mv(mask1, haystack_isL ? mask0101 : mask0001);
    __ mul(first, first, mask1);
    uint64_t mask7f7f = UCONST64(0x7f7f7f7f7f7f7f7f);
    uint64_t mask7fff = UCONST64(0x7fff7fff7fff7fff);
    __ mv(mask2, haystack_isL ? mask7f7f : mask7fff);
    if (needle_isL != haystack_isL) {
      __ mv(tmp, ch1);
    }
    __ subi(haystack_len, haystack_len, wordSize / haystack_chr_size - 1);
    __ blez(haystack_len, L_SMALL);

    if (needle_isL != haystack_isL) {
      __ inflate_lo32(ch1, tmp, match_mask, trailing_zeros);
    }
    // xorr, sub, orr, notr, andr
    // compare and set match_mask[i] with 0x80/0x8000 (Latin1/UTF16) if ch2[i] == first[i]
    // eg:
    // first:        aa aa aa aa aa aa aa aa
    // ch2:          aa aa li nx jd ka aa aa
    // match_mask:   80 80 00 00 00 00 80 80
    __ compute_match_mask(ch2, first, match_mask, mask1, mask2);

    // search first char of needle, if success, goto L_HAS_ZERO;
    __ bnez(match_mask, L_HAS_ZERO);
    __ subi(haystack_len, haystack_len, wordSize / haystack_chr_size);
    __ addi(result, result, wordSize / haystack_chr_size);
    __ addi(haystack, haystack, wordSize);
    __ bltz(haystack_len, L_POST_LOOP);

    __ bind(L_LOOP);
    __ ld(ch2, Address(haystack));
    __ compute_match_mask(ch2, first, match_mask, mask1, mask2);
    __ bnez(match_mask, L_HAS_ZERO);

    __ bind(L_LOOP_PROCEED);
    __ subi(haystack_len, haystack_len, wordSize / haystack_chr_size);
    __ addi(haystack, haystack, wordSize);
    __ addi(result, result, wordSize / haystack_chr_size);
    __ bgez(haystack_len, L_LOOP);

    __ bind(L_POST_LOOP);
    __ mv(ch2, -wordSize / haystack_chr_size);
    __ ble(haystack_len, ch2, NOMATCH); // no extra characters to check
    __ ld(ch2, Address(haystack));
    __ slli(haystack_len, haystack_len, LogBitsPerByte + haystack_chr_shift);
    __ neg(haystack_len, haystack_len);
    __ xorr(ch2, first, ch2);
    __ sub(match_mask, ch2, mask1);
    __ orr(ch2, ch2, mask2);
    __ mv(trailing_zeros, -1); // all bits set
    __ j(L_SMALL_PROCEED);

    __ align(OptoLoopAlignment);
    __ bind(L_SMALL);
    __ slli(haystack_len, haystack_len, LogBitsPerByte + haystack_chr_shift);
    __ neg(haystack_len, haystack_len);
    if (needle_isL != haystack_isL) {
      __ inflate_lo32(ch1, tmp, match_mask, trailing_zeros);
    }
    __ xorr(ch2, first, ch2);
    __ sub(match_mask, ch2, mask1);
    __ orr(ch2, ch2, mask2);
    __ mv(trailing_zeros, -1); // all bits set

    __ bind(L_SMALL_PROCEED);
    __ srl(trailing_zeros, trailing_zeros, haystack_len); // mask. zeroes on useless bits.
    __ notr(ch2, ch2);
    __ andr(match_mask, match_mask, ch2);
    __ andr(match_mask, match_mask, trailing_zeros); // clear useless bits and check
    __ beqz(match_mask, NOMATCH);

    __ bind(L_SMALL_HAS_ZERO_LOOP);
    // count bits of trailing zero chars
    __ ctzc_bits(trailing_zeros, match_mask, haystack_isL, ch2, tmp);
    __ addi(trailing_zeros, trailing_zeros, haystack_isL ? 7 : 15);
    __ mv(ch2, wordSize / haystack_chr_size);
    __ ble(needle_len, ch2, L_SMALL_CMP_LOOP_LAST_CMP2);
    __ compute_index(haystack, trailing_zeros, match_mask, result, ch2, tmp, haystack_isL);
    __ mv(trailing_zeros, wordSize / haystack_chr_size);
    __ bne(ch1, ch2, L_SMALL_CMP_LOOP_NOMATCH);

    __ bind(L_SMALL_CMP_LOOP);
    __ shadd(first, trailing_zeros, needle, first, needle_chr_shift);
    __ shadd(ch2, trailing_zeros, haystack, ch2, haystack_chr_shift);
    needle_isL ? __ lbu(first, Address(first)) : __ lhu(first, Address(first));
    haystack_isL ? __ lbu(ch2, Address(ch2)) : __ lhu(ch2, Address(ch2));
    __ addi(trailing_zeros, trailing_zeros, 1);
    __ bge(trailing_zeros, needle_len, L_SMALL_CMP_LOOP_LAST_CMP);
    __ beq(first, ch2, L_SMALL_CMP_LOOP);

    __ bind(L_SMALL_CMP_LOOP_NOMATCH);
    __ beqz(match_mask, NOMATCH);
    // count bits of trailing zero chars
    __ ctzc_bits(trailing_zeros, match_mask, haystack_isL, tmp, ch2);
    __ addi(trailing_zeros, trailing_zeros, haystack_isL ? 7 : 15);
    __ addi(result, result, 1);
    __ addi(haystack, haystack, haystack_chr_size);
    __ j(L_SMALL_HAS_ZERO_LOOP);

    __ align(OptoLoopAlignment);
    __ bind(L_SMALL_CMP_LOOP_LAST_CMP);
    __ bne(first, ch2, L_SMALL_CMP_LOOP_NOMATCH);
    __ j(DONE);

    __ align(OptoLoopAlignment);
    __ bind(L_SMALL_CMP_LOOP_LAST_CMP2);
    __ compute_index(haystack, trailing_zeros, match_mask, result, ch2, tmp, haystack_isL);
    __ bne(ch1, ch2, L_SMALL_CMP_LOOP_NOMATCH);
    __ j(DONE);

    __ align(OptoLoopAlignment);
    __ bind(L_HAS_ZERO);
    // count bits of trailing zero chars
    __ ctzc_bits(trailing_zeros, match_mask, haystack_isL, tmp, ch2);
    __ addi(trailing_zeros, trailing_zeros, haystack_isL ? 7 : 15);
    __ slli(needle_len, needle_len, BitsPerByte * wordSize / 2);
    __ orr(haystack_len, haystack_len, needle_len); // restore needle_len(32bits)
    __ subi(result, result, 1); // array index from 0, so result -= 1

    __ bind(L_HAS_ZERO_LOOP);
    __ mv(needle_len, wordSize / haystack_chr_size);
    __ srli(ch2, haystack_len, BitsPerByte * wordSize / 2);
    __ bge(needle_len, ch2, L_CMP_LOOP_LAST_CMP2);
    // load next 8 bytes from haystack, and increase result index
    __ compute_index(haystack, trailing_zeros, match_mask, result, ch2, tmp, haystack_isL);
    __ addi(result, result, 1);
    __ mv(trailing_zeros, wordSize / haystack_chr_size);
    __ bne(ch1, ch2, L_CMP_LOOP_NOMATCH);

    // compare one char
    __ bind(L_CMP_LOOP);
    __ shadd(needle_len, trailing_zeros, needle, needle_len, needle_chr_shift);
    needle_isL ? __ lbu(needle_len, Address(needle_len)) : __ lhu(needle_len, Address(needle_len));
    __ shadd(ch2, trailing_zeros, haystack, ch2, haystack_chr_shift);
    haystack_isL ? __ lbu(ch2, Address(ch2)) : __ lhu(ch2, Address(ch2));
    __ addi(trailing_zeros, trailing_zeros, 1); // next char index
    __ srli(tmp, haystack_len, BitsPerByte * wordSize / 2);
    __ bge(trailing_zeros, tmp, L_CMP_LOOP_LAST_CMP);
    __ beq(needle_len, ch2, L_CMP_LOOP);

    __ bind(L_CMP_LOOP_NOMATCH);
    __ beqz(match_mask, L_HAS_ZERO_LOOP_NOMATCH);
    // count bits of trailing zero chars
    __ ctzc_bits(trailing_zeros, match_mask, haystack_isL, needle_len, ch2);
    __ addi(trailing_zeros, trailing_zeros, haystack_isL ? 7 : 15);
    __ addi(haystack, haystack, haystack_chr_size);
    __ j(L_HAS_ZERO_LOOP);

    __ align(OptoLoopAlignment);
    __ bind(L_CMP_LOOP_LAST_CMP);
    __ bne(needle_len, ch2, L_CMP_LOOP_NOMATCH);
    __ j(DONE);

    __ align(OptoLoopAlignment);
    __ bind(L_CMP_LOOP_LAST_CMP2);
    __ compute_index(haystack, trailing_zeros, match_mask, result, ch2, tmp, haystack_isL);
    __ addi(result, result, 1);
    __ bne(ch1, ch2, L_CMP_LOOP_NOMATCH);
    __ j(DONE);

    __ align(OptoLoopAlignment);
    __ bind(L_HAS_ZERO_LOOP_NOMATCH);
    // 1) Restore "result" index. Index was wordSize/str2_chr_size * N until
    // L_HAS_ZERO block. Byte octet was analyzed in L_HAS_ZERO_LOOP,
    // so, result was increased at max by wordSize/str2_chr_size - 1, so,
    // respective high bit wasn't changed. L_LOOP_PROCEED will increase
    // result by analyzed characters value, so, we can just reset lower bits
    // in result here. Clear 2 lower bits for UU/UL and 3 bits for LL
    // 2) restore needle_len and haystack_len values from "compressed" haystack_len
    // 3) advance haystack value to represent next haystack octet. result & 7/3 is
    // index of last analyzed substring inside current octet. So, haystack in at
    // respective start address. We need to advance it to next octet
    __ andi(match_mask, result, wordSize / haystack_chr_size - 1);
    __ srli(needle_len, haystack_len, BitsPerByte * wordSize / 2);
    __ andi(result, result, haystack_isL ? -8 : -4);
    __ slli(tmp, match_mask, haystack_chr_shift);
    __ sub(haystack, haystack, tmp);
    __ sext(haystack_len, haystack_len, 32);
    __ j(L_LOOP_PROCEED);

    __ align(OptoLoopAlignment);
    __ bind(NOMATCH);
    __ mv(result, -1);

    __ bind(DONE);
    __ pop_reg(spilled_regs, sp);
    __ ret();
    return entry;
  }

  void generate_string_indexof_stubs()
  {
    StubRoutines::riscv::_string_indexof_linear_ll = generate_string_indexof_linear(StubGenStubId::string_indexof_linear_ll_id);
    StubRoutines::riscv::_string_indexof_linear_uu = generate_string_indexof_linear(StubGenStubId::string_indexof_linear_uu_id);
    StubRoutines::riscv::_string_indexof_linear_ul = generate_string_indexof_linear(StubGenStubId::string_indexof_linear_ul_id);
  }

#ifdef COMPILER2
  void generate_lookup_secondary_supers_table_stub() {
    StubGenStubId stub_id = StubGenStubId::lookup_secondary_supers_table_id;
    StubCodeMark mark(this, stub_id);

    const Register
      r_super_klass  = x10,
      r_array_base   = x11,
      r_array_length = x12,
      r_array_index  = x13,
      r_sub_klass    = x14,
      result         = x15,
      r_bitmap       = x16;

    for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) {
      StubRoutines::_lookup_secondary_supers_table_stubs[slot] = __ pc();
      Label L_success;
      __ enter();
      __ lookup_secondary_supers_table_const(r_sub_klass, r_super_klass, result,
                                             r_array_base, r_array_length, r_array_index,
                                             r_bitmap, slot, /*stub_is_near*/true);
      __ leave();
      __ ret();
    }
  }

  // Slow path implementation for UseSecondarySupersTable.
  address generate_lookup_secondary_supers_table_slow_path_stub() {
    StubGenStubId stub_id = StubGenStubId::lookup_secondary_supers_table_slow_path_id;
    StubCodeMark mark(this, stub_id);

    address start = __ pc();
    const Register
      r_super_klass  = x10,        // argument
      r_array_base   = x11,        // argument
      temp1          = x12,        // tmp
      r_array_index  = x13,        // argument
      result         = x15,        // argument
      r_bitmap       = x16;        // argument


    __ lookup_secondary_supers_table_slow_path(r_super_klass, r_array_base, r_array_index, r_bitmap, result, temp1);
    __ ret();

    return start;
  }

  address generate_mulAdd()
  {
    __ align(CodeEntryAlignment);
    StubGenStubId stub_id = StubGenStubId::mulAdd_id;
    StubCodeMark mark(this, stub_id);

    address entry = __ pc();

    const Register out     = x10;
    const Register in      = x11;
    const Register offset  = x12;
    const Register len     = x13;
    const Register k       = x14;
    const Register tmp     = x28;

    BLOCK_COMMENT("Entry:");
    __ enter();
    __ mul_add(out, in, offset, len, k, tmp);
    __ leave();
    __ ret();

    return entry;
  }

  /**
   *  Arguments:
   *
   *  Input:
   *    c_rarg0   - x address
   *    c_rarg1   - x length
   *    c_rarg2   - y address
   *    c_rarg3   - y length
   *    c_rarg4   - z address
   */
  address generate_multiplyToLen()
  {
    __ align(CodeEntryAlignment);
    StubGenStubId stub_id = StubGenStubId::multiplyToLen_id;
    StubCodeMark mark(this, stub_id);
    address entry = __ pc();

    const Register x     = x10;
    const Register xlen  = x11;
    const Register y     = x12;
    const Register ylen  = x13;
    const Register z     = x14;

    const Register tmp0  = x15;
    const Register tmp1  = x16;
    const Register tmp2  = x17;
    const Register tmp3  = x7;
    const Register tmp4  = x28;
    const Register tmp5  = x29;
    const Register tmp6  = x30;
    const Register tmp7  = x31;

    BLOCK_COMMENT("Entry:");
    __ enter(); // required for proper stackwalking of RuntimeStub frame
    __ multiply_to_len(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7);
    __ leave(); // required for proper stackwalking of RuntimeStub frame
    __ ret();

    return entry;
  }

  address generate_squareToLen()
  {
    __ align(CodeEntryAlignment);
    StubGenStubId stub_id = StubGenStubId::squareToLen_id;
    StubCodeMark mark(this, stub_id);
    address entry = __ pc();

    const Register x     = x10;
    const Register xlen  = x11;
    const Register z     = x12;
    const Register y     = x14; // == x
    const Register ylen  = x15; // == xlen

    const Register tmp0  = x13; // zlen, unused
    const Register tmp1  = x16;
    const Register tmp2  = x17;
    const Register tmp3  = x7;
    const Register tmp4  = x28;
    const Register tmp5  = x29;
    const Register tmp6  = x30;
    const Register tmp7  = x31;

    BLOCK_COMMENT("Entry:");
    __ enter();
    __ mv(y, x);
    __ mv(ylen, xlen);
    __ multiply_to_len(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7);
    __ leave();
    __ ret();

    return entry;
  }

  // Arguments:
  //
  // Input:
  //   c_rarg0   - newArr address
  //   c_rarg1   - oldArr address
  //   c_rarg2   - newIdx
  //   c_rarg3   - shiftCount
  //   c_rarg4   - numIter
  //
  address generate_bigIntegerLeftShift() {
    __ align(CodeEntryAlignment);
    StubGenStubId stub_id = StubGenStubId::bigIntegerLeftShiftWorker_id;
    StubCodeMark mark(this, stub_id);
    address entry = __ pc();

    Label loop, exit;

    Register newArr        = c_rarg0;
    Register oldArr        = c_rarg1;
    Register newIdx        = c_rarg2;
    Register shiftCount    = c_rarg3;
    Register numIter       = c_rarg4;

    Register shiftRevCount = c_rarg5;
    Register oldArrNext    = t1;

    __ beqz(numIter, exit);
    __ shadd(newArr, newIdx, newArr, t0, 2);

    __ mv(shiftRevCount, 32);
    __ sub(shiftRevCount, shiftRevCount, shiftCount);

    __ bind(loop);
    __ addi(oldArrNext, oldArr, 4);
    __ vsetvli(t0, numIter, Assembler::e32, Assembler::m4);
    __ vle32_v(v0, oldArr);
    __ vle32_v(v4, oldArrNext);
    __ vsll_vx(v0, v0, shiftCount);
    __ vsrl_vx(v4, v4, shiftRevCount);
    __ vor_vv(v0, v0, v4);
    __ vse32_v(v0, newArr);
    __ sub(numIter, numIter, t0);
    __ shadd(oldArr, t0, oldArr, t1, 2);
    __ shadd(newArr, t0, newArr, t1, 2);
    __ bnez(numIter, loop);

    __ bind(exit);
    __ ret();

    return entry;
  }

  // Arguments:
  //
  // Input:
  //   c_rarg0   - newArr address
  //   c_rarg1   - oldArr address
  //   c_rarg2   - newIdx
  //   c_rarg3   - shiftCount
  //   c_rarg4   - numIter
  //
  address generate_bigIntegerRightShift() {
    __ align(CodeEntryAlignment);
    StubGenStubId stub_id = StubGenStubId::bigIntegerRightShiftWorker_id;
    StubCodeMark mark(this, stub_id);
    address entry = __ pc();

    Label loop, exit;

    Register newArr        = c_rarg0;
    Register oldArr        = c_rarg1;
    Register newIdx        = c_rarg2;
    Register shiftCount    = c_rarg3;
    Register numIter       = c_rarg4;
    Register idx           = numIter;

    Register shiftRevCount = c_rarg5;
    Register oldArrNext    = c_rarg6;
    Register newArrCur     = t0;
    Register oldArrCur     = t1;

    __ beqz(idx, exit);
    __ shadd(newArr, newIdx, newArr, t0, 2);

    __ mv(shiftRevCount, 32);
    __ sub(shiftRevCount, shiftRevCount, shiftCount);

    __ bind(loop);
    __ vsetvli(t0, idx, Assembler::e32, Assembler::m4);
    __ sub(idx, idx, t0);
    __ shadd(oldArrNext, idx, oldArr, t1, 2);
    __ shadd(newArrCur, idx, newArr, t1, 2);
    __ addi(oldArrCur, oldArrNext, 4);
    __ vle32_v(v0, oldArrCur);
    __ vle32_v(v4, oldArrNext);
    __ vsrl_vx(v0, v0, shiftCount);
    __ vsll_vx(v4, v4, shiftRevCount);
    __ vor_vv(v0, v0, v4);
    __ vse32_v(v0, newArrCur);
    __ bnez(idx, loop);

    __ bind(exit);
    __ ret();

    return entry;
  }
#endif

#ifdef COMPILER2
  class MontgomeryMultiplyGenerator : public MacroAssembler {

    Register Pa_base, Pb_base, Pn_base, Pm_base, inv, Rlen, Ra, Rb, Rm, Rn,
      Pa, Pb, Pn, Pm, Rhi_ab, Rlo_ab, Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2, Ri, Rj;

    RegSet _toSave;
    bool _squaring;

  public:
    MontgomeryMultiplyGenerator (Assembler *as, bool squaring)
      : MacroAssembler(as->code()), _squaring(squaring) {

      // Register allocation

      RegSetIterator<Register> regs = RegSet::range(x10, x26).begin();
      Pa_base = *regs;       // Argument registers
      if (squaring) {
        Pb_base = Pa_base;
      } else {
        Pb_base = *++regs;
      }
      Pn_base = *++regs;
      Rlen= *++regs;
      inv = *++regs;
      Pm_base = *++regs;

                        // Working registers:
      Ra =  *++regs;    // The current digit of a, b, n, and m.
      Rb =  *++regs;
      Rm =  *++regs;
      Rn =  *++regs;

      Pa =  *++regs;      // Pointers to the current/next digit of a, b, n, and m.
      Pb =  *++regs;
      Pm =  *++regs;
      Pn =  *++regs;

      tmp0 =  *++regs;    // Three registers which form a
      tmp1 =  *++regs;    // triple-precision accumuator.
      tmp2 =  *++regs;

      Ri =  x6;         // Inner and outer loop indexes.
      Rj =  x7;

      Rhi_ab = x28;     // Product registers: low and high parts
      Rlo_ab = x29;     // of a*b and m*n.
      Rhi_mn = x30;
      Rlo_mn = x31;

      // x18 and up are callee-saved.
      _toSave = RegSet::range(x18, *regs) + Pm_base;
    }

  private:
    void save_regs() {
      push_reg(_toSave, sp);
    }

    void restore_regs() {
      pop_reg(_toSave, sp);
    }

    template <typename T>
    void unroll_2(Register count, T block) {
      Label loop, end, odd;
      beqz(count, end);
      test_bit(t0, count, 0);
      bnez(t0, odd);
      align(16);
      bind(loop);
      (this->*block)();
      bind(odd);
      (this->*block)();
      subi(count, count, 2);
      bgtz(count, loop);
      bind(end);
    }

    template <typename T>
    void unroll_2(Register count, T block, Register d, Register s, Register tmp) {
      Label loop, end, odd;
      beqz(count, end);
      test_bit(tmp, count, 0);
      bnez(tmp, odd);
      align(16);
      bind(loop);
      (this->*block)(d, s, tmp);
      bind(odd);
      (this->*block)(d, s, tmp);
      subi(count, count, 2);
      bgtz(count, loop);
      bind(end);
    }

    void pre1(RegisterOrConstant i) {
      block_comment("pre1");
      // Pa = Pa_base;
      // Pb = Pb_base + i;
      // Pm = Pm_base;
      // Pn = Pn_base + i;
      // Ra = *Pa;
      // Rb = *Pb;
      // Rm = *Pm;
      // Rn = *Pn;
      if (i.is_register()) {
        slli(t0, i.as_register(), LogBytesPerWord);
      } else {
        mv(t0, i.as_constant());
        slli(t0, t0, LogBytesPerWord);
      }

      mv(Pa, Pa_base);
      add(Pb, Pb_base, t0);
      mv(Pm, Pm_base);
      add(Pn, Pn_base, t0);

      ld(Ra, Address(Pa));
      ld(Rb, Address(Pb));
      ld(Rm, Address(Pm));
      ld(Rn, Address(Pn));

      // Zero the m*n result.
      mv(Rhi_mn, zr);
      mv(Rlo_mn, zr);
    }

    // The core multiply-accumulate step of a Montgomery
    // multiplication.  The idea is to schedule operations as a
    // pipeline so that instructions with long latencies (loads and
    // multiplies) have time to complete before their results are
    // used.  This most benefits in-order implementations of the
    // architecture but out-of-order ones also benefit.
    void step() {
      block_comment("step");
      // MACC(Ra, Rb, tmp0, tmp1, tmp2);
      // Ra = *++Pa;
      // Rb = *--Pb;
      mulhu(Rhi_ab, Ra, Rb);
      mul(Rlo_ab, Ra, Rb);
      addi(Pa, Pa, wordSize);
      ld(Ra, Address(Pa));
      subi(Pb, Pb, wordSize);
      ld(Rb, Address(Pb));
      acc(Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2); // The pending m*n from the
                                            // previous iteration.
      // MACC(Rm, Rn, tmp0, tmp1, tmp2);
      // Rm = *++Pm;
      // Rn = *--Pn;
      mulhu(Rhi_mn, Rm, Rn);
      mul(Rlo_mn, Rm, Rn);
      addi(Pm, Pm, wordSize);
      ld(Rm, Address(Pm));
      subi(Pn, Pn, wordSize);
      ld(Rn, Address(Pn));
      acc(Rhi_ab, Rlo_ab, tmp0, tmp1, tmp2);
    }

    void post1() {
      block_comment("post1");

      // MACC(Ra, Rb, tmp0, tmp1, tmp2);
      // Ra = *++Pa;
      // Rb = *--Pb;
      mulhu(Rhi_ab, Ra, Rb);
      mul(Rlo_ab, Ra, Rb);
      acc(Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2);  // The pending m*n
      acc(Rhi_ab, Rlo_ab, tmp0, tmp1, tmp2);

      // *Pm = Rm = tmp0 * inv;
      mul(Rm, tmp0, inv);
      sd(Rm, Address(Pm));

      // MACC(Rm, Rn, tmp0, tmp1, tmp2);
      // tmp0 = tmp1; tmp1 = tmp2; tmp2 = 0;
      mulhu(Rhi_mn, Rm, Rn);

#ifndef PRODUCT
      // assert(m[i] * n[0] + tmp0 == 0, "broken Montgomery multiply");
      {
        mul(Rlo_mn, Rm, Rn);
        add(Rlo_mn, tmp0, Rlo_mn);
        Label ok;
        beqz(Rlo_mn, ok);
        stop("broken Montgomery multiply");
        bind(ok);
      }
#endif
      // We have very carefully set things up so that
      // m[i]*n[0] + tmp0 == 0 (mod b), so we don't have to calculate
      // the lower half of Rm * Rn because we know the result already:
      // it must be -tmp0.  tmp0 + (-tmp0) must generate a carry iff
      // tmp0 != 0.  So, rather than do a mul and an cad we just set
      // the carry flag iff tmp0 is nonzero.
      //
      // mul(Rlo_mn, Rm, Rn);
      // cad(zr, tmp0, Rlo_mn);
      subi(t0, tmp0, 1);
      sltu(t0, t0, tmp0); // Set carry iff tmp0 is nonzero
      cadc(tmp0, tmp1, Rhi_mn, t0);
      adc(tmp1, tmp2, zr, t0);
      mv(tmp2, zr);
    }

    void pre2(Register i, Register len) {
      block_comment("pre2");
      // Pa = Pa_base + i-len;
      // Pb = Pb_base + len;
      // Pm = Pm_base + i-len;
      // Pn = Pn_base + len;

      sub(Rj, i, len);
      // Rj == i-len

      // Ra as temp register
      slli(Ra, Rj, LogBytesPerWord);
      add(Pa, Pa_base, Ra);
      add(Pm, Pm_base, Ra);
      slli(Ra, len, LogBytesPerWord);
      add(Pb, Pb_base, Ra);
      add(Pn, Pn_base, Ra);

      // Ra = *++Pa;
      // Rb = *--Pb;
      // Rm = *++Pm;
      // Rn = *--Pn;
      addi(Pa, Pa, wordSize);
      ld(Ra, Address(Pa));
      subi(Pb, Pb, wordSize);
      ld(Rb, Address(Pb));
      addi(Pm, Pm, wordSize);
      ld(Rm, Address(Pm));
      subi(Pn, Pn, wordSize);
      ld(Rn, Address(Pn));

      mv(Rhi_mn, zr);
      mv(Rlo_mn, zr);
    }

    void post2(Register i, Register len) {
      block_comment("post2");
      sub(Rj, i, len);

      cad(tmp0, tmp0, Rlo_mn, t0); // The pending m*n, low part

      // As soon as we know the least significant digit of our result,
      // store it.
      // Pm_base[i-len] = tmp0;
      // Rj as temp register
      slli(Rj, Rj, LogBytesPerWord);
      add(Rj, Pm_base, Rj);
      sd(tmp0, Address(Rj));

      // tmp0 = tmp1; tmp1 = tmp2; tmp2 = 0;
      cadc(tmp0, tmp1, Rhi_mn, t0); // The pending m*n, high part
      adc(tmp1, tmp2, zr, t0);
      mv(tmp2, zr);
    }

    // A carry in tmp0 after Montgomery multiplication means that we
    // should subtract multiples of n from our result in m.  We'll
    // keep doing that until there is no carry.
    void normalize(Register len) {
      block_comment("normalize");
      // while (tmp0)
      //   tmp0 = sub(Pm_base, Pn_base, tmp0, len);
      Label loop, post, again;
      Register cnt = tmp1, i = tmp2; // Re-use registers; we're done with them now
      beqz(tmp0, post); {
        bind(again); {
          mv(i, zr);
          mv(cnt, len);
          slli(Rn, i, LogBytesPerWord);
          add(Rm, Pm_base, Rn);
          ld(Rm, Address(Rm));
          add(Rn, Pn_base, Rn);
          ld(Rn, Address(Rn));
          mv(t0, 1); // set carry flag, i.e. no borrow
          align(16);
          bind(loop); {
            notr(Rn, Rn);
            add(Rm, Rm, t0);
            add(Rm, Rm, Rn);
            sltu(t0, Rm, Rn);
            slli(Rn, i, LogBytesPerWord); // Rn as temp register
            add(Rn, Pm_base, Rn);
            sd(Rm, Address(Rn));
            addi(i, i, 1);
            slli(Rn, i, LogBytesPerWord);
            add(Rm, Pm_base, Rn);
            ld(Rm, Address(Rm));
            add(Rn, Pn_base, Rn);
            ld(Rn, Address(Rn));
            subi(cnt, cnt, 1);
          } bnez(cnt, loop);
          subi(tmp0, tmp0, 1);
          add(tmp0, tmp0, t0);
        } bnez(tmp0, again);
      } bind(post);
    }

    // Move memory at s to d, reversing words.
    //    Increments d to end of copied memory
    //    Destroys tmp1, tmp2
    //    Preserves len
    //    Leaves s pointing to the address which was in d at start
    void reverse(Register d, Register s, Register len, Register tmp1, Register tmp2) {
      assert(tmp1->encoding() < x28->encoding(), "register corruption");
      assert(tmp2->encoding() < x28->encoding(), "register corruption");

      shadd(s, len, s, tmp1, LogBytesPerWord);
      mv(tmp1, len);
      unroll_2(tmp1,  &MontgomeryMultiplyGenerator::reverse1, d, s, tmp2);
      slli(tmp1, len, LogBytesPerWord);
      sub(s, d, tmp1);
    }
    // [63...0] -> [31...0][63...32]
    void reverse1(Register d, Register s, Register tmp) {
      subi(s, s, wordSize);
      ld(tmp, Address(s));
      ror(tmp, tmp, 32, t0);
      sd(tmp, Address(d));
      addi(d, d, wordSize);
    }

    void step_squaring() {
      // An extra ACC
      step();
      acc(Rhi_ab, Rlo_ab, tmp0, tmp1, tmp2);
    }

    void last_squaring(Register i) {
      Label dont;
      // if ((i & 1) == 0) {
      test_bit(t0, i, 0);
      bnez(t0, dont); {
        // MACC(Ra, Rb, tmp0, tmp1, tmp2);
        // Ra = *++Pa;
        // Rb = *--Pb;
        mulhu(Rhi_ab, Ra, Rb);
        mul(Rlo_ab, Ra, Rb);
        acc(Rhi_ab, Rlo_ab, tmp0, tmp1, tmp2);
      } bind(dont);
    }

    void extra_step_squaring() {
      acc(Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2);  // The pending m*n

      // MACC(Rm, Rn, tmp0, tmp1, tmp2);
      // Rm = *++Pm;
      // Rn = *--Pn;
      mulhu(Rhi_mn, Rm, Rn);
      mul(Rlo_mn, Rm, Rn);
      addi(Pm, Pm, wordSize);
      ld(Rm, Address(Pm));
      subi(Pn, Pn, wordSize);
      ld(Rn, Address(Pn));
    }

    void post1_squaring() {
      acc(Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2);  // The pending m*n

      // *Pm = Rm = tmp0 * inv;
      mul(Rm, tmp0, inv);
      sd(Rm, Address(Pm));

      // MACC(Rm, Rn, tmp0, tmp1, tmp2);
      // tmp0 = tmp1; tmp1 = tmp2; tmp2 = 0;
      mulhu(Rhi_mn, Rm, Rn);

#ifndef PRODUCT
      // assert(m[i] * n[0] + tmp0 == 0, "broken Montgomery multiply");
      {
        mul(Rlo_mn, Rm, Rn);
        add(Rlo_mn, tmp0, Rlo_mn);
        Label ok;
        beqz(Rlo_mn, ok); {
          stop("broken Montgomery multiply");
        } bind(ok);
      }
#endif
      // We have very carefully set things up so that
      // m[i]*n[0] + tmp0 == 0 (mod b), so we don't have to calculate
      // the lower half of Rm * Rn because we know the result already:
      // it must be -tmp0.  tmp0 + (-tmp0) must generate a carry iff
      // tmp0 != 0.  So, rather than do a mul and a cad we just set
      // the carry flag iff tmp0 is nonzero.
      //
      // mul(Rlo_mn, Rm, Rn);
      // cad(zr, tmp, Rlo_mn);
      subi(t0, tmp0, 1);
      sltu(t0, t0, tmp0); // Set carry iff tmp0 is nonzero
      cadc(tmp0, tmp1, Rhi_mn, t0);
      adc(tmp1, tmp2, zr, t0);
      mv(tmp2, zr);
    }

    // use t0 as carry
    void acc(Register Rhi, Register Rlo,
             Register tmp0, Register tmp1, Register tmp2) {
      cad(tmp0, tmp0, Rlo, t0);
      cadc(tmp1, tmp1, Rhi, t0);
      adc(tmp2, tmp2, zr, t0);
    }

  public:
    /**
     * Fast Montgomery multiplication.  The derivation of the
     * algorithm is in A Cryptographic Library for the Motorola
     * DSP56000, Dusse and Kaliski, Proc. EUROCRYPT 90, pp. 230-237.
     *
     * Arguments:
     *
     * Inputs for multiplication:
     *   c_rarg0   - int array elements a
     *   c_rarg1   - int array elements b
     *   c_rarg2   - int array elements n (the modulus)
     *   c_rarg3   - int length
     *   c_rarg4   - int inv
     *   c_rarg5   - int array elements m (the result)
     *
     * Inputs for squaring:
     *   c_rarg0   - int array elements a
     *   c_rarg1   - int array elements n (the modulus)
     *   c_rarg2   - int length
     *   c_rarg3   - int inv
     *   c_rarg4   - int array elements m (the result)
     *
     */
    address generate_multiply() {
      Label argh, nothing;
      bind(argh);
      stop("MontgomeryMultiply total_allocation must be <= 8192");

      align(CodeEntryAlignment);
      address entry = pc();

      beqz(Rlen, nothing);

      enter();

      // Make room.
      mv(Ra, 512);
      bgt(Rlen, Ra, argh);
      slli(Ra, Rlen, exact_log2(4 * sizeof(jint)));
      sub(Ra, sp, Ra);
      andi(sp, Ra, -2 * wordSize);

      srliw(Rlen, Rlen, 1);  // length in longwords = len/2

      {
        // Copy input args, reversing as we go.  We use Ra as a
        // temporary variable.
        reverse(Ra, Pa_base, Rlen, Ri, Rj);
        if (!_squaring)
          reverse(Ra, Pb_base, Rlen, Ri, Rj);
        reverse(Ra, Pn_base, Rlen, Ri, Rj);
      }

      // Push all call-saved registers and also Pm_base which we'll need
      // at the end.
      save_regs();

#ifndef PRODUCT
      // assert(inv * n[0] == -1UL, "broken inverse in Montgomery multiply");
      {
        ld(Rn, Address(Pn_base));
        mul(Rlo_mn, Rn, inv);
        mv(t0, -1);
        Label ok;
        beq(Rlo_mn, t0, ok);
        stop("broken inverse in Montgomery multiply");
        bind(ok);
      }
#endif

      mv(Pm_base, Ra);

      mv(tmp0, zr);
      mv(tmp1, zr);
      mv(tmp2, zr);

      block_comment("for (int i = 0; i < len; i++) {");
      mv(Ri, zr); {
        Label loop, end;
        bge(Ri, Rlen, end);

        bind(loop);
        pre1(Ri);

        block_comment("  for (j = i; j; j--) {"); {
          mv(Rj, Ri);
          unroll_2(Rj, &MontgomeryMultiplyGenerator::step);
        } block_comment("  } // j");

        post1();
        addiw(Ri, Ri, 1);
        blt(Ri, Rlen, loop);
        bind(end);
        block_comment("} // i");
      }

      block_comment("for (int i = len; i < 2*len; i++) {");
      mv(Ri, Rlen); {
        Label loop, end;
        slli(t0, Rlen, 1);
        bge(Ri, t0, end);

        bind(loop);
        pre2(Ri, Rlen);

        block_comment("  for (j = len*2-i-1; j; j--) {"); {
          slliw(Rj, Rlen, 1);
          subw(Rj, Rj, Ri);
          subiw(Rj, Rj, 1);
          unroll_2(Rj, &MontgomeryMultiplyGenerator::step);
        } block_comment("  } // j");

        post2(Ri, Rlen);
        addiw(Ri, Ri, 1);
        slli(t0, Rlen, 1);
        blt(Ri, t0, loop);
        bind(end);
      }
      block_comment("} // i");

      normalize(Rlen);

      mv(Ra, Pm_base);  // Save Pm_base in Ra
      restore_regs();  // Restore caller's Pm_base

      // Copy our result into caller's Pm_base
      reverse(Pm_base, Ra, Rlen, Ri, Rj);

      leave();
      bind(nothing);
      ret();

      return entry;
    }

    /**
     *
     * Arguments:
     *
     * Inputs:
     *   c_rarg0   - int array elements a
     *   c_rarg1   - int array elements n (the modulus)
     *   c_rarg2   - int length
     *   c_rarg3   - int inv
     *   c_rarg4   - int array elements m (the result)
     *
     */
    address generate_square() {
      Label argh;
      bind(argh);
      stop("MontgomeryMultiply total_allocation must be <= 8192");

      align(CodeEntryAlignment);
      address entry = pc();

      enter();

      // Make room.
      mv(Ra, 512);
      bgt(Rlen, Ra, argh);
      slli(Ra, Rlen, exact_log2(4 * sizeof(jint)));
      sub(Ra, sp, Ra);
      andi(sp, Ra, -2 * wordSize);

      srliw(Rlen, Rlen, 1);  // length in longwords = len/2

      {
        // Copy input args, reversing as we go.  We use Ra as a
        // temporary variable.
        reverse(Ra, Pa_base, Rlen, Ri, Rj);
        reverse(Ra, Pn_base, Rlen, Ri, Rj);
      }

      // Push all call-saved registers and also Pm_base which we'll need
      // at the end.
      save_regs();

      mv(Pm_base, Ra);

      mv(tmp0, zr);
      mv(tmp1, zr);
      mv(tmp2, zr);

      block_comment("for (int i = 0; i < len; i++) {");
      mv(Ri, zr); {
        Label loop, end;
        bind(loop);
        bge(Ri, Rlen, end);

        pre1(Ri);

        block_comment("for (j = (i+1)/2; j; j--) {"); {
          addi(Rj, Ri, 1);
          srliw(Rj, Rj, 1);
          unroll_2(Rj, &MontgomeryMultiplyGenerator::step_squaring);
        } block_comment("  } // j");

        last_squaring(Ri);

        block_comment("  for (j = i/2; j; j--) {"); {
          srliw(Rj, Ri, 1);
          unroll_2(Rj, &MontgomeryMultiplyGenerator::extra_step_squaring);
        } block_comment("  } // j");

        post1_squaring();
        addi(Ri, Ri, 1);
        blt(Ri, Rlen, loop);

        bind(end);
        block_comment("} // i");
      }

      block_comment("for (int i = len; i < 2*len; i++) {");
      mv(Ri, Rlen); {
        Label loop, end;
        bind(loop);
        slli(t0, Rlen, 1);
        bge(Ri, t0, end);

        pre2(Ri, Rlen);

        block_comment("  for (j = (2*len-i-1)/2; j; j--) {"); {
          slli(Rj, Rlen, 1);
          sub(Rj, Rj, Ri);
          subi(Rj, Rj, 1);
          srliw(Rj, Rj, 1);
          unroll_2(Rj, &MontgomeryMultiplyGenerator::step_squaring);
        } block_comment("  } // j");

        last_squaring(Ri);

        block_comment("  for (j = (2*len-i)/2; j; j--) {"); {
          slli(Rj, Rlen, 1);
          sub(Rj, Rj, Ri);
          srliw(Rj, Rj, 1);
          unroll_2(Rj, &MontgomeryMultiplyGenerator::extra_step_squaring);
        } block_comment("  } // j");

        post2(Ri, Rlen);
        addi(Ri, Ri, 1);
        slli(t0, Rlen, 1);
        blt(Ri, t0, loop);

        bind(end);
        block_comment("} // i");
      }

      normalize(Rlen);

      mv(Ra, Pm_base);  // Save Pm_base in Ra
      restore_regs();  // Restore caller's Pm_base

      // Copy our result into caller's Pm_base
      reverse(Pm_base, Ra, Rlen, Ri, Rj);

      leave();
      ret();

      return entry;
    }
  };

#endif // COMPILER2

  address generate_cont_thaw(Continuation::thaw_kind kind) {
    bool return_barrier = Continuation::is_thaw_return_barrier(kind);
    bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);

    address start = __ pc();

    if (return_barrier) {
      __ ld(sp, Address(xthread, JavaThread::cont_entry_offset()));
    }

#ifndef PRODUCT
    {
      Label OK;
      __ ld(t0, Address(xthread, JavaThread::cont_entry_offset()));
      __ beq(sp, t0, OK);
      __ stop("incorrect sp");
      __ bind(OK);
    }
#endif

    if (return_barrier) {
      // preserve possible return value from a method returning to the return barrier
      __ subi(sp, sp, 2 * wordSize);
      __ fsd(f10, Address(sp, 0 * wordSize));
      __ sd(x10, Address(sp, 1 * wordSize));
    }

    __ mv(c_rarg1, (return_barrier ? 1 : 0));
    __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), xthread, c_rarg1);
    __ mv(t1, x10); // x10 contains the size of the frames to thaw, 0 if overflow or no more frames

    if (return_barrier) {
      // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
      __ ld(x10, Address(sp, 1 * wordSize));
      __ fld(f10, Address(sp, 0 * wordSize));
      __ addi(sp, sp, 2 * wordSize);
    }

#ifndef PRODUCT
    {
      Label OK;
      __ ld(t0, Address(xthread, JavaThread::cont_entry_offset()));
      __ beq(sp, t0, OK);
      __ stop("incorrect sp");
      __ bind(OK);
    }
#endif

    Label thaw_success;
    // t1 contains the size of the frames to thaw, 0 if overflow or no more frames
    __ bnez(t1, thaw_success);
    __ j(RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
    __ bind(thaw_success);

    // make room for the thawed frames
    __ sub(t0, sp, t1);
    __ andi(sp, t0, -16); // align

    if (return_barrier) {
      // save original return value -- again
      __ subi(sp, sp, 2 * wordSize);
      __ fsd(f10, Address(sp, 0 * wordSize));
      __ sd(x10, Address(sp, 1 * wordSize));
    }

    // If we want, we can templatize thaw by kind, and have three different entries
    __ mv(c_rarg1, kind);

    __ call_VM_leaf(Continuation::thaw_entry(), xthread, c_rarg1);
    __ mv(t1, x10); // x10 is the sp of the yielding frame

    if (return_barrier) {
      // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
      __ ld(x10, Address(sp, 1 * wordSize));
      __ fld(f10, Address(sp, 0 * wordSize));
      __ addi(sp, sp, 2 * wordSize);
    } else {
      __ mv(x10, zr); // return 0 (success) from doYield
    }

    // we're now on the yield frame (which is in an address above us b/c sp has been pushed down)
    __ mv(fp, t1);
    __ subi(sp, t1, 2 * wordSize); // now pointing to fp spill

    if (return_barrier_exception) {
      __ ld(c_rarg1, Address(fp, -1 * wordSize)); // return address
      __ verify_oop(x10);
      __ mv(x9, x10); // save return value contaning the exception oop in callee-saved x9

      __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), xthread, c_rarg1);

      // see OptoRuntime::generate_exception_blob: x10 -- exception oop, x13 -- exception pc

      __ mv(x11, x10); // the exception handler
      __ mv(x10, x9); // restore return value contaning the exception oop
      __ verify_oop(x10);

      __ leave();
      __ mv(x13, ra);
      __ jr(x11); // the exception handler
    } else {
      // We're "returning" into the topmost thawed frame; see Thaw::push_return_frame
      __ leave();
      __ ret();
    }

    return start;
  }

  address generate_cont_thaw() {
    if (!Continuations::enabled()) return nullptr;

    StubGenStubId stub_id = StubGenStubId::cont_thaw_id;
    StubCodeMark mark(this, stub_id);
    address start = __ pc();
    generate_cont_thaw(Continuation::thaw_top);
    return start;
  }

  address generate_cont_returnBarrier() {
    if (!Continuations::enabled()) return nullptr;

    // TODO: will probably need multiple return barriers depending on return type
    StubGenStubId stub_id = StubGenStubId::cont_returnBarrier_id;
    StubCodeMark mark(this, stub_id);
    address start = __ pc();

    generate_cont_thaw(Continuation::thaw_return_barrier);

    return start;
  }

  address generate_cont_returnBarrier_exception() {
    if (!Continuations::enabled()) return nullptr;

    StubGenStubId stub_id = StubGenStubId::cont_returnBarrierExc_id;
    StubCodeMark mark(this, stub_id);
    address start = __ pc();

    generate_cont_thaw(Continuation::thaw_return_barrier_exception);

    return start;
  }

  address generate_cont_preempt_stub() {
    if (!Continuations::enabled()) return nullptr;
    StubGenStubId stub_id = StubGenStubId::cont_preempt_id;
    StubCodeMark mark(this, stub_id);
    address start = __ pc();

    __ reset_last_Java_frame(true);

    // Set sp to enterSpecial frame, i.e. remove all frames copied into the heap.
    __ ld(sp, Address(xthread, JavaThread::cont_entry_offset()));

    Label preemption_cancelled;
    __ lbu(t0, Address(xthread, JavaThread::preemption_cancelled_offset()));
    __ bnez(t0, preemption_cancelled);

    // Remove enterSpecial frame from the stack and return to Continuation.run() to unmount.
    SharedRuntime::continuation_enter_cleanup(_masm);
    __ leave();
    __ ret();

    // We acquired the monitor after freezing the frames so call thaw to continue execution.
    __ bind(preemption_cancelled);
    __ sb(zr, Address(xthread, JavaThread::preemption_cancelled_offset()));
    __ la(fp, Address(sp, checked_cast<int32_t>(ContinuationEntry::size() + 2 * wordSize)));
    __ la(t1, ExternalAddress(ContinuationEntry::thaw_call_pc_address()));
    __ ld(t1, Address(t1));
    __ jr(t1);

    return start;
  }

#if COMPILER2_OR_JVMCI

#undef __
#define __ this->

  class Sha2Generator : public MacroAssembler {
    StubCodeGenerator* _cgen;
   public:
      Sha2Generator(MacroAssembler* masm, StubCodeGenerator* cgen) : MacroAssembler(masm->code()), _cgen(cgen) {}
      address generate_sha256_implCompress(StubGenStubId stub_id) {
        return generate_sha2_implCompress(Assembler::e32, stub_id);
      }
      address generate_sha512_implCompress(StubGenStubId stub_id) {
        return generate_sha2_implCompress(Assembler::e64, stub_id);
      }
   private:

    void vleXX_v(Assembler::SEW vset_sew, VectorRegister vr, Register sr) {
      if (vset_sew == Assembler::e32) __ vle32_v(vr, sr);
      else                            __ vle64_v(vr, sr);
    }

    void vseXX_v(Assembler::SEW vset_sew, VectorRegister vr, Register sr) {
      if (vset_sew == Assembler::e32) __ vse32_v(vr, sr);
      else                            __ vse64_v(vr, sr);
    }

    // Overview of the logic in each "quad round".
    //
    // The code below repeats 16/20 times the logic implementing four rounds
    // of the SHA-256/512 core loop as documented by NIST. 16/20 "quad rounds"
    // to implementing the 64/80 single rounds.
    //
    //    // Load four word (u32/64) constants (K[t+3], K[t+2], K[t+1], K[t+0])
    //    // Output:
    //    //   vTmp1 = {K[t+3], K[t+2], K[t+1], K[t+0]}
    //    vl1reXX.v vTmp1, ofs
    //
    //    // Increment word constant address by stride (16/32 bytes, 4*4B/8B, 128b/256b)
    //    addi ofs, ofs, 16/32
    //
    //    // Add constants to message schedule words:
    //    //  Input
    //    //    vTmp1 = {K[t+3], K[t+2], K[t+1], K[t+0]}
    //    //    vW0 = {W[t+3], W[t+2], W[t+1], W[t+0]}; // Vt0 = W[3:0];
    //    //  Output
    //    //    vTmp0 = {W[t+3]+K[t+3], W[t+2]+K[t+2], W[t+1]+K[t+1], W[t+0]+K[t+0]}
    //    vadd.vv vTmp0, vTmp1, vW0
    //
    //    //  2 rounds of working variables updates.
    //    //     vState1[t+4] <- vState1[t], vState0[t], vTmp0[t]
    //    //  Input:
    //    //    vState1 = {c[t],d[t],g[t],h[t]}   " = vState1[t] "
    //    //    vState0 = {a[t],b[t],e[t],f[t]}
    //    //    vTmp0 = {W[t+3]+K[t+3], W[t+2]+K[t+2], W[t+1]+K[t+1], W[t+0]+K[t+0]}
    //    //  Output:
    //    //    vState1 = {f[t+2],e[t+2],b[t+2],a[t+2]}  " = vState0[t+2] "
    //    //        = {h[t+4],g[t+4],d[t+4],c[t+4]}  " = vState1[t+4] "
    //    vsha2cl.vv vState1, vState0, vTmp0
    //
    //    //  2 rounds of working variables updates.
    //    //     vState0[t+4] <- vState0[t], vState0[t+2], vTmp0[t]
    //    //  Input
    //    //   vState0 = {a[t],b[t],e[t],f[t]}       " = vState0[t] "
    //    //       = {h[t+2],g[t+2],d[t+2],c[t+2]}   " = vState1[t+2] "
    //    //   vState1 = {f[t+2],e[t+2],b[t+2],a[t+2]}   " = vState0[t+2] "
    //    //   vTmp0 = {W[t+3]+K[t+3], W[t+2]+K[t+2], W[t+1]+K[t+1], W[t+0]+K[t+0]}
    //    //  Output:
    //    //   vState0 = {f[t+4],e[t+4],b[t+4],a[t+4]}   " = vState0[t+4] "
    //    vsha2ch.vv vState0, vState1, vTmp0
    //
    //    // Combine 2QW into 1QW
    //    //
    //    // To generate the next 4 words, "new_vW0"/"vTmp0" from vW0-vW3, vsha2ms needs
    //    //     vW0[0..3], vW1[0], vW2[1..3], vW3[0, 2..3]
    //    // and it can only take 3 vectors as inputs. Hence we need to combine
    //    // vW1[0] and vW2[1..3] in a single vector.
    //    //
    //    // vmerge Vt4, Vt1, Vt2, V0
    //    // Input
    //    //  V0 = mask // first word from vW2, 1..3 words from vW1
    //    //  vW2 = {Wt-8, Wt-7, Wt-6, Wt-5}
    //    //  vW1 = {Wt-12, Wt-11, Wt-10, Wt-9}
    //    // Output
    //    //  Vt4 = {Wt-12, Wt-7, Wt-6, Wt-5}
    //    vmerge.vvm vTmp0, vW2, vW1, v0
    //
    //    // Generate next Four Message Schedule Words (hence allowing for 4 more rounds)
    //    // Input
    //    //  vW0 = {W[t+ 3], W[t+ 2], W[t+ 1], W[t+ 0]}     W[ 3: 0]
    //    //  vW3 = {W[t+15], W[t+14], W[t+13], W[t+12]}     W[15:12]
    //    //  vTmp0 = {W[t+11], W[t+10], W[t+ 9], W[t+ 4]}     W[11: 9,4]
    //    // Output (next four message schedule words)
    //    //  vW0 = {W[t+19],  W[t+18],  W[t+17],  W[t+16]}  W[19:16]
    //    vsha2ms.vv vW0, vTmp0, vW3
    //
    // BEFORE
    //  vW0 - vW3 hold the message schedule words (initially the block words)
    //    vW0 = W[ 3: 0]   "oldest"
    //    vW1 = W[ 7: 4]
    //    vW2 = W[11: 8]
    //    vW3 = W[15:12]   "newest"
    //
    //  vt6 - vt7 hold the working state variables
    //    vState0 = {a[t],b[t],e[t],f[t]}   // initially {H5,H4,H1,H0}
    //    vState1 = {c[t],d[t],g[t],h[t]}   // initially {H7,H6,H3,H2}
    //
    // AFTER
    //  vW0 - vW3 hold the message schedule words (initially the block words)
    //    vW1 = W[ 7: 4]   "oldest"
    //    vW2 = W[11: 8]
    //    vW3 = W[15:12]
    //    vW0 = W[19:16]   "newest"
    //
    //  vState0 and vState1 hold the working state variables
    //    vState0 = {a[t+4],b[t+4],e[t+4],f[t+4]}
    //    vState1 = {c[t+4],d[t+4],g[t+4],h[t+4]}
    //
    //  The group of vectors vW0,vW1,vW2,vW3 is "rotated" by one in each quad-round,
    //  hence the uses of those vectors rotate in each round, and we get back to the
    //  initial configuration every 4 quad-rounds. We could avoid those changes at
    //  the cost of moving those vectors at the end of each quad-rounds.
    void sha2_quad_round(Assembler::SEW vset_sew, VectorRegister rot1, VectorRegister rot2, VectorRegister rot3, VectorRegister rot4,
                         Register scalarconst, VectorRegister vtemp, VectorRegister vtemp2, VectorRegister v_abef, VectorRegister v_cdgh,
                         bool gen_words = true, bool step_const = true) {
      __ vleXX_v(vset_sew, vtemp, scalarconst);
      if (step_const) {
        __ addi(scalarconst, scalarconst, vset_sew == Assembler::e32 ? 16 : 32);
      }
      __ vadd_vv(vtemp2, vtemp, rot1);
      __ vsha2cl_vv(v_cdgh, v_abef, vtemp2);
      __ vsha2ch_vv(v_abef, v_cdgh, vtemp2);
      if (gen_words) {
        __ vmerge_vvm(vtemp2, rot3, rot2);
        __ vsha2ms_vv(rot1, vtemp2, rot4);
      }
    }

    // Arguments:
    //
    // Inputs:
    //   c_rarg0   - byte[]  source+offset
    //   c_rarg1   - int[]   SHA.state
    //   c_rarg2   - int     offset
    //   c_rarg3   - int     limit
    //
    address generate_sha2_implCompress(Assembler::SEW vset_sew, StubGenStubId stub_id) {
      alignas(64) static const uint32_t round_consts_256[64] = {
        0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
        0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
        0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
        0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
        0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
        0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
        0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
        0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
        0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
        0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
        0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
        0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
        0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
        0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
        0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
        0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
      };
      alignas(64) static const uint64_t round_consts_512[80] = {
        0x428a2f98d728ae22l, 0x7137449123ef65cdl, 0xb5c0fbcfec4d3b2fl,
        0xe9b5dba58189dbbcl, 0x3956c25bf348b538l, 0x59f111f1b605d019l,
        0x923f82a4af194f9bl, 0xab1c5ed5da6d8118l, 0xd807aa98a3030242l,
        0x12835b0145706fbel, 0x243185be4ee4b28cl, 0x550c7dc3d5ffb4e2l,
        0x72be5d74f27b896fl, 0x80deb1fe3b1696b1l, 0x9bdc06a725c71235l,
        0xc19bf174cf692694l, 0xe49b69c19ef14ad2l, 0xefbe4786384f25e3l,
        0x0fc19dc68b8cd5b5l, 0x240ca1cc77ac9c65l, 0x2de92c6f592b0275l,
        0x4a7484aa6ea6e483l, 0x5cb0a9dcbd41fbd4l, 0x76f988da831153b5l,
        0x983e5152ee66dfabl, 0xa831c66d2db43210l, 0xb00327c898fb213fl,
        0xbf597fc7beef0ee4l, 0xc6e00bf33da88fc2l, 0xd5a79147930aa725l,
        0x06ca6351e003826fl, 0x142929670a0e6e70l, 0x27b70a8546d22ffcl,
        0x2e1b21385c26c926l, 0x4d2c6dfc5ac42aedl, 0x53380d139d95b3dfl,
        0x650a73548baf63del, 0x766a0abb3c77b2a8l, 0x81c2c92e47edaee6l,
        0x92722c851482353bl, 0xa2bfe8a14cf10364l, 0xa81a664bbc423001l,
        0xc24b8b70d0f89791l, 0xc76c51a30654be30l, 0xd192e819d6ef5218l,
        0xd69906245565a910l, 0xf40e35855771202al, 0x106aa07032bbd1b8l,
        0x19a4c116b8d2d0c8l, 0x1e376c085141ab53l, 0x2748774cdf8eeb99l,
        0x34b0bcb5e19b48a8l, 0x391c0cb3c5c95a63l, 0x4ed8aa4ae3418acbl,
        0x5b9cca4f7763e373l, 0x682e6ff3d6b2b8a3l, 0x748f82ee5defb2fcl,
        0x78a5636f43172f60l, 0x84c87814a1f0ab72l, 0x8cc702081a6439ecl,
        0x90befffa23631e28l, 0xa4506cebde82bde9l, 0xbef9a3f7b2c67915l,
        0xc67178f2e372532bl, 0xca273eceea26619cl, 0xd186b8c721c0c207l,
        0xeada7dd6cde0eb1el, 0xf57d4f7fee6ed178l, 0x06f067aa72176fbal,
        0x0a637dc5a2c898a6l, 0x113f9804bef90dael, 0x1b710b35131c471bl,
        0x28db77f523047d84l, 0x32caab7b40c72493l, 0x3c9ebe0a15c9bebcl,
        0x431d67c49c100d4cl, 0x4cc5d4becb3e42b6l, 0x597f299cfc657e2al,
        0x5fcb6fab3ad6faecl, 0x6c44198c4a475817l
      };
      const int const_add = vset_sew == Assembler::e32 ? 16 : 32;

      bool multi_block;
      switch (stub_id) {
      case sha256_implCompress_id:
        assert (vset_sew == Assembler::e32, "wrong macroassembler for stub");
        multi_block = false;
        break;
      case sha256_implCompressMB_id:
        assert (vset_sew == Assembler::e32, "wrong macroassembler for stub");
        multi_block = true;
        break;
      case sha512_implCompress_id:
        assert (vset_sew == Assembler::e64, "wrong macroassembler for stub");
        multi_block = false;
        break;
      case sha512_implCompressMB_id:
        assert (vset_sew == Assembler::e64, "wrong macroassembler for stub");
        multi_block = true;
        break;
      default:
        ShouldNotReachHere();
      };
      __ align(CodeEntryAlignment);
      StubCodeMark mark(_cgen, stub_id);
      address start = __ pc();

      Register buf   = c_rarg0;
      Register state = c_rarg1;
      Register ofs   = c_rarg2;
      Register limit = c_rarg3;
      Register consts =  t2; // caller saved
      Register state_c = x28; // caller saved
      VectorRegister vindex = v2;
      VectorRegister vW0 = v4;
      VectorRegister vW1 = v6;
      VectorRegister vW2 = v8;
      VectorRegister vW3 = v10;
      VectorRegister vState0 = v12;
      VectorRegister vState1 = v14;
      VectorRegister vHash0  = v16;
      VectorRegister vHash1  = v18;
      VectorRegister vTmp0   = v20;
      VectorRegister vTmp1   = v22;

      Label multi_block_loop;

      __ enter();

      address constant_table = vset_sew == Assembler::e32 ? (address)round_consts_256 : (address)round_consts_512;
      la(consts, ExternalAddress(constant_table));

      // Register use in this function:
      //
      // VECTORS
      //  vW0 - vW3 (512/1024-bits / 4*128/256 bits / 4*4*32/65 bits), hold the message
      //             schedule words (Wt). They start with the message block
      //             content (W0 to W15), then further words in the message
      //             schedule generated via vsha2ms from previous Wt.
      //   Initially:
      //     vW0 = W[  3:0] = { W3,  W2,  W1,  W0}
      //     vW1 = W[  7:4] = { W7,  W6,  W5,  W4}
      //     vW2 = W[ 11:8] = {W11, W10,  W9,  W8}
      //     vW3 = W[15:12] = {W15, W14, W13, W12}
      //
      //  vState0 - vState1 hold the working state variables (a, b, ..., h)
      //    vState0 = {f[t],e[t],b[t],a[t]}
      //    vState1 = {h[t],g[t],d[t],c[t]}
      //   Initially:
      //    vState0 = {H5i-1, H4i-1, H1i-1 , H0i-1}
      //    vState1 = {H7i-i, H6i-1, H3i-1 , H2i-1}
      //
      //  v0 = masks for vrgather/vmerge. Single value during the 16 rounds.
      //
      //  vTmp0 = temporary, Wt+Kt
      //  vTmp1 = temporary, Kt
      //
      //  vHash0/vHash1 = hold the initial values of the hash, byte-swapped.
      //
      // During most of the function the vector state is configured so that each
      // vector is interpreted as containing four 32/64 bits (e32/e64) elements (128/256 bits).

      // vsha2ch/vsha2cl uses EGW of 4*SEW.
      // SHA256 SEW = e32, EGW = 128-bits
      // SHA512 SEW = e64, EGW = 256-bits
      //
      // VLEN is required to be at least 128.
      // For the case of VLEN=128 and SHA512 we need LMUL=2 to work with 4*e64 (EGW = 256)
      //
      // m1: LMUL=1/2
      // ta: tail agnostic (don't care about those lanes)
      // ma: mask agnostic (don't care about those lanes)
      // x0 is not written, we known the number of vector elements.

      if (vset_sew == Assembler::e64 && MaxVectorSize == 16) { // SHA512 and VLEN = 128
        __ vsetivli(x0, 4, vset_sew, Assembler::m2, Assembler::ma, Assembler::ta);
      } else {
        __ vsetivli(x0, 4, vset_sew, Assembler::m1, Assembler::ma, Assembler::ta);
      }

      int64_t indexes = vset_sew == Assembler::e32 ? 0x00041014ul : 0x00082028ul;
      __ li(t0, indexes);
      __ vmv_v_x(vindex, t0);

      // Step-over a,b, so we are pointing to c.
      // const_add is equal to 4x state variable, div by 2 is thus 2, a,b
      __ addi(state_c, state, const_add/2);

      // Use index-load to get {f,e,b,a},{h,g,d,c}
      __ vluxei8_v(vState0, state, vindex);
      __ vluxei8_v(vState1, state_c, vindex);

      __ bind(multi_block_loop);

      // Capture the initial H values in vHash0 and vHash1 to allow for computing
      // the resulting H', since H' = H+{a',b',c',...,h'}.
      __ vmv_v_v(vHash0, vState0);
      __ vmv_v_v(vHash1, vState1);

      // Load the 512/1024-bits of the message block in vW0-vW3 and perform
      // an endian swap on each 4/8 bytes element.
      //
      // If Zvkb is not implemented one can use vrgather
      // with an index sequence to byte-swap.
      //  sequence = [3 2 1 0   7 6 5 4  11 10 9 8   15 14 13 12]
      //   <https://oeis.org/A004444> gives us "N ^ 3" as a nice formula to generate
      //  this sequence. 'vid' gives us the N.
      __ vleXX_v(vset_sew, vW0, buf);
      __ vrev8_v(vW0, vW0);
      __ addi(buf, buf, const_add);
      __ vleXX_v(vset_sew, vW1, buf);
      __ vrev8_v(vW1, vW1);
      __ addi(buf, buf, const_add);
      __ vleXX_v(vset_sew, vW2, buf);
      __ vrev8_v(vW2, vW2);
      __ addi(buf, buf, const_add);
      __ vleXX_v(vset_sew, vW3, buf);
      __ vrev8_v(vW3, vW3);
      __ addi(buf, buf, const_add);

      // Set v0 up for the vmerge that replaces the first word (idx==0)
      __ vid_v(v0);
      __ vmseq_vi(v0, v0, 0x0);  // v0.mask[i] = (i == 0 ? 1 : 0)

      VectorRegister rotation_regs[] = {vW0, vW1, vW2, vW3};
      int rot_pos = 0;
      // Quad-round #0 (+0, vW0->vW1->vW2->vW3) ... #11 (+3, vW3->vW0->vW1->vW2)
      const int qr_end = vset_sew == Assembler::e32 ? 12 : 16;
      for (int i = 0; i < qr_end; i++) {
        sha2_quad_round(vset_sew,
                   rotation_regs[(rot_pos + 0) & 0x3],
                   rotation_regs[(rot_pos + 1) & 0x3],
                   rotation_regs[(rot_pos + 2) & 0x3],
                   rotation_regs[(rot_pos + 3) & 0x3],
                   consts,
                   vTmp1, vTmp0, vState0, vState1);
        ++rot_pos;
      }
      // Quad-round #12 (+0, vW0->vW1->vW2->vW3) ... #15 (+3, vW3->vW0->vW1->vW2)
      // Note that we stop generating new message schedule words (Wt, vW0-13)
      // as we already generated all the words we end up consuming (i.e., W[63:60]).
      const int qr_c_end = qr_end + 4;
      for (int i = qr_end; i < qr_c_end; i++) {
        sha2_quad_round(vset_sew,
                   rotation_regs[(rot_pos + 0) & 0x3],
                   rotation_regs[(rot_pos + 1) & 0x3],
                   rotation_regs[(rot_pos + 2) & 0x3],
                   rotation_regs[(rot_pos + 3) & 0x3],
                   consts,
                   vTmp1, vTmp0, vState0, vState1, false, i < (qr_c_end-1));
        ++rot_pos;
      }

      //--------------------------------------------------------------------------------
      // Compute the updated hash value H'
      //   H' = H + {h',g',...,b',a'}
      //      = {h,g,...,b,a} + {h',g',...,b',a'}
      //      = {h+h',g+g',...,b+b',a+a'}

      // H' = H+{a',b',c',...,h'}
      __ vadd_vv(vState0, vHash0, vState0);
      __ vadd_vv(vState1, vHash1, vState1);

      if (multi_block) {
        int total_adds = vset_sew == Assembler::e32 ? 240 : 608;
        __ subi(consts, consts, total_adds);
        __ addi(ofs, ofs, vset_sew == Assembler::e32 ? 64 : 128);
        __ ble(ofs, limit, multi_block_loop);
        __ mv(c_rarg0, ofs); // return ofs
      }

      // Store H[0..8] = {a,b,c,d,e,f,g,h} from
      //  vState0 = {f,e,b,a}
      //  vState1 = {h,g,d,c}
      __ vsuxei8_v(vState0, state,   vindex);
      __ vsuxei8_v(vState1, state_c, vindex);

      __ leave();
      __ ret();

      return start;
    }
  };

#undef __
#define __ _masm->

  // Set of L registers that correspond to a contiguous memory area.
  // Each 64-bit register typically corresponds to 2 32-bit integers.
  template <uint L>
  class RegCache {
  private:
    MacroAssembler *_masm;
    Register _regs[L];

  public:
    RegCache(MacroAssembler *masm, RegSet rs): _masm(masm) {
      assert(rs.size() == L, "%u registers are used to cache %u 4-byte data", rs.size(), 2 * L);
      auto it = rs.begin();
      for (auto &r: _regs) {
        r = *it;
        ++it;
      }
    }

    // generate load for the i'th register
    void gen_load(uint i, Register base) {
      assert(i < L, "invalid i: %u", i);
      __ ld(_regs[i], Address(base, 8 * i));
    }

    // add i'th 32-bit integer to dest
    void add_u32(const Register dest, uint i, const Register rtmp = t0) {
      assert(i < 2 * L, "invalid i: %u", i);

      if (is_even(i)) {
        // Use the bottom 32 bits. No need to mask off the top 32 bits
        // as addw will do the right thing.
        __ addw(dest, dest, _regs[i / 2]);
      } else {
        // Use the top 32 bits by right-shifting them.
        __ srli(rtmp, _regs[i / 2], 32);
        __ addw(dest, dest, rtmp);
      }
    }
  };

  typedef RegCache<8> BufRegCache;

  // a += value + x + ac;
  // a = Integer.rotateLeft(a, s) + b;
  void m5_FF_GG_HH_II_epilogue(BufRegCache& reg_cache,
                               Register a, Register b, Register c, Register d,
                               int k, int s, int t,
                               Register value) {
    // a += ac
    __ addw(a, a, t, t1);

    // a += x;
    reg_cache.add_u32(a, k);
    // a += value;
    __ addw(a, a, value);

    // a = Integer.rotateLeft(a, s) + b;
    __ rolw(a, a, s);
    __ addw(a, a, b);
  }

  // a += ((b & c) | ((~b) & d)) + x + ac;
  // a = Integer.rotateLeft(a, s) + b;
  void md5_FF(BufRegCache& reg_cache,
              Register a, Register b, Register c, Register d,
              int k, int s, int t,
              Register rtmp1, Register rtmp2) {
    // rtmp1 = b & c
    __ andr(rtmp1, b, c);

    // rtmp2 = (~b) & d
    __ andn(rtmp2, d, b);

    // rtmp1 = (b & c) | ((~b) & d)
    __ orr(rtmp1, rtmp1, rtmp2);

    m5_FF_GG_HH_II_epilogue(reg_cache, a, b, c, d, k, s, t, rtmp1);
  }

  // a += ((b & d) | (c & (~d))) + x + ac;
  // a = Integer.rotateLeft(a, s) + b;
  void md5_GG(BufRegCache& reg_cache,
              Register a, Register b, Register c, Register d,
              int k, int s, int t,
              Register rtmp1, Register rtmp2) {
    // rtmp1 = b & d
    __ andr(rtmp1, b, d);

    // rtmp2 = c & (~d)
    __ andn(rtmp2, c, d);

    // rtmp1 = (b & d) | (c & (~d))
    __ orr(rtmp1, rtmp1, rtmp2);

    m5_FF_GG_HH_II_epilogue(reg_cache, a, b, c, d, k, s, t, rtmp1);
  }

  // a += ((b ^ c) ^ d) + x + ac;
  // a = Integer.rotateLeft(a, s) + b;
  void md5_HH(BufRegCache& reg_cache,
              Register a, Register b, Register c, Register d,
              int k, int s, int t,
              Register rtmp1, Register rtmp2) {
    // rtmp1 = (b ^ c) ^ d
    __ xorr(rtmp2, b, c);
    __ xorr(rtmp1, rtmp2, d);

    m5_FF_GG_HH_II_epilogue(reg_cache, a, b, c, d, k, s, t, rtmp1);
  }

  // a += (c ^ (b | (~d))) + x + ac;
  // a = Integer.rotateLeft(a, s) + b;
  void md5_II(BufRegCache& reg_cache,
              Register a, Register b, Register c, Register d,
              int k, int s, int t,
              Register rtmp1, Register rtmp2) {
    // rtmp1 = c ^ (b | (~d))
    __ orn(rtmp2, b, d);
    __ xorr(rtmp1, c, rtmp2);

    m5_FF_GG_HH_II_epilogue(reg_cache, a, b, c, d, k, s, t, rtmp1);
  }

  // Arguments:
  //
  // Inputs:
  //   c_rarg0   - byte[]  source+offset
  //   c_rarg1   - int[]   SHA.state
  //   c_rarg2   - int     offset  (multi_block == True)
  //   c_rarg3   - int     limit   (multi_block == True)
  //
  // Registers:
  //    x0   zero  (zero)
  //    x1     ra  (return address)
  //    x2     sp  (stack pointer)
  //    x3     gp  (global pointer)
  //    x4     tp  (thread pointer)
  //    x5     t0  (tmp register)
  //    x6     t1  (tmp register)
  //    x7     t2  state0
  //    x8  f0/s0  (frame pointer)
  //    x9     s1
  //   x10     a0  rtmp1 / c_rarg0
  //   x11     a1  rtmp2 / c_rarg1
  //   x12     a2  a     / c_rarg2
  //   x13     a3  b     / c_rarg3
  //   x14     a4  c
  //   x15     a5  d
  //   x16     a6  buf
  //   x17     a7  state
  //   x18     s2  ofs     [saved-reg]  (multi_block == True)
  //   x19     s3  limit   [saved-reg]  (multi_block == True)
  //   x20     s4  state1  [saved-reg]
  //   x21     s5  state2  [saved-reg]
  //   x22     s6  state3  [saved-reg]
  //   x23     s7
  //   x24     s8  buf0    [saved-reg]
  //   x25     s9  buf1    [saved-reg]
  //   x26    s10  buf2    [saved-reg]
  //   x27    s11  buf3    [saved-reg]
  //   x28     t3  buf4
  //   x29     t4  buf5
  //   x30     t5  buf6
  //   x31     t6  buf7
  address generate_md5_implCompress(StubGenStubId stub_id) {
    __ align(CodeEntryAlignment);
    bool multi_block;
    switch (stub_id) {
    case md5_implCompress_id:
      multi_block = false;
      break;
    case md5_implCompressMB_id:
      multi_block = true;
      break;
    default:
      ShouldNotReachHere();
    };
    StubCodeMark mark(this, stub_id);
    address start = __ pc();

    // rotation constants
    const int S11 = 7;
    const int S12 = 12;
    const int S13 = 17;
    const int S14 = 22;
    const int S21 = 5;
    const int S22 = 9;
    const int S23 = 14;
    const int S24 = 20;
    const int S31 = 4;
    const int S32 = 11;
    const int S33 = 16;
    const int S34 = 23;
    const int S41 = 6;
    const int S42 = 10;
    const int S43 = 15;
    const int S44 = 21;

    const int64_t mask32 = 0xffffffff;

    Register buf_arg   = c_rarg0; // a0
    Register state_arg = c_rarg1; // a1
    Register ofs_arg   = c_rarg2; // a2
    Register limit_arg = c_rarg3; // a3

    // we'll copy the args to these registers to free up a0-a3
    // to use for other values manipulated by instructions
    // that can be compressed
    Register buf       = x16; // a6
    Register state     = x17; // a7
    Register ofs       = x18; // s2
    Register limit     = x19; // s3

    // using x12->15 to allow compressed instructions
    Register a         = x12; // a2
    Register b         = x13; // a3
    Register c         = x14; // a4
    Register d         = x15; // a5

    Register state0    =  x7; // t2
    Register state1    = x20; // s4
    Register state2    = x21; // s5
    Register state3    = x22; // s6

    // using x10->x11 to allow compressed instructions
    Register rtmp1     = x10; // a0
    Register rtmp2     = x11; // a1

    RegSet reg_cache_saved_regs = RegSet::of(x24, x25, x26, x27); // s8, s9, s10, s11
    RegSet reg_cache_regs;
    reg_cache_regs += reg_cache_saved_regs;
    reg_cache_regs += RegSet::of(t3, t4, t5, t6);
    BufRegCache reg_cache(_masm, reg_cache_regs);

    RegSet saved_regs;
    if (multi_block) {
      saved_regs += RegSet::of(ofs, limit);
    }
    saved_regs += RegSet::of(state1, state2, state3);
    saved_regs += reg_cache_saved_regs;

    __ push_reg(saved_regs, sp);

    __ mv(buf, buf_arg);
    __ mv(state, state_arg);
    if (multi_block) {
      __ mv(ofs, ofs_arg);
      __ mv(limit, limit_arg);
    }

    // to minimize the number of memory operations:
    // read the 4 state 4-byte values in pairs, with a single ld,
    // and split them into 2 registers.
    //
    // And, as the core algorithm of md5 works on 32-bits words, so
    // in the following code, it does not care about the content of
    // higher 32-bits in state[x]. Based on this observation,
    // we can apply further optimization, which is to just ignore the
    // higher 32-bits in state0/state2, rather than set the higher
    // 32-bits of state0/state2 to zero explicitly with extra instructions.
    __ ld(state0, Address(state));
    __ srli(state1, state0, 32);
    __ ld(state2, Address(state, 8));
    __ srli(state3, state2, 32);

    Label md5_loop;
    __ BIND(md5_loop);

    __ mv(a, state0);
    __ mv(b, state1);
    __ mv(c, state2);
    __ mv(d, state3);

    // Round 1
    reg_cache.gen_load(0, buf);
    md5_FF(reg_cache, a, b, c, d,  0, S11, 0xd76aa478, rtmp1, rtmp2);
    md5_FF(reg_cache, d, a, b, c,  1, S12, 0xe8c7b756, rtmp1, rtmp2);
    reg_cache.gen_load(1, buf);
    md5_FF(reg_cache, c, d, a, b,  2, S13, 0x242070db, rtmp1, rtmp2);
    md5_FF(reg_cache, b, c, d, a,  3, S14, 0xc1bdceee, rtmp1, rtmp2);
    reg_cache.gen_load(2, buf);
    md5_FF(reg_cache, a, b, c, d,  4, S11, 0xf57c0faf, rtmp1, rtmp2);
    md5_FF(reg_cache, d, a, b, c,  5, S12, 0x4787c62a, rtmp1, rtmp2);
    reg_cache.gen_load(3, buf);
    md5_FF(reg_cache, c, d, a, b,  6, S13, 0xa8304613, rtmp1, rtmp2);
    md5_FF(reg_cache, b, c, d, a,  7, S14, 0xfd469501, rtmp1, rtmp2);
    reg_cache.gen_load(4, buf);
    md5_FF(reg_cache, a, b, c, d,  8, S11, 0x698098d8, rtmp1, rtmp2);
    md5_FF(reg_cache, d, a, b, c,  9, S12, 0x8b44f7af, rtmp1, rtmp2);
    reg_cache.gen_load(5, buf);
    md5_FF(reg_cache, c, d, a, b, 10, S13, 0xffff5bb1, rtmp1, rtmp2);
    md5_FF(reg_cache, b, c, d, a, 11, S14, 0x895cd7be, rtmp1, rtmp2);
    reg_cache.gen_load(6, buf);
    md5_FF(reg_cache, a, b, c, d, 12, S11, 0x6b901122, rtmp1, rtmp2);
    md5_FF(reg_cache, d, a, b, c, 13, S12, 0xfd987193, rtmp1, rtmp2);
    reg_cache.gen_load(7, buf);
    md5_FF(reg_cache, c, d, a, b, 14, S13, 0xa679438e, rtmp1, rtmp2);
    md5_FF(reg_cache, b, c, d, a, 15, S14, 0x49b40821, rtmp1, rtmp2);

    // Round 2
    md5_GG(reg_cache, a, b, c, d,  1, S21, 0xf61e2562, rtmp1, rtmp2);
    md5_GG(reg_cache, d, a, b, c,  6, S22, 0xc040b340, rtmp1, rtmp2);
    md5_GG(reg_cache, c, d, a, b, 11, S23, 0x265e5a51, rtmp1, rtmp2);
    md5_GG(reg_cache, b, c, d, a,  0, S24, 0xe9b6c7aa, rtmp1, rtmp2);
    md5_GG(reg_cache, a, b, c, d,  5, S21, 0xd62f105d, rtmp1, rtmp2);
    md5_GG(reg_cache, d, a, b, c, 10, S22, 0x02441453, rtmp1, rtmp2);
    md5_GG(reg_cache, c, d, a, b, 15, S23, 0xd8a1e681, rtmp1, rtmp2);
    md5_GG(reg_cache, b, c, d, a,  4, S24, 0xe7d3fbc8, rtmp1, rtmp2);
    md5_GG(reg_cache, a, b, c, d,  9, S21, 0x21e1cde6, rtmp1, rtmp2);
    md5_GG(reg_cache, d, a, b, c, 14, S22, 0xc33707d6, rtmp1, rtmp2);
    md5_GG(reg_cache, c, d, a, b,  3, S23, 0xf4d50d87, rtmp1, rtmp2);
    md5_GG(reg_cache, b, c, d, a,  8, S24, 0x455a14ed, rtmp1, rtmp2);
    md5_GG(reg_cache, a, b, c, d, 13, S21, 0xa9e3e905, rtmp1, rtmp2);
    md5_GG(reg_cache, d, a, b, c,  2, S22, 0xfcefa3f8, rtmp1, rtmp2);
    md5_GG(reg_cache, c, d, a, b,  7, S23, 0x676f02d9, rtmp1, rtmp2);
    md5_GG(reg_cache, b, c, d, a, 12, S24, 0x8d2a4c8a, rtmp1, rtmp2);

    // Round 3
    md5_HH(reg_cache, a, b, c, d,  5, S31, 0xfffa3942, rtmp1, rtmp2);
    md5_HH(reg_cache, d, a, b, c,  8, S32, 0x8771f681, rtmp1, rtmp2);
    md5_HH(reg_cache, c, d, a, b, 11, S33, 0x6d9d6122, rtmp1, rtmp2);
    md5_HH(reg_cache, b, c, d, a, 14, S34, 0xfde5380c, rtmp1, rtmp2);
    md5_HH(reg_cache, a, b, c, d,  1, S31, 0xa4beea44, rtmp1, rtmp2);
    md5_HH(reg_cache, d, a, b, c,  4, S32, 0x4bdecfa9, rtmp1, rtmp2);
    md5_HH(reg_cache, c, d, a, b,  7, S33, 0xf6bb4b60, rtmp1, rtmp2);
    md5_HH(reg_cache, b, c, d, a, 10, S34, 0xbebfbc70, rtmp1, rtmp2);
    md5_HH(reg_cache, a, b, c, d, 13, S31, 0x289b7ec6, rtmp1, rtmp2);
    md5_HH(reg_cache, d, a, b, c,  0, S32, 0xeaa127fa, rtmp1, rtmp2);
    md5_HH(reg_cache, c, d, a, b,  3, S33, 0xd4ef3085, rtmp1, rtmp2);
    md5_HH(reg_cache, b, c, d, a,  6, S34, 0x04881d05, rtmp1, rtmp2);
    md5_HH(reg_cache, a, b, c, d,  9, S31, 0xd9d4d039, rtmp1, rtmp2);
    md5_HH(reg_cache, d, a, b, c, 12, S32, 0xe6db99e5, rtmp1, rtmp2);
    md5_HH(reg_cache, c, d, a, b, 15, S33, 0x1fa27cf8, rtmp1, rtmp2);
    md5_HH(reg_cache, b, c, d, a,  2, S34, 0xc4ac5665, rtmp1, rtmp2);

    // Round 4
    md5_II(reg_cache, a, b, c, d,  0, S41, 0xf4292244, rtmp1, rtmp2);
    md5_II(reg_cache, d, a, b, c,  7, S42, 0x432aff97, rtmp1, rtmp2);
    md5_II(reg_cache, c, d, a, b, 14, S43, 0xab9423a7, rtmp1, rtmp2);
    md5_II(reg_cache, b, c, d, a,  5, S44, 0xfc93a039, rtmp1, rtmp2);
    md5_II(reg_cache, a, b, c, d, 12, S41, 0x655b59c3, rtmp1, rtmp2);
    md5_II(reg_cache, d, a, b, c,  3, S42, 0x8f0ccc92, rtmp1, rtmp2);
    md5_II(reg_cache, c, d, a, b, 10, S43, 0xffeff47d, rtmp1, rtmp2);
    md5_II(reg_cache, b, c, d, a,  1, S44, 0x85845dd1, rtmp1, rtmp2);
    md5_II(reg_cache, a, b, c, d,  8, S41, 0x6fa87e4f, rtmp1, rtmp2);
    md5_II(reg_cache, d, a, b, c, 15, S42, 0xfe2ce6e0, rtmp1, rtmp2);
    md5_II(reg_cache, c, d, a, b,  6, S43, 0xa3014314, rtmp1, rtmp2);
    md5_II(reg_cache, b, c, d, a, 13, S44, 0x4e0811a1, rtmp1, rtmp2);
    md5_II(reg_cache, a, b, c, d,  4, S41, 0xf7537e82, rtmp1, rtmp2);
    md5_II(reg_cache, d, a, b, c, 11, S42, 0xbd3af235, rtmp1, rtmp2);
    md5_II(reg_cache, c, d, a, b,  2, S43, 0x2ad7d2bb, rtmp1, rtmp2);
    md5_II(reg_cache, b, c, d, a,  9, S44, 0xeb86d391, rtmp1, rtmp2);

    __ addw(state0, state0, a);
    __ addw(state1, state1, b);
    __ addw(state2, state2, c);
    __ addw(state3, state3, d);

    if (multi_block) {
      __ addi(buf, buf, 64);
      __ addi(ofs, ofs, 64);
      // if (ofs <= limit) goto m5_loop
      __ bge(limit, ofs, md5_loop);
      __ mv(c_rarg0, ofs); // return ofs
    }

    // to minimize the number of memory operations:
    // write back the 4 state 4-byte values in pairs, with a single sd
    __ mv(t0, mask32);
    __ andr(state0, state0, t0);
    __ slli(state1, state1, 32);
    __ orr(state0, state0, state1);
    __ sd(state0, Address(state));
    __ andr(state2, state2, t0);
    __ slli(state3, state3, 32);
    __ orr(state2, state2, state3);
    __ sd(state2, Address(state, 8));

    __ pop_reg(saved_regs, sp);
    __ ret();

    return (address) start;
  }

  /**
   * Perform the quarter round calculations on values contained within four vector registers.
   *
   * @param aVec the SIMD register containing only the "a" values
   * @param bVec the SIMD register containing only the "b" values
   * @param cVec the SIMD register containing only the "c" values
   * @param dVec the SIMD register containing only the "d" values
   * @param tmp_vr temporary vector register holds intermedia values.
   */
  void chacha20_quarter_round(VectorRegister aVec, VectorRegister bVec,
                          VectorRegister cVec, VectorRegister dVec, VectorRegister tmp_vr) {
    // a += b, d ^= a, d <<<= 16
    __ vadd_vv(aVec, aVec, bVec);
    __ vxor_vv(dVec, dVec, aVec);
    __ vrole32_vi(dVec, 16, tmp_vr);

    // c += d, b ^= c, b <<<= 12
    __ vadd_vv(cVec, cVec, dVec);
    __ vxor_vv(bVec, bVec, cVec);
    __ vrole32_vi(bVec, 12, tmp_vr);

    // a += b, d ^= a, d <<<= 8
    __ vadd_vv(aVec, aVec, bVec);
    __ vxor_vv(dVec, dVec, aVec);
    __ vrole32_vi(dVec, 8, tmp_vr);

    // c += d, b ^= c, b <<<= 7
    __ vadd_vv(cVec, cVec, dVec);
    __ vxor_vv(bVec, bVec, cVec);
    __ vrole32_vi(bVec, 7, tmp_vr);
  }

  /**
   * int com.sun.crypto.provider.ChaCha20Cipher.implChaCha20Block(int[] initState, byte[] result)
   *
   *  Input arguments:
   *  c_rarg0   - state, the starting state
   *  c_rarg1   - key_stream, the array that will hold the result of the ChaCha20 block function
   *
   *  Implementation Note:
   *   Parallelization is achieved by loading individual state elements into vectors for N blocks.
   *   N depends on single vector register length.
   */
  address generate_chacha20Block() {
    Label L_Rounds;

    __ align(CodeEntryAlignment);
    StubGenStubId stub_id = StubGenStubId::chacha20Block_id;
    StubCodeMark mark(this, stub_id);
    address start = __ pc();
    __ enter();

    const int states_len = 16;
    const int step = 4;
    const Register state = c_rarg0;
    const Register key_stream = c_rarg1;
    const Register tmp_addr = t0;
    const Register length = t1;

    // Organize vector registers in an array that facilitates
    // putting repetitive opcodes into loop structures below.
    const VectorRegister work_vrs[16] = {
      v0, v1, v2,  v3,  v4,  v5,  v6,  v7,
      v8, v9, v10, v11, v12, v13, v14, v15
    };
    const VectorRegister tmp_vr = v16;
    const VectorRegister counter_vr = v17;

    {
      // Put 16 here, as com.sun.crypto.providerChaCha20Cipher.KS_MAX_LEN is 1024
      // in java level.
      __ vsetivli(length, 16, Assembler::e32, Assembler::m1);
    }

    // Load from source state.
    // Every element in source state is duplicated to all elements in the corresponding vector.
    __ mv(tmp_addr, state);
    for (int i = 0; i < states_len; i += 1) {
      __ vlse32_v(work_vrs[i], tmp_addr, zr);
      __ addi(tmp_addr, tmp_addr, step);
    }
    // Adjust counter for every individual block.
    __ vid_v(counter_vr);
    __ vadd_vv(work_vrs[12], work_vrs[12], counter_vr);

    // Perform 10 iterations of the 8 quarter round set
    {
      const Register loop = t2; // share t2 with other non-overlapping usages.
      __ mv(loop, 10);
      __ BIND(L_Rounds);

      chacha20_quarter_round(work_vrs[0], work_vrs[4], work_vrs[8],  work_vrs[12], tmp_vr);
      chacha20_quarter_round(work_vrs[1], work_vrs[5], work_vrs[9],  work_vrs[13], tmp_vr);
      chacha20_quarter_round(work_vrs[2], work_vrs[6], work_vrs[10], work_vrs[14], tmp_vr);
      chacha20_quarter_round(work_vrs[3], work_vrs[7], work_vrs[11], work_vrs[15], tmp_vr);

      chacha20_quarter_round(work_vrs[0], work_vrs[5], work_vrs[10], work_vrs[15], tmp_vr);
      chacha20_quarter_round(work_vrs[1], work_vrs[6], work_vrs[11], work_vrs[12], tmp_vr);
      chacha20_quarter_round(work_vrs[2], work_vrs[7], work_vrs[8],  work_vrs[13], tmp_vr);
      chacha20_quarter_round(work_vrs[3], work_vrs[4], work_vrs[9],  work_vrs[14], tmp_vr);

      __ subi(loop, loop, 1);
      __ bnez(loop, L_Rounds);
    }

    // Add the original state into the end working state.
    // We do this by first duplicating every element in source state array to the corresponding
    // vector, then adding it to the post-loop working state.
    __ mv(tmp_addr, state);
    for (int i = 0; i < states_len; i += 1) {
      __ vlse32_v(tmp_vr, tmp_addr, zr);
      __ addi(tmp_addr, tmp_addr, step);
      __ vadd_vv(work_vrs[i], work_vrs[i], tmp_vr);
    }
    // Add the counter overlay onto work_vrs[12] at the end.
    __ vadd_vv(work_vrs[12], work_vrs[12], counter_vr);

    // Store result to key stream.
    {
      const Register stride = t2; // share t2 with other non-overlapping usages.
      // Every block occupies 64 bytes, so we use 64 as stride of the vector store.
      __ mv(stride, 64);
      for (int i = 0; i < states_len; i += 1) {
        __ vsse32_v(work_vrs[i], key_stream, stride);
        __ addi(key_stream, key_stream, step);
      }
    }

    // Return length of output key_stream
    __ slli(c_rarg0, length, 6);

    __ leave();
    __ ret();

    return (address) start;
  }


  // ------------------------ SHA-1 intrinsic ------------------------

  // K't =
  //    5a827999, 0  <= t <= 19
  //    6ed9eba1, 20 <= t <= 39
  //    8f1bbcdc, 40 <= t <= 59
  //    ca62c1d6, 60 <= t <= 79
  void sha1_prepare_k(Register cur_k, int round) {
    assert(round >= 0 && round < 80, "must be");

    static const int64_t ks[] = {0x5a827999, 0x6ed9eba1, 0x8f1bbcdc, 0xca62c1d6};
    if ((round % 20) == 0) {
      __ mv(cur_k, ks[round/20]);
    }
  }

  // W't =
  //    M't,                                      0 <=  t <= 15
  //    ROTL'1(W't-3 ^ W't-8 ^ W't-14 ^ W't-16),  16 <= t <= 79
  void sha1_prepare_w(Register cur_w, Register ws[], Register buf, int round) {
    assert(round >= 0 && round < 80, "must be");

    if (round < 16) {
      // in the first 16 rounds, in ws[], every register contains 2 W't, e.g.
      //   in ws[0], high part contains W't-0, low part contains W't-1,
      //   in ws[1], high part contains W't-2, low part contains W't-3,
      //   ...
      //   in ws[7], high part contains W't-14, low part contains W't-15.

      if ((round % 2) == 0) {
        __ ld(ws[round/2], Address(buf, (round/2) * 8));
        // reverse bytes, as SHA-1 is defined in big-endian.
        __ revb(ws[round/2], ws[round/2]);
        __ srli(cur_w, ws[round/2], 32);
      } else {
        __ mv(cur_w, ws[round/2]);
      }

      return;
    }

    if ((round % 2) == 0) {
      int idx = 16;
      // W't = ROTL'1(W't-3 ^ W't-8 ^ W't-14 ^ W't-16),  16 <= t <= 79
      __ srli(t1, ws[(idx-8)/2], 32);
      __ xorr(t0, ws[(idx-3)/2], t1);

      __ srli(t1, ws[(idx-14)/2], 32);
      __ srli(cur_w, ws[(idx-16)/2], 32);
      __ xorr(cur_w, cur_w, t1);

      __ xorr(cur_w, cur_w, t0);
      __ rolw(cur_w, cur_w, 1, t0);

      // copy the cur_w value to ws[8].
      // now, valid w't values are at:
      //  w0:       ws[0]'s lower 32 bits
      //  w1 ~ w14: ws[1] ~ ws[7]
      //  w15:      ws[8]'s higher 32 bits
      __ slli(ws[idx/2], cur_w, 32);

      return;
    }

    int idx = 17;
    // W't = ROTL'1(W't-3 ^ W't-8 ^ W't-14 ^ W't-16),  16 <= t <= 79
    __ srli(t1, ws[(idx-3)/2], 32);
    __ xorr(t0, t1, ws[(idx-8)/2]);

    __ xorr(cur_w, ws[(idx-16)/2], ws[(idx-14)/2]);

    __ xorr(cur_w, cur_w, t0);
    __ rolw(cur_w, cur_w, 1, t0);

    // copy the cur_w value to ws[8]
    __ zext(cur_w, cur_w, 32);
    __ orr(ws[idx/2], ws[idx/2], cur_w);

    // shift the w't registers, so they start from ws[0] again.
    // now, valid w't values are at:
    //  w0 ~ w15: ws[0] ~ ws[7]
    Register ws_0 = ws[0];
    for (int i = 0; i < 16/2; i++) {
      ws[i] = ws[i+1];
    }
    ws[8] = ws_0;
  }

  // f't(x, y, z) =
  //    Ch(x, y, z)     = (x & y) ^ (~x & z)            , 0  <= t <= 19
  //    Parity(x, y, z) = x ^ y ^ z                     , 20 <= t <= 39
  //    Maj(x, y, z)    = (x & y) ^ (x & z) ^ (y & z)   , 40 <= t <= 59
  //    Parity(x, y, z) = x ^ y ^ z                     , 60 <= t <= 79
  void sha1_f(Register dst, Register x, Register y, Register z, int round) {
    assert(round >= 0 && round < 80, "must be");
    assert_different_registers(dst, x, y, z, t0, t1);

    if (round < 20) {
      // (x & y) ^ (~x & z)
      __ andr(t0, x, y);
      __ andn(dst, z, x);
      __ xorr(dst, dst, t0);
    } else if (round >= 40 && round < 60) {
      // (x & y) ^ (x & z) ^ (y & z)
      __ andr(t0, x, y);
      __ andr(t1, x, z);
      __ andr(dst, y, z);
      __ xorr(dst, dst, t0);
      __ xorr(dst, dst, t1);
    } else {
      // x ^ y ^ z
      __ xorr(dst, x, y);
      __ xorr(dst, dst, z);
    }
  }

  // T = ROTL'5(a) + f't(b, c, d) + e + K't + W't
  // e = d
  // d = c
  // c = ROTL'30(b)
  // b = a
  // a = T
  void sha1_process_round(Register a, Register b, Register c, Register d, Register e,
                          Register cur_k, Register cur_w, Register tmp, int round) {
    assert(round >= 0 && round < 80, "must be");
    assert_different_registers(a, b, c, d, e, cur_w, cur_k, tmp, t0);

    // T = ROTL'5(a) + f't(b, c, d) + e + K't + W't

    // cur_w will be recalculated at the beginning of each round,
    // so, we can reuse it as a temp register here.
    Register tmp2 = cur_w;

    // reuse e as a temporary register, as we will mv new value into it later
    Register tmp3 = e;
    __ add(tmp2, cur_k, tmp2);
    __ add(tmp3, tmp3, tmp2);
    __ rolw(tmp2, a, 5, t0);

    sha1_f(tmp, b, c, d, round);

    __ add(tmp2, tmp2, tmp);
    __ add(tmp2, tmp2, tmp3);

    // e = d
    // d = c
    // c = ROTL'30(b)
    // b = a
    // a = T
    __ mv(e, d);
    __ mv(d, c);

    __ rolw(c, b, 30);
    __ mv(b, a);
    __ mv(a, tmp2);
  }

  // H(i)0 = a + H(i-1)0
  // H(i)1 = b + H(i-1)1
  // H(i)2 = c + H(i-1)2
  // H(i)3 = d + H(i-1)3
  // H(i)4 = e + H(i-1)4
  void sha1_calculate_im_hash(Register a, Register b, Register c, Register d, Register e,
                              Register prev_ab, Register prev_cd, Register prev_e) {
    assert_different_registers(a, b, c, d, e, prev_ab, prev_cd, prev_e);

    __ add(a, a, prev_ab);
    __ srli(prev_ab, prev_ab, 32);
    __ add(b, b, prev_ab);

    __ add(c, c, prev_cd);
    __ srli(prev_cd, prev_cd, 32);
    __ add(d, d, prev_cd);

    __ add(e, e, prev_e);
  }

  void sha1_preserve_prev_abcde(Register a, Register b, Register c, Register d, Register e,
                                Register prev_ab, Register prev_cd, Register prev_e) {
    assert_different_registers(a, b, c, d, e, prev_ab, prev_cd, prev_e, t0);

    __ slli(t0, b, 32);
    __ zext(prev_ab, a, 32);
    __ orr(prev_ab, prev_ab, t0);

    __ slli(t0, d, 32);
    __ zext(prev_cd, c, 32);
    __ orr(prev_cd, prev_cd, t0);

    __ mv(prev_e, e);
  }

  // Intrinsic for:
  //   void sun.security.provider.SHA.implCompress0(byte[] buf, int ofs)
  //   void sun.security.provider.DigestBase.implCompressMultiBlock0(byte[] b, int ofs, int limit)
  //
  // Arguments:
  //
  // Inputs:
  //   c_rarg0: byte[]  src array + offset
  //   c_rarg1: int[]   SHA.state
  //   - - - - - - below are only for implCompressMultiBlock0 - - - - - -
  //   c_rarg2: int     offset
  //   c_rarg3: int     limit
  //
  // Outputs:
  //   - - - - - - below are only for implCompressMultiBlock0 - - - - - -
  //   c_rarg0: int offset, when (multi_block == true)
  //
  address generate_sha1_implCompress(StubGenStubId stub_id) {
      bool multi_block;
      switch (stub_id) {
      case sha1_implCompress_id:
        multi_block = false;
        break;
      case sha1_implCompressMB_id:
        multi_block = true;
        break;
      default:
        ShouldNotReachHere();
      };
    __ align(CodeEntryAlignment);
    StubCodeMark mark(this, stub_id);

    address start = __ pc();
    __ enter();

    RegSet saved_regs = RegSet::range(x18, x27);
    if (multi_block) {
      // use x9 as src below.
      saved_regs += RegSet::of(x9);
    }
    __ push_reg(saved_regs, sp);

    // c_rarg0 - c_rarg3: x10 - x13
    Register buf    = c_rarg0;
    Register state  = c_rarg1;
    Register offset = c_rarg2;
    Register limit  = c_rarg3;
    // use src to contain the original start point of the array.
    Register src    = x9;

    if (multi_block) {
      __ sub(limit, limit, offset);
      __ add(limit, limit, buf);
      __ sub(src, buf, offset);
    }

    // [args-reg]:  x14 - x17
    // [temp-reg]:  x28 - x31
    // [saved-reg]: x18 - x27

    // h0/1/2/3/4
    const Register a = x14, b = x15, c = x16, d = x17, e = x28;
    // w0, w1, ... w15
    // put two adjecent w's in one register:
    //    one at high word part, another at low word part
    // at different round (even or odd), w't value reside in different items in ws[].
    // w0 ~ w15, either reside in
    //    ws[0] ~ ws[7], where
    //      w0 at higher 32 bits of ws[0],
    //      w1 at lower 32 bits of ws[0],
    //      ...
    //      w14 at higher 32 bits of ws[7],
    //      w15 at lower 32 bits of ws[7].
    // or, reside in
    //    w0:       ws[0]'s lower 32 bits
    //    w1 ~ w14: ws[1] ~ ws[7]
    //    w15:      ws[8]'s higher 32 bits
    Register ws[9] = {x29, x30, x31, x18,
                      x19, x20, x21, x22,
                      x23}; // auxiliary register for calculating w's value
    // current k't's value
    const Register cur_k = x24;
    // current w't's value
    const Register cur_w = x25;
    // values of a, b, c, d, e in the previous round
    const Register prev_ab = x26, prev_cd = x27;
    const Register prev_e = offset; // reuse offset/c_rarg2

    // load 5 words state into a, b, c, d, e.
    //
    // To minimize the number of memory operations, we apply following
    // optimization: read the states (a/b/c/d) of 4-byte values in pairs,
    // with a single ld, and split them into 2 registers.
    //
    // And, as the core algorithm of SHA-1 works on 32-bits words, so
    // in the following code, it does not care about the content of
    // higher 32-bits in a/b/c/d/e. Based on this observation,
    // we can apply further optimization, which is to just ignore the
    // higher 32-bits in a/c/e, rather than set the higher
    // 32-bits of a/c/e to zero explicitly with extra instructions.
    __ ld(a, Address(state, 0));
    __ srli(b, a, 32);
    __ ld(c, Address(state, 8));
    __ srli(d, c, 32);
    __ lw(e, Address(state, 16));

    Label L_sha1_loop;
    if (multi_block) {
      __ BIND(L_sha1_loop);
    }

    sha1_preserve_prev_abcde(a, b, c, d, e, prev_ab, prev_cd, prev_e);

    for (int round = 0; round < 80; round++) {
      // prepare K't value
      sha1_prepare_k(cur_k, round);

      // prepare W't value
      sha1_prepare_w(cur_w, ws, buf, round);

      // one round process
      sha1_process_round(a, b, c, d, e, cur_k, cur_w, t2, round);
    }

    // compute the intermediate hash value
    sha1_calculate_im_hash(a, b, c, d, e, prev_ab, prev_cd, prev_e);

    if (multi_block) {
      int64_t block_bytes = 16 * 4;
      __ addi(buf, buf, block_bytes);

      __ bge(limit, buf, L_sha1_loop, true);
    }

    // store back the state.
    __ zext(a, a, 32);
    __ slli(b, b, 32);
    __ orr(a, a, b);
    __ sd(a, Address(state, 0));
    __ zext(c, c, 32);
    __ slli(d, d, 32);
    __ orr(c, c, d);
    __ sd(c, Address(state, 8));
    __ sw(e, Address(state, 16));

    // return offset
    if (multi_block) {
      __ sub(c_rarg0, buf, src);
    }

    __ pop_reg(saved_regs, sp);

    __ leave();
    __ ret();

    return (address) start;
  }

  /**
   * vector registers:
   *   input VectorRegister's:  intputV1-V3, for m2 they could be v2, v4, v6, for m1 they could be v1, v2, v3
   *   index VectorRegister's:  idxV1-V4, for m2 they could be v8, v10, v12, v14, for m1 they could be v4, v5, v6, v7
   *   output VectorRegister's: outputV1-V4, for m2 they could be v16, v18, v20, v22, for m1 they could be v8, v9, v10, v11
   *
   * NOTE: each field will occupy a vector register group
   */
  void base64_vector_encode_round(Register src, Register dst, Register codec,
                    Register size, Register stepSrc, Register stepDst,
                    VectorRegister inputV1, VectorRegister inputV2, VectorRegister inputV3,
                    VectorRegister idxV1, VectorRegister idxV2, VectorRegister idxV3, VectorRegister idxV4,
                    VectorRegister outputV1, VectorRegister outputV2, VectorRegister outputV3, VectorRegister outputV4,
                    Assembler::LMUL lmul) {
    // set vector register type/len
    __ vsetvli(x0, size, Assembler::e8, lmul);

    // segmented load src into v registers: mem(src) => vr(3)
    __ vlseg3e8_v(inputV1, src);

    // src = src + register_group_len_bytes * 3
    __ add(src, src, stepSrc);

    // encoding
    //   1. compute index into lookup table: vr(3) => vr(4)
    __ vsrl_vi(idxV1, inputV1, 2);

    __ vsrl_vi(idxV2, inputV2, 2);
    __ vsll_vi(inputV1, inputV1, 6);
    __ vor_vv(idxV2, idxV2, inputV1);
    __ vsrl_vi(idxV2, idxV2, 2);

    __ vsrl_vi(idxV3, inputV3, 4);
    __ vsll_vi(inputV2, inputV2, 4);
    __ vor_vv(idxV3, inputV2, idxV3);
    __ vsrl_vi(idxV3, idxV3, 2);

    __ vsll_vi(idxV4, inputV3, 2);
    __ vsrl_vi(idxV4, idxV4, 2);

    //   2. indexed load: vr(4) => vr(4)
    __ vluxei8_v(outputV1, codec, idxV1);
    __ vluxei8_v(outputV2, codec, idxV2);
    __ vluxei8_v(outputV3, codec, idxV3);
    __ vluxei8_v(outputV4, codec, idxV4);

    // segmented store encoded data in v registers back to dst: vr(4) => mem(dst)
    __ vsseg4e8_v(outputV1, dst);

    // dst = dst + register_group_len_bytes * 4
    __ add(dst, dst, stepDst);
  }

  /**
   *  void j.u.Base64.Encoder.encodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, boolean isURL)
   *
   *  Input arguments:
   *  c_rarg0   - src, source array
   *  c_rarg1   - sp, src start offset
   *  c_rarg2   - sl, src end offset
   *  c_rarg3   - dst, dest array
   *  c_rarg4   - dp, dst start offset
   *  c_rarg5   - isURL, Base64 or URL character set
   */
  address generate_base64_encodeBlock() {
    alignas(64) static const char toBase64[64] = {
      'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
      'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
      'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
      'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
      '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'
    };

    alignas(64) static const char toBase64URL[64] = {
      'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
      'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
      'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
      'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
      '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_'
    };

    __ align(CodeEntryAlignment);
    StubGenStubId stub_id = StubGenStubId::base64_encodeBlock_id;
    StubCodeMark mark(this, stub_id);
    address start = __ pc();
    __ enter();

    Register src    = c_rarg0;
    Register soff   = c_rarg1;
    Register send   = c_rarg2;
    Register dst    = c_rarg3;
    Register doff   = c_rarg4;
    Register isURL  = c_rarg5;

    Register codec  = c_rarg6;
    Register length = c_rarg7; // total length of src data in bytes

    Label ProcessData, Exit;

    // length should be multiple of 3
    __ sub(length, send, soff);
    // real src/dst to process data
    __ add(src, src, soff);
    __ add(dst, dst, doff);

    // load the codec base address
    __ la(codec, ExternalAddress((address) toBase64));
    __ beqz(isURL, ProcessData);
    __ la(codec, ExternalAddress((address) toBase64URL));
    __ BIND(ProcessData);

    // vector version
    if (UseRVV) {
      Label ProcessM2, ProcessM1, ProcessScalar;

      Register size      = soff;
      Register stepSrcM1 = send;
      Register stepSrcM2 = doff;
      Register stepDst   = isURL;

      __ mv(size, MaxVectorSize * 2);
      __ mv(stepSrcM1, MaxVectorSize * 3);
      __ slli(stepSrcM2, stepSrcM1, 1);
      __ mv(stepDst, MaxVectorSize * 2 * 4);

      __ blt(length, stepSrcM2, ProcessM1);

      __ BIND(ProcessM2);
      base64_vector_encode_round(src, dst, codec,
                    size, stepSrcM2, stepDst,
                    v2, v4, v6,         // inputs
                    v8, v10, v12, v14,  // indexes
                    v16, v18, v20, v22, // outputs
                    Assembler::m2);

      __ sub(length, length, stepSrcM2);
      __ bge(length, stepSrcM2, ProcessM2);

      __ BIND(ProcessM1);
      __ blt(length, stepSrcM1, ProcessScalar);

      __ srli(size, size, 1);
      __ srli(stepDst, stepDst, 1);
      base64_vector_encode_round(src, dst, codec,
                    size, stepSrcM1, stepDst,
                    v1, v2, v3,         // inputs
                    v4, v5, v6, v7,     // indexes
                    v8, v9, v10, v11,   // outputs
                    Assembler::m1);
      __ sub(length, length, stepSrcM1);

      __ BIND(ProcessScalar);
    }

    // scalar version
    {
      Register byte1 = soff, byte0 = send, byte2 = doff;
      Register combined24Bits = isURL;

      __ beqz(length, Exit);

      Label ScalarLoop;
      __ BIND(ScalarLoop);
      {
        // plain:   [byte0[7:0] : byte1[7:0] : byte2[7:0]] =>
        // encoded: [byte0[7:2] : byte0[1:0]+byte1[7:4] : byte1[3:0]+byte2[7:6] : byte2[5:0]]

        // load 3 bytes src data
        __ lbu(byte0, Address(src, 0));
        __ lbu(byte1, Address(src, 1));
        __ lbu(byte2, Address(src, 2));
        __ addi(src, src, 3);

        // construct 24 bits from 3 bytes
        __ slliw(byte0, byte0, 16);
        __ slliw(byte1, byte1, 8);
        __ orr(combined24Bits, byte0, byte1);
        __ orr(combined24Bits, combined24Bits, byte2);

        // get codec index and encode(ie. load from codec by index)
        __ slliw(byte0, combined24Bits, 8);
        __ srliw(byte0, byte0, 26);
        __ add(byte0, codec, byte0);
        __ lbu(byte0, byte0);

        __ slliw(byte1, combined24Bits, 14);
        __ srliw(byte1, byte1, 26);
        __ add(byte1, codec, byte1);
        __ lbu(byte1, byte1);

        __ slliw(byte2, combined24Bits, 20);
        __ srliw(byte2, byte2, 26);
        __ add(byte2, codec, byte2);
        __ lbu(byte2, byte2);

        __ andi(combined24Bits, combined24Bits, 0x3f);
        __ add(combined24Bits, codec, combined24Bits);
        __ lbu(combined24Bits, combined24Bits);

        // store 4 bytes encoded data
        __ sb(byte0, Address(dst, 0));
        __ sb(byte1, Address(dst, 1));
        __ sb(byte2, Address(dst, 2));
        __ sb(combined24Bits, Address(dst, 3));

        __ subi(length, length, 3);
        __ addi(dst, dst, 4);
        // loop back
        __ bnez(length, ScalarLoop);
      }
    }

    __ BIND(Exit);

    __ leave();
    __ ret();

    return (address) start;
  }

  /**
   * vector registers:
   * input VectorRegister's:  intputV1-V4, for m2 they could be v2, v4, v6, for m1 they could be v2, v4, v6, v8
   * index VectorRegister's:  idxV1-V3, for m2 they could be v8, v10, v12, v14, for m1 they could be v10, v12, v14, v16
   * output VectorRegister's: outputV1-V4, for m2 they could be v16, v18, v20, v22, for m1 they could be v18, v20, v22
   *
   * NOTE: each field will occupy a single vector register group
   */
  void base64_vector_decode_round(Register src, Register dst, Register codec,
                    Register size, Register stepSrc, Register stepDst, Register failedIdx,
                    VectorRegister inputV1, VectorRegister inputV2, VectorRegister inputV3, VectorRegister inputV4,
                    VectorRegister idxV1, VectorRegister idxV2, VectorRegister idxV3, VectorRegister idxV4,
                    VectorRegister outputV1, VectorRegister outputV2, VectorRegister outputV3,
                    Assembler::LMUL lmul) {
    // set vector register type/len
    __ vsetvli(x0, size, Assembler::e8, lmul, Assembler::ma, Assembler::ta);

    // segmented load src into v registers: mem(src) => vr(4)
    __ vlseg4e8_v(inputV1, src);

    // src = src + register_group_len_bytes * 4
    __ add(src, src, stepSrc);

    // decoding
    //   1. indexed load: vr(4) => vr(4)
    __ vluxei8_v(idxV1, codec, inputV1);
    __ vluxei8_v(idxV2, codec, inputV2);
    __ vluxei8_v(idxV3, codec, inputV3);
    __ vluxei8_v(idxV4, codec, inputV4);

    //   2. check wrong data
    __ vor_vv(outputV1, idxV1, idxV2);
    __ vor_vv(outputV2, idxV3, idxV4);
    __ vor_vv(outputV1, outputV1, outputV2);
    __ vmseq_vi(v0, outputV1, -1);
    __ vfirst_m(failedIdx, v0);
    Label NoFailure, FailureAtIdx0;
    // valid value can only be -1 when < 0
    __ bltz(failedIdx, NoFailure);
    // when the first data (at index 0) fails, no need to process data anymore
    __ beqz(failedIdx, FailureAtIdx0);
    __ vsetvli(x0, failedIdx, Assembler::e8, lmul, Assembler::mu, Assembler::tu);
    __ slli(stepDst, failedIdx, 1);
    __ add(stepDst, failedIdx, stepDst);
    __ BIND(NoFailure);

    //   3. compute the decoded data: vr(4) => vr(3)
    __ vsll_vi(idxV1, idxV1, 2);
    __ vsrl_vi(outputV1, idxV2, 4);
    __ vor_vv(outputV1, outputV1, idxV1);

    __ vsll_vi(idxV2, idxV2, 4);
    __ vsrl_vi(outputV2, idxV3, 2);
    __ vor_vv(outputV2, outputV2, idxV2);

    __ vsll_vi(idxV3, idxV3, 6);
    __ vor_vv(outputV3, idxV4, idxV3);

    // segmented store encoded data in v registers back to dst: vr(3) => mem(dst)
    __ vsseg3e8_v(outputV1, dst);

    // dst = dst + register_group_len_bytes * 3
    __ add(dst, dst, stepDst);
    __ BIND(FailureAtIdx0);
  }

  /**
   * int j.u.Base64.Decoder.decodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, boolean isURL, boolean isMIME)
   *
   *  Input arguments:
   *  c_rarg0   - src, source array
   *  c_rarg1   - sp, src start offset
   *  c_rarg2   - sl, src end offset
   *  c_rarg3   - dst, dest array
   *  c_rarg4   - dp, dst start offset
   *  c_rarg5   - isURL, Base64 or URL character set
   *  c_rarg6   - isMIME, Decoding MIME block
   */
  address generate_base64_decodeBlock() {

    static const uint8_t fromBase64[256] = {
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,  62u, 255u, 255u, 255u,  63u,
        52u,  53u,  54u,  55u,  56u,  57u,  58u,  59u,  60u,  61u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u,   0u,   1u,   2u,   3u,   4u,   5u,   6u,   7u,   8u,   9u,  10u,  11u,  12u,  13u,  14u,
        15u,  16u,  17u,  18u,  19u,  20u,  21u,  22u,  23u,  24u,  25u, 255u, 255u, 255u, 255u, 255u,
        255u,  26u,  27u,  28u,  29u,  30u,  31u,  32u,  33u,  34u,  35u,  36u,  37u,  38u,  39u,  40u,
        41u,  42u,  43u,  44u,  45u,  46u,  47u,  48u,  49u,  50u,  51u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
    };

    static const uint8_t fromBase64URL[256] = {
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,  62u, 255u, 255u,
        52u,  53u,  54u,  55u,  56u,  57u,  58u,  59u,  60u,  61u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u,   0u,   1u,   2u,   3u,   4u,   5u,   6u,   7u,   8u,   9u,  10u,  11u,  12u,  13u,  14u,
        15u,  16u,  17u,  18u,  19u,  20u,  21u,  22u,  23u,  24u,  25u, 255u, 255u, 255u, 255u,  63u,
        255u,  26u,  27u,  28u,  29u,  30u,  31u,  32u,  33u,  34u,  35u,  36u,  37u,  38u,  39u,  40u,
        41u,  42u,  43u,  44u,  45u,  46u,  47u,  48u,  49u,  50u,  51u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
        255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
    };

    __ align(CodeEntryAlignment);
    StubGenStubId stub_id = StubGenStubId::base64_decodeBlock_id;
    StubCodeMark mark(this, stub_id);
    address start = __ pc();
    __ enter();

    Register src    = c_rarg0;
    Register soff   = c_rarg1;
    Register send   = c_rarg2;
    Register dst    = c_rarg3;
    Register doff   = c_rarg4;
    Register isURL  = c_rarg5;
    Register isMIME = c_rarg6;

    Register codec     = c_rarg7;
    Register dstBackup = t6;
    Register length    = t3;     // total length of src data in bytes

    Label ProcessData, Exit;
    Label ProcessScalar, ScalarLoop;

    // passed in length (send - soff) is guaranteed to be > 4,
    // and in this intrinsic we only process data of length in multiple of 4,
    // it's not guaranteed to be multiple of 4 by java level, so do it explicitly
    __ sub(length, send, soff);
    __ andi(length, length, -4);
    // real src/dst to process data
    __ add(src, src, soff);
    __ add(dst, dst, doff);
    // backup of dst, used to calculate the return value at exit
    __ mv(dstBackup, dst);

    // load the codec base address
    __ la(codec, ExternalAddress((address) fromBase64));
    __ beqz(isURL, ProcessData);
    __ la(codec, ExternalAddress((address) fromBase64URL));
    __ BIND(ProcessData);

    // vector version
    if (UseRVV) {
      // for MIME case, it has a default length limit of 76 which could be
      // different(smaller) from (send - soff), so in MIME case, we go through
      // the scalar code path directly.
      __ bnez(isMIME, ScalarLoop);

      Label ProcessM1, ProcessM2;

      Register failedIdx = soff;
      Register stepSrcM1 = send;
      Register stepSrcM2 = doff;
      Register stepDst   = isURL;
      Register size      = t4;

      __ mv(size, MaxVectorSize * 2);
      __ mv(stepSrcM1, MaxVectorSize * 4);
      __ slli(stepSrcM2, stepSrcM1, 1);
      __ mv(stepDst, MaxVectorSize * 2 * 3);

      __ blt(length, stepSrcM2, ProcessM1);


      // Assembler::m2
      __ BIND(ProcessM2);
      base64_vector_decode_round(src, dst, codec,
                    size, stepSrcM2, stepDst, failedIdx,
                    v2, v4, v6, v8,      // inputs
                    v10, v12, v14, v16,  // indexes
                    v18, v20, v22,       // outputs
                    Assembler::m2);
      __ sub(length, length, stepSrcM2);

      // error check
      // valid value of failedIdx can only be -1 when < 0
      __ bgez(failedIdx, Exit);

      __ bge(length, stepSrcM2, ProcessM2);


      // Assembler::m1
      __ BIND(ProcessM1);
      __ blt(length, stepSrcM1, ProcessScalar);

      __ srli(size, size, 1);
      __ srli(stepDst, stepDst, 1);
      base64_vector_decode_round(src, dst, codec,
                    size, stepSrcM1, stepDst, failedIdx,
                    v1, v2, v3, v4,      // inputs
                    v5, v6, v7, v8,      // indexes
                    v9, v10, v11,        // outputs
                    Assembler::m1);
      __ sub(length, length, stepSrcM1);

      // error check
      // valid value of failedIdx can only be -1 when < 0
      __ bgez(failedIdx, Exit);

      __ BIND(ProcessScalar);
      __ beqz(length, Exit);
    }

    // scalar version
    {
      Register byte0 = soff, byte1 = send, byte2 = doff, byte3 = isURL;
      Register combined32Bits = t4;

      // encoded:   [byte0[5:0] : byte1[5:0] : byte2[5:0]] : byte3[5:0]] =>
      // plain:     [byte0[5:0]+byte1[5:4] : byte1[3:0]+byte2[5:2] : byte2[1:0]+byte3[5:0]]
      __ BIND(ScalarLoop);

      // load 4 bytes encoded src data
      __ lbu(byte0, Address(src, 0));
      __ lbu(byte1, Address(src, 1));
      __ lbu(byte2, Address(src, 2));
      __ lbu(byte3, Address(src, 3));
      __ addi(src, src, 4);

      // get codec index and decode (ie. load from codec by index)
      __ add(byte0, codec, byte0);
      __ add(byte1, codec, byte1);
      __ lb(byte0, Address(byte0, 0));
      __ lb(byte1, Address(byte1, 0));
      __ add(byte2, codec, byte2);
      __ add(byte3, codec, byte3);
      __ lb(byte2, Address(byte2, 0));
      __ lb(byte3, Address(byte3, 0));
      __ slliw(byte0, byte0, 18);
      __ slliw(byte1, byte1, 12);
      __ orr(byte0, byte0, byte1);
      __ orr(byte0, byte0, byte3);
      __ slliw(byte2, byte2, 6);
      // For performance consideration, `combined32Bits` is constructed for 2 purposes at the same time,
      //  1. error check below
      //  2. decode below
      __ orr(combined32Bits, byte0, byte2);

      // error check
      __ bltz(combined32Bits, Exit);

      // store 3 bytes decoded data
      __ sraiw(byte0, combined32Bits, 16);
      __ sraiw(byte1, combined32Bits, 8);
      __ sb(byte0, Address(dst, 0));
      __ sb(byte1, Address(dst, 1));
      __ sb(combined32Bits, Address(dst, 2));

      __ subi(length, length, 4);
      __ addi(dst, dst, 3);
      // loop back
      __ bnez(length, ScalarLoop);
    }

    __ BIND(Exit);
    __ sub(c_rarg0, dst, dstBackup);

    __ leave();
    __ ret();

    return (address) start;
  }

  void adler32_process_bytes(Register buff, Register s1, Register s2, VectorRegister vtable,
    VectorRegister vzero, VectorRegister vbytes, VectorRegister vs1acc, VectorRegister vs2acc,
    Register temp0, Register temp1, Register temp2,  Register temp3,
    VectorRegister vtemp1, VectorRegister vtemp2, int step, Assembler::LMUL lmul) {

    assert((lmul == Assembler::m4 && step == 64) ||
           (lmul == Assembler::m2 && step == 32) ||
           (lmul == Assembler::m1 && step == 16),
           "LMUL should be aligned with step: m4 and 64, m2 and 32 or m1 and 16");
    // Below is function for calculating Adler32 checksum with 64-, 32- or 16-byte step. LMUL=m4, m2 or m1 is used.
    // The results are in v12, v13, ..., v22, v23. Example below is for 64-byte step case.
    // We use b1, b2, ..., b64 to denote the 64 bytes loaded in each iteration.
    // In non-vectorized code, we update s1 and s2 as:
    //   s1 <- s1 + b1
    //   s2 <- s2 + s1
    //   s1 <- s1 + b2
    //   s2 <- s2 + b1
    //   ...
    //   s1 <- s1 + b64
    //   s2 <- s2 + s1
    // Putting above assignments together, we have:
    //   s1_new = s1 + b1 + b2 + ... + b64
    //   s2_new = s2 + (s1 + b1) + (s1 + b1 + b2) + ... + (s1 + b1 + b2 + ... + b64) =
    //          = s2 + s1 * 64 + (b1 * 64 + b2 * 63 + ... + b64 * 1) =
    //          = s2 + s1 * 64 + (b1, b2, ... b64) dot (64, 63, ... 1)

    __ mv(temp3, step);
    // Load data
    __ vsetvli(temp0, temp3, Assembler::e8, lmul);
    __ vle8_v(vbytes, buff);
    __ addi(buff, buff, step);

    // Upper bound reduction sum for s1_new:
    // 0xFF * 64 = 0x3FC0, so:
    // 1. Need to do vector-widening reduction sum
    // 2. It is safe to perform sign-extension during vmv.x.s with 16-bits elements
    __ vwredsumu_vs(vs1acc, vbytes, vzero);
    // Multiplication for s2_new
    __ vwmulu_vv(vs2acc, vtable, vbytes);

    // s2 = s2 + s1 * log2(step)
    __ slli(temp1, s1, exact_log2(step));
    __ add(s2, s2, temp1);

    // Summing up calculated results for s2_new
    if (MaxVectorSize > 16) {
      __ vsetvli(temp0, temp3, Assembler::e16, lmul);
    } else {
      // Half of vector-widening multiplication result is in successor of vs2acc
      // group for vlen == 16, in which case we need to double vector register
      // group width in order to reduction sum all of them
      Assembler::LMUL lmulx2 = (lmul == Assembler::m1) ? Assembler::m2 :
                               (lmul == Assembler::m2) ? Assembler::m4 : Assembler::m8;
      __ vsetvli(temp0, temp3, Assembler::e16, lmulx2);
    }
    // Upper bound for reduction sum:
    // 0xFF * (64 + 63 + ... + 2 + 1) = 0x817E0 max for whole register group, so:
    // 1. Need to do vector-widening reduction sum
    // 2. It is safe to perform sign-extension during vmv.x.s with 32-bits elements
    __ vwredsumu_vs(vtemp1, vs2acc, vzero);

    // Extracting results for:
    // s1_new
    __ vmv_x_s(temp0, vs1acc);
    __ add(s1, s1, temp0);
    // s2_new
    __ vsetvli(temp0, temp3, Assembler::e32, Assembler::m1);
    __ vmv_x_s(temp1, vtemp1);
    __ add(s2, s2, temp1);
  }

  /***
   *  int java.util.zip.Adler32.updateBytes(int adler, byte[] b, int off, int len)
   *
   *  Arguments:
   *
   *  Inputs:
   *   c_rarg0   - int   adler
   *   c_rarg1   - byte* buff (b + off)
   *   c_rarg2   - int   len
   *
   *  Output:
   *   c_rarg0   - int adler result
   */
  address generate_updateBytesAdler32() {
    __ align(CodeEntryAlignment);
    StubGenStubId stub_id = StubGenStubId::updateBytesAdler32_id;
    StubCodeMark mark(this, stub_id);
    address start = __ pc();

    Label L_nmax, L_nmax_loop, L_nmax_loop_entry, L_by16, L_by16_loop,
      L_by16_loop_unroll, L_by1_loop, L_do_mod, L_combine, L_by1;

    // Aliases
    Register adler  = c_rarg0;
    Register s1     = c_rarg0;
    Register s2     = c_rarg3;
    Register buff   = c_rarg1;
    Register len    = c_rarg2;
    Register nmax  = c_rarg4;
    Register base  = c_rarg5;
    Register count = c_rarg6;
    Register temp0 = t3;
    Register temp1 = t4;
    Register temp2 = t5;
    Register temp3 = t6;

    VectorRegister vzero = v31;
    VectorRegister vbytes = v8; // group: v8, v9, v10, v11
    VectorRegister vs1acc = v12; // group: v12, v13, v14, v15
    VectorRegister vs2acc = v16; // group: v16, v17, v18, v19, v20, v21, v22, v23
    VectorRegister vtable_64 = v24; // group: v24, v25, v26, v27
    VectorRegister vtable_32 = v4; // group: v4, v5
    VectorRegister vtable_16 = v30;
    VectorRegister vtemp1 = v28;
    VectorRegister vtemp2 = v29;

    // Max number of bytes we can process before having to take the mod
    // 0x15B0 is 5552 in decimal, the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1
    const uint64_t BASE = 0xfff1;
    const uint64_t NMAX = 0x15B0;

    // Loops steps
    int step_64 = 64;
    int step_32 = 32;
    int step_16 = 16;
    int step_1  = 1;

    __ enter(); // Required for proper stackwalking of RuntimeStub frame
    __ mv(temp1, 64);
    __ vsetvli(temp0, temp1, Assembler::e8, Assembler::m4);

    // Generating accumulation coefficients for further calculations
    // vtable_64:
    __ vid_v(vtemp1);
    __ vrsub_vx(vtable_64, vtemp1, temp1);
    // vtable_64 group now contains { 0x40, 0x3f, 0x3e, ..., 0x3, 0x2, 0x1 }

    // vtable_32:
    __ mv(temp1, 32);
    __ vsetvli(temp0, temp1, Assembler::e8, Assembler::m2);
    __ vid_v(vtemp1);
    __ vrsub_vx(vtable_32, vtemp1, temp1);
    // vtable_32 group now contains { 0x20, 0x1f, 0x1e, ..., 0x3, 0x2, 0x1 }

    __ vsetivli(temp0, 16, Assembler::e8, Assembler::m1);
    // vtable_16:
    __ mv(temp1, 16);
    __ vid_v(vtemp1);
    __ vrsub_vx(vtable_16, vtemp1, temp1);
    // vtable_16 now contains { 0x10, 0xf, 0xe, ..., 0x3, 0x2, 0x1 }

    __ vmv_v_i(vzero, 0);

    __ mv(base, BASE);
    __ mv(nmax, NMAX);

    // s1 is initialized to the lower 16 bits of adler
    // s2 is initialized to the upper 16 bits of adler
    __ srliw(s2, adler, 16); // s2 = ((adler >> 16) & 0xffff)
    __ zext(s1, adler, 16); // s1 = (adler & 0xffff)

    // The pipelined loop needs at least 16 elements for 1 iteration
    // It does check this, but it is more effective to skip to the cleanup loop
    __ mv(temp0, step_16);
    __ bgeu(len, temp0, L_nmax);
    __ beqz(len, L_combine);

    // Jumping to L_by1_loop
    __ subi(len, len, step_1);
    __ j(L_by1_loop);

  __ bind(L_nmax);
    __ sub(len, len, nmax);
    __ subi(count, nmax, 16);
    __ bltz(len, L_by16);

  // Align L_nmax loop by 64
  __ bind(L_nmax_loop_entry);
    __ subi(count, count, 32);

  __ bind(L_nmax_loop);
    adler32_process_bytes(buff, s1, s2, vtable_64, vzero,
      vbytes, vs1acc, vs2acc, temp0, temp1, temp2, temp3,
      vtemp1, vtemp2, step_64, Assembler::m4);
    __ subi(count, count, step_64);
    __ bgtz(count, L_nmax_loop);

    // There are three iterations left to do
    adler32_process_bytes(buff, s1, s2, vtable_32, vzero,
      vbytes, vs1acc, vs2acc, temp0, temp1, temp2, temp3,
      vtemp1, vtemp2, step_32, Assembler::m2);
    adler32_process_bytes(buff, s1, s2, vtable_16, vzero,
      vbytes, vs1acc, vs2acc, temp0, temp1, temp2, temp3,
      vtemp1, vtemp2, step_16, Assembler::m1);

    // s1 = s1 % BASE
    __ remuw(s1, s1, base);
    // s2 = s2 % BASE
    __ remuw(s2, s2, base);

    __ sub(len, len, nmax);
    __ subi(count, nmax, 16);
    __ bgez(len, L_nmax_loop_entry);

  __ bind(L_by16);
    __ add(len, len, count);
    __ bltz(len, L_by1);
    // Trying to unroll
    __ mv(temp3, step_64);
    __ blt(len, temp3, L_by16_loop);

  __ bind(L_by16_loop_unroll);
    adler32_process_bytes(buff, s1, s2, vtable_64, vzero,
      vbytes, vs1acc, vs2acc, temp0, temp1, temp2, temp3,
      vtemp1, vtemp2, step_64, Assembler::m4);
    __ subi(len, len, step_64);
    // By now the temp3 should still be 64
    __ bge(len, temp3, L_by16_loop_unroll);

  __ bind(L_by16_loop);
    adler32_process_bytes(buff, s1, s2, vtable_16, vzero,
      vbytes, vs1acc, vs2acc, temp0, temp1, temp2, temp3,
      vtemp1, vtemp2, step_16, Assembler::m1);
    __ subi(len, len, step_16);
    __ bgez(len, L_by16_loop);

  __ bind(L_by1);
    __ addi(len, len, 15);
    __ bltz(len, L_do_mod);

  __ bind(L_by1_loop);
    __ lbu(temp0, Address(buff, 0));
    __ addi(buff, buff, step_1);
    __ add(s1, temp0, s1);
    __ add(s2, s2, s1);
    __ subi(len, len, step_1);
    __ bgez(len, L_by1_loop);

  __ bind(L_do_mod);
    // s1 = s1 % BASE
    __ remuw(s1, s1, base);
    // s2 = s2 % BASE
    __ remuw(s2, s2, base);

    // Combine lower bits and higher bits
    // adler = s1 | (s2 << 16)
  __ bind(L_combine);
    __ slli(s2, s2, 16);
    __ orr(s1, s1, s2);

    __ leave(); // Required for proper stackwalking of RuntimeStub frame
    __ ret();

    return start;
  }

#endif // COMPILER2_OR_JVMCI

  // x10 = input (float16)
  // f10 = result (float)
  // t1  = temporary register
  address generate_float16ToFloat() {
    __ align(CodeEntryAlignment);
    StubGenStubId stub_id = StubGenStubId::hf2f_id;
    StubCodeMark mark(this, stub_id);
    address entry = __ pc();
    BLOCK_COMMENT("float16ToFloat:");

    FloatRegister dst = f10;
    Register src = x10;
    Label NaN_SLOW;

    assert(VM_Version::supports_float16_float_conversion(), "must");

    // On riscv, NaN needs a special process as fcvt does not work in that case.
    // On riscv, Inf does not need a special process as fcvt can handle it correctly.
    // but we consider to get the slow path to process NaN and Inf at the same time,
    // as both of them are rare cases, and if we try to get the slow path to handle
    // only NaN case it would sacrifise the performance for normal cases,
    // i.e. non-NaN and non-Inf cases.

    // check whether it's a NaN or +/- Inf.
    __ mv(t0, 0x7c00);
    __ andr(t1, src, t0);
    // jump to stub processing NaN and Inf cases.
    __ beq(t0, t1, NaN_SLOW);

    // non-NaN or non-Inf cases, just use built-in instructions.
    __ fmv_h_x(dst, src);
    __ fcvt_s_h(dst, dst);
    __ ret();

    __ bind(NaN_SLOW);
    // following instructions mainly focus on NaN, as riscv does not handle
    // NaN well with fcvt, but the code also works for Inf at the same time.

    // construct a NaN in 32 bits from the NaN in 16 bits,
    // we need the payloads of non-canonical NaNs to be preserved.
    __ mv(t1, 0x7f800000);
    // sign-bit was already set via sign-extension if necessary.
    __ slli(t0, src, 13);
    __ orr(t1, t0, t1);
    __ fmv_w_x(dst, t1);

    __ ret();
    return entry;
  }

  // f10 = input (float)
  // x10 = result (float16)
  // f11 = temporary float register
  // t1  = temporary register
  address generate_floatToFloat16() {
    __ align(CodeEntryAlignment);
    StubGenStubId stub_id = StubGenStubId::f2hf_id;
    StubCodeMark mark(this, stub_id);
    address entry = __ pc();
    BLOCK_COMMENT("floatToFloat16:");

    Register dst = x10;
    FloatRegister src = f10, ftmp = f11;
    Label NaN_SLOW;

    assert(VM_Version::supports_float16_float_conversion(), "must");

    // On riscv, NaN needs a special process as fcvt does not work in that case.

    // check whether it's a NaN.
    // replace fclass with feq as performance optimization.
    __ feq_s(t0, src, src);
    // jump to stub processing NaN cases.
    __ beqz(t0, NaN_SLOW);

    // non-NaN cases, just use built-in instructions.
    __ fcvt_h_s(ftmp, src);
    __ fmv_x_h(dst, ftmp);
    __ ret();

    __ bind(NaN_SLOW);

    __ float_to_float16_NaN(dst, src, t0, t1);

    __ ret();
    return entry;
  }

#ifdef COMPILER2

static const int64_t right_2_bits = right_n_bits(2);
static const int64_t right_3_bits = right_n_bits(3);

  // In sun.security.util.math.intpoly.IntegerPolynomial1305, integers
  // are represented as long[5], with BITS_PER_LIMB = 26.
  // Pack five 26-bit limbs into three 64-bit registers.
  void poly1305_pack_26(Register dest0, Register dest1, Register dest2, Register src, Register tmp1, Register tmp2) {
    assert_different_registers(dest0, dest1, dest2, src, tmp1, tmp2);

    // The goal is to have 128-bit value in dest2:dest1:dest0
    __ ld(dest0, Address(src, 0));    // 26 bits in dest0

    __ ld(tmp1, Address(src, sizeof(jlong)));
    __ slli(tmp1, tmp1, 26);
    __ add(dest0, dest0, tmp1);       // 52 bits in dest0

    __ ld(tmp2, Address(src, 2 * sizeof(jlong)));
    __ slli(tmp1, tmp2, 52);
    __ add(dest0, dest0, tmp1);       // dest0 is full

    __ srli(dest1, tmp2, 12);         // 14-bit in dest1

    __ ld(tmp1, Address(src, 3 * sizeof(jlong)));
    __ slli(tmp1, tmp1, 14);
    __ add(dest1, dest1, tmp1);       // 40-bit in dest1

    __ ld(tmp1, Address(src, 4 * sizeof(jlong)));
    __ slli(tmp2, tmp1, 40);
    __ add(dest1, dest1, tmp2);       // dest1 is full

    if (dest2->is_valid()) {
      __ srli(tmp1, tmp1, 24);
      __ mv(dest2, tmp1);               // 2 bits in dest2
    } else {
#ifdef ASSERT
      Label OK;
      __ srli(tmp1, tmp1, 24);
      __ beq(zr, tmp1, OK);           // 2 bits
      __ stop("high bits of Poly1305 integer should be zero");
      __ should_not_reach_here();
      __ bind(OK);
#endif
    }
  }

  // As above, but return only a 128-bit integer, packed into two
  // 64-bit registers.
  void poly1305_pack_26(Register dest0, Register dest1, Register src, Register tmp1, Register tmp2) {
    poly1305_pack_26(dest0, dest1, noreg, src, tmp1, tmp2);
  }

  // U_2:U_1:U_0: += (U_2 >> 2) * 5
  void poly1305_reduce(Register U_2, Register U_1, Register U_0, Register tmp1, Register tmp2) {
    assert_different_registers(U_2, U_1, U_0, tmp1, tmp2);

    // First, U_2:U_1:U_0 += (U_2 >> 2)
    __ srli(tmp1, U_2, 2);
    __ cad(U_0, U_0, tmp1, tmp2); // Add tmp1 to U_0 with carry output to tmp2
    __ andi(U_2, U_2, right_2_bits); // Clear U_2 except for the lowest two bits
    __ cad(U_1, U_1, tmp2, tmp2); // Add carry to U_1 with carry output to tmp2
    __ add(U_2, U_2, tmp2);

    // Second, U_2:U_1:U_0 += (U_2 >> 2) << 2
    __ slli(tmp1, tmp1, 2);
    __ cad(U_0, U_0, tmp1, tmp2); // Add tmp1 to U_0 with carry output to tmp2
    __ cad(U_1, U_1, tmp2, tmp2); // Add carry to U_1 with carry output to tmp2
    __ add(U_2, U_2, tmp2);
  }

  // Poly1305, RFC 7539
  // void com.sun.crypto.provider.Poly1305.processMultipleBlocks(byte[] input, int offset, int length, long[] aLimbs, long[] rLimbs)

  // Arguments:
  //    c_rarg0:   input_start -- where the input is stored
  //    c_rarg1:   length
  //    c_rarg2:   acc_start -- where the output will be stored
  //    c_rarg3:   r_start -- where the randomly generated 128-bit key is stored

  // See https://loup-vaillant.fr/tutorials/poly1305-design for a
  // description of the tricks used to simplify and accelerate this
  // computation.

  address generate_poly1305_processBlocks() {
    __ align(CodeEntryAlignment);
    StubGenStubId stub_id = StubGenStubId::poly1305_processBlocks_id;
    StubCodeMark mark(this, stub_id);
    address start = __ pc();
    __ enter();
    Label here;

    RegSet saved_regs = RegSet::range(x18, x21);
    RegSetIterator<Register> regs = (RegSet::range(x14, x31) - RegSet::range(x22, x27)).begin();
    __ push_reg(saved_regs, sp);

    // Arguments
    const Register input_start = c_rarg0, length = c_rarg1, acc_start = c_rarg2, r_start = c_rarg3;

    // R_n is the 128-bit randomly-generated key, packed into two
    // registers. The caller passes this key to us as long[5], with
    // BITS_PER_LIMB = 26.
    const Register R_0 = *regs, R_1 = *++regs;
    poly1305_pack_26(R_0, R_1, r_start, t1, t2);

    // RR_n is (R_n >> 2) * 5
    const Register RR_0 = *++regs, RR_1 = *++regs;
    __ srli(t1, R_0, 2);
    __ shadd(RR_0, t1, t1, t2, 2);
    __ srli(t1, R_1, 2);
    __ shadd(RR_1, t1, t1, t2, 2);

    // U_n is the current checksum
    const Register U_0 = *++regs, U_1 = *++regs, U_2 = *++regs;
    poly1305_pack_26(U_0, U_1, U_2, acc_start, t1, t2);

    static constexpr int BLOCK_LENGTH = 16;
    Label DONE, LOOP;

    __ mv(t1, BLOCK_LENGTH);
    __ blt(length, t1, DONE); {
      __ bind(LOOP);

      // S_n is to be the sum of U_n and the next block of data
      const Register S_0 = *++regs, S_1 = *++regs, S_2 = *++regs;
      __ ld(S_0, Address(input_start, 0));
      __ ld(S_1, Address(input_start, wordSize));

      __ cad(S_0, S_0, U_0, t1); // Add U_0 to S_0 with carry output to t1
      __ cadc(S_1, S_1, U_1, t1); // Add U_1 with carry to S_1 with carry output to t1
      __ add(S_2, U_2, t1);

      __ addi(S_2, S_2, 1);

      const Register U_0HI = *++regs, U_1HI = *++regs;

      // NB: this logic depends on some of the special properties of
      // Poly1305 keys. In particular, because we know that the top
      // four bits of R_0 and R_1 are zero, we can add together
      // partial products without any risk of needing to propagate a
      // carry out.
      __ wide_mul(U_0, U_0HI, S_0, R_0);
      __ wide_madd(U_0, U_0HI, S_1, RR_1, t1, t2);
      __ wide_madd(U_0, U_0HI, S_2, RR_0, t1, t2);

      __ wide_mul(U_1, U_1HI, S_0, R_1);
      __ wide_madd(U_1, U_1HI, S_1, R_0, t1, t2);
      __ wide_madd(U_1, U_1HI, S_2, RR_1, t1, t2);

      __ andi(U_2, R_0, right_2_bits);
      __ mul(U_2, S_2, U_2);

      // Partial reduction mod 2**130 - 5
      __ cad(U_1, U_1, U_0HI, t1); // Add U_0HI to U_1 with carry output to t1
      __ adc(U_2, U_2, U_1HI, t1);
      // Sum is now in U_2:U_1:U_0.

      // U_2:U_1:U_0: += (U_2 >> 2) * 5
      poly1305_reduce(U_2, U_1, U_0, t1, t2);

      __ subi(length, length, BLOCK_LENGTH);
      __ addi(input_start, input_start, BLOCK_LENGTH);
      __ mv(t1, BLOCK_LENGTH);
      __ bge(length, t1, LOOP);
    }

    // Further reduce modulo 2^130 - 5
    poly1305_reduce(U_2, U_1, U_0, t1, t2);

    // Unpack the sum into five 26-bit limbs and write to memory.
    // First 26 bits is the first limb
    __ slli(t1, U_0, 38); // Take lowest 26 bits
    __ srli(t1, t1, 38);
    __ sd(t1, Address(acc_start)); // First 26-bit limb

    // 27-52 bits of U_0 is the second limb
    __ slli(t1, U_0, 12); // Take next 27-52 bits
    __ srli(t1, t1, 38);
    __ sd(t1, Address(acc_start, sizeof (jlong))); // Second 26-bit limb

    // Getting 53-64 bits of U_0 and 1-14 bits of U_1 in one register
    __ srli(t1, U_0, 52);
    __ slli(t2, U_1, 50);
    __ srli(t2, t2, 38);
    __ add(t1, t1, t2);
    __ sd(t1, Address(acc_start, 2 * sizeof (jlong))); // Third 26-bit limb

    // Storing 15-40 bits of U_1
    __ slli(t1, U_1, 24); // Already used up 14 bits
    __ srli(t1, t1, 38); // Clear all other bits from t1
    __ sd(t1, Address(acc_start, 3 * sizeof (jlong))); // Fourth 26-bit limb

    // Storing 41-64 bits of U_1 and first three bits from U_2 in one register
    __ srli(t1, U_1, 40);
    __ andi(t2, U_2, right_3_bits);
    __ slli(t2, t2, 24);
    __ add(t1, t1, t2);
    __ sd(t1, Address(acc_start, 4 * sizeof (jlong))); // Fifth 26-bit limb

    __ bind(DONE);
    __ pop_reg(saved_regs, sp);
    __ leave(); // Required for proper stackwalking
    __ ret();

    return start;
  }

#endif // COMPILER2

  /**
   *  Arguments:
   *
   * Inputs:
   *   c_rarg0   - int crc
   *   c_rarg1   - byte* buf
   *   c_rarg2   - int length
   *
   * Output:
   *   c_rarg0   - int crc result
   */
  address generate_updateBytesCRC32() {
    assert(UseCRC32Intrinsics, "what are we doing here?");

    __ align(CodeEntryAlignment);
    StubGenStubId stub_id = StubGenStubId::updateBytesCRC32_id;
    StubCodeMark mark(this, stub_id);

    address start = __ pc();

    // input parameters
    const Register crc    = c_rarg0;  // crc
    const Register buf    = c_rarg1;  // source java byte array address
    const Register len    = c_rarg2;  // length

    BLOCK_COMMENT("Entry:");
    __ enter(); // required for proper stackwalking of RuntimeStub frame

    __ kernel_crc32(crc, buf, len,
                    c_rarg3, c_rarg4, c_rarg5, c_rarg6, // tmp's for tables
                    c_rarg7, t2, t3, t4, t5, t6);       // misc tmps

    __ leave(); // required for proper stackwalking of RuntimeStub frame
    __ ret();

    return start;
  }

  // exception handler for upcall stubs
  address generate_upcall_stub_exception_handler() {
    StubGenStubId stub_id = StubGenStubId::upcall_stub_exception_handler_id;
    StubCodeMark mark(this, stub_id);
    address start = __ pc();

    // Native caller has no idea how to handle exceptions,
    // so we just crash here. Up to callee to catch exceptions.
    __ verify_oop(x10); // return a exception oop in a0
    __ rt_call(CAST_FROM_FN_PTR(address, UpcallLinker::handle_uncaught_exception));
    __ should_not_reach_here();

    return start;
  }

  // load Method* target of MethodHandle
  // j_rarg0 = jobject receiver
  // xmethod = Method* result
  address generate_upcall_stub_load_target() {

    StubGenStubId stub_id = StubGenStubId::upcall_stub_load_target_id;
    StubCodeMark mark(this, stub_id);
    address start = __ pc();

    __ resolve_global_jobject(j_rarg0, t0, t1);
      // Load target method from receiver
    __ load_heap_oop(xmethod, Address(j_rarg0, java_lang_invoke_MethodHandle::form_offset()), t0, t1);
    __ load_heap_oop(xmethod, Address(xmethod, java_lang_invoke_LambdaForm::vmentry_offset()), t0, t1);
    __ load_heap_oop(xmethod, Address(xmethod, java_lang_invoke_MemberName::method_offset()), t0, t1);
    __ access_load_at(T_ADDRESS, IN_HEAP, xmethod,
                      Address(xmethod, java_lang_invoke_ResolvedMethodName::vmtarget_offset()),
                      noreg, noreg);
    __ sd(xmethod, Address(xthread, JavaThread::callee_target_offset())); // just in case callee is deoptimized

    __ ret();

    return start;
  }

#undef __

  // Initialization
  void generate_initial_stubs() {
    // Generate initial stubs and initializes the entry points

    // entry points that exist in all platforms Note: This is code
    // that could be shared among different platforms - however the
    // benefit seems to be smaller than the disadvantage of having a
    // much more complicated generator structure. See also comment in
    // stubRoutines.hpp.

    StubRoutines::_forward_exception_entry = generate_forward_exception();

    if (UnsafeMemoryAccess::_table == nullptr) {
      UnsafeMemoryAccess::create_table(8 + 4); // 8 for copyMemory; 4 for setMemory
    }

    StubRoutines::_call_stub_entry =
      generate_call_stub(StubRoutines::_call_stub_return_address);

    // is referenced by megamorphic call
    StubRoutines::_catch_exception_entry = generate_catch_exception();

    if (UseCRC32Intrinsics) {
      // set table address before stub generation which use it
      StubRoutines::_crc_table_adr = (address)StubRoutines::riscv::_crc_table;
      StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
    }

    if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
        vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
      StubRoutines::_hf2f = generate_float16ToFloat();
      StubRoutines::_f2hf = generate_floatToFloat16();
    }
  }

  void generate_continuation_stubs() {
    // Continuation stubs:
    StubRoutines::_cont_thaw             = generate_cont_thaw();
    StubRoutines::_cont_returnBarrier    = generate_cont_returnBarrier();
    StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
    StubRoutines::_cont_preempt_stub     = generate_cont_preempt_stub();
  }

  void generate_final_stubs() {
    // support for verify_oop (must happen after universe_init)
    if (VerifyOops) {
      StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
    }

    // arraycopy stubs used by compilers
    generate_arraycopy_stubs();

    StubRoutines::_method_entry_barrier = generate_method_entry_barrier();

#ifdef COMPILER2
    if (UseSecondarySupersTable) {
      StubRoutines::_lookup_secondary_supers_table_slow_path_stub = generate_lookup_secondary_supers_table_slow_path_stub();
      if (!InlineSecondarySupersTest) {
        generate_lookup_secondary_supers_table_stub();
      }
    }
#endif // COMPILER2

    StubRoutines::_upcall_stub_exception_handler = generate_upcall_stub_exception_handler();
    StubRoutines::_upcall_stub_load_target = generate_upcall_stub_load_target();

    StubRoutines::riscv::set_completed();
  }

  void generate_compiler_stubs() {
#ifdef COMPILER2
    if (UseMulAddIntrinsic) {
      StubRoutines::_mulAdd = generate_mulAdd();
    }

    if (UseMultiplyToLenIntrinsic) {
      StubRoutines::_multiplyToLen = generate_multiplyToLen();
    }

    if (UseSquareToLenIntrinsic) {
      StubRoutines::_squareToLen = generate_squareToLen();
    }

    if (UseMontgomeryMultiplyIntrinsic) {
      StubGenStubId stub_id = StubGenStubId::montgomeryMultiply_id;
      StubCodeMark mark(this, stub_id);
      MontgomeryMultiplyGenerator g(_masm, /*squaring*/false);
      StubRoutines::_montgomeryMultiply = g.generate_multiply();
    }

    if (UseMontgomerySquareIntrinsic) {
      StubGenStubId stub_id = StubGenStubId::montgomerySquare_id;
      StubCodeMark mark(this, stub_id);
      MontgomeryMultiplyGenerator g(_masm, /*squaring*/true);
      StubRoutines::_montgomerySquare = g.generate_square();
    }

    if (UseAESIntrinsics) {
      StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
      StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
    }

    if (UsePoly1305Intrinsics) {
      StubRoutines::_poly1305_processBlocks = generate_poly1305_processBlocks();
    }

    if (UseRVV) {
      StubRoutines::_bigIntegerLeftShiftWorker = generate_bigIntegerLeftShift();
      StubRoutines::_bigIntegerRightShiftWorker = generate_bigIntegerRightShift();
    }

    if (UseSHA256Intrinsics) {
      Sha2Generator sha2(_masm, this);
      StubRoutines::_sha256_implCompress   = sha2.generate_sha256_implCompress(StubGenStubId::sha256_implCompress_id);
      StubRoutines::_sha256_implCompressMB = sha2.generate_sha256_implCompress(StubGenStubId::sha256_implCompressMB_id);
    }

    if (UseSHA512Intrinsics) {
      Sha2Generator sha2(_masm, this);
      StubRoutines::_sha512_implCompress   = sha2.generate_sha512_implCompress(StubGenStubId::sha512_implCompress_id);
      StubRoutines::_sha512_implCompressMB = sha2.generate_sha512_implCompress(StubGenStubId::sha512_implCompressMB_id);
    }

    if (UseMD5Intrinsics) {
      StubRoutines::_md5_implCompress   = generate_md5_implCompress(StubGenStubId::md5_implCompress_id);
      StubRoutines::_md5_implCompressMB = generate_md5_implCompress(StubGenStubId::md5_implCompressMB_id);
    }

    if (UseChaCha20Intrinsics) {
      StubRoutines::_chacha20Block = generate_chacha20Block();
    }

    if (UseSHA1Intrinsics) {
      StubRoutines::_sha1_implCompress     = generate_sha1_implCompress(StubGenStubId::sha1_implCompress_id);
      StubRoutines::_sha1_implCompressMB   = generate_sha1_implCompress(StubGenStubId::sha1_implCompressMB_id);
    }

    if (UseBASE64Intrinsics) {
      StubRoutines::_base64_encodeBlock = generate_base64_encodeBlock();
      StubRoutines::_base64_decodeBlock = generate_base64_decodeBlock();
    }

    if (UseAdler32Intrinsics) {
      StubRoutines::_updateBytesAdler32 = generate_updateBytesAdler32();
    }

    generate_compare_long_strings();

    generate_string_indexof_stubs();

#endif // COMPILER2
  }

 public:
  StubGenerator(CodeBuffer* code, StubGenBlobId blob_id) : StubCodeGenerator(code, blob_id) {
    switch(blob_id) {
    case initial_id:
      generate_initial_stubs();
      break;
     case continuation_id:
      generate_continuation_stubs();
      break;
    case compiler_id:
      generate_compiler_stubs();
      break;
    case final_id:
      generate_final_stubs();
      break;
    default:
      fatal("unexpected blob id: %d", blob_id);
      break;
    };
  }
}; // end class declaration

void StubGenerator_generate(CodeBuffer* code, StubGenBlobId blob_id) {
  StubGenerator g(code, blob_id);
}