| /* | |
| * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved. | |
| * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. | |
| * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |
| * | |
| * This code is free software; you can redistribute it and/or modify it | |
| * under the terms of the GNU General Public License version 2 only, as | |
| * published by the Free Software Foundation. | |
| * | |
| * This code is distributed in the hope that it will be useful, but WITHOUT | |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| * version 2 for more details (a copy is included in the LICENSE file that | |
| * accompanied this code). | |
| * | |
| * You should have received a copy of the GNU General Public License version | |
| * 2 along with this work; if not, write to the Free Software Foundation, | |
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| * | |
| * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA | |
| * or visit www.oracle.com if you need additional information or have any | |
| * questions. | |
| * | |
| */ | |
| #include "asm/macroAssembler.inline.hpp" | |
| #include "classfile/javaClasses.hpp" | |
| #include "compiler/disassembler.hpp" | |
| #include "compiler/compiler_globals.hpp" | |
| #include "gc/shared/barrierSetAssembler.hpp" | |
| #include "interpreter/bytecodeHistogram.hpp" | |
| #include "interpreter/interpreter.hpp" | |
| #include "interpreter/interpreterRuntime.hpp" | |
| #include "interpreter/interp_masm.hpp" | |
| #include "interpreter/templateInterpreterGenerator.hpp" | |
| #include "interpreter/templateTable.hpp" | |
| #include "interpreter/bytecodeTracer.hpp" | |
| #include "memory/resourceArea.hpp" | |
| #include "oops/arrayOop.hpp" | |
| #include "oops/method.hpp" | |
| #include "oops/methodCounters.hpp" | |
| #include "oops/methodData.hpp" | |
| #include "oops/oop.inline.hpp" | |
| #include "oops/resolvedIndyEntry.hpp" | |
| #include "oops/resolvedMethodEntry.hpp" | |
| #include "prims/jvmtiExport.hpp" | |
| #include "prims/jvmtiThreadState.hpp" | |
| #include "runtime/arguments.hpp" | |
| #include "runtime/deoptimization.hpp" | |
| #include "runtime/frame.inline.hpp" | |
| #include "runtime/globals.hpp" | |
| #include "runtime/jniHandles.hpp" | |
| #include "runtime/sharedRuntime.hpp" | |
| #include "runtime/stubRoutines.hpp" | |
| #include "runtime/synchronizer.hpp" | |
| #include "runtime/timer.hpp" | |
| #include "runtime/vframeArray.hpp" | |
| #include "utilities/checkedCast.hpp" | |
| #include "utilities/debug.hpp" | |
| #include "utilities/powerOfTwo.hpp" | |
| #include <sys/types.h> | |
| // Size of interpreter code. Increase if too small. Interpreter will | |
| // fail with a guarantee ("not enough space for interpreter generation"); | |
| // if too small. | |
| // Run with +PrintInterpreter to get the VM to print out the size. | |
| // Max size with JVMTI | |
| int TemplateInterpreter::InterpreterCodeSize = 200 * 1024; | |
| #define __ Disassembler::hook<InterpreterMacroAssembler>(__FILE__, __LINE__, _masm)-> | |
| address TemplateInterpreterGenerator::generate_slow_signature_handler() { | |
| address entry = __ pc(); | |
| __ andr(esp, esp, -16); | |
| __ mov(c_rarg3, esp); | |
| // rmethod | |
| // rlocals | |
| // c_rarg3: first stack arg - wordSize | |
| // adjust sp | |
| __ sub(sp, c_rarg3, 18 * wordSize); | |
| __ str(lr, Address(__ pre(sp, -2 * wordSize))); | |
| __ call_VM(noreg, | |
| CAST_FROM_FN_PTR(address, | |
| InterpreterRuntime::slow_signature_handler), | |
| rmethod, rlocals, c_rarg3); | |
| // r0: result handler | |
| // Stack layout: | |
| // rsp: return address <- sp | |
| // 1 garbage | |
| // 8 integer args (if static first is unused) | |
| // 1 float/double identifiers | |
| // 8 double args | |
| // stack args <- esp | |
| // garbage | |
| // expression stack bottom | |
| // bcp (null) | |
| // ... | |
| // Restore LR | |
| __ ldr(lr, Address(__ post(sp, 2 * wordSize))); | |
| // Do FP first so we can use c_rarg3 as temp | |
| __ ldrw(c_rarg3, Address(sp, 9 * wordSize)); // float/double identifiers | |
| for (int i = 0; i < Argument::n_float_register_parameters_c; i++) { | |
| const FloatRegister r = as_FloatRegister(i); | |
| Label d, done; | |
| __ tbnz(c_rarg3, i, d); | |
| __ ldrs(r, Address(sp, (10 + i) * wordSize)); | |
| __ b(done); | |
| __ bind(d); | |
| __ ldrd(r, Address(sp, (10 + i) * wordSize)); | |
| __ bind(done); | |
| } | |
| // c_rarg0 contains the result from the call of | |
| // InterpreterRuntime::slow_signature_handler so we don't touch it | |
| // here. It will be loaded with the JNIEnv* later. | |
| __ ldr(c_rarg1, Address(sp, 1 * wordSize)); | |
| for (int i = c_rarg2->encoding(); i <= c_rarg7->encoding(); i += 2) { | |
| Register rm = as_Register(i), rn = as_Register(i+1); | |
| __ ldp(rm, rn, Address(sp, i * wordSize)); | |
| } | |
| __ add(sp, sp, 18 * wordSize); | |
| __ ret(lr); | |
| return entry; | |
| } | |
| // | |
| // Various method entries | |
| // | |
| address TemplateInterpreterGenerator::generate_math_entry(AbstractInterpreter::MethodKind kind) { | |
| // rmethod: Method* | |
| // r19_sender_sp: sender sp | |
| // esp: args | |
| // These don't need a safepoint check because they aren't virtually | |
| // callable. We won't enter these intrinsics from compiled code. | |
| // If in the future we added an intrinsic which was virtually callable | |
| // we'd have to worry about how to safepoint so that this code is used. | |
| // mathematical functions inlined by compiler | |
| // (interpreter must provide identical implementation | |
| // in order to avoid monotonicity bugs when switching | |
| // from interpreter to compiler in the middle of some | |
| // computation) | |
| // | |
| // stack: | |
| // [ arg ] <-- esp | |
| // [ arg ] | |
| // retaddr in lr | |
| address entry_point = nullptr; | |
| Register continuation = lr; | |
| switch (kind) { | |
| case Interpreter::java_lang_math_abs: | |
| entry_point = __ pc(); | |
| __ ldrd(v0, Address(esp)); | |
| __ fabsd(v0, v0); | |
| __ mov(sp, r19_sender_sp); // Restore caller's SP | |
| break; | |
| case Interpreter::java_lang_math_sqrt: | |
| entry_point = __ pc(); | |
| __ ldrd(v0, Address(esp)); | |
| __ fsqrtd(v0, v0); | |
| __ mov(sp, r19_sender_sp); | |
| break; | |
| case Interpreter::java_lang_math_sin : | |
| case Interpreter::java_lang_math_cos : | |
| case Interpreter::java_lang_math_tan : | |
| case Interpreter::java_lang_math_log : | |
| case Interpreter::java_lang_math_log10 : | |
| case Interpreter::java_lang_math_exp : | |
| entry_point = __ pc(); | |
| __ ldrd(v0, Address(esp)); | |
| __ mov(sp, r19_sender_sp); | |
| __ mov(r23, lr); | |
| continuation = r23; // The first free callee-saved register | |
| generate_transcendental_entry(kind, 1); | |
| break; | |
| case Interpreter::java_lang_math_pow : | |
| entry_point = __ pc(); | |
| __ mov(r23, lr); | |
| continuation = r23; | |
| __ ldrd(v0, Address(esp, 2 * Interpreter::stackElementSize)); | |
| __ ldrd(v1, Address(esp)); | |
| __ mov(sp, r19_sender_sp); | |
| generate_transcendental_entry(kind, 2); | |
| break; | |
| case Interpreter::java_lang_math_fmaD : | |
| if (UseFMA) { | |
| entry_point = __ pc(); | |
| __ ldrd(v0, Address(esp, 4 * Interpreter::stackElementSize)); | |
| __ ldrd(v1, Address(esp, 2 * Interpreter::stackElementSize)); | |
| __ ldrd(v2, Address(esp)); | |
| __ fmaddd(v0, v0, v1, v2); | |
| __ mov(sp, r19_sender_sp); // Restore caller's SP | |
| } | |
| break; | |
| case Interpreter::java_lang_math_fmaF : | |
| if (UseFMA) { | |
| entry_point = __ pc(); | |
| __ ldrs(v0, Address(esp, 2 * Interpreter::stackElementSize)); | |
| __ ldrs(v1, Address(esp, Interpreter::stackElementSize)); | |
| __ ldrs(v2, Address(esp)); | |
| __ fmadds(v0, v0, v1, v2); | |
| __ mov(sp, r19_sender_sp); // Restore caller's SP | |
| } | |
| break; | |
| default: | |
| ; | |
| } | |
| if (entry_point) { | |
| __ br(continuation); | |
| } | |
| return entry_point; | |
| } | |
| // double trigonometrics and transcendentals | |
| // static jdouble dsin(jdouble x); | |
| // static jdouble dcos(jdouble x); | |
| // static jdouble dtan(jdouble x); | |
| // static jdouble dlog(jdouble x); | |
| // static jdouble dlog10(jdouble x); | |
| // static jdouble dexp(jdouble x); | |
| // static jdouble dpow(jdouble x, jdouble y); | |
| void TemplateInterpreterGenerator::generate_transcendental_entry(AbstractInterpreter::MethodKind kind, int fpargs) { | |
| address fn; | |
| switch (kind) { | |
| case Interpreter::java_lang_math_sin : | |
| if (StubRoutines::dsin() == nullptr) { | |
| fn = CAST_FROM_FN_PTR(address, SharedRuntime::dsin); | |
| } else { | |
| fn = CAST_FROM_FN_PTR(address, StubRoutines::dsin()); | |
| } | |
| break; | |
| case Interpreter::java_lang_math_cos : | |
| if (StubRoutines::dcos() == nullptr) { | |
| fn = CAST_FROM_FN_PTR(address, SharedRuntime::dcos); | |
| } else { | |
| fn = CAST_FROM_FN_PTR(address, StubRoutines::dcos()); | |
| } | |
| break; | |
| case Interpreter::java_lang_math_tan : | |
| if (StubRoutines::dtan() == nullptr) { | |
| fn = CAST_FROM_FN_PTR(address, SharedRuntime::dtan); | |
| } else { | |
| fn = CAST_FROM_FN_PTR(address, StubRoutines::dtan()); | |
| } | |
| break; | |
| case Interpreter::java_lang_math_log : | |
| if (StubRoutines::dlog() == nullptr) { | |
| fn = CAST_FROM_FN_PTR(address, SharedRuntime::dlog); | |
| } else { | |
| fn = CAST_FROM_FN_PTR(address, StubRoutines::dlog()); | |
| } | |
| break; | |
| case Interpreter::java_lang_math_log10 : | |
| if (StubRoutines::dlog10() == nullptr) { | |
| fn = CAST_FROM_FN_PTR(address, SharedRuntime::dlog10); | |
| } else { | |
| fn = CAST_FROM_FN_PTR(address, StubRoutines::dlog10()); | |
| } | |
| break; | |
| case Interpreter::java_lang_math_exp : | |
| if (StubRoutines::dexp() == nullptr) { | |
| fn = CAST_FROM_FN_PTR(address, SharedRuntime::dexp); | |
| } else { | |
| fn = CAST_FROM_FN_PTR(address, StubRoutines::dexp()); | |
| } | |
| break; | |
| case Interpreter::java_lang_math_pow : | |
| if (StubRoutines::dpow() == nullptr) { | |
| fn = CAST_FROM_FN_PTR(address, SharedRuntime::dpow); | |
| } else { | |
| fn = CAST_FROM_FN_PTR(address, StubRoutines::dpow()); | |
| } | |
| break; | |
| default: | |
| ShouldNotReachHere(); | |
| fn = nullptr; // unreachable | |
| } | |
| __ mov(rscratch1, fn); | |
| __ blr(rscratch1); | |
| } | |
| address TemplateInterpreterGenerator::generate_Float_float16ToFloat_entry() { | |
| assert(VM_Version::supports_float16(), "this intrinsic is not supported"); | |
| // r19_sender_sp: sender sp | |
| // stack: | |
| // [ arg ] <-- esp | |
| // [ arg ] | |
| // retaddr in lr | |
| // result in v0 | |
| address entry_point = __ pc(); | |
| __ ldrw(c_rarg0, Address(esp)); | |
| __ flt16_to_flt(v0, c_rarg0, v1); | |
| __ mov(sp, r19_sender_sp); // Restore caller's SP | |
| __ br(lr); | |
| return entry_point; | |
| } | |
| address TemplateInterpreterGenerator::generate_Float_floatToFloat16_entry() { | |
| assert(VM_Version::supports_float16(), "this intrinsic is not supported"); | |
| // r19_sender_sp: sender sp | |
| // stack: | |
| // [ arg ] <-- esp | |
| // [ arg ] | |
| // retaddr in lr | |
| // result in c_rarg0 | |
| address entry_point = __ pc(); | |
| __ ldrs(v0, Address(esp)); | |
| __ flt_to_flt16(c_rarg0, v0, v1); | |
| __ mov(sp, r19_sender_sp); // Restore caller's SP | |
| __ br(lr); | |
| return entry_point; | |
| } | |
| // Abstract method entry | |
| // Attempt to execute abstract method. Throw exception | |
| address TemplateInterpreterGenerator::generate_abstract_entry(void) { | |
| // rmethod: Method* | |
| // r19_sender_sp: sender SP | |
| address entry_point = __ pc(); | |
| // abstract method entry | |
| // pop return address, reset last_sp to null | |
| __ empty_expression_stack(); | |
| __ restore_bcp(); // bcp must be correct for exception handler (was destroyed) | |
| __ restore_locals(); // make sure locals pointer is correct as well (was destroyed) | |
| // throw exception | |
| __ call_VM(noreg, CAST_FROM_FN_PTR(address, | |
| InterpreterRuntime::throw_AbstractMethodErrorWithMethod), | |
| rmethod); | |
| // the call_VM checks for exception, so we should never return here. | |
| __ should_not_reach_here(); | |
| return entry_point; | |
| } | |
| address TemplateInterpreterGenerator::generate_StackOverflowError_handler() { | |
| address entry = __ pc(); | |
| #ifdef ASSERT | |
| { | |
| Label L; | |
| __ ldr(rscratch1, Address(rfp, | |
| frame::interpreter_frame_monitor_block_top_offset * | |
| wordSize)); | |
| __ lea(rscratch1, Address(rfp, rscratch1, Address::lsl(Interpreter::logStackElementSize))); | |
| __ mov(rscratch2, sp); | |
| __ cmp(rscratch1, rscratch2); // maximal rsp for current rfp (stack | |
| // grows negative) | |
| __ br(Assembler::HS, L); // check if frame is complete | |
| __ stop ("interpreter frame not set up"); | |
| __ bind(L); | |
| } | |
| #endif // ASSERT | |
| // Restore bcp under the assumption that the current frame is still | |
| // interpreted | |
| __ restore_bcp(); | |
| // expression stack must be empty before entering the VM if an | |
| // exception happened | |
| __ empty_expression_stack(); | |
| // throw exception | |
| __ call_VM(noreg, | |
| CAST_FROM_FN_PTR(address, | |
| InterpreterRuntime::throw_StackOverflowError)); | |
| return entry; | |
| } | |
| address TemplateInterpreterGenerator::generate_ArrayIndexOutOfBounds_handler() { | |
| address entry = __ pc(); | |
| // expression stack must be empty before entering the VM if an | |
| // exception happened | |
| __ empty_expression_stack(); | |
| // setup parameters | |
| // ??? convention: expect aberrant index in register r1 | |
| __ movw(c_rarg2, r1); | |
| // ??? convention: expect array in register r3 | |
| __ mov(c_rarg1, r3); | |
| __ call_VM(noreg, | |
| CAST_FROM_FN_PTR(address, | |
| InterpreterRuntime:: | |
| throw_ArrayIndexOutOfBoundsException), | |
| c_rarg1, c_rarg2); | |
| return entry; | |
| } | |
| address TemplateInterpreterGenerator::generate_ClassCastException_handler() { | |
| address entry = __ pc(); | |
| // object is at TOS | |
| __ pop(c_rarg1); | |
| // expression stack must be empty before entering the VM if an | |
| // exception happened | |
| __ empty_expression_stack(); | |
| __ call_VM(noreg, | |
| CAST_FROM_FN_PTR(address, | |
| InterpreterRuntime:: | |
| throw_ClassCastException), | |
| c_rarg1); | |
| return entry; | |
| } | |
| address TemplateInterpreterGenerator::generate_exception_handler_common( | |
| const char* name, const char* message, bool pass_oop) { | |
| assert(!pass_oop || message == nullptr, "either oop or message but not both"); | |
| address entry = __ pc(); | |
| if (pass_oop) { | |
| // object is at TOS | |
| __ pop(c_rarg2); | |
| } | |
| // expression stack must be empty before entering the VM if an | |
| // exception happened | |
| __ empty_expression_stack(); | |
| // setup parameters | |
| __ lea(c_rarg1, Address((address)name)); | |
| if (pass_oop) { | |
| __ call_VM(r0, CAST_FROM_FN_PTR(address, | |
| InterpreterRuntime:: | |
| create_klass_exception), | |
| c_rarg1, c_rarg2); | |
| } else { | |
| // kind of lame ExternalAddress can't take null because | |
| // external_word_Relocation will assert. | |
| if (message != nullptr) { | |
| __ lea(c_rarg2, Address((address)message)); | |
| } else { | |
| __ mov(c_rarg2, NULL_WORD); | |
| } | |
| __ call_VM(r0, | |
| CAST_FROM_FN_PTR(address, InterpreterRuntime::create_exception), | |
| c_rarg1, c_rarg2); | |
| } | |
| // throw exception | |
| __ b(address(Interpreter::throw_exception_entry())); | |
| return entry; | |
| } | |
| address TemplateInterpreterGenerator::generate_return_entry_for(TosState state, int step, size_t index_size) { | |
| address entry = __ pc(); | |
| // Restore stack bottom in case i2c adjusted stack | |
| __ ldr(rscratch1, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize)); | |
| __ lea(esp, Address(rfp, rscratch1, Address::lsl(Interpreter::logStackElementSize))); | |
| // and null it as marker that esp is now tos until next java call | |
| __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize)); | |
| __ restore_bcp(); | |
| __ restore_locals(); | |
| __ restore_constant_pool_cache(); | |
| __ get_method(rmethod); | |
| if (state == atos) { | |
| Register obj = r0; | |
| Register mdp = r1; | |
| Register tmp = r2; | |
| __ profile_return_type(mdp, obj, tmp); | |
| } | |
| const Register cache = r1; | |
| const Register index = r2; | |
| if (index_size == sizeof(u4)) { | |
| __ load_resolved_indy_entry(cache, index); | |
| __ load_unsigned_short(cache, Address(cache, in_bytes(ResolvedIndyEntry::num_parameters_offset()))); | |
| __ add(esp, esp, cache, Assembler::LSL, 3); | |
| } else { | |
| // Pop N words from the stack | |
| assert(index_size == sizeof(u2), "Can only be u2"); | |
| __ load_method_entry(cache, index); | |
| __ load_unsigned_short(cache, Address(cache, in_bytes(ResolvedMethodEntry::num_parameters_offset()))); | |
| __ add(esp, esp, cache, Assembler::LSL, 3); | |
| } | |
| // Restore machine SP | |
| __ restore_sp_after_call(); | |
| __ check_and_handle_popframe(rthread); | |
| __ check_and_handle_earlyret(rthread); | |
| __ get_dispatch(); | |
| __ dispatch_next(state, step); | |
| return entry; | |
| } | |
| address TemplateInterpreterGenerator::generate_deopt_entry_for(TosState state, | |
| int step, | |
| address continuation) { | |
| address entry = __ pc(); | |
| __ restore_bcp(); | |
| __ restore_locals(); | |
| __ restore_constant_pool_cache(); | |
| __ get_method(rmethod); | |
| __ get_dispatch(); | |
| __ restore_sp_after_call(); // Restore SP to extended SP | |
| // Restore expression stack pointer | |
| __ ldr(rscratch1, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize)); | |
| __ lea(esp, Address(rfp, rscratch1, Address::lsl(Interpreter::logStackElementSize))); | |
| // null last_sp until next java call | |
| __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize)); | |
| #if INCLUDE_JVMCI | |
| // Check if we need to take lock at entry of synchronized method. This can | |
| // only occur on method entry so emit it only for vtos with step 0. | |
| if (EnableJVMCI && state == vtos && step == 0) { | |
| Label L; | |
| __ ldrb(rscratch1, Address(rthread, JavaThread::pending_monitorenter_offset())); | |
| __ cbz(rscratch1, L); | |
| // Clear flag. | |
| __ strb(zr, Address(rthread, JavaThread::pending_monitorenter_offset())); | |
| // Take lock. | |
| lock_method(); | |
| __ bind(L); | |
| } else { | |
| #ifdef ASSERT | |
| if (EnableJVMCI) { | |
| Label L; | |
| __ ldrb(rscratch1, Address(rthread, JavaThread::pending_monitorenter_offset())); | |
| __ cbz(rscratch1, L); | |
| __ stop("unexpected pending monitor in deopt entry"); | |
| __ bind(L); | |
| } | |
| #endif | |
| } | |
| #endif | |
| // handle exceptions | |
| { | |
| Label L; | |
| __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset())); | |
| __ cbz(rscratch1, L); | |
| __ call_VM(noreg, | |
| CAST_FROM_FN_PTR(address, | |
| InterpreterRuntime::throw_pending_exception)); | |
| __ should_not_reach_here(); | |
| __ bind(L); | |
| } | |
| if (continuation == nullptr) { | |
| __ dispatch_next(state, step); | |
| } else { | |
| __ jump_to_entry(continuation); | |
| } | |
| return entry; | |
| } | |
| address TemplateInterpreterGenerator::generate_result_handler_for( | |
| BasicType type) { | |
| address entry = __ pc(); | |
| switch (type) { | |
| case T_BOOLEAN: __ c2bool(r0); break; | |
| case T_CHAR : __ uxth(r0, r0); break; | |
| case T_BYTE : __ sxtb(r0, r0); break; | |
| case T_SHORT : __ sxth(r0, r0); break; | |
| case T_INT : __ uxtw(r0, r0); break; // FIXME: We almost certainly don't need this | |
| case T_LONG : /* nothing to do */ break; | |
| case T_VOID : /* nothing to do */ break; | |
| case T_FLOAT : /* nothing to do */ break; | |
| case T_DOUBLE : /* nothing to do */ break; | |
| case T_OBJECT : | |
| // retrieve result from frame | |
| __ ldr(r0, Address(rfp, frame::interpreter_frame_oop_temp_offset*wordSize)); | |
| // and verify it | |
| __ verify_oop(r0); | |
| break; | |
| default : ShouldNotReachHere(); | |
| } | |
| __ ret(lr); // return from result handler | |
| return entry; | |
| } | |
| address TemplateInterpreterGenerator::generate_safept_entry_for( | |
| TosState state, | |
| address runtime_entry) { | |
| address entry = __ pc(); | |
| __ push(state); | |
| __ push_cont_fastpath(rthread); | |
| __ call_VM(noreg, runtime_entry); | |
| __ pop_cont_fastpath(rthread); | |
| __ membar(Assembler::AnyAny); | |
| __ dispatch_via(vtos, Interpreter::_normal_table.table_for(vtos)); | |
| return entry; | |
| } | |
| address TemplateInterpreterGenerator::generate_cont_resume_interpreter_adapter() { | |
| if (!Continuations::enabled()) return nullptr; | |
| address start = __ pc(); | |
| __ restore_bcp(); | |
| __ restore_locals(); | |
| // Restore constant pool cache | |
| __ ldr(rcpool, Address(rfp, frame::interpreter_frame_cache_offset * wordSize)); | |
| // Restore Java expression stack pointer | |
| __ ldr(rscratch1, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize)); | |
| __ lea(esp, Address(rfp, rscratch1, Address::lsl(Interpreter::logStackElementSize))); | |
| // and null it as marker that esp is now tos until next java call | |
| __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize)); | |
| // Restore machine SP | |
| __ ldr(rscratch1, Address(rfp, frame::interpreter_frame_extended_sp_offset * wordSize)); | |
| __ lea(sp, Address(rfp, rscratch1, Address::lsl(LogBytesPerWord))); | |
| // Restore method | |
| __ ldr(rmethod, Address(rfp, frame::interpreter_frame_method_offset * wordSize)); | |
| // Restore dispatch | |
| uint64_t offset; | |
| __ adrp(rdispatch, ExternalAddress((address)Interpreter::dispatch_table()), offset); | |
| __ add(rdispatch, rdispatch, offset); | |
| __ ret(lr); | |
| return start; | |
| } | |
| // Helpers for commoning out cases in the various type of method entries. | |
| // | |
| // increment invocation count & check for overflow | |
| // | |
| // Note: checking for negative value instead of overflow | |
| // so we have a 'sticky' overflow test | |
| // | |
| // rmethod: method | |
| // | |
| void TemplateInterpreterGenerator::generate_counter_incr(Label* overflow) { | |
| Label done; | |
| // Note: In tiered we increment either counters in Method* or in MDO depending if we're profiling or not. | |
| int increment = InvocationCounter::count_increment; | |
| Label no_mdo; | |
| if (ProfileInterpreter) { | |
| // Are we profiling? | |
| __ ldr(r0, Address(rmethod, Method::method_data_offset())); | |
| __ cbz(r0, no_mdo); | |
| // Increment counter in the MDO | |
| const Address mdo_invocation_counter(r0, in_bytes(MethodData::invocation_counter_offset()) + | |
| in_bytes(InvocationCounter::counter_offset())); | |
| const Address mask(r0, in_bytes(MethodData::invoke_mask_offset())); | |
| __ increment_mask_and_jump(mdo_invocation_counter, increment, mask, rscratch1, rscratch2, false, Assembler::EQ, overflow); | |
| __ b(done); | |
| } | |
| __ bind(no_mdo); | |
| // Increment counter in MethodCounters | |
| const Address invocation_counter(rscratch2, | |
| MethodCounters::invocation_counter_offset() + | |
| InvocationCounter::counter_offset()); | |
| __ get_method_counters(rmethod, rscratch2, done); | |
| const Address mask(rscratch2, in_bytes(MethodCounters::invoke_mask_offset())); | |
| __ increment_mask_and_jump(invocation_counter, increment, mask, rscratch1, r1, false, Assembler::EQ, overflow); | |
| __ bind(done); | |
| } | |
| void TemplateInterpreterGenerator::generate_counter_overflow(Label& do_continue) { | |
| // Asm interpreter on entry | |
| // On return (i.e. jump to entry_point) [ back to invocation of interpreter ] | |
| // Everything as it was on entry | |
| // InterpreterRuntime::frequency_counter_overflow takes two | |
| // arguments, the first (thread) is passed by call_VM, the second | |
| // indicates if the counter overflow occurs at a backwards branch | |
| // (null bcp). We pass zero for it. The call returns the address | |
| // of the verified entry point for the method or null if the | |
| // compilation did not complete (either went background or bailed | |
| // out). | |
| __ mov(c_rarg1, 0); | |
| __ call_VM(noreg, | |
| CAST_FROM_FN_PTR(address, | |
| InterpreterRuntime::frequency_counter_overflow), | |
| c_rarg1); | |
| __ b(do_continue); | |
| } | |
| // See if we've got enough room on the stack for locals plus overhead | |
| // below JavaThread::stack_overflow_limit(). If not, throw a StackOverflowError | |
| // without going through the signal handler, i.e., reserved and yellow zones | |
| // will not be made usable. The shadow zone must suffice to handle the | |
| // overflow. | |
| // The expression stack grows down incrementally, so the normal guard | |
| // page mechanism will work for that. | |
| // | |
| // NOTE: Since the additional locals are also always pushed (wasn't | |
| // obvious in generate_method_entry) so the guard should work for them | |
| // too. | |
| // | |
| // Args: | |
| // r3: number of additional locals this frame needs (what we must check) | |
| // rmethod: Method* | |
| // | |
| // Kills: | |
| // r0 | |
| void TemplateInterpreterGenerator::generate_stack_overflow_check(void) { | |
| // monitor entry size: see picture of stack set | |
| // (generate_method_entry) and frame_amd64.hpp | |
| const int entry_size = frame::interpreter_frame_monitor_size_in_bytes(); | |
| // total overhead size: entry_size + (saved rbp through expr stack | |
| // bottom). be sure to change this if you add/subtract anything | |
| // to/from the overhead area | |
| const int overhead_size = | |
| -(frame::interpreter_frame_initial_sp_offset * wordSize) + entry_size; | |
| const size_t page_size = os::vm_page_size(); | |
| Label after_frame_check; | |
| // see if the frame is greater than one page in size. If so, | |
| // then we need to verify there is enough stack space remaining | |
| // for the additional locals. | |
| // | |
| // Note that we use SUBS rather than CMP here because the immediate | |
| // field of this instruction may overflow. SUBS can cope with this | |
| // because it is a macro that will expand to some number of MOV | |
| // instructions and a register operation. | |
| __ subs(rscratch1, r3, (page_size - overhead_size) / Interpreter::stackElementSize); | |
| __ br(Assembler::LS, after_frame_check); | |
| // compute rsp as if this were going to be the last frame on | |
| // the stack before the red zone | |
| // locals + overhead, in bytes | |
| __ mov(r0, overhead_size); | |
| __ add(r0, r0, r3, Assembler::LSL, Interpreter::logStackElementSize); // 2 slots per parameter. | |
| const Address stack_limit(rthread, JavaThread::stack_overflow_limit_offset()); | |
| __ ldr(rscratch1, stack_limit); | |
| #ifdef ASSERT | |
| Label limit_okay; | |
| // Verify that thread stack limit is non-zero. | |
| __ cbnz(rscratch1, limit_okay); | |
| __ stop("stack overflow limit is zero"); | |
| __ bind(limit_okay); | |
| #endif | |
| // Add stack limit to locals. | |
| __ add(r0, r0, rscratch1); | |
| // Check against the current stack bottom. | |
| __ cmp(sp, r0); | |
| __ br(Assembler::HI, after_frame_check); | |
| // Remove the incoming args, peeling the machine SP back to where it | |
| // was in the caller. This is not strictly necessary, but unless we | |
| // do so the stack frame may have a garbage FP; this ensures a | |
| // correct call stack that we can always unwind. The ANDR should be | |
| // unnecessary because the sender SP in r19 is always aligned, but | |
| // it doesn't hurt. | |
| __ andr(sp, r19_sender_sp, -16); | |
| // Note: the restored frame is not necessarily interpreted. | |
| // Use the shared runtime version of the StackOverflowError. | |
| assert(SharedRuntime::throw_StackOverflowError_entry() != nullptr, "stub not yet generated"); | |
| __ far_jump(RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry())); | |
| // all done with frame size check | |
| __ bind(after_frame_check); | |
| } | |
| // Allocate monitor and lock method (asm interpreter) | |
| // | |
| // Args: | |
| // rmethod: Method* | |
| // rlocals: locals | |
| // | |
| // Kills: | |
| // r0 | |
| // c_rarg0, c_rarg1, c_rarg2, c_rarg3, ...(param regs) | |
| // rscratch1, rscratch2 (scratch regs) | |
| void TemplateInterpreterGenerator::lock_method() { | |
| // synchronize method | |
| const Address access_flags(rmethod, Method::access_flags_offset()); | |
| const Address monitor_block_top( | |
| rfp, | |
| frame::interpreter_frame_monitor_block_top_offset * wordSize); | |
| const int entry_size = frame::interpreter_frame_monitor_size_in_bytes(); | |
| #ifdef ASSERT | |
| { | |
| Label L; | |
| __ ldrh(r0, access_flags); | |
| __ tst(r0, JVM_ACC_SYNCHRONIZED); | |
| __ br(Assembler::NE, L); | |
| __ stop("method doesn't need synchronization"); | |
| __ bind(L); | |
| } | |
| #endif // ASSERT | |
| // get synchronization object | |
| { | |
| Label done; | |
| __ ldrh(r0, access_flags); | |
| __ tst(r0, JVM_ACC_STATIC); | |
| // get receiver (assume this is frequent case) | |
| __ ldr(r0, Address(rlocals, Interpreter::local_offset_in_bytes(0))); | |
| __ br(Assembler::EQ, done); | |
| __ load_mirror(r0, rmethod, r5, rscratch2); | |
| #ifdef ASSERT | |
| { | |
| Label L; | |
| __ cbnz(r0, L); | |
| __ stop("synchronization object is null"); | |
| __ bind(L); | |
| } | |
| #endif // ASSERT | |
| __ bind(done); | |
| } | |
| // add space for monitor & lock | |
| __ check_extended_sp(); | |
| __ sub(sp, sp, entry_size); // add space for a monitor entry | |
| __ sub(esp, esp, entry_size); | |
| __ sub(rscratch1, sp, rfp); | |
| __ asr(rscratch1, rscratch1, Interpreter::logStackElementSize); | |
| __ str(rscratch1, Address(rfp, frame::interpreter_frame_extended_sp_offset * wordSize)); | |
| __ sub(rscratch1, esp, rfp); | |
| __ asr(rscratch1, rscratch1, Interpreter::logStackElementSize); | |
| __ str(rscratch1, monitor_block_top); // set new monitor block top | |
| // store object | |
| __ str(r0, Address(esp, BasicObjectLock::obj_offset())); | |
| __ mov(c_rarg1, esp); // object address | |
| __ lock_object(c_rarg1); | |
| } | |
| // Generate a fixed interpreter frame. This is identical setup for | |
| // interpreted methods and for native methods hence the shared code. | |
| // | |
| // Args: | |
| // lr: return address | |
| // rmethod: Method* | |
| // rlocals: pointer to locals | |
| // rcpool: cp cache | |
| // stack_pointer: previous sp | |
| void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) { | |
| // Save ConstMethod* in r5_const_method for later use to avoid loading multiple times | |
| Register r5_const_method = r5; | |
| __ ldr(r5_const_method, Address(rmethod, Method::const_offset())); | |
| // initialize fixed part of activation frame | |
| if (native_call) { | |
| __ sub(esp, sp, 14 * wordSize); | |
| __ mov(rbcp, zr); | |
| __ mov(rscratch1, frame::interpreter_frame_initial_sp_offset); | |
| __ stp(rscratch1, zr, Address(__ pre(sp, -14 * wordSize))); | |
| // add 2 zero-initialized slots for native calls | |
| __ stp(zr, zr, Address(sp, 12 * wordSize)); | |
| } else { | |
| __ sub(esp, sp, 12 * wordSize); | |
| __ add(rbcp, r5_const_method, in_bytes(ConstMethod::codes_offset())); // get codebase | |
| __ mov(rscratch1, frame::interpreter_frame_initial_sp_offset); | |
| __ stp(rscratch1, rbcp, Address(__ pre(sp, -12 * wordSize))); | |
| } | |
| if (ProfileInterpreter) { | |
| Label method_data_continue; | |
| __ ldr(rscratch1, Address(rmethod, Method::method_data_offset())); | |
| __ cbz(rscratch1, method_data_continue); | |
| __ lea(rscratch1, Address(rscratch1, in_bytes(MethodData::data_offset()))); | |
| __ bind(method_data_continue); | |
| __ stp(rscratch1, rmethod, Address(sp, 6 * wordSize)); // save Method* and mdp (method data pointer) | |
| } else { | |
| __ stp(zr, rmethod, Address(sp, 6 * wordSize)); // save Method* (no mdp) | |
| } | |
| __ protect_return_address(); | |
| __ stp(rfp, lr, Address(sp, 10 * wordSize)); | |
| __ lea(rfp, Address(sp, 10 * wordSize)); | |
| // Save ConstantPool* in r11_constants for later use to avoid loading multiple times | |
| Register r11_constants = r11; | |
| __ ldr(r11_constants, Address(r5_const_method, ConstMethod::constants_offset())); | |
| __ ldr(rcpool, Address(r11_constants, ConstantPool::cache_offset())); | |
| __ sub(rscratch1, rlocals, rfp); | |
| __ lsr(rscratch1, rscratch1, Interpreter::logStackElementSize); // rscratch1 = rlocals - fp(); | |
| // Store relativized rlocals, see frame::interpreter_frame_locals(). | |
| __ stp(rscratch1, rcpool, Address(sp, 2 * wordSize)); | |
| // set sender sp | |
| // leave last_sp as null | |
| __ stp(zr, r19_sender_sp, Address(sp, 8 * wordSize)); | |
| // Get mirror. Resolve ConstantPool* -> InstanceKlass* -> Java mirror. | |
| __ ldr(r10, Address(r11_constants, ConstantPool::pool_holder_offset())); | |
| __ ldr(r10, Address(r10, in_bytes(Klass::java_mirror_offset()))); | |
| __ resolve_oop_handle(r10, rscratch1, rscratch2); | |
| if (! native_call) { | |
| __ ldrh(rscratch1, Address(r5_const_method, ConstMethod::max_stack_offset())); | |
| __ add(rscratch1, rscratch1, MAX2(3, Method::extra_stack_entries())); | |
| __ sub(rscratch1, sp, rscratch1, ext::uxtw, 3); | |
| __ andr(rscratch1, rscratch1, -16); | |
| __ sub(rscratch2, rscratch1, rfp); | |
| __ asr(rscratch2, rscratch2, Interpreter::logStackElementSize); | |
| // Store extended SP and mirror | |
| __ stp(r10, rscratch2, Address(sp, 4 * wordSize)); | |
| // Move SP out of the way | |
| __ mov(sp, rscratch1); | |
| } else { | |
| // Make sure there is room for the exception oop pushed in case method throws | |
| // an exception (see TemplateInterpreterGenerator::generate_throw_exception()) | |
| __ sub(rscratch1, sp, 2 * wordSize); | |
| __ sub(rscratch2, rscratch1, rfp); | |
| __ asr(rscratch2, rscratch2, Interpreter::logStackElementSize); | |
| __ stp(r10, rscratch2, Address(sp, 4 * wordSize)); | |
| __ mov(sp, rscratch1); | |
| } | |
| } | |
| // End of helpers | |
| // Various method entries | |
| //------------------------------------------------------------------------------------------------------------------------ | |
| // | |
| // | |
| // Method entry for java.lang.ref.Reference.get. | |
| address TemplateInterpreterGenerator::generate_Reference_get_entry(void) { | |
| // Code: _aload_0, _getfield, _areturn | |
| // parameter size = 1 | |
| // | |
| // The code that gets generated by this routine is split into 2 parts: | |
| // 1. The "intrinsified" code for G1 (or any SATB based GC), | |
| // 2. The slow path - which is an expansion of the regular method entry. | |
| // | |
| // Notes:- | |
| // * In the G1 code we do not check whether we need to block for | |
| // a safepoint. If G1 is enabled then we must execute the specialized | |
| // code for Reference.get (except when the Reference object is null) | |
| // so that we can log the value in the referent field with an SATB | |
| // update buffer. | |
| // If the code for the getfield template is modified so that the | |
| // G1 pre-barrier code is executed when the current method is | |
| // Reference.get() then going through the normal method entry | |
| // will be fine. | |
| // * The G1 code can, however, check the receiver object (the instance | |
| // of java.lang.Reference) and jump to the slow path if null. If the | |
| // Reference object is null then we obviously cannot fetch the referent | |
| // and so we don't need to call the G1 pre-barrier. Thus we can use the | |
| // regular method entry code to generate the NPE. | |
| // | |
| // This code is based on generate_accessor_entry. | |
| // | |
| // rmethod: Method* | |
| // r19_sender_sp: senderSP must preserve for slow path, set SP to it on fast path | |
| // LR is live. It must be saved around calls. | |
| address entry = __ pc(); | |
| const int referent_offset = java_lang_ref_Reference::referent_offset(); | |
| Label slow_path; | |
| const Register local_0 = c_rarg0; | |
| // Check if local 0 != null | |
| // If the receiver is null then it is OK to jump to the slow path. | |
| __ ldr(local_0, Address(esp, 0)); | |
| __ cbz(local_0, slow_path); | |
| // Load the value of the referent field. | |
| const Address field_address(local_0, referent_offset); | |
| BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); | |
| bs->load_at(_masm, IN_HEAP | ON_WEAK_OOP_REF, T_OBJECT, local_0, field_address, /*tmp1*/ rscratch1, /*tmp2*/ rscratch2); | |
| // areturn | |
| __ andr(sp, r19_sender_sp, -16); // done with stack | |
| __ ret(lr); | |
| // generate a vanilla interpreter entry as the slow path | |
| __ bind(slow_path); | |
| __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::zerolocals)); | |
| return entry; | |
| } | |
| /** | |
| * Method entry for static native methods: | |
| * int java.util.zip.CRC32.update(int crc, int b) | |
| */ | |
| address TemplateInterpreterGenerator::generate_CRC32_update_entry() { | |
| assert(UseCRC32Intrinsics, "this intrinsic is not supported"); | |
| address entry = __ pc(); | |
| // rmethod: Method* | |
| // r19_sender_sp: senderSP must preserved for slow path | |
| // esp: args | |
| Label slow_path; | |
| // If we need a safepoint check, generate full interpreter entry. | |
| __ safepoint_poll(slow_path, false /* at_return */, false /* acquire */, false /* in_nmethod */); | |
| // We don't generate local frame and don't align stack because | |
| // we call stub code and there is no safepoint on this path. | |
| // Load parameters | |
| const Register crc = c_rarg0; // crc | |
| const Register val = c_rarg1; // source java byte value | |
| const Register tbl = c_rarg2; // scratch | |
| // Arguments are reversed on java expression stack | |
| __ ldrw(val, Address(esp, 0)); // byte value | |
| __ ldrw(crc, Address(esp, wordSize)); // Initial CRC | |
| uint64_t offset; | |
| __ adrp(tbl, ExternalAddress(StubRoutines::crc_table_addr()), offset); | |
| __ add(tbl, tbl, offset); | |
| __ mvnw(crc, crc); // ~crc | |
| __ update_byte_crc32(crc, val, tbl); | |
| __ mvnw(crc, crc); // ~crc | |
| // result in c_rarg0 | |
| __ andr(sp, r19_sender_sp, -16); | |
| __ ret(lr); | |
| // generate a vanilla native entry as the slow path | |
| __ bind(slow_path); | |
| __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::native)); | |
| return entry; | |
| } | |
| /** | |
| * Method entry for static native methods: | |
| * int java.util.zip.CRC32.updateBytes(int crc, byte[] b, int off, int len) | |
| * int java.util.zip.CRC32.updateByteBuffer(int crc, long buf, int off, int len) | |
| */ | |
| address TemplateInterpreterGenerator::generate_CRC32_updateBytes_entry(AbstractInterpreter::MethodKind kind) { | |
| assert(UseCRC32Intrinsics, "this intrinsic is not supported"); | |
| address entry = __ pc(); | |
| // rmethod,: Method* | |
| // r19_sender_sp: senderSP must preserved for slow path | |
| Label slow_path; | |
| // If we need a safepoint check, generate full interpreter entry. | |
| __ safepoint_poll(slow_path, false /* at_return */, false /* acquire */, false /* in_nmethod */); | |
| // We don't generate local frame and don't align stack because | |
| // we call stub code and there is no safepoint on this path. | |
| // Load parameters | |
| const Register crc = c_rarg0; // crc | |
| const Register buf = c_rarg1; // source java byte array address | |
| const Register len = c_rarg2; // length | |
| const Register off = len; // offset (never overlaps with 'len') | |
| // Arguments are reversed on java expression stack | |
| // Calculate address of start element | |
| if (kind == Interpreter::java_util_zip_CRC32_updateByteBuffer) { | |
| __ ldr(buf, Address(esp, 2*wordSize)); // long buf | |
| __ ldrw(off, Address(esp, wordSize)); // offset | |
| __ add(buf, buf, off); // + offset | |
| __ ldrw(crc, Address(esp, 4*wordSize)); // Initial CRC | |
| } else { | |
| __ ldr(buf, Address(esp, 2*wordSize)); // byte[] array | |
| __ add(buf, buf, arrayOopDesc::base_offset_in_bytes(T_BYTE)); // + header size | |
| __ ldrw(off, Address(esp, wordSize)); // offset | |
| __ add(buf, buf, off); // + offset | |
| __ ldrw(crc, Address(esp, 3*wordSize)); // Initial CRC | |
| } | |
| // Can now load 'len' since we're finished with 'off' | |
| __ ldrw(len, Address(esp, 0x0)); // Length | |
| __ andr(sp, r19_sender_sp, -16); // Restore the caller's SP | |
| // We are frameless so we can just jump to the stub. | |
| __ b(CAST_FROM_FN_PTR(address, StubRoutines::updateBytesCRC32())); | |
| // generate a vanilla native entry as the slow path | |
| __ bind(slow_path); | |
| __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::native)); | |
| return entry; | |
| } | |
| /** | |
| * Method entry for intrinsic-candidate (non-native) methods: | |
| * int java.util.zip.CRC32C.updateBytes(int crc, byte[] b, int off, int end) | |
| * int java.util.zip.CRC32C.updateDirectByteBuffer(int crc, long buf, int off, int end) | |
| * Unlike CRC32, CRC32C does not have any methods marked as native | |
| * CRC32C also uses an "end" variable instead of the length variable CRC32 uses | |
| */ | |
| address TemplateInterpreterGenerator::generate_CRC32C_updateBytes_entry(AbstractInterpreter::MethodKind kind) { | |
| assert(UseCRC32CIntrinsics, "this intrinsic is not supported"); | |
| address entry = __ pc(); | |
| // Prepare jump to stub using parameters from the stack | |
| const Register crc = c_rarg0; // initial crc | |
| const Register buf = c_rarg1; // source java byte array address | |
| const Register len = c_rarg2; // len argument to the kernel | |
| const Register end = len; // index of last element to process | |
| const Register off = crc; // offset | |
| __ ldrw(end, Address(esp)); // int end | |
| __ ldrw(off, Address(esp, wordSize)); // int offset | |
| __ sub(len, end, off); | |
| __ ldr(buf, Address(esp, 2*wordSize)); // byte[] buf | long buf | |
| __ add(buf, buf, off); // + offset | |
| if (kind == Interpreter::java_util_zip_CRC32C_updateDirectByteBuffer) { | |
| __ ldrw(crc, Address(esp, 4*wordSize)); // long crc | |
| } else { | |
| __ add(buf, buf, arrayOopDesc::base_offset_in_bytes(T_BYTE)); // + header size | |
| __ ldrw(crc, Address(esp, 3*wordSize)); // long crc | |
| } | |
| __ andr(sp, r19_sender_sp, -16); // Restore the caller's SP | |
| // Jump to the stub. | |
| __ b(CAST_FROM_FN_PTR(address, StubRoutines::updateBytesCRC32C())); | |
| return entry; | |
| } | |
| void TemplateInterpreterGenerator::bang_stack_shadow_pages(bool native_call) { | |
| // See more discussion in stackOverflow.hpp. | |
| const int shadow_zone_size = checked_cast<int>(StackOverflow::stack_shadow_zone_size()); | |
| const int page_size = (int)os::vm_page_size(); | |
| const int n_shadow_pages = shadow_zone_size / page_size; | |
| #ifdef ASSERT | |
| Label L_good_limit; | |
| __ ldr(rscratch1, Address(rthread, JavaThread::shadow_zone_safe_limit())); | |
| __ cbnz(rscratch1, L_good_limit); | |
| __ stop("shadow zone safe limit is not initialized"); | |
| __ bind(L_good_limit); | |
| Label L_good_watermark; | |
| __ ldr(rscratch1, Address(rthread, JavaThread::shadow_zone_growth_watermark())); | |
| __ cbnz(rscratch1, L_good_watermark); | |
| __ stop("shadow zone growth watermark is not initialized"); | |
| __ bind(L_good_watermark); | |
| #endif | |
| Label L_done; | |
| __ ldr(rscratch1, Address(rthread, JavaThread::shadow_zone_growth_watermark())); | |
| __ cmp(sp, rscratch1); | |
| __ br(Assembler::HI, L_done); | |
| for (int p = 1; p <= n_shadow_pages; p++) { | |
| __ sub(rscratch2, sp, p*page_size); | |
| __ str(zr, Address(rscratch2)); | |
| } | |
| // Record the new watermark, but only if the update is above the safe limit. | |
| // Otherwise, the next time around the check above would pass the safe limit. | |
| __ ldr(rscratch1, Address(rthread, JavaThread::shadow_zone_safe_limit())); | |
| __ cmp(sp, rscratch1); | |
| __ br(Assembler::LS, L_done); | |
| __ mov(rscratch1, sp); | |
| __ str(rscratch1, Address(rthread, JavaThread::shadow_zone_growth_watermark())); | |
| __ bind(L_done); | |
| } | |
| // Interpreter stub for calling a native method. (asm interpreter) | |
| // This sets up a somewhat different looking stack for calling the | |
| // native method than the typical interpreter frame setup. | |
| address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { | |
| // determine code generation flags | |
| bool inc_counter = UseCompiler || CountCompiledCalls; | |
| // r1: Method* | |
| // rscratch1: sender sp | |
| address entry_point = __ pc(); | |
| const Address constMethod (rmethod, Method::const_offset()); | |
| const Address access_flags (rmethod, Method::access_flags_offset()); | |
| const Address size_of_parameters(r2, ConstMethod:: | |
| size_of_parameters_offset()); | |
| // get parameter size (always needed) | |
| __ ldr(r2, constMethod); | |
| __ load_unsigned_short(r2, size_of_parameters); | |
| // Native calls don't need the stack size check since they have no | |
| // expression stack and the arguments are already on the stack and | |
| // we only add a handful of words to the stack. | |
| // rmethod: Method* | |
| // r2: size of parameters | |
| // rscratch1: sender sp | |
| // for natives the size of locals is zero | |
| // compute beginning of parameters (rlocals) | |
| __ add(rlocals, esp, r2, ext::uxtx, 3); | |
| __ add(rlocals, rlocals, -wordSize); | |
| // Pull SP back to minimum size: this avoids holes in the stack | |
| __ andr(sp, esp, -16); | |
| // initialize fixed part of activation frame | |
| generate_fixed_frame(true); | |
| // make sure method is native & not abstract | |
| #ifdef ASSERT | |
| __ ldrh(r0, access_flags); | |
| { | |
| Label L; | |
| __ tst(r0, JVM_ACC_NATIVE); | |
| __ br(Assembler::NE, L); | |
| __ stop("tried to execute non-native method as native"); | |
| __ bind(L); | |
| } | |
| { | |
| Label L; | |
| __ tst(r0, JVM_ACC_ABSTRACT); | |
| __ br(Assembler::EQ, L); | |
| __ stop("tried to execute abstract method in interpreter"); | |
| __ bind(L); | |
| } | |
| #endif | |
| // Since at this point in the method invocation the exception | |
| // handler would try to exit the monitor of synchronized methods | |
| // which hasn't been entered yet, we set the thread local variable | |
| // _do_not_unlock_if_synchronized to true. The remove_activation | |
| // will check this flag. | |
| const Address do_not_unlock_if_synchronized(rthread, | |
| in_bytes(JavaThread::do_not_unlock_if_synchronized_offset())); | |
| __ mov(rscratch2, true); | |
| __ strb(rscratch2, do_not_unlock_if_synchronized); | |
| // increment invocation count & check for overflow | |
| Label invocation_counter_overflow; | |
| if (inc_counter) { | |
| generate_counter_incr(&invocation_counter_overflow); | |
| } | |
| Label continue_after_compile; | |
| __ bind(continue_after_compile); | |
| bang_stack_shadow_pages(true); | |
| // reset the _do_not_unlock_if_synchronized flag | |
| __ strb(zr, do_not_unlock_if_synchronized); | |
| // check for synchronized methods | |
| // Must happen AFTER invocation_counter check and stack overflow check, | |
| // so method is not locked if overflows. | |
| if (synchronized) { | |
| lock_method(); | |
| } else { | |
| // no synchronization necessary | |
| #ifdef ASSERT | |
| { | |
| Label L; | |
| __ ldrh(r0, access_flags); | |
| __ tst(r0, JVM_ACC_SYNCHRONIZED); | |
| __ br(Assembler::EQ, L); | |
| __ stop("method needs synchronization"); | |
| __ bind(L); | |
| } | |
| #endif | |
| } | |
| // start execution | |
| #ifdef ASSERT | |
| { | |
| Label L; | |
| const Address monitor_block_top(rfp, | |
| frame::interpreter_frame_monitor_block_top_offset * wordSize); | |
| __ ldr(rscratch1, monitor_block_top); | |
| __ lea(rscratch1, Address(rfp, rscratch1, Address::lsl(Interpreter::logStackElementSize))); | |
| __ cmp(esp, rscratch1); | |
| __ br(Assembler::EQ, L); | |
| __ stop("broken stack frame setup in interpreter 1"); | |
| __ bind(L); | |
| } | |
| #endif | |
| // jvmti support | |
| __ notify_method_entry(); | |
| // work registers | |
| const Register t = r17; | |
| const Register result_handler = r19; | |
| // allocate space for parameters | |
| __ ldr(t, Address(rmethod, Method::const_offset())); | |
| __ load_unsigned_short(t, Address(t, ConstMethod::size_of_parameters_offset())); | |
| __ sub(rscratch1, esp, t, ext::uxtx, Interpreter::logStackElementSize); | |
| __ andr(sp, rscratch1, -16); | |
| __ mov(esp, rscratch1); | |
| // get signature handler | |
| { | |
| Label L; | |
| __ ldr(t, Address(rmethod, Method::signature_handler_offset())); | |
| __ cbnz(t, L); | |
| __ call_VM(noreg, | |
| CAST_FROM_FN_PTR(address, | |
| InterpreterRuntime::prepare_native_call), | |
| rmethod); | |
| __ ldr(t, Address(rmethod, Method::signature_handler_offset())); | |
| __ bind(L); | |
| } | |
| // call signature handler | |
| assert(InterpreterRuntime::SignatureHandlerGenerator::from() == rlocals, | |
| "adjust this code"); | |
| assert(InterpreterRuntime::SignatureHandlerGenerator::to() == sp, | |
| "adjust this code"); | |
| assert(InterpreterRuntime::SignatureHandlerGenerator::temp() == rscratch1, | |
| "adjust this code"); | |
| // The generated handlers do not touch rmethod (the method). | |
| // However, large signatures cannot be cached and are generated | |
| // each time here. The slow-path generator can do a GC on return, | |
| // so we must reload it after the call. | |
| __ blr(t); | |
| __ get_method(rmethod); // slow path can do a GC, reload rmethod | |
| // result handler is in r0 | |
| // set result handler | |
| __ mov(result_handler, r0); | |
| // Save it in the frame in case of preemption; we cannot rely on callee saved registers. | |
| __ str(r0, Address(rfp, frame::interpreter_frame_result_handler_offset * wordSize)); | |
| // pass mirror handle if static call | |
| { | |
| Label L; | |
| __ ldrh(t, Address(rmethod, Method::access_flags_offset())); | |
| __ tbz(t, exact_log2(JVM_ACC_STATIC), L); | |
| // get mirror | |
| __ load_mirror(t, rmethod, r10, rscratch2); | |
| // copy mirror into activation frame | |
| __ str(t, Address(rfp, frame::interpreter_frame_oop_temp_offset * wordSize)); | |
| // pass handle to mirror | |
| __ add(c_rarg1, rfp, frame::interpreter_frame_oop_temp_offset * wordSize); | |
| __ bind(L); | |
| } | |
| // get native function entry point in r10 | |
| { | |
| Label L; | |
| __ ldr(r10, Address(rmethod, Method::native_function_offset())); | |
| ExternalAddress unsatisfied(SharedRuntime::native_method_throw_unsatisfied_link_error_entry()); | |
| __ lea(rscratch2, unsatisfied); | |
| __ ldr(rscratch2, rscratch2); | |
| __ cmp(r10, rscratch2); | |
| __ br(Assembler::NE, L); | |
| __ call_VM(noreg, | |
| CAST_FROM_FN_PTR(address, | |
| InterpreterRuntime::prepare_native_call), | |
| rmethod); | |
| __ get_method(rmethod); | |
| __ ldr(r10, Address(rmethod, Method::native_function_offset())); | |
| __ bind(L); | |
| } | |
| // pass JNIEnv | |
| __ add(c_rarg0, rthread, in_bytes(JavaThread::jni_environment_offset())); | |
| // It is enough that the pc() points into the right code | |
| // segment. It does not have to be the correct return pc. | |
| // For convenience we use the pc we want to resume to in | |
| // case of preemption on Object.wait. | |
| Label native_return; | |
| __ set_last_Java_frame(esp, rfp, native_return, rscratch1); | |
| // change thread state | |
| #ifdef ASSERT | |
| { | |
| Label L; | |
| __ ldrw(t, Address(rthread, JavaThread::thread_state_offset())); | |
| __ cmp(t, (u1)_thread_in_Java); | |
| __ br(Assembler::EQ, L); | |
| __ stop("Wrong thread state in native stub"); | |
| __ bind(L); | |
| } | |
| #endif | |
| // Change state to native | |
| __ mov(rscratch1, _thread_in_native); | |
| __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset())); | |
| __ stlrw(rscratch1, rscratch2); | |
| __ push_cont_fastpath(); | |
| // Call the native method. | |
| __ blr(r10); | |
| __ pop_cont_fastpath(); | |
| __ get_method(rmethod); | |
| // result potentially in r0 or v0 | |
| // Restore cpu control state after JNI call | |
| __ restore_cpu_control_state_after_jni(rscratch1, rscratch2); | |
| // make room for the pushes we're about to do | |
| __ sub(rscratch1, esp, 4 * wordSize); | |
| __ andr(sp, rscratch1, -16); | |
| // NOTE: The order of these pushes is known to frame::interpreter_frame_result | |
| // in order to extract the result of a method call. If the order of these | |
| // pushes change or anything else is added to the stack then the code in | |
| // interpreter_frame_result must also change. | |
| __ push(dtos); | |
| __ push(ltos); | |
| __ verify_sve_vector_length(); | |
| // change thread state | |
| __ mov(rscratch1, _thread_in_native_trans); | |
| __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset())); | |
| __ stlrw(rscratch1, rscratch2); | |
| // Force this write out before the read below | |
| if (!UseSystemMemoryBarrier) { | |
| __ dmb(Assembler::ISH); | |
| } | |
| // check for safepoint operation in progress and/or pending suspend requests | |
| { | |
| Label L, Continue; | |
| // No need for acquire as Java threads always disarm themselves. | |
| __ safepoint_poll(L, true /* at_return */, false /* acquire */, false /* in_nmethod */); | |
| __ ldrw(rscratch2, Address(rthread, JavaThread::suspend_flags_offset())); | |
| __ cbz(rscratch2, Continue); | |
| __ bind(L); | |
| // Don't use call_VM as it will see a possible pending exception | |
| // and forward it and never return here preventing us from | |
| // clearing _last_native_pc down below. So we do a runtime call by | |
| // hand. | |
| // | |
| __ mov(c_rarg0, rthread); | |
| __ lea(rscratch2, RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans))); | |
| __ blr(rscratch2); | |
| __ get_method(rmethod); | |
| __ reinit_heapbase(); | |
| __ bind(Continue); | |
| } | |
| // change thread state | |
| __ mov(rscratch1, _thread_in_Java); | |
| __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset())); | |
| __ stlrw(rscratch1, rscratch2); | |
| if (LockingMode != LM_LEGACY) { | |
| // Check preemption for Object.wait() | |
| Label not_preempted; | |
| __ ldr(rscratch1, Address(rthread, JavaThread::preempt_alternate_return_offset())); | |
| __ cbz(rscratch1, not_preempted); | |
| __ str(zr, Address(rthread, JavaThread::preempt_alternate_return_offset())); | |
| __ br(rscratch1); | |
| __ bind(native_return); | |
| __ restore_after_resume(true /* is_native */); | |
| // reload result_handler | |
| __ ldr(result_handler, Address(rfp, frame::interpreter_frame_result_handler_offset*wordSize)); | |
| __ bind(not_preempted); | |
| } else { | |
| // any pc will do so just use this one for LM_LEGACY to keep code together. | |
| __ bind(native_return); | |
| } | |
| // reset_last_Java_frame | |
| __ reset_last_Java_frame(true); | |
| if (CheckJNICalls) { | |
| // clear_pending_jni_exception_check | |
| __ str(zr, Address(rthread, JavaThread::pending_jni_exception_check_fn_offset())); | |
| } | |
| // reset handle block | |
| __ ldr(t, Address(rthread, JavaThread::active_handles_offset())); | |
| __ str(zr, Address(t, JNIHandleBlock::top_offset())); | |
| // If result is an oop unbox and store it in frame where gc will see it | |
| // and result handler will pick it up | |
| { | |
| Label no_oop; | |
| __ adr(t, ExternalAddress(AbstractInterpreter::result_handler(T_OBJECT))); | |
| __ cmp(t, result_handler); | |
| __ br(Assembler::NE, no_oop); | |
| // Unbox oop result, e.g. JNIHandles::resolve result. | |
| __ pop(ltos); | |
| __ resolve_jobject(r0, t, rscratch2); | |
| __ str(r0, Address(rfp, frame::interpreter_frame_oop_temp_offset*wordSize)); | |
| // keep stack depth as expected by pushing oop which will eventually be discarded | |
| __ push(ltos); | |
| __ bind(no_oop); | |
| } | |
| { | |
| Label no_reguard; | |
| __ lea(rscratch1, Address(rthread, in_bytes(JavaThread::stack_guard_state_offset()))); | |
| __ ldrw(rscratch1, Address(rscratch1)); | |
| __ cmp(rscratch1, (u1)StackOverflow::stack_guard_yellow_reserved_disabled); | |
| __ br(Assembler::NE, no_reguard); | |
| __ push_call_clobbered_registers(); | |
| __ mov(c_rarg0, rthread); | |
| __ lea(rscratch2, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages))); | |
| __ blr(rscratch2); | |
| __ pop_call_clobbered_registers(); | |
| __ bind(no_reguard); | |
| } | |
| // The method register is junk from after the thread_in_native transition | |
| // until here. Also can't call_VM until the bcp has been | |
| // restored. Need bcp for throwing exception below so get it now. | |
| __ get_method(rmethod); | |
| // restore bcp to have legal interpreter frame, i.e., bci == 0 <=> | |
| // rbcp == code_base() | |
| __ ldr(rbcp, Address(rmethod, Method::const_offset())); // get ConstMethod* | |
| __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset())); // get codebase | |
| // handle exceptions (exception handling will handle unlocking!) | |
| { | |
| Label L; | |
| __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset())); | |
| __ cbz(rscratch1, L); | |
| // Note: At some point we may want to unify this with the code | |
| // used in call_VM_base(); i.e., we should use the | |
| // StubRoutines::forward_exception code. For now this doesn't work | |
| // here because the rsp is not correctly set at this point. | |
| __ MacroAssembler::call_VM(noreg, | |
| CAST_FROM_FN_PTR(address, | |
| InterpreterRuntime::throw_pending_exception)); | |
| __ should_not_reach_here(); | |
| __ bind(L); | |
| } | |
| // do unlocking if necessary | |
| { | |
| Label L; | |
| __ ldrh(t, Address(rmethod, Method::access_flags_offset())); | |
| __ tbz(t, exact_log2(JVM_ACC_SYNCHRONIZED), L); | |
| // the code below should be shared with interpreter macro | |
| // assembler implementation | |
| { | |
| Label unlock; | |
| // BasicObjectLock will be first in list, since this is a | |
| // synchronized method. However, need to check that the object | |
| // has not been unlocked by an explicit monitorexit bytecode. | |
| // monitor expect in c_rarg1 for slow unlock path | |
| __ lea (c_rarg1, Address(rfp, // address of first monitor | |
| (intptr_t)(frame::interpreter_frame_initial_sp_offset * | |
| wordSize - sizeof(BasicObjectLock)))); | |
| __ ldr(t, Address(c_rarg1, BasicObjectLock::obj_offset())); | |
| __ cbnz(t, unlock); | |
| // Entry already unlocked, need to throw exception | |
| __ MacroAssembler::call_VM(noreg, | |
| CAST_FROM_FN_PTR(address, | |
| InterpreterRuntime::throw_illegal_monitor_state_exception)); | |
| __ should_not_reach_here(); | |
| __ bind(unlock); | |
| __ unlock_object(c_rarg1); | |
| } | |
| __ bind(L); | |
| } | |
| #if INCLUDE_JFR | |
| __ enter_jfr_critical_section(); | |
| // This poll test is to uphold the invariant that a JFR sampled frame | |
| // must not return to its caller without a prior safepoint poll check. | |
| // The earlier poll check in this routine is insufficient for this purpose | |
| // because the thread has transitioned back to Java. | |
| Label slow_path; | |
| Label fast_path; | |
| __ safepoint_poll(slow_path, true /* at_return */, false /* acquire */, false /* in_nmethod */); | |
| __ br(Assembler::AL, fast_path); | |
| __ bind(slow_path); | |
| __ push(dtos); | |
| __ push(ltos); | |
| __ set_last_Java_frame(esp, rfp, __ pc(), rscratch1); | |
| __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind), rthread); | |
| __ reset_last_Java_frame(true); | |
| __ pop(ltos); | |
| __ pop(dtos); | |
| __ bind(fast_path); | |
| #endif // INCLUDE_JFR | |
| // jvmti support | |
| // Note: This must happen _after_ handling/throwing any exceptions since | |
| // the exception handler code notifies the runtime of method exits | |
| // too. If this happens before, method entry/exit notifications are | |
| // not properly paired (was bug - gri 11/22/99). | |
| __ notify_method_exit(vtos, InterpreterMacroAssembler::NotifyJVMTI); | |
| // restore potential result in r0:d0, call result handler to | |
| // restore potential result in ST0 & handle result | |
| __ pop(ltos); | |
| __ pop(dtos); | |
| __ blr(result_handler); | |
| // remove activation | |
| __ ldr(esp, Address(rfp, | |
| frame::interpreter_frame_sender_sp_offset * | |
| wordSize)); // get sender sp | |
| // remove frame anchor | |
| __ leave(); | |
| JFR_ONLY(__ leave_jfr_critical_section();) | |
| // restore sender sp | |
| __ mov(sp, esp); | |
| __ ret(lr); | |
| if (inc_counter) { | |
| // Handle overflow of counter and compile method | |
| __ bind(invocation_counter_overflow); | |
| generate_counter_overflow(continue_after_compile); | |
| } | |
| return entry_point; | |
| } | |
| // | |
| // Generic interpreted method entry to (asm) interpreter | |
| // | |
| address TemplateInterpreterGenerator::generate_normal_entry(bool synchronized) { | |
| // determine code generation flags | |
| bool inc_counter = UseCompiler || CountCompiledCalls; | |
| // rscratch1: sender sp | |
| address entry_point = __ pc(); | |
| const Address constMethod(rmethod, Method::const_offset()); | |
| const Address access_flags(rmethod, Method::access_flags_offset()); | |
| const Address size_of_parameters(r3, | |
| ConstMethod::size_of_parameters_offset()); | |
| const Address size_of_locals(r3, ConstMethod::size_of_locals_offset()); | |
| // get parameter size (always needed) | |
| // need to load the const method first | |
| __ ldr(r3, constMethod); | |
| __ load_unsigned_short(r2, size_of_parameters); | |
| // r2: size of parameters | |
| __ load_unsigned_short(r3, size_of_locals); // get size of locals in words | |
| __ sub(r3, r3, r2); // r3 = no. of additional locals | |
| // see if we've got enough room on the stack for locals plus overhead. | |
| generate_stack_overflow_check(); | |
| // compute beginning of parameters (rlocals) | |
| __ add(rlocals, esp, r2, ext::uxtx, 3); | |
| __ sub(rlocals, rlocals, wordSize); | |
| __ mov(rscratch1, esp); | |
| // r3 - # of additional locals | |
| // allocate space for locals | |
| // explicitly initialize locals | |
| // Initializing memory allocated for locals in the same direction as | |
| // the stack grows to ensure page initialization order according | |
| // to windows-aarch64 stack page growth requirement (see | |
| // https://docs.microsoft.com/en-us/cpp/build/arm64-windows-abi-conventions?view=msvc-160#stack) | |
| { | |
| Label exit, loop; | |
| __ ands(zr, r3, r3); | |
| __ br(Assembler::LE, exit); // do nothing if r3 <= 0 | |
| __ bind(loop); | |
| __ str(zr, Address(__ pre(rscratch1, -wordSize))); | |
| __ sub(r3, r3, 1); // until everything initialized | |
| __ cbnz(r3, loop); | |
| __ bind(exit); | |
| } | |
| // Padding between locals and fixed part of activation frame to ensure | |
| // SP is always 16-byte aligned. | |
| __ andr(sp, rscratch1, -16); | |
| // And the base dispatch table | |
| __ get_dispatch(); | |
| // initialize fixed part of activation frame | |
| generate_fixed_frame(false); | |
| // make sure method is not native & not abstract | |
| #ifdef ASSERT | |
| __ ldrh(r0, access_flags); | |
| { | |
| Label L; | |
| __ tst(r0, JVM_ACC_NATIVE); | |
| __ br(Assembler::EQ, L); | |
| __ stop("tried to execute native method as non-native"); | |
| __ bind(L); | |
| } | |
| { | |
| Label L; | |
| __ tst(r0, JVM_ACC_ABSTRACT); | |
| __ br(Assembler::EQ, L); | |
| __ stop("tried to execute abstract method in interpreter"); | |
| __ bind(L); | |
| } | |
| #endif | |
| // Since at this point in the method invocation the exception | |
| // handler would try to exit the monitor of synchronized methods | |
| // which hasn't been entered yet, we set the thread local variable | |
| // _do_not_unlock_if_synchronized to true. The remove_activation | |
| // will check this flag. | |
| const Address do_not_unlock_if_synchronized(rthread, | |
| in_bytes(JavaThread::do_not_unlock_if_synchronized_offset())); | |
| __ mov(rscratch2, true); | |
| __ strb(rscratch2, do_not_unlock_if_synchronized); | |
| Register mdp = r3; | |
| __ profile_parameters_type(mdp, r1, r2); | |
| // increment invocation count & check for overflow | |
| Label invocation_counter_overflow; | |
| if (inc_counter) { | |
| generate_counter_incr(&invocation_counter_overflow); | |
| } | |
| Label continue_after_compile; | |
| __ bind(continue_after_compile); | |
| bang_stack_shadow_pages(false); | |
| // reset the _do_not_unlock_if_synchronized flag | |
| __ strb(zr, do_not_unlock_if_synchronized); | |
| // check for synchronized methods | |
| // Must happen AFTER invocation_counter check and stack overflow check, | |
| // so method is not locked if overflows. | |
| if (synchronized) { | |
| // Allocate monitor and lock method | |
| lock_method(); | |
| } else { | |
| // no synchronization necessary | |
| #ifdef ASSERT | |
| { | |
| Label L; | |
| __ ldrh(r0, access_flags); | |
| __ tst(r0, JVM_ACC_SYNCHRONIZED); | |
| __ br(Assembler::EQ, L); | |
| __ stop("method needs synchronization"); | |
| __ bind(L); | |
| } | |
| #endif | |
| } | |
| // start execution | |
| #ifdef ASSERT | |
| { | |
| Label L; | |
| const Address monitor_block_top (rfp, | |
| frame::interpreter_frame_monitor_block_top_offset * wordSize); | |
| __ ldr(rscratch1, monitor_block_top); | |
| __ lea(rscratch1, Address(rfp, rscratch1, Address::lsl(Interpreter::logStackElementSize))); | |
| __ cmp(esp, rscratch1); | |
| __ br(Assembler::EQ, L); | |
| __ stop("broken stack frame setup in interpreter 2"); | |
| __ bind(L); | |
| } | |
| #endif | |
| // jvmti support | |
| __ notify_method_entry(); | |
| __ dispatch_next(vtos); | |
| // invocation counter overflow | |
| if (inc_counter) { | |
| // Handle overflow of counter and compile method | |
| __ bind(invocation_counter_overflow); | |
| generate_counter_overflow(continue_after_compile); | |
| } | |
| return entry_point; | |
| } | |
| // Method entry for java.lang.Thread.currentThread | |
| address TemplateInterpreterGenerator::generate_currentThread() { | |
| address entry_point = __ pc(); | |
| __ ldr(r0, Address(rthread, JavaThread::vthread_offset())); | |
| __ resolve_oop_handle(r0, rscratch1, rscratch2); | |
| __ ret(lr); | |
| return entry_point; | |
| } | |
| //----------------------------------------------------------------------------- | |
| // Exceptions | |
| void TemplateInterpreterGenerator::generate_throw_exception() { | |
| // Entry point in previous activation (i.e., if the caller was | |
| // interpreted) | |
| Interpreter::_rethrow_exception_entry = __ pc(); | |
| // Restore sp to interpreter_frame_last_sp even though we are going | |
| // to empty the expression stack for the exception processing. | |
| __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize)); | |
| // r0: exception | |
| // r3: return address/pc that threw exception | |
| __ restore_bcp(); // rbcp points to call/send | |
| __ restore_locals(); | |
| __ restore_constant_pool_cache(); | |
| __ reinit_heapbase(); // restore rheapbase as heapbase. | |
| __ get_dispatch(); | |
| // Entry point for exceptions thrown within interpreter code | |
| Interpreter::_throw_exception_entry = __ pc(); | |
| // If we came here via a NullPointerException on the receiver of a | |
| // method, rmethod may be corrupt. | |
| __ get_method(rmethod); | |
| // expression stack is undefined here | |
| // r0: exception | |
| // rbcp: exception bcp | |
| __ verify_oop(r0); | |
| __ mov(c_rarg1, r0); | |
| // expression stack must be empty before entering the VM in case of | |
| // an exception | |
| __ empty_expression_stack(); | |
| // find exception handler address and preserve exception oop | |
| __ call_VM(r3, | |
| CAST_FROM_FN_PTR(address, | |
| InterpreterRuntime::exception_handler_for_exception), | |
| c_rarg1); | |
| // Restore machine SP | |
| __ restore_sp_after_call(); | |
| // r0: exception handler entry point | |
| // r3: preserved exception oop | |
| // rbcp: bcp for exception handler | |
| __ push_ptr(r3); // push exception which is now the only value on the stack | |
| __ br(r0); // jump to exception handler (may be _remove_activation_entry!) | |
| // If the exception is not handled in the current frame the frame is | |
| // removed and the exception is rethrown (i.e. exception | |
| // continuation is _rethrow_exception). | |
| // | |
| // Note: At this point the bci is still the bxi for the instruction | |
| // which caused the exception and the expression stack is | |
| // empty. Thus, for any VM calls at this point, GC will find a legal | |
| // oop map (with empty expression stack). | |
| // | |
| // JVMTI PopFrame support | |
| // | |
| Interpreter::_remove_activation_preserving_args_entry = __ pc(); | |
| __ empty_expression_stack(); | |
| __ restore_bcp(); // We could have returned from deoptimizing this frame, so restore rbcp. | |
| // Set the popframe_processing bit in pending_popframe_condition | |
| // indicating that we are currently handling popframe, so that | |
| // call_VMs that may happen later do not trigger new popframe | |
| // handling cycles. | |
| __ ldrw(r3, Address(rthread, JavaThread::popframe_condition_offset())); | |
| __ orr(r3, r3, JavaThread::popframe_processing_bit); | |
| __ strw(r3, Address(rthread, JavaThread::popframe_condition_offset())); | |
| { | |
| // Check to see whether we are returning to a deoptimized frame. | |
| // (The PopFrame call ensures that the caller of the popped frame is | |
| // either interpreted or compiled and deoptimizes it if compiled.) | |
| // In this case, we can't call dispatch_next() after the frame is | |
| // popped, but instead must save the incoming arguments and restore | |
| // them after deoptimization has occurred. | |
| // | |
| // Note that we don't compare the return PC against the | |
| // deoptimization blob's unpack entry because of the presence of | |
| // adapter frames in C2. | |
| Label caller_not_deoptimized; | |
| __ ldr(c_rarg1, Address(rfp, frame::return_addr_offset * wordSize)); | |
| // This is a return address, so requires authenticating for PAC. | |
| __ authenticate_return_address(c_rarg1); | |
| __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, | |
| InterpreterRuntime::interpreter_contains), c_rarg1); | |
| __ cbnz(r0, caller_not_deoptimized); | |
| // Compute size of arguments for saving when returning to | |
| // deoptimized caller | |
| __ get_method(r0); | |
| __ ldr(r0, Address(r0, Method::const_offset())); | |
| __ load_unsigned_short(r0, Address(r0, in_bytes(ConstMethod:: | |
| size_of_parameters_offset()))); | |
| __ lsl(r0, r0, Interpreter::logStackElementSize); | |
| __ restore_locals(); // XXX do we need this? | |
| __ sub(rlocals, rlocals, r0); | |
| __ add(rlocals, rlocals, wordSize); | |
| // Save these arguments | |
| __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, | |
| Deoptimization:: | |
| popframe_preserve_args), | |
| rthread, r0, rlocals); | |
| __ remove_activation(vtos, | |
| /* throw_monitor_exception */ false, | |
| /* install_monitor_exception */ false, | |
| /* notify_jvmdi */ false); | |
| // Inform deoptimization that it is responsible for restoring | |
| // these arguments | |
| __ mov(rscratch1, JavaThread::popframe_force_deopt_reexecution_bit); | |
| __ strw(rscratch1, Address(rthread, JavaThread::popframe_condition_offset())); | |
| // Continue in deoptimization handler | |
| __ ret(lr); | |
| __ bind(caller_not_deoptimized); | |
| } | |
| __ remove_activation(vtos, | |
| /* throw_monitor_exception */ false, | |
| /* install_monitor_exception */ false, | |
| /* notify_jvmdi */ false); | |
| // Restore the last_sp and null it out | |
| __ ldr(rscratch1, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize)); | |
| __ lea(esp, Address(rfp, rscratch1, Address::lsl(Interpreter::logStackElementSize))); | |
| __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize)); | |
| __ restore_bcp(); | |
| __ restore_locals(); | |
| __ restore_constant_pool_cache(); | |
| __ get_method(rmethod); | |
| __ get_dispatch(); | |
| // The method data pointer was incremented already during | |
| // call profiling. We have to restore the mdp for the current bcp. | |
| if (ProfileInterpreter) { | |
| __ set_method_data_pointer_for_bcp(); | |
| } | |
| // Clear the popframe condition flag | |
| __ strw(zr, Address(rthread, JavaThread::popframe_condition_offset())); | |
| assert(JavaThread::popframe_inactive == 0, "fix popframe_inactive"); | |
| #if INCLUDE_JVMTI | |
| { | |
| Label L_done; | |
| __ ldrb(rscratch1, Address(rbcp, 0)); | |
| __ cmpw(rscratch1, Bytecodes::_invokestatic); | |
| __ br(Assembler::NE, L_done); | |
| // The member name argument must be restored if _invokestatic is re-executed after a PopFrame call. | |
| // Detect such a case in the InterpreterRuntime function and return the member name argument, or null. | |
| __ ldr(c_rarg0, Address(rlocals, 0)); | |
| __ call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::member_name_arg_or_null), c_rarg0, rmethod, rbcp); | |
| __ cbz(r0, L_done); | |
| __ str(r0, Address(esp, 0)); | |
| __ bind(L_done); | |
| } | |
| #endif // INCLUDE_JVMTI | |
| // Restore machine SP | |
| __ restore_sp_after_call(); | |
| __ dispatch_next(vtos); | |
| // end of PopFrame support | |
| Interpreter::_remove_activation_entry = __ pc(); | |
| // preserve exception over this code sequence | |
| __ pop_ptr(r0); | |
| __ str(r0, Address(rthread, JavaThread::vm_result_oop_offset())); | |
| // remove the activation (without doing throws on illegalMonitorExceptions) | |
| __ remove_activation(vtos, false, true, false); | |
| // restore exception | |
| __ get_vm_result_oop(r0, rthread); | |
| // In between activations - previous activation type unknown yet | |
| // compute continuation point - the continuation point expects the | |
| // following registers set up: | |
| // | |
| // r0: exception | |
| // lr: return address/pc that threw exception | |
| // esp: expression stack of caller | |
| // rfp: fp of caller | |
| __ stp(r0, lr, Address(__ pre(sp, -2 * wordSize))); // save exception & return address | |
| __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, | |
| SharedRuntime::exception_handler_for_return_address), | |
| rthread, lr); | |
| __ mov(r1, r0); // save exception handler | |
| __ ldp(r0, lr, Address(__ post(sp, 2 * wordSize))); // restore exception & return address | |
| // We might be returning to a deopt handler that expects r3 to | |
| // contain the exception pc | |
| __ mov(r3, lr); | |
| // Note that an "issuing PC" is actually the next PC after the call | |
| __ br(r1); // jump to exception | |
| // handler of caller | |
| } | |
| // | |
| // JVMTI ForceEarlyReturn support | |
| // | |
| address TemplateInterpreterGenerator::generate_earlyret_entry_for(TosState state) { | |
| address entry = __ pc(); | |
| __ restore_bcp(); | |
| __ restore_locals(); | |
| __ empty_expression_stack(); | |
| __ load_earlyret_value(state); | |
| __ ldr(rscratch1, Address(rthread, JavaThread::jvmti_thread_state_offset())); | |
| Address cond_addr(rscratch1, JvmtiThreadState::earlyret_state_offset()); | |
| // Clear the earlyret state | |
| assert(JvmtiThreadState::earlyret_inactive == 0, "should be"); | |
| __ str(zr, cond_addr); | |
| __ remove_activation(state, | |
| false, /* throw_monitor_exception */ | |
| false, /* install_monitor_exception */ | |
| true); /* notify_jvmdi */ | |
| __ ret(lr); | |
| return entry; | |
| } // end of ForceEarlyReturn support | |
| //----------------------------------------------------------------------------- | |
| // Helper for vtos entry point generation | |
| void TemplateInterpreterGenerator::set_vtos_entry_points(Template* t, | |
| address& bep, | |
| address& cep, | |
| address& sep, | |
| address& aep, | |
| address& iep, | |
| address& lep, | |
| address& fep, | |
| address& dep, | |
| address& vep) { | |
| assert(t->is_valid() && t->tos_in() == vtos, "illegal template"); | |
| Label L; | |
| aep = __ pc(); // atos entry point | |
| __ push_ptr(); | |
| __ b(L); | |
| fep = __ pc(); // ftos entry point | |
| __ push_f(); | |
| __ b(L); | |
| dep = __ pc(); // dtos entry point | |
| __ push_d(); | |
| __ b(L); | |
| lep = __ pc(); // ltos entry point | |
| __ push_l(); | |
| __ b(L); | |
| bep = cep = sep = iep = __ pc(); // [bcsi]tos entry point | |
| __ push_i(); | |
| vep = __ pc(); // vtos entry point | |
| __ bind(L); | |
| generate_and_dispatch(t); | |
| } | |
| //----------------------------------------------------------------------------- | |
| // Non-product code | |
| #ifndef PRODUCT | |
| address TemplateInterpreterGenerator::generate_trace_code(TosState state) { | |
| address entry = __ pc(); | |
| __ protect_return_address(); | |
| __ push(lr); | |
| __ push(state); | |
| __ push(RegSet::range(r0, r15), sp); | |
| __ mov(c_rarg2, r0); // Pass itos | |
| __ call_VM(noreg, | |
| CAST_FROM_FN_PTR(address, InterpreterRuntime::trace_bytecode), | |
| c_rarg1, c_rarg2, c_rarg3); | |
| __ pop(RegSet::range(r0, r15), sp); | |
| __ pop(state); | |
| __ pop(lr); | |
| __ authenticate_return_address(); | |
| __ ret(lr); // return from result handler | |
| return entry; | |
| } | |
| void TemplateInterpreterGenerator::count_bytecode() { | |
| __ mov(r10, (address) &BytecodeCounter::_counter_value); | |
| __ atomic_add(noreg, 1, r10); | |
| } | |
| void TemplateInterpreterGenerator::histogram_bytecode(Template* t) { | |
| __ mov(r10, (address) &BytecodeHistogram::_counters[t->bytecode()]); | |
| __ atomic_addw(noreg, 1, r10); | |
| } | |
| void TemplateInterpreterGenerator::histogram_bytecode_pair(Template* t) { | |
| // Calculate new index for counter: | |
| // _index = (_index >> log2_number_of_codes) | | |
| // (bytecode << log2_number_of_codes); | |
| Register index_addr = rscratch1; | |
| Register index = rscratch2; | |
| __ mov(index_addr, (address) &BytecodePairHistogram::_index); | |
| __ ldrw(index, index_addr); | |
| __ mov(r10, | |
| ((int)t->bytecode()) << BytecodePairHistogram::log2_number_of_codes); | |
| __ orrw(index, r10, index, Assembler::LSR, | |
| BytecodePairHistogram::log2_number_of_codes); | |
| __ strw(index, index_addr); | |
| // Bump bucket contents: | |
| // _counters[_index] ++; | |
| Register counter_addr = rscratch1; | |
| __ mov(r10, (address) &BytecodePairHistogram::_counters); | |
| __ lea(counter_addr, Address(r10, index, Address::lsl(LogBytesPerInt))); | |
| __ atomic_addw(noreg, 1, counter_addr); | |
| } | |
| void TemplateInterpreterGenerator::trace_bytecode(Template* t) { | |
| // Call a little run-time stub to avoid blow-up for each bytecode. | |
| // The run-time runtime saves the right registers, depending on | |
| // the tosca in-state for the given template. | |
| assert(Interpreter::trace_code(t->tos_in()) != nullptr, | |
| "entry must have been generated"); | |
| __ bl(RuntimeAddress(Interpreter::trace_code(t->tos_in()))); | |
| __ reinit_heapbase(); | |
| } | |
| void TemplateInterpreterGenerator::stop_interpreter_at() { | |
| Label L; | |
| __ push(rscratch1); | |
| __ mov(rscratch1, (address) &BytecodeCounter::_counter_value); | |
| __ ldr(rscratch1, Address(rscratch1)); | |
| __ mov(rscratch2, StopInterpreterAt); | |
| __ cmp(rscratch1, rscratch2); | |
| __ br(Assembler::NE, L); | |
| __ brk(0); | |
| __ bind(L); | |
| __ pop(rscratch1); | |
| } | |
| #endif // !PRODUCT |