blob: cc07e0e4076157fabd951a411219f9e819a4439e [file]
//
// Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2020, 2025, Arm Limited. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
// under the terms of the GNU General Public License version 2 only, as
// published by the Free Software Foundation.
//
// This code is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// version 2 for more details (a copy is included in the LICENSE file that
// accompanied this code).
//
// You should have received a copy of the GNU General Public License version
// 2 along with this work; if not, write to the Free Software Foundation,
// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
//
// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
// or visit www.oracle.com if you need additional information or have any
// questions.
//
//
dnl Generate the warning
// This file is automatically generated by running "m4 aarch64_vector_ad.m4". Do not edit!
dnl
// AArch64 VECTOR Architecture Description File
dnl
dnl OPERAND_VMEMORYA_IMMEDIATE_OFFSET($1, $2 )
dnl OPERAND_VMEMORYA_IMMEDIATE_OFFSET(imm_type_abbr, imm_type)
define(`OPERAND_VMEMORYA_IMMEDIATE_OFFSET', `
operand vmemA_imm$1Offset4() %{
// (esize / msize) = 1
predicate(Address::offset_ok_for_sve_immed(n->get_$2(), 4,
Matcher::scalable_vector_reg_size(T_BYTE)));
match(Con$1);
op_cost(0);
format %{ %}
interface(CONST_INTER);
%}')dnl
dnl
dnl OPERAND_VMEMORYA_INDIRECT_OFFSET($1 )
dnl OPERAND_VMEMORYA_INDIRECT_OFFSET(imm_type_abbr)
define(`OPERAND_VMEMORYA_INDIRECT_OFFSET', `
operand vmemA_indOff$1`'4(iRegP reg, vmemA_imm$1Offset4 off) %{
constraint(ALLOC_IN_RC(ptr_reg));
match(AddP reg off);
op_cost(0);
format %{ "[$reg, $off]" %}
interface(MEMORY_INTER) %{
base($reg);
`index'(0xffffffff);
scale(0x0);
disp($off);
%}
%}')dnl
dnl
// 4 bit signed offset -- for predicated load/store
OPERAND_VMEMORYA_IMMEDIATE_OFFSET(I, int)
OPERAND_VMEMORYA_IMMEDIATE_OFFSET(L, long)
OPERAND_VMEMORYA_INDIRECT_OFFSET(I)
OPERAND_VMEMORYA_INDIRECT_OFFSET(L)
// The indOff of vmemA is valid only when the vector element (load to/store from)
// size equals to memory element (load from/store to) size.
opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);
source_hpp %{
// Assert that the given node is not a variable shift.
bool assert_not_var_shift(const Node* n);
Assembler::SIMD_Arrangement get_arrangement(const Node* n);
Assembler::SIMD_RegVariant get_reg_variant(const Node* n);
%}
source %{
typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
PRegister Pg, const Address &adr);
// Predicated load/store, with optional ptrue to all elements of given predicate register.
static void loadStoreA_predicated(C2_MacroAssembler* masm, bool is_store, FloatRegister reg,
PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
int opcode, Register base, int index, int size, int disp) {
sve_mem_insn_predicate insn;
int mesize = type2aelembytes(mem_elem_bt);
if (index == -1) {
assert(size == 0, "unsupported address mode: scale size = %d", size);
switch(mesize) {
case 1:
insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
break;
case 2:
insn = is_store ? &C2_MacroAssembler::sve_st1h : &C2_MacroAssembler::sve_ld1h;
break;
case 4:
insn = is_store ? &C2_MacroAssembler::sve_st1w : &C2_MacroAssembler::sve_ld1w;
break;
case 8:
insn = is_store ? &C2_MacroAssembler::sve_st1d : &C2_MacroAssembler::sve_ld1d;
break;
default:
assert(false, "unsupported");
ShouldNotReachHere();
}
int imm4 = disp / mesize / Matcher::scalable_vector_reg_size(vector_elem_bt);
(masm->*insn)(reg, Assembler::elemType_to_regVariant(vector_elem_bt), pg, Address(base, imm4));
} else {
assert(false, "unimplemented");
ShouldNotReachHere();
}
}
bool Matcher::match_rule_supported_auto_vectorization(int opcode, int vlen, BasicType bt) {
if (UseSVE == 0) {
// These operations are not profitable to be vectorized on NEON, because no direct
// NEON instructions support them. But the match rule support for them is profitable for
// Vector API intrinsics.
if ((opcode == Op_VectorCastD2X && bt == T_INT) ||
(opcode == Op_VectorCastL2X && bt == T_FLOAT) ||
(opcode == Op_CountLeadingZerosV && bt == T_LONG) ||
(opcode == Op_CountTrailingZerosV && bt == T_LONG) ||
// The implementations of Op_AddReductionVD/F in Neon are for the Vector API only.
// They are not suitable for auto-vectorization because the result would not conform
// to the JLS, Section Evaluation Order.
opcode == Op_AddReductionVD || opcode == Op_AddReductionVF ||
opcode == Op_MulReductionVD || opcode == Op_MulReductionVF ||
opcode == Op_MulVL) {
return false;
}
}
return match_rule_supported_vector(opcode, vlen, bt);
}
// Identify extra cases that we might want to provide match rules for vector nodes and
// other intrinsics guarded with vector length (vlen) and element type (bt).
bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
if (!match_rule_supported(opcode)) {
return false;
}
int length_in_bytes = vlen * type2aelembytes(bt);
if (UseSVE == 0 && length_in_bytes > FloatRegister::neon_vl) {
return false;
}
// Check whether specific Op is supported.
// Fail fast, otherwise fall through to common vector_size_supported() check.
switch (opcode) {
case Op_AndVMask:
case Op_OrVMask:
case Op_XorVMask:
case Op_MaskAll:
case Op_VectorMaskGen:
case Op_LoadVectorMasked:
case Op_StoreVectorMasked:
case Op_StoreVectorScatter:
case Op_StoreVectorScatterMasked:
case Op_PopulateIndex:
case Op_CompressM:
case Op_CompressV:
if (UseSVE == 0) {
return false;
}
break;
case Op_LoadVectorGather:
case Op_LoadVectorGatherMasked:
if (UseSVE == 0 || is_subword_type(bt)) {
return false;
}
break;
case Op_MulAddVS2VI:
if (length_in_bytes != 16) {
return false;
}
break;
case Op_MulReductionVD:
case Op_MulReductionVF:
case Op_MulReductionVI:
case Op_MulReductionVL:
// No vector multiply reduction instructions, but we do
// emit scalar instructions for 64/128-bit vectors.
if (length_in_bytes != 8 && length_in_bytes != 16) {
return false;
}
break;
case Op_VectorMaskCmp:
if (length_in_bytes < 8) {
return false;
}
break;
case Op_ExpandV:
if (UseSVE < 2 || is_subword_type(bt)) {
return false;
}
break;
case Op_VectorMaskToLong:
if (UseSVE > 0 && vlen > 64) {
return false;
}
break;
case Op_VectorLongToMask:
if (vlen > 64 || !VM_Version::supports_svebitperm()) {
return false;
}
break;
case Op_CompressBitsV:
case Op_ExpandBitsV:
if (!VM_Version::supports_svebitperm()) {
return false;
}
break;
case Op_AddVHF:
case Op_SubVHF:
case Op_MulVHF:
case Op_DivVHF:
case Op_MinVHF:
case Op_MaxVHF:
case Op_SqrtVHF:
// FEAT_FP16 is enabled if both "fphp" and "asimdhp" features are supported.
// Only the Neon instructions need this check. SVE supports half-precision floats
// by default.
if (UseSVE == 0 && !is_feat_fp16_supported()) {
return false;
}
break;
case Op_FmaVHF:
// UseFMA flag needs to be checked along with FEAT_FP16
if (!UseFMA || (UseSVE == 0 && !is_feat_fp16_supported())) {
return false;
}
break;
default:
break;
}
return vector_size_supported(bt, vlen);
}
bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
// Only SVE supports masked operations.
if (UseSVE == 0) {
return false;
}
// If an opcode does not support the masked version,
// unpredicated node with VectorBlend node will be used instead.
switch(opcode) {
case Op_VectorRearrange:
case Op_MulReductionVD:
case Op_MulReductionVF:
case Op_MulReductionVI:
case Op_MulReductionVL:
case Op_CompressBitsV:
case Op_ExpandBitsV:
return false;
case Op_SaturatingAddV:
case Op_SaturatingSubV:
// Only SVE2 supports the predicated saturating instructions.
if (UseSVE < 2) {
return false;
}
break;
// We use Op_LoadVectorMasked to implement the predicated Op_LoadVector.
// Hence we turn to check whether Op_LoadVectorMasked is supported. The
// same as vector store/gather/scatter.
case Op_LoadVector:
opcode = Op_LoadVectorMasked;
break;
case Op_StoreVector:
opcode = Op_StoreVectorMasked;
break;
case Op_LoadVectorGather:
opcode = Op_LoadVectorGatherMasked;
break;
case Op_StoreVectorScatter:
opcode = Op_StoreVectorScatterMasked;
break;
// Currently, the masked versions of the following 8 Float16 operations are disabled.
// When the support for Float16 vector classes is added in VectorAPI and the masked
// Float16 IR can be generated, these masked operations will be enabled and relevant
// backend support added.
case Op_AddVHF:
case Op_SubVHF:
case Op_MulVHF:
case Op_DivVHF:
case Op_MaxVHF:
case Op_MinVHF:
case Op_SqrtVHF:
case Op_FmaVHF:
return false;
default:
break;
}
return match_rule_supported_vector(opcode, vlen, bt);
}
bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* vt) {
// Only SVE has partial vector operations
if (UseSVE == 0) {
return false;
}
switch(node->Opcode()) {
case Op_VectorLoadMask:
case Op_VectorMaskCmp:
case Op_LoadVectorGather:
case Op_StoreVectorScatter:
case Op_AddReductionVF:
case Op_AddReductionVD:
case Op_AndReductionV:
case Op_OrReductionV:
case Op_XorReductionV:
// Mask is needed for partial Op_VectorMaskFirstTrue, because when the
// input predicate is all-false, the result should be the vector length
// instead of the vector register size.
case Op_VectorMaskFirstTrue:
return true;
case Op_MaskAll:
return !node->in(1)->is_Con();
case Op_LoadVector:
case Op_StoreVector:
// We use NEON load/store instructions if the vector length is <= 128 bits.
return vt->length_in_bytes() > 16;
case Op_AddReductionVI:
case Op_AddReductionVL:
// We may prefer using NEON instructions rather than SVE partial operations.
return !VM_Version::use_neon_for_vector(vt->length_in_bytes());
case Op_MinReductionV:
case Op_MaxReductionV:
// For BYTE/SHORT/INT/FLOAT/DOUBLE types, we may prefer using NEON
// instructions rather than SVE partial operations.
return vt->element_basic_type() == T_LONG ||
!VM_Version::use_neon_for_vector(vt->length_in_bytes());
default:
// For other ops whose vector size is smaller than the max vector size, a
// full-sized unpredicated operation does not impact the final vector result.
return false;
}
}
bool Matcher::vector_rearrange_requires_load_shuffle(BasicType elem_bt, int vlen) {
return false;
}
// Assert that the given node is not a variable shift.
bool assert_not_var_shift(const Node* n) {
assert(!n->as_ShiftV()->is_var_shift(), "illegal variable shift");
return true;
}
Assembler::SIMD_Arrangement get_arrangement(const Node* n) {
BasicType bt = Matcher::vector_element_basic_type(n);
uint length_in_bytes = Matcher::vector_length_in_bytes(n);
return Assembler::esize2arrangement((uint)type2aelembytes(bt),
/* isQ */ length_in_bytes == 16);
}
Assembler::SIMD_RegVariant get_reg_variant(const Node* n) {
BasicType bt = Matcher::vector_element_basic_type(n);
return Assembler::elemType_to_regVariant(bt);
}
%}
// All VECTOR instructions
// ------------------------------ Vector load/store ----------------------------
dnl
dnl VECTOR_LOAD_STORE($1, $2, $3, $4, $5 )
dnl VECTOR_LOAD_STORE(type, nbytes, arg_name, nbits, size)
define(`VECTOR_LOAD_STORE', `
// ifelse(load, $1, Load, Store) Vector ($4 bits)
instruct $1V$2(vReg $3, vmem$2 mem) %{
predicate(`n->as_'ifelse(load, $1, Load, Store)Vector()->memory_size() == $2);
match(Set ifelse(load, $1, dst (LoadVector mem), mem (StoreVector mem src)));
format %{ "$1V$2 ifelse(load, $1, `$dst, $mem', `$mem, $src')\t# vector ($4 bits)" %}
ins_encode( `aarch64_enc_'ifelse(load, $1, ldr, str)v$5($3, mem) );
ins_pipe(pipe_slow);
%}')dnl
dnl
VECTOR_LOAD_STORE(load, 2, dst, 16, H)
VECTOR_LOAD_STORE(store, 2, src, 16, H)
VECTOR_LOAD_STORE(load, 4, dst, 32, S)
VECTOR_LOAD_STORE(store, 4, src, 32, S)
VECTOR_LOAD_STORE(load, 8, dst, 64, D)
VECTOR_LOAD_STORE(store, 8, src, 64, D)
VECTOR_LOAD_STORE(load, 16, dst, 128, Q)
VECTOR_LOAD_STORE(store, 16, src, 128, Q)
// Load Vector (> 128 bits)
instruct loadV(vReg dst, vmemA mem) %{
predicate(n->as_LoadVector()->memory_size() > 16);
match(Set dst (LoadVector mem));
format %{ "loadV $dst, $mem\t# vector (sve)" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
loadStoreA_predicated(masm, /* is_store */ false,
$dst$$FloatRegister, ptrue, bt, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
ins_pipe(pipe_slow);
%}
// Store Vector (> 128 bits)
instruct storeV(vReg src, vmemA mem) %{
predicate(n->as_StoreVector()->memory_size() > 16);
match(Set mem (StoreVector mem src));
format %{ "storeV $mem, $src\t# vector (sve)" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this, $src);
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
loadStoreA_predicated(masm, /* is_store */ true,
$src$$FloatRegister, ptrue, bt, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
ins_pipe(pipe_slow);
%}
// vector load/store - predicated
instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst (LoadVectorMasked mem pg));
format %{ "loadV_masked $dst, $pg, $mem" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
loadStoreA_predicated(masm, /* is_store */ false, $dst$$FloatRegister,
$pg$$PRegister, bt, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
ins_pipe(pipe_slow);
%}
instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set mem (StoreVectorMasked mem (Binary src pg)));
format %{ "storeV_masked $mem, $pg, $src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
loadStoreA_predicated(masm, /* is_store */ true, $src$$FloatRegister,
$pg$$PRegister, bt, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
ins_pipe(pipe_slow);
%}
// vector load const
instruct vloadcon(vReg dst, immI0 src) %{
match(Set dst (VectorLoadConst src));
format %{ "vloadcon $dst, $src\t# load/generate iota indices" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
if (UseSVE == 0) {
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes <= 16, "must be");
// The iota indices are ordered by type B/S/I/L/F/D, and the offset between two types is 16.
int offset = exact_log2(type2aelembytes(bt)) << 4;
if (is_floating_point_type(bt)) {
offset += 32;
}
__ lea(rscratch1, ExternalAddress(StubRoutines::aarch64::vector_iota_indices() + offset));
if (length_in_bytes == 16) {
__ ldrq($dst$$FloatRegister, rscratch1);
} else {
__ ldrd($dst$$FloatRegister, rscratch1);
}
} else {
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_index($dst$$FloatRegister, size, 0, 1);
if (is_floating_point_type(bt)) {
__ sve_scvtf($dst$$FloatRegister, size, ptrue, $dst$$FloatRegister, size);
}
}
%}
ins_pipe(pipe_slow);
%}
dnl
dnl BINARY_OP($1, $2, $3, $4, $5 )
dnl BINARY_OP(rule_name, op_name, insn_neon, insn_sve, size)
define(`BINARY_OP', `
instruct $1(vReg dst, vReg src1, vReg src2) %{
match(Set dst ($2 src1 src2));
format %{ "$1 $dst, $src1, $src2" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ $3($dst$$FloatRegister, get_arrangement(this),
$src1$$FloatRegister, $src2$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ $4($dst$$FloatRegister, __ $5, $src1$$FloatRegister, $src2$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl BINARY_OP_PREDICATE($1, $2, $3, $4 )
dnl BINARY_OP_PREDICATE(rule_name, op_name, insn, size)
define(`BINARY_OP_PREDICATE', `
instruct $1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
format %{ "$1_masked $dst_src1, $pg, $dst_src1, $src2" %}
ins_encode %{
__ $3($dst_src1$$FloatRegister, __ $4, $pg$$PRegister, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VADD_IMM($1, $2, $3 )
dnl VADD_IMM(type, imm_type, size)
define(`VADD_IMM', `
instruct vaddImm$1(vReg dst_src, $2 con) %{
predicate(UseSVE > 0);
match(Set dst_src (AddV$1 dst_src (Replicate con)));
format %{ "vaddImm$1 $dst_src, $dst_src, $con" %}
ins_encode %{
int val = (int)$con$$constant;
if (val > 0) {
__ sve_add($dst_src$$FloatRegister, __ $3, val);
} else {
__ sve_sub($dst_src$$FloatRegister, __ $3, -val);
}
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// ------------------------------ Vector add -----------------------------------
// vector add
BINARY_OP(vaddB, AddVB, addv, sve_add, B)
BINARY_OP(vaddS, AddVS, addv, sve_add, H)
BINARY_OP(vaddI, AddVI, addv, sve_add, S)
BINARY_OP(vaddL, AddVL, addv, sve_add, D)
BINARY_OP(vaddHF, AddVHF, fadd, sve_fadd, H)
BINARY_OP(vaddF, AddVF, fadd, sve_fadd, S)
BINARY_OP(vaddD, AddVD, fadd, sve_fadd, D)
// vector add - predicated
BINARY_OP_PREDICATE(vaddB, AddVB, sve_add, B)
BINARY_OP_PREDICATE(vaddS, AddVS, sve_add, H)
BINARY_OP_PREDICATE(vaddI, AddVI, sve_add, S)
BINARY_OP_PREDICATE(vaddL, AddVL, sve_add, D)
BINARY_OP_PREDICATE(vaddF, AddVF, sve_fadd, S)
BINARY_OP_PREDICATE(vaddD, AddVD, sve_fadd, D)
// vector add reg imm (unpredicated)
VADD_IMM(B, immBAddSubV, B)
VADD_IMM(S, immIAddSubV, H)
VADD_IMM(I, immIAddSubV, S)
VADD_IMM(L, immLAddSubV, D)
// ------------------------------ Vector sub -----------------------------------
// vector sub
BINARY_OP(vsubB, SubVB, subv, sve_sub, B)
BINARY_OP(vsubS, SubVS, subv, sve_sub, H)
BINARY_OP(vsubI, SubVI, subv, sve_sub, S)
BINARY_OP(vsubL, SubVL, subv, sve_sub, D)
BINARY_OP(vsubHF, SubVHF, fsub, sve_fsub, H)
BINARY_OP(vsubF, SubVF, fsub, sve_fsub, S)
BINARY_OP(vsubD, SubVD, fsub, sve_fsub, D)
// vector sub - predicated
BINARY_OP_PREDICATE(vsubB, SubVB, sve_sub, B)
BINARY_OP_PREDICATE(vsubS, SubVS, sve_sub, H)
BINARY_OP_PREDICATE(vsubI, SubVI, sve_sub, S)
BINARY_OP_PREDICATE(vsubL, SubVL, sve_sub, D)
BINARY_OP_PREDICATE(vsubF, SubVF, sve_fsub, S)
BINARY_OP_PREDICATE(vsubD, SubVD, sve_fsub, D)
dnl
dnl BINARY_OP_NEON_SVE_PAIRWISE($1, $2, $3, $4, $5 )
dnl BINARY_OP_NEON_SVE_PAIRWISE(rule_name, op_name, insn_neon, insn_sve, size)
define(`BINARY_OP_NEON_SVE_PAIRWISE', `
instruct $1_neon(vReg dst, vReg src1, vReg src2) %{
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
match(Set dst ($2 src1 src2));
format %{ "$1_neon $dst, $src1, $src2" %}
ins_encode %{
__ $3($dst$$FloatRegister, get_arrangement(this),
$src1$$FloatRegister, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct $1_sve(vReg dst_src1, vReg src2) %{
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
match(Set dst_src1 ($2 dst_src1 src2));
format %{ "$1_sve $dst_src1, $dst_src1, $src2" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
__ $4($dst_src1$$FloatRegister, __ $5, ptrue, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// ------------------------------ Vector mul -----------------------------------
// vector mul - BYTE, CHAR, SHORT, INT
BINARY_OP_NEON_SVE_PAIRWISE(vmulB, MulVB, mulv, sve_mul, B)
BINARY_OP_NEON_SVE_PAIRWISE(vmulS, MulVS, mulv, sve_mul, H)
BINARY_OP_NEON_SVE_PAIRWISE(vmulI, MulVI, mulv, sve_mul, S)
// vector mul - LONG
instruct vmulL_neon(vReg dst, vReg src1, vReg src2) %{
predicate(UseSVE == 0);
match(Set dst (MulVL src1 src2));
format %{ "vmulL_neon $dst, $src1, $src2\t# 2L" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == 16, "must be");
__ umov(rscratch1, $src1$$FloatRegister, __ D, 0);
__ umov(rscratch2, $src2$$FloatRegister, __ D, 0);
__ mul(rscratch2, rscratch2, rscratch1);
__ mov($dst$$FloatRegister, __ D, 0, rscratch2);
__ umov(rscratch1, $src1$$FloatRegister, __ D, 1);
__ umov(rscratch2, $src2$$FloatRegister, __ D, 1);
__ mul(rscratch2, rscratch2, rscratch1);
__ mov($dst$$FloatRegister, __ D, 1, rscratch2);
%}
ins_pipe(pipe_slow);
%}
instruct vmulL_sve(vReg dst_src1, vReg src2) %{
predicate(UseSVE > 0);
match(Set dst_src1 (MulVL dst_src1 src2));
format %{ "vmulL_sve $dst_src1, $dst_src1, $src2" %}
ins_encode %{
__ sve_mul($dst_src1$$FloatRegister, __ D, ptrue, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// vector mul - floating-point
BINARY_OP(vmulHF, MulVHF, fmul, sve_fmul, H)
BINARY_OP(vmulF, MulVF, fmul, sve_fmul, S)
BINARY_OP(vmulD, MulVD, fmul, sve_fmul, D)
// vector mul - predicated
BINARY_OP_PREDICATE(vmulB, MulVB, sve_mul, B)
BINARY_OP_PREDICATE(vmulS, MulVS, sve_mul, H)
BINARY_OP_PREDICATE(vmulI, MulVI, sve_mul, S)
BINARY_OP_PREDICATE(vmulL, MulVL, sve_mul, D)
BINARY_OP_PREDICATE(vmulF, MulVF, sve_fmul, S)
BINARY_OP_PREDICATE(vmulD, MulVD, sve_fmul, D)
// ------------------------------ Vector float div -----------------------------
// vector float div
BINARY_OP_NEON_SVE_PAIRWISE(vdivHF, DivVHF, fdiv, sve_fdiv, H)
BINARY_OP_NEON_SVE_PAIRWISE(vdivF, DivVF, fdiv, sve_fdiv, S)
BINARY_OP_NEON_SVE_PAIRWISE(vdivD, DivVD, fdiv, sve_fdiv, D)
// vector float div - predicated
BINARY_OP_PREDICATE(vdivF, DivVF, sve_fdiv, S)
BINARY_OP_PREDICATE(vdivD, DivVD, sve_fdiv, D)
dnl
dnl BITWISE_OP_IMM($1, $2, $3, $4, $5, $6 )
dnl BITWISE_OP_IMM(rule_name, type, op_name, insn, size, basic_type)
define(`BITWISE_OP_IMM', `
instruct $1(vReg dst_src, imm$2Log con) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == $6);
match(Set dst_src ($3 dst_src (Replicate con)));
format %{ "$1 $dst_src, $dst_src, $con" %}
ins_encode %{
__ $4($dst_src$$FloatRegister, __ $5, (uint64_t)($con$$constant));
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl BITWISE_OPS($1, $2, $3, $4 )
dnl BITWISE_OPS(rule_name, op_name, insn_neon, insn_sve)
define(`BITWISE_OPS',
`// vector $1
instruct v$1(vReg dst, vReg src1, vReg src2) %{
match(Set dst ($2 src1 src2));
format %{ "v$1 $dst, $src1, $src2" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ $3($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src1$$FloatRegister, $src2$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ $4($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}
// vector $1 - predicated
instruct v$1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
format %{ "v$1_masked $dst_src1, $pg, $dst_src1, $src2" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ $4($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// vector $1 reg imm (unpredicated)
BITWISE_OP_IMM(v$1ImmB, B, $2, $4, B, T_BYTE)
BITWISE_OP_IMM(v$1ImmS, S, $2, $4, H, T_SHORT)
BITWISE_OP_IMM(v$1ImmI, I, $2, $4, S, T_INT)
BITWISE_OP_IMM(v$1ImmL, L, $2, $4, D, T_LONG)')dnl
// ------------------------------ Vector and -----------------------------------
BITWISE_OPS(and, AndV, andr, sve_and)
// ------------------------------ Vector or ------------------------------------
BITWISE_OPS(or, OrV, orr, sve_orr)
// ------------------------------ Vector xor -----------------------------------
BITWISE_OPS(xor, XorV, eor, sve_eor)
// vector eor3 (unpredicated)
instruct veor3_neon(vReg dst, vReg src1, vReg src2, vReg src3) %{
predicate(VM_Version::supports_sha3() &&
VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
match(Set dst (XorV src1 (XorV src2 src3)));
format %{ "veor3_neon $dst, $src1, $src2, $src3" %}
ins_encode %{
__ eor3($dst$$FloatRegister, __ T16B, $src1$$FloatRegister,
$src2$$FloatRegister, $src3$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct veor3_sve(vReg dst_src1, vReg src2, vReg src3) %{
predicate(UseSVE == 2 && !VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
match(Set dst_src1 (XorV dst_src1 (XorV src2 src3)));
format %{ "veor3_sve $dst_src1, $dst_src1, $src2, $src3" %}
ins_encode %{
__ sve_eor3($dst_src1$$FloatRegister, $src2$$FloatRegister, $src3$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Vector not -----------------------------------
dnl
dnl VECTOR_NOT($1 )
dnl VECTOR_NOT(type)
define(`VECTOR_NOT', `
instruct vnot$1`'(vReg dst, vReg src, imm$1_M1 m1) %{
match(Set dst (XorV src (Replicate m1)));
format %{ "vnot$1 $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ notr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_not($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// vector not
VECTOR_NOT(I)
VECTOR_NOT(L)
dnl
dnl VECTOR_NOT_PREDICATE($1 )
dnl VECTOR_NOT_PREDICATE(type)
define(`VECTOR_NOT_PREDICATE', `
instruct vnot$1_masked`'(vReg dst_src, imm$1_M1 m1, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src (XorV (Binary dst_src (Replicate m1)) pg));
format %{ "vnot$1_masked $dst_src, $pg, $dst_src" %}
ins_encode %{
__ sve_not($dst_src$$FloatRegister, get_reg_variant(this),
$pg$$PRegister, $dst_src$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// vector not - predicated
VECTOR_NOT_PREDICATE(I)
VECTOR_NOT_PREDICATE(L)
dnl
// ------------------------------ Vector and_not -------------------------------
dnl
dnl VECTOR_AND_NOT($1 )
dnl VECTOR_AND_NOT(type)
define(`VECTOR_AND_NOT', `
instruct vand_not$1`'(vReg dst, vReg src1, vReg src2, imm$1_M1 m1) %{
match(Set dst (AndV src1 (XorV src2 (Replicate m1))));
format %{ "vand_not$1 $dst, $src1, $src2" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ bic($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src1$$FloatRegister, $src2$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_bic($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// vector and_not
VECTOR_AND_NOT(I)
VECTOR_AND_NOT(L)
dnl
dnl VECTOR_AND_NOT_PREDICATE($1 )
dnl VECTOR_AND_NOT_PREDICATE(type)
define(`VECTOR_AND_NOT_PREDICATE', `
instruct vand_not$1_masked`'(vReg dst_src1, vReg src2, imm$1_M1 m1, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (Replicate m1))) pg));
format %{ "vand_not$1_masked $dst_src1, $pg, $dst_src1, $src2" %}
ins_encode %{
__ sve_bic($dst_src1$$FloatRegister, get_reg_variant(this),
$pg$$PRegister, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// vector and_not - predicated
VECTOR_AND_NOT_PREDICATE(I)
VECTOR_AND_NOT_PREDICATE(L)
dnl
dnl VECTOR_SATURATING_OP($1, $2, $3 )
dnl VECTOR_SATURATING_OP(prefix, op, op_name)
define(`VECTOR_SATURATING_OP', `
instruct v$1$2(vReg dst, vReg src1, vReg src2) %{
predicate(ifelse($1, sq, `!',`')n->as_SaturatingVector()->is_unsigned());
match(Set dst ($3 src1 src2));
format %{ "v$1$2 $dst, $src1, $src2" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ $1$2v($dst$$FloatRegister, get_arrangement(this),
$src1$$FloatRegister, $src2$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_$1$2($dst$$FloatRegister, __ elemType_to_regVariant(bt),
$src1$$FloatRegister, $src2$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl
dnl VECTOR_SATURATING_PREDICATE($1, $2, $3 )
dnl VECTOR_SATURATING_PREDICATE(prefix, op, op_name)
define(`VECTOR_SATURATING_PREDICATE', `
instruct v$1$2_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
predicate(UseSVE == 2 && ifelse($1, sq, `!',`')n->as_SaturatingVector()->is_unsigned());
match(Set dst_src1 ($3 (Binary dst_src1 src2) pg));
format %{ "v$1$2_masked $dst_src1, $pg, $dst_src1, $src2" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_$1$2($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// ------------------------- Vector saturating add -----------------------------
// Signed saturating add
VECTOR_SATURATING_OP(sq, add, SaturatingAddV)
VECTOR_SATURATING_PREDICATE(sq, add, SaturatingAddV)
// Unsigned saturating add
VECTOR_SATURATING_OP(uq, add, SaturatingAddV)
VECTOR_SATURATING_PREDICATE(uq, add, SaturatingAddV)
// ------------------------- Vector saturating sub -----------------------------
// Signed saturating sub
VECTOR_SATURATING_OP(sq, sub, SaturatingSubV)
VECTOR_SATURATING_PREDICATE(sq, sub, SaturatingSubV)
// Unsigned saturating sub
VECTOR_SATURATING_OP(uq, sub, SaturatingSubV)
VECTOR_SATURATING_PREDICATE(uq, sub, SaturatingSubV)
dnl
dnl UNARY_OP($1, $2, $3, $4, $5 )
dnl UNARY_OP(rule_name, op_name, insn_neon, insn_sve, size)
define(`UNARY_OP', `
instruct $1(vReg dst, vReg src) %{
match(Set dst ($2 src));
format %{ "$1 $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ $3($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ $4($dst$$FloatRegister, __ $5, ptrue, $src$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl UNARY_OP_PREDICATE($1, $2, $3 )
dnl UNARY_OP_PREDICATE(rule_name, op_name, insn)
define(`UNARY_OP_PREDICATE', `
instruct $1_masked(vReg dst_src, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src ($2 dst_src pg));
format %{ "$1_masked $dst_src, $pg, $dst_src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ $3($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $dst_src$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl UNARY_OP_PREDICATE_WITH_SIZE($1, $2, $3, $4 )
dnl UNARY_OP_PREDICATE_WITH_SIZE(rule_name, op_name, insn, size)
define(`UNARY_OP_PREDICATE_WITH_SIZE', `
instruct $1_masked(vReg dst_src, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src ($2 dst_src pg));
format %{ "$1_masked $dst_src, $pg, $dst_src" %}
ins_encode %{
__ $3($dst_src$$FloatRegister, __ $4, $pg$$PRegister, $dst_src$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// ------------------------------ Vector abs -----------------------------------
// vector abs
UNARY_OP(vabsB, AbsVB, absr, sve_abs, B)
UNARY_OP(vabsS, AbsVS, absr, sve_abs, H)
UNARY_OP(vabsI, AbsVI, absr, sve_abs, S)
UNARY_OP(vabsL, AbsVL, absr, sve_abs, D)
UNARY_OP(vabsF, AbsVF, fabs, sve_fabs, S)
UNARY_OP(vabsD, AbsVD, fabs, sve_fabs, D)
// vector abs - predicated
UNARY_OP_PREDICATE_WITH_SIZE(vabsB, AbsVB, sve_abs, B)
UNARY_OP_PREDICATE_WITH_SIZE(vabsS, AbsVS, sve_abs, H)
UNARY_OP_PREDICATE_WITH_SIZE(vabsI, AbsVI, sve_abs, S)
UNARY_OP_PREDICATE_WITH_SIZE(vabsL, AbsVL, sve_abs, D)
UNARY_OP_PREDICATE_WITH_SIZE(vabsF, AbsVF, sve_fabs, S)
UNARY_OP_PREDICATE_WITH_SIZE(vabsD, AbsVD, sve_fabs, D)
// ------------------------------ Vector fabd ----------------------------------
// vector fabs diff
instruct vfabd_neon(vReg dst, vReg src1, vReg src2) %{
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
match(Set dst (AbsVF (SubVF src1 src2)));
match(Set dst (AbsVD (SubVD src1 src2)));
format %{ "vfabd_neon $dst, $src1, $src2" %}
ins_encode %{
__ fabd($dst$$FloatRegister, get_arrangement(this),
$src1$$FloatRegister, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct vfabd_sve(vReg dst_src1, vReg src2) %{
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
match(Set dst_src1 (AbsVF (SubVF dst_src1 src2)));
match(Set dst_src1 (AbsVD (SubVD dst_src1 src2)));
format %{ "vfabd_sve $dst_src1, $dst_src1, $src2" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_fabd($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
ptrue, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// vector fabs diff - predicated
instruct vfabd_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src1 (AbsVF (SubVF (Binary dst_src1 src2) pg) pg));
match(Set dst_src1 (AbsVD (SubVD (Binary dst_src1 src2) pg) pg));
format %{ "vfabd_masked $dst_src1, $pg, $dst_src1, $src2" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_fabd($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Vector neg -----------------------------------
// vector neg
instruct vnegI(vReg dst, vReg src) %{
match(Set dst (NegVI src));
format %{ "vnegI $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ negr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_neg($dst$$FloatRegister, __ elemType_to_regVariant(bt),
ptrue, $src$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}
UNARY_OP(vnegL, NegVL, negr, sve_neg, D)
UNARY_OP(vnegF, NegVF, fneg, sve_fneg, S)
UNARY_OP(vnegD, NegVD, fneg, sve_fneg, D)
// vector neg - predicated
UNARY_OP_PREDICATE(vnegI, NegVI, sve_neg)
UNARY_OP_PREDICATE_WITH_SIZE(vnegL, NegVL, sve_neg, D)
UNARY_OP_PREDICATE_WITH_SIZE(vnegF, NegVF, sve_fneg, S)
UNARY_OP_PREDICATE_WITH_SIZE(vnegD, NegVD, sve_fneg, D)
// ------------------------------ Vector sqrt ----------------------------------
// vector sqrt
UNARY_OP(vsqrtHF, SqrtVHF, fsqrt, sve_fsqrt, H)
UNARY_OP(vsqrtF, SqrtVF, fsqrt, sve_fsqrt, S)
UNARY_OP(vsqrtD, SqrtVD, fsqrt, sve_fsqrt, D)
// vector sqrt - predicated
UNARY_OP_PREDICATE_WITH_SIZE(vsqrtF, SqrtVF, sve_fsqrt, S)
UNARY_OP_PREDICATE_WITH_SIZE(vsqrtD, SqrtVD, sve_fsqrt, D)
dnl
dnl VMINMAX_L_NEON($1, $2 , $3 )
dnl VMINMAX_L_NEON(type, op_name, sign)
define(`VMINMAX_L_NEON', `
instruct v$3$1L_neon(vReg dst, vReg src1, vReg src2) %{
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst ($2 src1 src2));
effect(TEMP_DEF dst);
format %{ "v$3$1L_neon $dst, $src1, $src2\t# 2L" %}
ins_encode %{
__ cm(Assembler::ifelse($3, u, HI, GT), $dst$$FloatRegister, __ T2D, $src1$$FloatRegister, $src2$$FloatRegister);
__ bsl($dst$$FloatRegister, __ T16B, ifelse($1, min, $src2, $src1)$$FloatRegister, ifelse(min, $1, $src1, $src2)$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VMINMAX_L_SVE($1, $2, $3 )
dnl VMINMAX_L_SVE(type, op_name, insn)
define(`VMINMAX_L_SVE', `
instruct v$1L_sve(vReg dst_src1, vReg src2) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst_src1 ($2 dst_src1 src2));
format %{ "v$1L_sve $dst_src1, $dst_src1, $src2" %}
ins_encode %{
__ $3($dst_src1$$FloatRegister, __ D, ptrue, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VMINMAX_NEON($1, $2, $3, $4 )
dnl VMINMAX_NEON(type, op_name, insn_fp, insn_integral)
define(`VMINMAX_NEON', `
instruct v$1_neon(vReg dst, vReg src1, vReg src2) %{
predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
match(Set dst ($2 src1 src2));
format %{ "v$1_neon $dst, $src1, $src2\t# B/S/I/F/D" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
if (is_floating_point_type(bt)) {
__ $3($dst$$FloatRegister, get_arrangement(this),
$src1$$FloatRegister, $src2$$FloatRegister);
} else {
assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
__ $4($dst$$FloatRegister, get_arrangement(this),
$src1$$FloatRegister, $src2$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VMINMAX_HF_NEON($1, $2, $3 )
dnl VMINMAX_HF_NEON(type, op_name, insn_fp)
define(`VMINMAX_HF_NEON', `
instruct v$1_HF_neon(vReg dst, vReg src1, vReg src2) %{
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
match(Set dst ($2 src1 src2));
format %{ "v$1_HF_neon $dst, $src1, $src2\t# Half float" %}
ins_encode %{
__ $3($dst$$FloatRegister, get_arrangement(this),
$src1$$FloatRegister, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VMINMAX_SVE($1, $2, $3, $4 )
dnl VMINMAX_SVE(type, op_name, insn_fp, insn_integral)
define(`VMINMAX_SVE', `
instruct v$1_sve(vReg dst_src1, vReg src2) %{
predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
match(Set dst_src1 ($2 dst_src1 src2));
format %{ "v$1_sve $dst_src1, $dst_src1, $src2\t# B/S/I/F/D" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this);
if (is_floating_point_type(bt)) {
__ $3($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
ptrue, $src2$$FloatRegister);
} else {
assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
__ $4($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
ptrue, $src2$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VMINMAX_HF_SVE($1, $2, $3 )
dnl VMINMAX_HF_SVE(type, op_name, insn_fp)
define(`VMINMAX_HF_SVE', `
instruct v$1_HF_sve(vReg dst_src1, vReg src2) %{
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
match(Set dst_src1 ($2 dst_src1 src2));
format %{ "v$1_HF_sve $dst_src1, $dst_src1, $src2\t# Half float" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
__ $3($dst_src1$$FloatRegister, __ H,
ptrue, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VMINMAX_PREDICATE($1, $2, $3, $4 )
dnl VMINMAX_PREDICATE(type, op_name, insn_fp, insn_integral)
define(`VMINMAX_PREDICATE', `
instruct v$1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
format %{ "v$1_masked $dst_src1, $pg, $dst_src1, $src2" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
if (is_floating_point_type(bt)) {
__ $3($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $src2$$FloatRegister);
} else {
assert(is_integral_type(bt), "unsupported type");
__ $4($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $src2$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VUMINMAX_NEON($1, $2, $3 )
dnl VUMINMAX_NEON(type, op_name, insn)
define(`VUMINMAX_NEON', `
instruct v$1_neon(vReg dst, vReg src1, vReg src2) %{
predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
match(Set dst ($2 src1 src2));
format %{ "v$1_neon $dst, $src1, $src2\t# B/S/I" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
__ $3($dst$$FloatRegister, get_arrangement(this),
$src1$$FloatRegister, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VUMINMAX_SVE($1, $2, $3 )
dnl VUMINMAX_SVE(type, op_name, insn)
define(`VUMINMAX_SVE', `
instruct v$1_sve(vReg dst_src1, vReg src2) %{
predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
match(Set dst_src1 ($2 dst_src1 src2));
format %{ "v$1_sve $dst_src1, $dst_src1, $src2\t# B/S/I" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this);
assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
__ $3($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
ptrue, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VUMINMAX_PREDICATE($1, $2, $3 )
dnl VUMINMAX_PREDICATE(type, op_name, insn)
define(`VUMINMAX_PREDICATE', `
instruct v$1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
format %{ "v$1_masked $dst_src1, $pg, $dst_src1, $src2" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
assert(is_integral_type(bt), "unsupported type");
__ $3($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// ------------------------------ Vector min -----------------------------------
// vector min - LONG
VMINMAX_L_NEON(min, MinV)
VMINMAX_L_SVE(min, MinV, sve_smin)
// vector min - B/S/I/HF/F/D
VMINMAX_NEON(min, MinV, fmin, minv)
VMINMAX_SVE(min, MinV, sve_fmin, sve_smin)
VMINMAX_HF_NEON(min, MinVHF, fmin)
VMINMAX_HF_SVE(min, MinVHF, sve_fmin)
// vector min - predicated
VMINMAX_PREDICATE(min, MinV, sve_fmin, sve_smin)
// vector unsigned min - LONG
VMINMAX_L_NEON(min, UMinV, u)
VMINMAX_L_SVE(umin, UMinV, sve_umin)
// vector unsigned min - B/S/I
VUMINMAX_NEON(umin, UMinV, uminv)
VUMINMAX_SVE(umin, UMinV, sve_umin)
// vector unsigned min - predicated
VUMINMAX_PREDICATE(umin, UMinV, sve_umin)
// ------------------------------ Vector max -----------------------------------
// vector max - LONG
VMINMAX_L_NEON(max, MaxV)
VMINMAX_L_SVE(max, MaxV, sve_smax)
// vector max - B/S/I/HF/F/D
VMINMAX_NEON(max, MaxV, fmax, maxv)
VMINMAX_SVE(max, MaxV, sve_fmax, sve_smax)
VMINMAX_HF_NEON(max, MaxVHF, fmax)
VMINMAX_HF_SVE(max, MaxVHF, sve_fmax)
// vector max - predicated
VMINMAX_PREDICATE(max, MaxV, sve_fmax, sve_smax)
// vector unsigned max - LONG
VMINMAX_L_NEON(max, UMaxV, u)
VMINMAX_L_SVE(umax, UMaxV, sve_umax)
// vector unsigned max - B/S/I
VUMINMAX_NEON(umax, UMaxV, umaxv)
VUMINMAX_SVE(umax, UMaxV, sve_umax)
// vector unsigned max - predicated
VUMINMAX_PREDICATE(umax, UMaxV, sve_umax)
// ------------------------------ MLA RELATED ----------------------------------
// vector mla
// dst_src1 = dst_src1 + src2 * src3
instruct vmla(vReg dst_src1, vReg src2, vReg src3) %{
match(Set dst_src1 (AddVB dst_src1 (MulVB src2 src3)));
match(Set dst_src1 (AddVS dst_src1 (MulVS src2 src3)));
match(Set dst_src1 (AddVI dst_src1 (MulVI src2 src3)));
format %{ "vmla $dst_src1, $src2, $src3" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ mlav($dst_src1$$FloatRegister, get_arrangement(this),
$src2$$FloatRegister, $src3$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_mla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}
instruct vmlaL(vReg dst_src1, vReg src2, vReg src3) %{
predicate(UseSVE > 0);
match(Set dst_src1 (AddVL dst_src1 (MulVL src2 src3)));
format %{ "vmlaL $dst_src1, $src2, $src3" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_mla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct vmla_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src1 (AddVB (Binary dst_src1 (MulVB src2 src3)) pg));
match(Set dst_src1 (AddVS (Binary dst_src1 (MulVS src2 src3)) pg));
match(Set dst_src1 (AddVI (Binary dst_src1 (MulVI src2 src3)) pg));
match(Set dst_src1 (AddVL (Binary dst_src1 (MulVL src2 src3)) pg));
format %{ "vmla_masked $dst_src1, $pg, $src2, $src3" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_mla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// vector fmla
// dst_src1 = src2 * src3 + dst_src1
instruct vfmla(vReg dst_src1, vReg src2, vReg src3) %{
match(Set dst_src1 (FmaVHF dst_src1 (Binary src2 src3)));
match(Set dst_src1 (FmaVF dst_src1 (Binary src2 src3)));
match(Set dst_src1 (FmaVD dst_src1 (Binary src2 src3)));
format %{ "vfmla $dst_src1, $src2, $src3" %}
ins_encode %{
assert(UseFMA, "Needs FMA instructions support.");
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ fmla($dst_src1$$FloatRegister, get_arrangement(this),
$src2$$FloatRegister, $src3$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_fmla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}
// vector fmad - predicated
// dst_src1 = dst_src1 * src2 + src3
instruct vfmad_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary src3 pg)));
match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary src3 pg)));
format %{ "vfmad_masked $dst_src1, $pg, $src2, $src3" %}
ins_encode %{
assert(UseFMA, "Needs FMA instructions support.");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_fmad($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// vector mls
// dst_src1 = dst_src1 - src2 * src3
instruct vmls(vReg dst_src1, vReg src2, vReg src3) %{
match(Set dst_src1 (SubVB dst_src1 (MulVB src2 src3)));
match(Set dst_src1 (SubVS dst_src1 (MulVS src2 src3)));
match(Set dst_src1 (SubVI dst_src1 (MulVI src2 src3)));
format %{ "vmls $dst_src1, $src2, $src3" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ mlsv($dst_src1$$FloatRegister, get_arrangement(this),
$src2$$FloatRegister, $src3$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_mls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}
instruct vmlsL(vReg dst_src1, vReg src2, vReg src3) %{
predicate(UseSVE > 0);
match(Set dst_src1 (SubVL dst_src1 (MulVL src2 src3)));
format %{ "vmlsL $dst_src1, $src2, $src3" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_mls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct vmls_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src1 (SubVB (Binary dst_src1 (MulVB src2 src3)) pg));
match(Set dst_src1 (SubVS (Binary dst_src1 (MulVS src2 src3)) pg));
match(Set dst_src1 (SubVI (Binary dst_src1 (MulVI src2 src3)) pg));
match(Set dst_src1 (SubVL (Binary dst_src1 (MulVL src2 src3)) pg));
format %{ "vmls_masked $dst_src1, $pg, $src2, $src3" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_mls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// vector fmls
// dst_src1 = src2 * (-src3) + dst_src1
// "(-src2) * src3 + dst_src1" has been idealized to "src3 * (-src2) + dst_src1"
instruct vfmls(vReg dst_src1, vReg src2, vReg src3) %{
match(Set dst_src1 (FmaVF dst_src1 (Binary src2 (NegVF src3))));
match(Set dst_src1 (FmaVD dst_src1 (Binary src2 (NegVD src3))));
format %{ "vfmls $dst_src1, $src2, $src3" %}
ins_encode %{
assert(UseFMA, "Needs FMA instructions support.");
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ fmls($dst_src1$$FloatRegister, get_arrangement(this),
$src2$$FloatRegister, $src3$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_fmls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}
// vector fmsb - predicated
// dst_src1 = dst_src1 * (-src2) + src3
instruct vfmsb_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary src3 pg)));
match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary src3 pg)));
format %{ "vfmsb_masked $dst_src1, $pg, $src2, $src3" %}
ins_encode %{
assert(UseFMA, "Needs FMA instructions support.");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_fmsb($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// vector fnmla (sve)
// dst_src1 = src2 * (-src3) - dst_src1
// "(-src2) * src3 - dst_src1" has been idealized to "src3 * (-src2) - dst_src1"
instruct vfnmla(vReg dst_src1, vReg src2, vReg src3) %{
predicate(UseSVE > 0);
match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 (NegVF src3))));
match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 (NegVD src3))));
format %{ "vfnmla $dst_src1, $src2, $src3" %}
ins_encode %{
assert(UseFMA, "Needs FMA instructions support.");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_fnmla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// vector fnmad - predicated
// dst_src1 = dst_src1 * (-src2) - src3
instruct vfnmad_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary (NegVF src3) pg)));
match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary (NegVD src3) pg)));
format %{ "vfnmad_masked $dst_src1, $pg, $src2, $src3" %}
ins_encode %{
assert(UseFMA, "Needs FMA instructions support.");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_fnmad($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// vector fnmls (sve)
// dst_src1 = src2 * src3 - dst_src1
instruct vfnmls(vReg dst_src1, vReg src2, vReg src3) %{
predicate(UseSVE > 0);
match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 src3)));
match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 src3)));
format %{ "vfnmls $dst_src1, $src2, $src3" %}
ins_encode %{
assert(UseFMA, "Needs FMA instructions support.");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_fnmls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// vector fnmsb - predicated
// dst_src1 = dst_src1 * src2 - src3
instruct vfnmsb_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary (NegVF src3) pg)));
match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary (NegVD src3) pg)));
format %{ "vfnmsb_masked $dst_src1, $pg, $src2, $src3" %}
ins_encode %{
assert(UseFMA, "Needs FMA instructions support.");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_fnmsb($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// MulAddVS2VI
// Vector Multiply-Add Shorts into Integer
instruct vmuladdS2I(vReg dst, vReg src1, vReg src2, vReg tmp) %{
predicate(Matcher::vector_length_in_bytes(n) == 16 &&
Matcher::vector_element_basic_type(n->in(1)) == T_SHORT);
match(Set dst (MulAddVS2VI src1 src2));
effect(TEMP_DEF dst, TEMP tmp);
format %{ "vmuladdS2I $dst, $src1, $src2\t# KILL $tmp" %}
ins_encode %{
__ smullv($tmp$$FloatRegister, __ T4H, $src1$$FloatRegister, $src2$$FloatRegister);
__ smullv($dst$$FloatRegister, __ T8H, $src1$$FloatRegister, $src2$$FloatRegister);
__ addpv($dst$$FloatRegister, __ T4S, $tmp$$FloatRegister, $dst$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Vector shift ---------------------------------
// Vector right shift in AArch64 ASIMD
//
// Right shifts with vector shift count on AArch64 ASIMD are implemented
// as left shift by negative shift count.
// There are two cases for vector shift count.
//
// Case 1: The vector shift count is from replication.
// | |
// LoadVector RShiftCntV
// | /
// RShiftVI
//
// Case 2: The vector shift count is from loading.
// This case isn't supported by middle-end now. But it's supported by
// panama/vectorIntrinsics(JEP 338: Vector API).
// | |
// LoadVector LoadVector
// | /
// RShiftVI
//
// The negate is conducted in RShiftCntV rule for case 1, whereas it's done in
// RShiftV* rules for case 2. Because there exists an optimization opportunity
// for case 1, that is, multiple neg instructions in inner loop can be hoisted
// to outer loop and merged into one neg instruction.
//
// Note that ShiftVNode::is_var_shift() indicates whether the vector shift
// count is a variable vector(case 2) or not(a vector generated by RShiftCntV,
// i.e. case 1).
// vector shift count
instruct vshiftcntL(vReg dst, iRegIorL2I cnt) %{
match(Set dst (LShiftCntV cnt));
format %{ "vshiftcntL $dst, $cnt" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ dup($dst$$FloatRegister, get_arrangement(this), $cnt$$Register);
} else {
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_dup($dst$$FloatRegister, __ elemType_to_regVariant(bt), $cnt$$Register);
}
%}
ins_pipe(pipe_slow);
%}
instruct vshiftcntR(vReg dst, iRegIorL2I cnt) %{
match(Set dst (RShiftCntV cnt));
format %{ "vshiftcntR $dst, $cnt" %}
ins_encode %{
if (UseSVE == 0) {
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes <= 16, "must be");
__ negw(rscratch1, $cnt$$Register);
__ dup($dst$$FloatRegister, get_arrangement(this), rscratch1);
} else {
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_dup($dst$$FloatRegister, __ elemType_to_regVariant(bt), $cnt$$Register);
}
%}
ins_pipe(pipe_slow);
%}
// vector shift left
instruct vlsl_neon(vReg dst, vReg src, vReg shift) %{
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
match(Set dst (LShiftVB src shift));
match(Set dst (LShiftVS src shift));
match(Set dst (LShiftVI src shift));
match(Set dst (LShiftVL src shift));
format %{ "vlsl_neon $dst, $src, $shift" %}
ins_encode %{
__ sshl($dst$$FloatRegister, get_arrangement(this),
$src$$FloatRegister, $shift$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct vlsl_sve(vReg dst_src, vReg shift) %{
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
match(Set dst_src (LShiftVB dst_src shift));
match(Set dst_src (LShiftVS dst_src shift));
match(Set dst_src (LShiftVI dst_src shift));
match(Set dst_src (LShiftVL dst_src shift));
format %{ "vlsl_sve $dst_src, $dst_src, $shift" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_lsl($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
ptrue, $shift$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
dnl
dnl VRSHIFT_NEON($1, $2, $3 )
dnl VRSHIFT_NEON(type, op_name, insn)
define(`VRSHIFT_NEON', `
instruct v$1_neon(vReg dst, vReg src, vReg shift) %{
predicate(UseSVE == 0 && !n->as_ShiftV()->is_var_shift());
match(Set dst ($2VB src shift));
match(Set dst ($2VS src shift));
match(Set dst ($2VI src shift));
match(Set dst ($2VL src shift));
format %{ "v$1_neon $dst, $src, $shift\t# not variable shift" %}
ins_encode %{
__ $3($dst$$FloatRegister, get_arrangement(this),
$src$$FloatRegister, $shift$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VRSHIFT_NEON_VAR($1, $2, $3 )
dnl VRSHIFT_NEON_VAR(type, op_name, insn)
define(`VRSHIFT_NEON_VAR', `
instruct v$1_neon_var(vReg dst, vReg src, vReg shift) %{
predicate(UseSVE == 0 && n->as_ShiftV()->is_var_shift());
match(Set dst ($2VB src shift));
match(Set dst ($2VS src shift));
match(Set dst ($2VI src shift));
match(Set dst ($2VL src shift));
effect(TEMP_DEF dst);
format %{ "v$1_neon_var $dst, $src, $shift\t# variable shift" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
__ negr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$shift$$FloatRegister);
__ $3($dst$$FloatRegister, get_arrangement(this),
$src$$FloatRegister, $dst$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VRSHIFT_SVE($1, $2, $3 )
dnl VRSHIFT_SVE(type, op_name, insn)
define(`VRSHIFT_SVE', `
instruct v$1_sve(vReg dst_src, vReg shift) %{
predicate(UseSVE > 0);
match(Set dst_src ($2VB dst_src shift));
match(Set dst_src ($2VS dst_src shift));
match(Set dst_src ($2VI dst_src shift));
match(Set dst_src ($2VL dst_src shift));
format %{ "v$1_sve $dst_src, $dst_src, $shift" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ $3($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
ptrue, $shift$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// vector shift right (arithmetic)
VRSHIFT_NEON(asr, RShift, sshl)
VRSHIFT_NEON_VAR(asr, RShift, sshl)
VRSHIFT_SVE(asr, RShift, sve_asr)
// vector shift right (logical)
VRSHIFT_NEON(lsr, URShift, ushl)
VRSHIFT_NEON_VAR(lsr, URShift, ushl)
VRSHIFT_SVE(lsr, URShift, sve_lsr)
// vector shift with imm
instruct vlsl_imm(vReg dst, vReg src, immI shift) %{
predicate(assert_not_var_shift(n));
match(Set dst (LShiftVB src (LShiftCntV shift)));
match(Set dst (LShiftVS src (LShiftCntV shift)));
match(Set dst (LShiftVI src (LShiftCntV shift)));
match(Set dst (LShiftVL src (LShiftCntV shift)));
format %{ "vlsl_imm $dst, $src, $shift" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
int con = (int)$shift$$constant;
if (is_subword_type(bt)) {
// Optimize for B/S
int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
if (con >= esize_in_bits) {
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ eor($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src$$FloatRegister, $src$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_eor($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
}
return;
}
}
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ shl($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_lsl($dst$$FloatRegister, __ elemType_to_regVariant(bt), $src$$FloatRegister, con);
}
%}
ins_pipe(pipe_slow);
%}
instruct vasr_imm(vReg dst, vReg src, immI_positive shift) %{
predicate(assert_not_var_shift(n));
match(Set dst (RShiftVB src (RShiftCntV shift)));
match(Set dst (RShiftVS src (RShiftCntV shift)));
match(Set dst (RShiftVI src (RShiftCntV shift)));
match(Set dst (RShiftVL src (RShiftCntV shift)));
format %{ "vasr_imm $dst, $src, $shift" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
int con = (int)$shift$$constant;
if (is_subword_type(bt)) {
// Refine con for B/S
int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
if (con >= esize_in_bits) con = esize_in_bits - 1;
}
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ sshr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_asr($dst$$FloatRegister, __ elemType_to_regVariant(bt), $src$$FloatRegister, con);
}
%}
ins_pipe(pipe_slow);
%}
instruct vlsr_imm(vReg dst, vReg src, immI_positive shift) %{
predicate(assert_not_var_shift(n));
match(Set dst (URShiftVB src (RShiftCntV shift)));
match(Set dst (URShiftVS src (RShiftCntV shift)));
match(Set dst (URShiftVI src (RShiftCntV shift)));
match(Set dst (URShiftVL src (RShiftCntV shift)));
format %{ "vlsr_imm $dst, $src, $shift" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
int con = (int)$shift$$constant;
if (is_subword_type(bt)) {
// Optimize for B/S
int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
if (con >= esize_in_bits) {
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ eor($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src$$FloatRegister, $src$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_eor($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
}
return;
}
}
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ ushr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_lsr($dst$$FloatRegister, __ elemType_to_regVariant(bt), $src$$FloatRegister, con);
}
%}
ins_pipe(pipe_slow);
%}
// shift right add with imm (vector length <= 128 bits only)
instruct vasra_imm(vReg dst, vReg src, immI_positive shift) %{
predicate(Matcher::vector_length_in_bytes(n) <= 16);
match(Set dst (AddVB dst (RShiftVB src (RShiftCntV shift))));
match(Set dst (AddVS dst (RShiftVS src (RShiftCntV shift))));
match(Set dst (AddVI dst (RShiftVI src (RShiftCntV shift))));
match(Set dst (AddVL dst (RShiftVL src (RShiftCntV shift))));
format %{ "vasra_imm $dst, $src, $shift" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
int con = (int)$shift$$constant;
if (is_subword_type(bt)) {
// Refine con for B/S
int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
if (con >= esize_in_bits) con = esize_in_bits - 1;
}
__ ssra($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
%}
ins_pipe(pipe_slow);
%}
instruct vlsra_imm(vReg dst, vReg src, immI_positive shift) %{
predicate(Matcher::vector_length_in_bytes(n) <= 16);
match(Set dst (AddVB dst (URShiftVB src (RShiftCntV shift))));
match(Set dst (AddVS dst (URShiftVS src (RShiftCntV shift))));
match(Set dst (AddVI dst (URShiftVI src (RShiftCntV shift))));
match(Set dst (AddVL dst (URShiftVL src (RShiftCntV shift))));
format %{ "vlsra_imm $dst, $src, $shift" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
int con = (int)$shift$$constant;
if (is_subword_type(bt)) { // for B/H
int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
if (con < esize_in_bits) {
__ usra($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
}
} else { // for S/D
assert(type2aelembytes(bt) == 4 || type2aelembytes(bt) == 8, "unsupported type");
__ usra($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
}
%}
ins_pipe(pipe_slow);
%}
dnl
dnl VSHIFT_PREDICATE($1, $2, $3 )
dnl VSHIFT_PREDICATE(type, op_name, insn)
define(`VSHIFT_PREDICATE', `
instruct v$1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src1 ($2VB (Binary dst_src1 src2) pg));
match(Set dst_src1 ($2VS (Binary dst_src1 src2) pg));
match(Set dst_src1 ($2VI (Binary dst_src1 src2) pg));
match(Set dst_src1 ($2VL (Binary dst_src1 src2) pg));
format %{ "v$1_masked $dst_src1, $pg, $dst_src1, $src2" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ $3($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VSHIFT_IMM_PREDICATE($1, $2, $3, $4, $5 )
dnl VSHIFT_IMM_PREDICATE(type, arg_type, op_name1, op_name2, insn)
define(`VSHIFT_IMM_PREDICATE', `
instruct v$1_imm_masked(vReg dst_src, $2 shift, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src ($3VB (Binary dst_src ($4 shift)) pg));
match(Set dst_src ($3VS (Binary dst_src ($4 shift)) pg));
match(Set dst_src ($3VI (Binary dst_src ($4 shift)) pg));
match(Set dst_src ($3VL (Binary dst_src ($4 shift)) pg));
format %{ "v$1_imm_masked $dst_src, $pg, $dst_src, $shift" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
int con = (int)$shift$$constant;
assert(con ifelse($1, lsl, >=, >) 0 && con < esize_in_bits, "invalid shift immediate");
__ $5($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, con);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// vector shift - predicated
VSHIFT_PREDICATE(lsl, LShift, sve_lsl)
VSHIFT_PREDICATE(asr, RShift, sve_asr)
VSHIFT_PREDICATE(lsr, URShift, sve_lsr)
// vector shift with imm - predicated
VSHIFT_IMM_PREDICATE(lsl, immI, LShift, LShiftCntV, sve_lsl)
VSHIFT_IMM_PREDICATE(asr, immI_positive, RShift, RShiftCntV, sve_asr)
VSHIFT_IMM_PREDICATE(lsr, immI_positive, URShift, RShiftCntV, sve_lsr)
// ------------------------------ Vector reduction add -------------------------
dnl
dnl REDUCE_ADD_INT_NEON_SVE_PAIRWISE($1, $2 )
dnl REDUCE_ADD_INT_NEON_SVE_PAIRWISE(type, arg_type)
define(`REDUCE_ADD_INT_NEON_SVE_PAIRWISE', `
instruct reduce_add$1_neon(iReg$1NoSp dst, $2 isrc, vReg vsrc, vReg tmp) %{
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))));
match(Set dst (AddReductionV$1 isrc vsrc));
effect(TEMP_DEF dst, TEMP tmp);
format %{ "reduce_add$1_neon $dst, $isrc, $vsrc\t# KILL $tmp" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
__ neon_reduce_add_integral($dst$$Register, bt,
$isrc$$Register, $vsrc$$FloatRegister,
length_in_bytes, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct reduce_add$1_sve(iReg$1NoSp dst, $2 isrc, vReg vsrc, vRegD tmp) %{
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))));
match(Set dst (AddReductionV$1 isrc vsrc));
effect(TEMP_DEF dst, TEMP tmp);
format %{ "reduce_add$1_sve $dst, $isrc, $vsrc\t# KILL $tmp" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
$isrc$$Register, $vsrc$$FloatRegister,
ptrue, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// reduction addI
REDUCE_ADD_INT_NEON_SVE_PAIRWISE(I, iRegIorL2I)
// reduction addL
REDUCE_ADD_INT_NEON_SVE_PAIRWISE(L, iRegL)
// reduction addF
instruct reduce_non_strict_order_add2F_neon(vRegF dst, vRegF fsrc, vReg vsrc) %{
// Non-strictly ordered floating-point add reduction for a 64-bits-long vector. This rule is
// intended for the VectorAPI (which allows for non-strictly ordered add reduction).
predicate(Matcher::vector_length(n->in(2)) == 2 && !n->as_Reduction()->requires_strict_order());
match(Set dst (AddReductionVF fsrc vsrc));
effect(TEMP_DEF dst);
format %{ "reduce_non_strict_order_add2F_neon $dst, $fsrc, $vsrc" %}
ins_encode %{
__ faddp($dst$$FloatRegister, $vsrc$$FloatRegister, __ S);
__ fadds($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct reduce_non_strict_order_add4F_neon(vRegF dst, vRegF fsrc, vReg vsrc, vReg tmp) %{
// Non-strictly ordered floating-point add reduction for 128-bits-long vector. This rule is
// intended for the VectorAPI (which allows for non-strictly ordered add reduction).
predicate(Matcher::vector_length(n->in(2)) == 4 && !n->as_Reduction()->requires_strict_order());
match(Set dst (AddReductionVF fsrc vsrc));
effect(TEMP_DEF dst, TEMP tmp);
format %{ "reduce_non_strict_order_add4F_neon $dst, $fsrc, $vsrc\t# KILL $tmp" %}
ins_encode %{
__ faddp($tmp$$FloatRegister, __ T4S, $vsrc$$FloatRegister, $vsrc$$FloatRegister);
__ faddp($dst$$FloatRegister, $tmp$$FloatRegister, __ S);
__ fadds($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
dnl
dnl REDUCE_ADD_FP_SVE($1, $2 )
dnl REDUCE_ADD_FP_SVE(type, size)
define(`REDUCE_ADD_FP_SVE', `
// This rule calculates the reduction result in strict order. Two cases will
// reach here:
// 1. Non strictly-ordered AddReductionV$1 when vector size > 128-bits. For example -
// AddReductionV$1 generated by Vector API. For vector size > 128-bits, it is more
// beneficial performance-wise to generate direct SVE instruction even if it is
// strictly ordered.
// 2. Strictly-ordered AddReductionV$1. For example - AddReductionV$1 generated by
// auto-vectorization on SVE machine.
instruct reduce_add$1_sve(vReg$1 dst_src1, vReg src2) %{
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) ||
n->as_Reduction()->requires_strict_order());
match(Set dst_src1 (AddReductionV$1 dst_src1 src2));
format %{ "reduce_add$1_sve $dst_src1, $dst_src1, $src2" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src2);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_fadda($dst_src1$$FloatRegister, __ $2, ptrue, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
REDUCE_ADD_FP_SVE(F, S)
// reduction addD
instruct reduce_non_strict_order_add2D_neon(vRegD dst, vRegD dsrc, vReg vsrc) %{
// Non-strictly ordered floating-point add reduction for doubles. This rule is
// intended for the VectorAPI (which allows for non-strictly ordered add reduction).
predicate(!n->as_Reduction()->requires_strict_order());
match(Set dst (AddReductionVD dsrc vsrc));
effect(TEMP_DEF dst);
format %{ "reduce_non_strict_order_add2D_neon $dst, $dsrc, $vsrc\t# 2D" %}
ins_encode %{
__ faddp($dst$$FloatRegister, $vsrc$$FloatRegister, __ D);
__ faddd($dst$$FloatRegister, $dst$$FloatRegister, $dsrc$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
REDUCE_ADD_FP_SVE(D, D)
// reduction add - predicated
dnl
dnl REDUCE_ADD_INT_PREDICATE($1, $2 )
dnl REDUCE_ADD_INT_PREDICATE(insn_name, op_name)
define(`REDUCE_ADD_INT_PREDICATE', `
instruct reduce_add$1_masked(iReg$1NoSp dst, $2 isrc, vReg vsrc, pRegGov pg, vRegD tmp) %{
predicate(UseSVE > 0);
match(Set dst (AddReductionV$1 (Binary isrc vsrc) pg));
effect(TEMP_DEF dst, TEMP tmp);
format %{ "reduce_add$1_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
$isrc$$Register, $vsrc$$FloatRegister,
$pg$$PRegister, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl REDUCE_ADD_FP_PREDICATE($1, $2 )
dnl REDUCE_ADD_FP_PREDICATE(insn_name, op_name)
define(`REDUCE_ADD_FP_PREDICATE', `
instruct reduce_add$1_masked(vReg$1 dst_src1, vReg src2, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src1 (AddReductionV$1 (Binary dst_src1 src2) pg));
format %{ "reduce_add$1_masked $dst_src1, $pg, $dst_src1, $src2" %}
ins_encode %{
__ sve_fadda($dst_src1$$FloatRegister, __ $2,
$pg$$PRegister, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
REDUCE_ADD_INT_PREDICATE(I, iRegIorL2I)
REDUCE_ADD_INT_PREDICATE(L, iRegL)
REDUCE_ADD_FP_PREDICATE(F, S)
REDUCE_ADD_FP_PREDICATE(D, D)
// ------------------------------ Vector reduction mul -------------------------
instruct reduce_mulI(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc,
vReg tmp1, vReg tmp2) %{
predicate(Matcher::vector_length_in_bytes(n->in(2)) == 8 ||
Matcher::vector_length_in_bytes(n->in(2)) == 16);
match(Set dst (MulReductionVI isrc vsrc));
effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
format %{ "reduce_mulI $dst, $isrc, $vsrc\t# vector (64/128 bits). KILL $tmp1, $tmp2" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
__ neon_reduce_mul_integral($dst$$Register, bt, $isrc$$Register,
$vsrc$$FloatRegister, length_in_bytes,
$tmp1$$FloatRegister, $tmp2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct reduce_mulL(iRegLNoSp dst, iRegL isrc, vReg vsrc) %{
predicate(Matcher::vector_length_in_bytes(n->in(2)) == 16);
match(Set dst (MulReductionVL isrc vsrc));
effect(TEMP_DEF dst);
format %{ "reduce_mulL $dst, $isrc, $vsrc\t# 2L" %}
ins_encode %{
__ neon_reduce_mul_integral($dst$$Register, T_LONG, $isrc$$Register,
$vsrc$$FloatRegister, 16, fnoreg, fnoreg);
%}
ins_pipe(pipe_slow);
%}
instruct reduce_mulF(vRegF dst, vRegF fsrc, vReg vsrc, vReg tmp) %{
predicate(Matcher::vector_length_in_bytes(n->in(2)) <= 16);
match(Set dst (MulReductionVF fsrc vsrc));
effect(TEMP_DEF dst, TEMP tmp);
format %{ "reduce_mulF $dst, $fsrc, $vsrc\t# 2F/4F. KILL $tmp" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
__ neon_reduce_mul_fp($dst$$FloatRegister, T_FLOAT, $fsrc$$FloatRegister,
$vsrc$$FloatRegister, length_in_bytes, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct reduce_mulD(vRegD dst, vRegD dsrc, vReg vsrc, vReg tmp) %{
predicate(Matcher::vector_length_in_bytes(n->in(2)) == 16);
match(Set dst (MulReductionVD dsrc vsrc));
effect(TEMP_DEF dst, TEMP tmp);
format %{ "reduce_mulD $dst, $dsrc, $vsrc\t# 2D. KILL $tmp" %}
ins_encode %{
__ neon_reduce_mul_fp($dst$$FloatRegister, T_DOUBLE, $dsrc$$FloatRegister,
$vsrc$$FloatRegister, 16, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
dnl
dnl REDUCE_BITWISE_OP_NEON($1, $2 $3 $4 )
dnl REDUCE_BITWISE_OP_NEON(insn_name, is_long, type, op_name)
define(`REDUCE_BITWISE_OP_NEON', `
instruct reduce_$1$2_neon(iReg$2NoSp dst, $3 isrc, vReg vsrc) %{
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) ifelse($2, L, ==, !=) T_LONG);
match(Set dst ($4 isrc vsrc));
effect(TEMP_DEF dst);
format %{ "reduce_$1$2_neon $dst, $isrc, $vsrc" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
__ neon_reduce_logical(this->ideal_Opcode(), $dst$$Register, bt,
$isrc$$Register, $vsrc$$FloatRegister, length_in_bytes);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl REDUCE_BITWISE_OP_SVE($1, $2 $3 $4 )
dnl REDUCE_BITWISE_OP_SVE(insn_name, is_long, type, op_name)
define(`REDUCE_BITWISE_OP_SVE', `
instruct reduce_$1$2_sve(iReg$2NoSp dst, $3 isrc, vReg vsrc, vRegD tmp) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) ifelse($2, L, ==, !=) T_LONG);
match(Set dst ($4 isrc vsrc));
effect(TEMP_DEF dst, TEMP tmp);
format %{ "reduce_$1$2_sve $dst, $isrc, $vsrc\t# KILL $tmp" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
$isrc$$Register, $vsrc$$FloatRegister,
ptrue, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl REDUCE_BITWISE_OP_PREDICATE($1, $2 $3 $4 )
dnl REDUCE_BITWISE_OP_PREDICATE(insn_name, is_long, type, op_name)
define(`REDUCE_BITWISE_OP_PREDICATE', `
instruct reduce_$1$2_masked(iReg$2NoSp dst, $3 isrc, vReg vsrc, pRegGov pg, vRegD tmp) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) ifelse($2, L, ==, !=) T_LONG);
match(Set dst ($4 (Binary isrc vsrc) pg));
effect(TEMP_DEF dst, TEMP tmp);
format %{ "reduce_$1$2_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
$isrc$$Register, $vsrc$$FloatRegister,
$pg$$PRegister, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// ------------------------------ Vector reduction and -------------------------
// reduction andI
REDUCE_BITWISE_OP_NEON(and, I, iRegIorL2I, AndReductionV)
REDUCE_BITWISE_OP_SVE(and, I, iRegIorL2I, AndReductionV)
// reduction andL
REDUCE_BITWISE_OP_NEON(and, L, iRegL, AndReductionV)
REDUCE_BITWISE_OP_SVE(and, L, iRegL, AndReductionV)
// reduction and - predicated
REDUCE_BITWISE_OP_PREDICATE(and, I, iRegIorL2I, AndReductionV)
REDUCE_BITWISE_OP_PREDICATE(and, L, iRegL, AndReductionV)
// ------------------------------ Vector reduction or --------------------------
// reduction orI
REDUCE_BITWISE_OP_NEON(or, I, iRegIorL2I, OrReductionV)
REDUCE_BITWISE_OP_SVE(or, I, iRegIorL2I, OrReductionV)
// reduction orL
REDUCE_BITWISE_OP_NEON(or, L, iRegL, OrReductionV)
REDUCE_BITWISE_OP_SVE(or, L, iRegL, OrReductionV)
// reduction or - predicated
REDUCE_BITWISE_OP_PREDICATE(or, I, iRegIorL2I, OrReductionV)
REDUCE_BITWISE_OP_PREDICATE(or, L, iRegL, OrReductionV)
// ------------------------------ Vector reduction xor -------------------------
// reduction xorI
REDUCE_BITWISE_OP_NEON(xor, I, iRegIorL2I, XorReductionV)
REDUCE_BITWISE_OP_SVE(xor, I, iRegIorL2I, XorReductionV)
// reduction xorL
REDUCE_BITWISE_OP_NEON(xor, L, iRegL, XorReductionV)
REDUCE_BITWISE_OP_SVE(xor, L, iRegL, XorReductionV)
// reduction xor - predicated
REDUCE_BITWISE_OP_PREDICATE(xor, I, iRegIorL2I, XorReductionV)
REDUCE_BITWISE_OP_PREDICATE(xor, L, iRegL, XorReductionV)
dnl
dnl REDUCE_MAXMIN_I_NEON($1, $2 )
dnl REDUCE_MAXMIN_I_NEON(type, op_name)
define(`REDUCE_MAXMIN_I_NEON', `
instruct reduce_$1I_neon(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc,
vReg tmp, rFlagsReg cr) %{
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) &&
(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
Matcher::vector_element_basic_type(n->in(2)) == T_INT));
match(Set dst ($2 isrc vsrc));
effect(TEMP_DEF dst, TEMP tmp, KILL cr);
format %{ "reduce_$1I_neon $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
__ neon_reduce_minmax_integral(this->ideal_Opcode(), $dst$$Register, bt,
$isrc$$Register, $vsrc$$FloatRegister,
length_in_bytes, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl REDUCE_MAXMIN_I_SVE($1, $2 )
dnl REDUCE_MAXMIN_I_SVE(type, op_name)
define(`REDUCE_MAXMIN_I_SVE', `
instruct reduce_$1I_sve(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc,
vRegD tmp, rFlagsReg cr) %{
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) &&
(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
Matcher::vector_element_basic_type(n->in(2)) == T_INT));
match(Set dst ($2 isrc vsrc));
effect(TEMP_DEF dst, TEMP tmp, KILL cr);
format %{ "reduce_$1I_sve $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
$isrc$$Register, $vsrc$$FloatRegister,
ptrue, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl REDUCE_MAXMIN_L_NEON($1, $2 )
dnl REDUCE_MAXMIN_L_NEON(type, op_name)
define(`REDUCE_MAXMIN_L_NEON', `
instruct reduce_$1L_neon(iRegLNoSp dst, iRegL isrc, vReg vsrc, rFlagsReg cr) %{
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
match(Set dst ($2 isrc vsrc));
effect(TEMP_DEF dst, KILL cr);
format %{ "reduce_$1L_neon $dst, $isrc, $vsrc\t# 2L. KILL cr" %}
ins_encode %{
__ neon_reduce_minmax_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
$isrc$$Register, $vsrc$$FloatRegister,
/* vector_length_in_bytes */ 16, fnoreg);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl REDUCE_MAXMIN_L_SVE($1, $2 )
dnl REDUCE_MAXMIN_L_SVE(type, op_name)
define(`REDUCE_MAXMIN_L_SVE', `
instruct reduce_$1L_sve(iRegLNoSp dst, iRegL isrc, vReg vsrc,
vRegD tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
match(Set dst ($2 isrc vsrc));
effect(TEMP_DEF dst, TEMP tmp, KILL cr);
format %{ "reduce_$1L_sve $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
$isrc$$Register, $vsrc$$FloatRegister,
ptrue, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl REDUCE_MAXMIN_F($1, $2, $3, $4, $5, $6 )
dnl REDUCE_MAXMIN_F(type, op_name, insn1, insn2, insn3, insn4)
define(`REDUCE_MAXMIN_F', `
instruct reduce_$1F(vRegF dst, vRegF fsrc, vReg vsrc) %{
predicate(Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT);
match(Set dst ($2 fsrc vsrc));
effect(TEMP_DEF dst);
format %{ "reduce_$1F $dst, $fsrc, $vsrc" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
if (length_in_bytes == 8) {
__ $3($dst$$FloatRegister, $vsrc$$FloatRegister, __ S);
} else {
__ $4($dst$$FloatRegister, __ T4S, $vsrc$$FloatRegister);
}
} else {
assert(UseSVE > 0, "must be sve");
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ $5($dst$$FloatRegister, __ S, ptrue, $vsrc$$FloatRegister);
}
__ $6($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl REDUCE_MAXMIN_D($1, $2, $3, $4, $5 )
dnl REDUCE_MAXMIN_D(type, op_name, insn1, insn2, insn3)
define(`REDUCE_MAXMIN_D', `
instruct reduce_$1D(vRegD dst, vRegD dsrc, vReg vsrc) %{
predicate(Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE);
match(Set dst ($2 dsrc vsrc));
effect(TEMP_DEF dst);
format %{ "reduce_$1D $dst, $dsrc, $vsrc" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ $3($dst$$FloatRegister, $vsrc$$FloatRegister, __ D);
} else {
assert(UseSVE > 0, "must be sve");
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ $4($dst$$FloatRegister, __ D, ptrue, $vsrc$$FloatRegister);
}
__ $5($dst$$FloatRegister, $dst$$FloatRegister, $dsrc$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl REDUCE_MAXMIN_INT_PREDICATE($1, $2, $3, $4 )
dnl REDUCE_MAXMIN_INT_PREDICATE(type, is_long, arg_type, op_name)
define(`REDUCE_MAXMIN_INT_PREDICATE', `
instruct reduce_$1$2_masked(iReg$2NoSp dst, $3 isrc, vReg vsrc, pRegGov pg,
vRegD tmp, rFlagsReg cr) %{
ifelse($2, I,
`predicate(UseSVE > 0 &&
(Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_BYTE ||
Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_SHORT ||
Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_INT));',
`predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_LONG);')
match(Set dst ($4 (Binary isrc vsrc) pg));
effect(TEMP_DEF dst, TEMP tmp, KILL cr);
format %{ "reduce_$1$2_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp, cr" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
$isrc$$Register, $vsrc$$FloatRegister,
$pg$$PRegister, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl REDUCE_MAXMIN_FP_PREDICATE($1, $2, $3, $4, $5, $6 )
dnl REDUCE_MAXMIN_FP_PREDICATE(type, is_float, arg_name, op_name, insn1, insn2)
define(`REDUCE_MAXMIN_FP_PREDICATE', `
instruct reduce_$1$2_masked(vReg$2 dst, vReg$2 $3, vReg vsrc, pRegGov pg) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == ifelse($2, F, T_FLOAT, T_DOUBLE));
match(Set dst ($4 (Binary $3 vsrc) pg));
effect(TEMP_DEF dst);
format %{ "reduce_$1$2_masked $dst, $$3, $pg, $vsrc" %}
ins_encode %{
__ $5($dst$$FloatRegister, __ ifelse($2, F, S, D), $pg$$PRegister, $vsrc$$FloatRegister);
__ $6($dst$$FloatRegister, $dst$$FloatRegister, $$3$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// ------------------------------ Vector reduction max -------------------------
// reduction maxI
REDUCE_MAXMIN_I_NEON(max, MaxReductionV)
REDUCE_MAXMIN_I_SVE(max, MaxReductionV)
// reduction maxL
REDUCE_MAXMIN_L_NEON(max, MaxReductionV)
REDUCE_MAXMIN_L_SVE(max, MaxReductionV)
// reduction maxF
REDUCE_MAXMIN_F(max, MaxReductionV, fmaxp, fmaxv, sve_fmaxv, fmaxs)
// reduction maxD
REDUCE_MAXMIN_D(max, MaxReductionV, fmaxp, sve_fmaxv, fmaxd)
// reduction max - predicated
REDUCE_MAXMIN_INT_PREDICATE(max, I, iRegIorL2I, MaxReductionV)
REDUCE_MAXMIN_INT_PREDICATE(max, L, iRegL, MaxReductionV)
REDUCE_MAXMIN_FP_PREDICATE(max, F, fsrc, MaxReductionV, sve_fmaxv, fmaxs)
REDUCE_MAXMIN_FP_PREDICATE(max, D, dsrc, MaxReductionV, sve_fmaxv, fmaxd)
// ------------------------------ Vector reduction min -------------------------
// reduction minI
REDUCE_MAXMIN_I_NEON(min, MinReductionV)
REDUCE_MAXMIN_I_SVE(min, MinReductionV)
// reduction minL
REDUCE_MAXMIN_L_NEON(min, MinReductionV)
REDUCE_MAXMIN_L_SVE(min, MinReductionV)
// reduction minF
REDUCE_MAXMIN_F(min, MinReductionV, fminp, fminv, sve_fminv, fmins)
// reduction minD
REDUCE_MAXMIN_D(min, MinReductionV, fminp, sve_fminv, fmind)
// reduction min - predicated
REDUCE_MAXMIN_INT_PREDICATE(min, I, iRegIorL2I, MinReductionV)
REDUCE_MAXMIN_INT_PREDICATE(min, L, iRegL, MinReductionV)
REDUCE_MAXMIN_FP_PREDICATE(min, F, fsrc, MinReductionV, sve_fminv, fmins)
REDUCE_MAXMIN_FP_PREDICATE(min, D, dsrc, MinReductionV, sve_fminv, fmind)
// ------------------------------ Vector reinterpret ---------------------------
instruct reinterpret_same_size(vReg dst_src) %{
predicate(Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
match(Set dst_src (VectorReinterpret dst_src));
ins_cost(0);
format %{ "reinterpret_same_size $dst_src\t# do nothing" %}
ins_encode(/* empty encoding */);
ins_pipe(pipe_class_empty);
%}
instruct reinterpret_resize_le128b(vReg dst, vReg src) %{
predicate(Matcher::vector_length_in_bytes(n) != Matcher::vector_length_in_bytes(n->in(1)) &&
Matcher::vector_length_in_bytes(n) <= 16 &&
Matcher::vector_length_in_bytes(n->in(1)) <= 16);
match(Set dst (VectorReinterpret src));
format %{ "reinterpret_resize_le128b $dst, $src\t# vector <= 128 bits." %}
ins_encode %{
uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
// The higher bits in "dst" register must be cleared to zero.
if ((length_in_bytes_src == 4 && length_in_bytes_dst == 8) ||
(length_in_bytes_src == 8 && length_in_bytes_dst == 4)) {
// Reinterpret between 32 bits and 64 bits
__ dup($dst$$FloatRegister, __ S, $src$$FloatRegister);
} else if ((length_in_bytes_src == 4 && length_in_bytes_dst == 16) ||
(length_in_bytes_src == 16 && length_in_bytes_dst == 4)) {
// Reinterpret between 32 bits and 128 bits
__ dup($dst$$FloatRegister, __ S, $src$$FloatRegister);
} else if ((length_in_bytes_src == 8 && length_in_bytes_dst == 16) ||
(length_in_bytes_src == 16 && length_in_bytes_dst == 8)) {
// Reinterpret between 64 bits and 128 bits
__ orr($dst$$FloatRegister, __ T8B, $src$$FloatRegister, $src$$FloatRegister);
} else {
assert(false, "invalid vector length");
ShouldNotReachHere();
}
%}
ins_pipe(pipe_slow);
%}
instruct reinterpret_resize_gt128b(vReg dst, vReg src, pReg ptmp, rFlagsReg cr) %{
predicate(Matcher::vector_length_in_bytes(n) != Matcher::vector_length_in_bytes(n->in(1)) &&
(Matcher::vector_length_in_bytes(n) > 16 ||
Matcher::vector_length_in_bytes(n->in(1)) > 16));
match(Set dst (VectorReinterpret src));
effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
format %{ "reinterpret_resize_gt128b $dst, $src\t# vector > 128 bits. KILL $ptmp, cr" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
length_in_bytes_src : length_in_bytes_dst;
assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
"invalid vector length");
__ sve_gen_mask_imm($ptmp$$PRegister, T_BYTE, length_in_bytes_resize);
__ sve_dup($dst$$FloatRegister, __ B, 0);
__ sve_sel($dst$$FloatRegister, __ B, $ptmp$$PRegister,
$src$$FloatRegister, $dst$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// ---------------------------- Vector zero extend --------------------------------
dnl VECTOR_ZERO_EXTEND($1, $2, $3, $4, $5 $6, $7, )
dnl VECTOR_ZERO_EXTEND(op_name, dst_bt, src_bt, dst_size, src_size, assertion, neon_comment)
define(`VECTOR_ZERO_EXTEND', `
instruct vzeroExt$1toX(vReg dst, vReg src) %{
match(Set dst (VectorUCast`$1'2X src));
format %{ "vzeroExt$1toX $dst, $src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
assert($6, "must be");
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
// $7
__ neon_vector_extend($dst$$FloatRegister, $2, length_in_bytes,
$src$$FloatRegister, $3, /* is_unsigned */ true);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_vector_extend($dst$$FloatRegister, __ $4,
$src$$FloatRegister, __ $5, /* is_unsigned */ true);
}
%}
ins_pipe(pipe_slow);
%}')dnl
VECTOR_ZERO_EXTEND(B, bt, T_BYTE, elemType_to_regVariant(bt), B, bt == T_SHORT || bt == T_INT || bt == T_LONG, `4B to 4S/4I, 8B to 8S')
VECTOR_ZERO_EXTEND(S, T_INT, T_SHORT, elemType_to_regVariant(bt), H, bt == T_INT || bt == T_LONG, `4S to 4I')
VECTOR_ZERO_EXTEND(I, T_LONG, T_INT, D, S, bt == T_LONG, `2I to 2L')
// ------------------------------ Vector cast ----------------------------------
// VectorCastB2X
instruct vcvtBtoX(vReg dst, vReg src) %{
match(Set dst (VectorCastB2X src));
format %{ "vcvtBtoX $dst, $src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
// 4B to 4S/4I/4F, 8B to 8S
__ neon_vector_extend($dst$$FloatRegister, bt == T_FLOAT ? T_INT : bt,
length_in_bytes, $src$$FloatRegister, T_BYTE);
if (bt == T_FLOAT) {
__ scvtfv(__ T4S, $dst$$FloatRegister, $dst$$FloatRegister);
}
} else {
assert(UseSVE > 0, "must be sve");
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_vector_extend($dst$$FloatRegister, size, $src$$FloatRegister, __ B);
if (is_floating_point_type(bt)) {
__ sve_scvtf($dst$$FloatRegister, size, ptrue, $dst$$FloatRegister, size);
}
}
%}
ins_pipe(pipe_slow);
%}
// VectorCastS2X
instruct vcvtStoB_neon(vReg dst, vReg src) %{
predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
match(Set dst (VectorCastS2X src));
format %{ "vcvtStoB_neon $dst, $src" %}
ins_encode %{
// 4S/8S to 4B/8B
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
__ neon_vector_narrow($dst$$FloatRegister, T_BYTE,
$src$$FloatRegister, T_SHORT, length_in_bytes);
%}
ins_pipe(pipe_slow);
%}
instruct vcvtStoB_sve(vReg dst, vReg src, vReg tmp) %{
predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
match(Set dst (VectorCastS2X src));
effect(TEMP tmp);
format %{ "vcvtStoB_sve $dst, $src\t# KILL $tmp" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
__ sve_vector_narrow($dst$$FloatRegister, __ B,
$src$$FloatRegister, __ H, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct vcvtStoX_extend(vReg dst, vReg src) %{
predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) >= 4);
match(Set dst (VectorCastS2X src));
format %{ "vcvtStoX_extend $dst, $src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
// 4S to 4I/4F
__ neon_vector_extend($dst$$FloatRegister, T_INT, length_in_bytes,
$src$$FloatRegister, T_SHORT);
if (bt == T_FLOAT) {
__ scvtfv(__ T4S, $dst$$FloatRegister, $dst$$FloatRegister);
}
} else {
assert(UseSVE > 0, "must be sve");
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_vector_extend($dst$$FloatRegister, size, $src$$FloatRegister, __ H);
if (is_floating_point_type(bt)) {
__ sve_scvtf($dst$$FloatRegister, size, ptrue, $dst$$FloatRegister, size);
}
}
%}
ins_pipe(pipe_slow);
%}
// VectorCastI2X
instruct vcvtItoX_narrow_neon(vReg dst, vReg src) %{
predicate((Matcher::vector_element_basic_type(n) == T_BYTE ||
Matcher::vector_element_basic_type(n) == T_SHORT) &&
VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
match(Set dst (VectorCastI2X src));
effect(TEMP_DEF dst);
format %{ "vcvtItoX_narrow_neon $dst, $src" %}
ins_encode %{
// 4I to 4B/4S
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
__ neon_vector_narrow($dst$$FloatRegister, bt,
$src$$FloatRegister, T_INT, length_in_bytes);
%}
ins_pipe(pipe_slow);
%}
instruct vcvtItoX_narrow_sve(vReg dst, vReg src, vReg tmp) %{
predicate((Matcher::vector_element_basic_type(n) == T_BYTE ||
Matcher::vector_element_basic_type(n) == T_SHORT) &&
!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
match(Set dst (VectorCastI2X src));
effect(TEMP_DEF dst, TEMP tmp);
format %{ "vcvtItoX_narrow_sve $dst, $src\t# KILL $tmp" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_vector_narrow($dst$$FloatRegister, __ elemType_to_regVariant(bt),
$src$$FloatRegister, __ S, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct vcvtItoX(vReg dst, vReg src) %{
predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) >= 4);
match(Set dst (VectorCastI2X src));
format %{ "vcvtItoX $dst, $src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (bt == T_FLOAT) {
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
// 2I/4I to 2F/4F
__ scvtfv(get_arrangement(this), $dst$$FloatRegister, $src$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_scvtf($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister, __ S);
}
} else {
assert(type2aelembytes(bt) == 8, "unsupported type");
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
// 2I to 2L/2D
__ neon_vector_extend($dst$$FloatRegister, T_LONG, length_in_bytes,
$src$$FloatRegister, T_INT);
if (bt == T_DOUBLE) {
__ scvtfv(__ T2D, $dst$$FloatRegister, $dst$$FloatRegister);
}
} else {
assert(UseSVE > 0, "must be sve");
__ sve_vector_extend($dst$$FloatRegister, __ D, $src$$FloatRegister, __ S);
if (bt == T_DOUBLE) {
__ sve_scvtf($dst$$FloatRegister, __ D, ptrue, $dst$$FloatRegister, __ D);
}
}
}
%}
ins_pipe(pipe_slow);
%}
// VectorCastL2X
instruct vcvtLtoI_neon(vReg dst, vReg src) %{
predicate(Matcher::vector_element_basic_type(n) == T_INT &&
VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
match(Set dst (VectorCastL2X src));
format %{ "vcvtLtoI_neon $dst, $src" %}
ins_encode %{
// 2L to 2I
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
__ neon_vector_narrow($dst$$FloatRegister, T_INT,
$src$$FloatRegister, T_LONG, length_in_bytes);
%}
ins_pipe(pipe_slow);
%}
instruct vcvtLtoI_sve(vReg dst, vReg src, vReg tmp) %{
predicate((Matcher::vector_element_basic_type(n) == T_INT &&
!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1)))) ||
Matcher::vector_element_basic_type(n) == T_BYTE ||
Matcher::vector_element_basic_type(n) == T_SHORT);
match(Set dst (VectorCastL2X src));
effect(TEMP_DEF dst, TEMP tmp);
format %{ "vcvtLtoI_sve $dst, $src\t# KILL $tmp" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_vector_narrow($dst$$FloatRegister, __ elemType_to_regVariant(bt),
$src$$FloatRegister, __ D, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct vcvtLtoF_neon(vReg dst, vReg src, vRegF tmp) %{
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
match(Set dst (VectorCastL2X src));
effect(TEMP_DEF dst, TEMP tmp);
format %{ "vcvtLtoF_neon $dst, $src\t# 2L to 2F. KILL $tmp" %}
ins_encode %{
// 2L to 2F
__ umov(rscratch1, $src$$FloatRegister, __ D, 0);
__ scvtfs($dst$$FloatRegister, rscratch1);
__ umov(rscratch1, $src$$FloatRegister, __ D, 1);
__ scvtfs($tmp$$FloatRegister, rscratch1);
__ ins($dst$$FloatRegister, __ S, $tmp$$FloatRegister, 1, 0);
%}
ins_pipe(pipe_slow);
%}
instruct vcvtLtoF_sve(vReg dst, vReg src, vReg tmp) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
match(Set dst (VectorCastL2X src));
effect(TEMP_DEF dst, TEMP tmp);
format %{ "vcvtLtoF_sve $dst, $src\t# KILL $tmp" %}
ins_encode %{
__ sve_scvtf($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister, __ D);
__ sve_vector_narrow($dst$$FloatRegister, __ S,
$dst$$FloatRegister, __ D, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct vcvtLtoD(vReg dst, vReg src) %{
predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
match(Set dst (VectorCastL2X src));
format %{ "vcvtLtoD $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
// 2L to 2D
__ scvtfv(__ T2D, $dst$$FloatRegister, $src$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_scvtf($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister, __ D);
}
%}
ins_pipe(pipe_slow);
%}
// VectorCastF2X
instruct vcvtFtoX_narrow_neon(vReg dst, vReg src) %{
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))) &&
(Matcher::vector_element_basic_type(n) == T_BYTE ||
Matcher::vector_element_basic_type(n) == T_SHORT));
match(Set dst (VectorCastF2X src));
effect(TEMP_DEF dst);
format %{ "vcvtFtoX_narrow_neon $dst, $src" %}
ins_encode %{
// 4F to 4B/4S
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
__ fcvtzs($dst$$FloatRegister, __ T4S, $src$$FloatRegister);
__ neon_vector_narrow($dst$$FloatRegister, bt,
$dst$$FloatRegister, T_INT, length_in_bytes);
%}
ins_pipe(pipe_slow);
%}
instruct vcvtFtoX_narrow_sve(vReg dst, vReg src, vReg tmp) %{
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))) &&
(Matcher::vector_element_basic_type(n) == T_BYTE ||
Matcher::vector_element_basic_type(n) == T_SHORT));
match(Set dst (VectorCastF2X src));
effect(TEMP_DEF dst, TEMP tmp);
format %{ "vcvtFtoX_narrow_sve $dst, $src\t# KILL $tmp" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_fcvtzs($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister, __ S);
__ sve_vector_narrow($dst$$FloatRegister, __ elemType_to_regVariant(bt),
$dst$$FloatRegister, __ S, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct vcvtFtoX(vReg dst, vReg src) %{
predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) >= 4);
match(Set dst (VectorCastF2X src));
format %{ "vcvtFtoX $dst, $src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (bt == T_INT) {
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
// 2F/4F to 2I/4I
__ fcvtzs($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_fcvtzs($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister, __ S);
}
} else if (bt == T_LONG) {
if (UseSVE == 0) {
// 2F to 2L
__ fcvtl($dst$$FloatRegister, __ T2D, $src$$FloatRegister, __ T2S);
__ fcvtzs($dst$$FloatRegister, __ T2D, $dst$$FloatRegister);
} else {
__ sve_vector_extend($dst$$FloatRegister, __ D, $src$$FloatRegister, __ S);
__ sve_fcvtzs($dst$$FloatRegister, __ D, ptrue, $dst$$FloatRegister, __ S);
}
} else {
assert(bt == T_DOUBLE, "unsupported type");
if (length_in_bytes == 16) {
// 2F to 2D
__ fcvtl($dst$$FloatRegister, __ T2D, $src$$FloatRegister, __ T2S);
} else {
assert(UseSVE > 0 && length_in_bytes > 16, "must be");
__ sve_vector_extend($dst$$FloatRegister, __ D, $src$$FloatRegister, __ S);
__ sve_fcvt($dst$$FloatRegister, __ D, ptrue, $dst$$FloatRegister, __ S);
}
}
%}
ins_pipe(pipe_slow);
%}
// VectorCastD2X
instruct vcvtDtoI_neon(vReg dst, vReg src) %{
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n) == T_INT);
match(Set dst (VectorCastD2X src));
effect(TEMP_DEF dst);
format %{ "vcvtDtoI_neon $dst, $src\t# 2D to 2I" %}
ins_encode %{
// 2D to 2I
__ ins($dst$$FloatRegister, __ D, $src$$FloatRegister, 0, 1);
// We can't use fcvtzs(vector, integer) instruction here because we need
// saturation arithmetic. See JDK-8276151.
__ fcvtzdw(rscratch1, $src$$FloatRegister);
__ fcvtzdw(rscratch2, $dst$$FloatRegister);
__ fmovs($dst$$FloatRegister, rscratch1);
__ mov($dst$$FloatRegister, __ S, 1, rscratch2);
%}
ins_pipe(pipe_slow);
%}
instruct vcvtDtoI_sve(vReg dst, vReg src, vReg tmp) %{
predicate(UseSVE > 0 &&
(Matcher::vector_element_basic_type(n) == T_BYTE ||
Matcher::vector_element_basic_type(n) == T_SHORT ||
Matcher::vector_element_basic_type(n) == T_INT));
match(Set dst (VectorCastD2X src));
effect(TEMP_DEF dst, TEMP tmp);
format %{ "vcvtDtoI_sve $dst, $src\t# KILL $tmp" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_fcvtzs($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister, __ D);
__ sve_vector_narrow($dst$$FloatRegister, __ elemType_to_regVariant(bt),
$dst$$FloatRegister, __ D, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct vcvtDtoL(vReg dst, vReg src) %{
predicate(Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst (VectorCastD2X src));
format %{ "vcvtDtoL $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
// 2D to 2L
__ fcvtzs($dst$$FloatRegister, __ T2D, $src$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_fcvtzs($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister, __ D);
}
%}
ins_pipe(pipe_slow);
%}
instruct vcvtDtoF_64b(vReg dst, vReg src) %{
predicate(Matcher::vector_element_basic_type(n) == T_FLOAT &&
Matcher::vector_length_in_bytes(n) == 8);
match(Set dst (VectorCastD2X src));
format %{ "vcvtDtoF_64b $dst, $src\t# 2D to 2F" %}
ins_encode %{
// 2D to 2F
__ fcvtn($dst$$FloatRegister, __ T2S, $src$$FloatRegister, __ T2D);
%}
ins_pipe(pipe_slow);
%}
instruct vcvtDtoF_gt64b(vReg dst, vReg src, vReg tmp) %{
predicate(Matcher::vector_element_basic_type(n) == T_FLOAT &&
Matcher::vector_length_in_bytes(n) > 8);
match(Set dst (VectorCastD2X src));
effect(TEMP_DEF dst, TEMP tmp);
format %{ "vcvtDtoF_gt64b $dst, $src\t# vector > 64 bits. KILL $tmp" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
__ sve_fcvt($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister, __ D);
__ sve_vector_narrow($dst$$FloatRegister, __ S,
$dst$$FloatRegister, __ D, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// VectorCastHF2F
instruct vcvtHFtoF(vReg dst, vReg src) %{
match(Set dst (VectorCastHF2F src));
format %{ "vcvtHFtoF $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
// 4HF to 4F
__ fcvtl($dst$$FloatRegister, __ T4S, $src$$FloatRegister, __ T4H);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_vector_extend($dst$$FloatRegister, __ S, $src$$FloatRegister, __ H);
__ sve_fcvt($dst$$FloatRegister, __ S, ptrue, $dst$$FloatRegister, __ H);
}
%}
ins_pipe(pipe_slow);
%}
// VectorCastF2HF
instruct vcvtFtoHF_neon(vReg dst, vReg src) %{
predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
match(Set dst (VectorCastF2HF src));
format %{ "vcvtFtoHF_neon $dst, $src\t# 4F to 4HF" %}
ins_encode %{
// 4F to 4HF
__ fcvtn($dst$$FloatRegister, __ T4H, $src$$FloatRegister, __ T4S);
%}
ins_pipe(pipe_slow);
%}
instruct vcvtFtoHF_sve(vReg dst, vReg src, vReg tmp) %{
predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
match(Set dst (VectorCastF2HF src));
effect(TEMP_DEF dst, TEMP tmp);
format %{ "vcvtFtoHF_sve $dst, $src\t# KILL $tmp" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
__ sve_fcvt($dst$$FloatRegister, __ H, ptrue, $src$$FloatRegister, __ S);
__ sve_vector_narrow($dst$$FloatRegister, __ H,
$dst$$FloatRegister, __ S, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Replicate ------------------------------------
dnl REPLICATE_INT($1, $2, $3 )
dnl REPLICATE_INT(type, arg_type, size)
define(`REPLICATE_INT', `
instruct replicate$1(vReg dst, $2 src) %{
match(Set dst (Replicate src));
format %{ "replicate$1 $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ dup($dst$$FloatRegister, get_arrangement(this), $src$$Register);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_dup($dst$$FloatRegister, get_reg_variant(this), $src$$Register);
}
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl REPLICATE_FP($1, $2 )
dnl REPLICATE_FP(type, size)
define(`REPLICATE_FP', `
instruct replicate$1(vReg dst, vReg$1 src) %{
match(Set dst (Replicate src));
format %{ "replicate$1 $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ dup($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_cpy($dst$$FloatRegister, __ $2, ptrue, $src$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// replicate from reg
REPLICATE_INT(I, iRegIorL2I)
REPLICATE_INT(L, iRegL)
REPLICATE_FP(F, S, T_FLOAT )
REPLICATE_FP(D, D, T_DOUBLE)
// Replicate a half-precision float value held in a floating point register
instruct replicateHF(vReg dst, vRegF src) %{
predicate(Matcher::vector_element_basic_type(n) == T_SHORT);
match(Set dst (Replicate src));
format %{ "replicateHF $dst, $src\t# replicate half-precision float" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ dup($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
} else { // length_in_bytes must be > 16 and SVE should be enabled
assert(UseSVE > 0, "must be sve");
__ sve_cpy($dst$$FloatRegister, __ H, ptrue, $src$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}
// replicate from imm
instruct replicateI_imm_le128b(vReg dst, immI con) %{
predicate(Matcher::vector_length_in_bytes(n) <= 16 &&
Matcher::is_non_long_integral_vector(n));
match(Set dst (Replicate con));
format %{ "replicateI_imm_le128b $dst, $con\t# vector <= 128 bits" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
int imm = (int)$con$$constant;
if (type2aelembytes(bt) == 1) {
// Refine imm for B
imm = imm & 0xff;
} else if (type2aelembytes(bt) == 2) {
// Refine imm for S
imm = imm & 0xffff;
}
__ mov($dst$$FloatRegister, get_arrangement(this), imm);
%}
ins_pipe(pipe_slow);
%}
instruct replicateB_imm8_gt128b(vReg dst, immI8 con) %{
predicate(Matcher::vector_length_in_bytes(n) > 16 &&
Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst (Replicate con));
format %{ "replicateB_imm8_gt128b $dst, $con\t# vector > 128 bits" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
__ sve_dup($dst$$FloatRegister, __ B, (int)($con$$constant));
%}
ins_pipe(pipe_slow);
%}
instruct replicateI_imm8_gt128b(vReg dst, immI8_shift8 con) %{
predicate(Matcher::vector_length_in_bytes(n) > 16 &&
(Matcher::vector_element_basic_type(n) == T_SHORT ||
Matcher::vector_element_basic_type(n) == T_INT));
match(Set dst (Replicate con));
format %{ "replicateI_imm8_gt128b $dst, $con\t# vector > 128 bits" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
__ sve_dup($dst$$FloatRegister, get_reg_variant(this), (int)($con$$constant));
%}
ins_pipe(pipe_slow);
%}
instruct replicateL_imm_128b(vReg dst, immL con) %{
predicate(Matcher::vector_length_in_bytes(n) == 16);
match(Set dst (Replicate con));
format %{ "replicateL_imm_128b $dst, $con\t# vector > 128 bits" %}
ins_encode %{
__ mov($dst$$FloatRegister, __ T2D, (uint64_t)($con$$constant));
%}
ins_pipe(pipe_slow);
%}
instruct replicateL_imm8_gt128b(vReg dst, immL8_shift8 con) %{
predicate(Matcher::vector_length_in_bytes(n) > 16);
match(Set dst (Replicate con));
format %{ "replicateL_imm8_gt128b $dst, $con\t# vector > 128 bits" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
__ sve_dup($dst$$FloatRegister, __ D, (int)($con$$constant));
%}
ins_pipe(pipe_slow);
%}
// Replicate a 16-bit half precision float value
instruct replicateHF_imm(vReg dst, immH con) %{
match(Set dst (Replicate con));
format %{ "replicateHF_imm $dst, $con\t# replicate immediate half-precision float" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
int imm = (int)($con$$constant) & 0xffff;
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ mov($dst$$FloatRegister, get_arrangement(this), imm);
} else { // length_in_bytes must be > 16 and SVE should be enabled
assert(UseSVE > 0, "must be sve");
__ sve_dup($dst$$FloatRegister, __ H, imm);
}
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Vector insert --------------------------------
// BYTE, SHORT, INT
instruct insertI_le128b(vReg dst, vReg src, iRegIorL2I val, immI idx) %{
predicate(Matcher::vector_length_in_bytes(n) <= 16 &&
(Matcher::vector_element_basic_type(n) == T_BYTE ||
Matcher::vector_element_basic_type(n) == T_SHORT ||
Matcher::vector_element_basic_type(n) == T_INT));
match(Set dst (VectorInsert (Binary src val) idx));
format %{ "insertI_le128b $dst, $src, $val, $idx\t# vector <= 128 bits" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if ($dst$$FloatRegister != $src$$FloatRegister) {
__ orr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src$$FloatRegister, $src$$FloatRegister);
}
__ mov($dst$$FloatRegister, __ elemType_to_regVariant(bt),
(int)($idx$$constant), $val$$Register);
%}
ins_pipe(pipe_slow);
%}
instruct insertI_index_lt32(vReg dst, vReg src, iRegIorL2I val, immI idx,
vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{
predicate(n->in(2)->get_int() < 32 &&
Matcher::vector_length_in_bytes(n) > 16 &&
(Matcher::vector_element_basic_type(n) == T_BYTE ||
Matcher::vector_element_basic_type(n) == T_SHORT ||
Matcher::vector_element_basic_type(n) == T_INT));
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP tmp, TEMP pgtmp, KILL cr);
format %{ "insertI_index_lt32 $dst, $src, $val, $idx\t# vector > 128 bits, index < 31. KILL $tmp, $pgtmp, cr" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this, $src);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_index($tmp$$FloatRegister, size, -16, 1);
__ sve_cmp(Assembler::EQ, $pgtmp$$PRegister, size, ptrue,
$tmp$$FloatRegister, (int)($idx$$constant) - 16);
if ($dst$$FloatRegister != $src$$FloatRegister) {
__ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
}
__ sve_cpy($dst$$FloatRegister, size, $pgtmp$$PRegister, $val$$Register);
%}
ins_pipe(pipe_slow);
%}
instruct insertI_index_ge32(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp1,
vReg tmp2, pRegGov pgtmp, rFlagsReg cr) %{
predicate(n->in(2)->get_int() >= 32 &&
(Matcher::vector_element_basic_type(n) == T_BYTE ||
Matcher::vector_element_basic_type(n) == T_SHORT ||
Matcher::vector_element_basic_type(n) == T_INT));
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP tmp1, TEMP tmp2, TEMP pgtmp, KILL cr);
format %{ "insertI_index_ge32 $dst, $src, $val, $idx\t# index >= 32. KILL $tmp1, $tmp2, $pgtmp, cr" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this, $src);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_index($tmp1$$FloatRegister, size, 0, 1);
__ sve_dup($tmp2$$FloatRegister, size, (int)($idx$$constant));
__ sve_cmp(Assembler::EQ, $pgtmp$$PRegister, size, ptrue,
$tmp1$$FloatRegister, $tmp2$$FloatRegister);
if ($dst$$FloatRegister != $src$$FloatRegister) {
__ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
}
__ sve_cpy($dst$$FloatRegister, size, $pgtmp$$PRegister, $val$$Register);
%}
ins_pipe(pipe_slow);
%}
// LONG
instruct insertL_128b(vReg dst, vReg src, iRegL val, immI idx) %{
predicate(Matcher::vector_length_in_bytes(n) == 16 &&
Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst (VectorInsert (Binary src val) idx));
format %{ "insertL_128b $dst, $src, $val, $idx\t# 2L" %}
ins_encode %{
if ($dst$$FloatRegister != $src$$FloatRegister) {
__ orr($dst$$FloatRegister, __ T16B, $src$$FloatRegister, $src$$FloatRegister);
}
__ mov($dst$$FloatRegister, __ D, (int)($idx$$constant), $val$$Register);
%}
ins_pipe(pipe_slow);
%}
instruct insertL_gt128b(vReg dst, vReg src, iRegL val, immI idx,
vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{
predicate(Matcher::vector_length_in_bytes(n) > 16 &&
Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP tmp, TEMP pgtmp, KILL cr);
format %{ "insertL_gt128b $dst, $src, $val, $idx\t# vector > 128 bits. KILL $tmp, $pgtmp, cr" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
__ sve_index($tmp$$FloatRegister, __ D, -16, 1);
__ sve_cmp(Assembler::EQ, $pgtmp$$PRegister, __ D, ptrue,
$tmp$$FloatRegister, (int)($idx$$constant) - 16);
if ($dst$$FloatRegister != $src$$FloatRegister) {
__ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
}
__ sve_cpy($dst$$FloatRegister, __ D, $pgtmp$$PRegister, $val$$Register);
%}
ins_pipe(pipe_slow);
%}
// FLOAT
instruct insertF_le128b(vReg dst, vReg src, vRegF val, immI idx) %{
predicate(Matcher::vector_length_in_bytes(n) <= 16 &&
Matcher::vector_element_basic_type(n) == T_FLOAT);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP_DEF dst);
format %{ "insertF_le128b $dst, $src, $val, $idx\t# vector <= 128 bits" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if ($dst$$FloatRegister != $src$$FloatRegister) {
__ orr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src$$FloatRegister, $src$$FloatRegister);
}
__ ins($dst$$FloatRegister, __ S, $val$$FloatRegister, (int)($idx$$constant), 0);
%}
ins_pipe(pipe_slow);
%}
instruct insertF_index_lt32(vReg dst, vReg src, vRegF val, immI idx,
pRegGov pgtmp, rFlagsReg cr) %{
predicate(n->in(2)->get_int() < 32 &&
Matcher::vector_length_in_bytes(n) > 16 &&
Matcher::vector_element_basic_type(n) == T_FLOAT);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
format %{ "insertF_index_lt32 $dst, $src, $val, $idx\t# vector > 128 bits, index < 32. KILL $pgtmp, cr" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
__ sve_index($dst$$FloatRegister, __ S, -16, 1);
__ sve_cmp(Assembler::EQ, $pgtmp$$PRegister, __ S, ptrue,
$dst$$FloatRegister, (int)($idx$$constant) - 16);
__ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
__ sve_cpy($dst$$FloatRegister, __ S, $pgtmp$$PRegister, $val$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct insertF_index_ge32(vReg dst, vReg src, vRegF val, immI idx, vReg tmp,
pRegGov pgtmp, rFlagsReg cr) %{
predicate(n->in(2)->get_int() >= 32 &&
Matcher::vector_element_basic_type(n) == T_FLOAT);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP_DEF dst, TEMP tmp, TEMP pgtmp, KILL cr);
format %{ "insertF_index_ge32 $dst, $src, $val, $idx\t# index >= 32. KILL $tmp, $pgtmp, cr" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
__ sve_index($tmp$$FloatRegister, __ S, 0, 1);
__ sve_dup($dst$$FloatRegister, __ S, (int)($idx$$constant));
__ sve_cmp(Assembler::EQ, $pgtmp$$PRegister, __ S, ptrue,
$tmp$$FloatRegister, $dst$$FloatRegister);
__ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
__ sve_cpy($dst$$FloatRegister, __ S, $pgtmp$$PRegister, $val$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// DOUBLE
instruct insertD_128b(vReg dst, vReg src, vRegD val, immI idx) %{
predicate(Matcher::vector_length_in_bytes(n) == 16 &&
Matcher::vector_element_basic_type(n) == T_DOUBLE);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP_DEF dst);
format %{ "insertD_128b $dst, $src, $val, $idx\t# 2D" %}
ins_encode %{
if ($dst$$FloatRegister != $src$$FloatRegister) {
__ orr($dst$$FloatRegister, __ T16B, $src$$FloatRegister, $src$$FloatRegister);
}
__ ins($dst$$FloatRegister, __ D, $val$$FloatRegister, (int)($idx$$constant), 0);
%}
ins_pipe(pipe_slow);
%}
instruct insertD_gt128b(vReg dst, vReg src, vRegD val, immI idx,
pRegGov pgtmp, rFlagsReg cr) %{
predicate(Matcher::vector_length_in_bytes(n) > 16 &&
Matcher::vector_element_basic_type(n) == T_DOUBLE);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
format %{ "insertD_gt128b $dst, $src, $val, $idx\t# vector > 128 bits. KILL $pgtmp, cr" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
__ sve_index($dst$$FloatRegister, __ D, -16, 1);
__ sve_cmp(Assembler::EQ, $pgtmp$$PRegister, __ D, ptrue,
$dst$$FloatRegister, (int)($idx$$constant) - 16);
__ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
__ sve_cpy($dst$$FloatRegister, __ D, $pgtmp$$PRegister, $val$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Extract --------------------------------------
dnl
dnl EXTRACT_INT_SMALL($1, $2, $3, $4, $5 )
dnl EXTRACT_INT_SMALL(type, index, arg_type, insn, size)
define(`EXTRACT_INT_SMALL', `
instruct extract$1_index_lt$2($3 dst, vReg src, immI idx) %{
predicate(n->in(2)->get_int() < $2);
match(Set dst (Extract$1 src idx));
format %{ "extract$1_index_lt$2 $dst, $src, $idx\t# index < $2" %}
ins_encode %{
__ $4($dst$$Register, $src$$FloatRegister, __ $5, (int)($idx$$constant));
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl
dnl EXTRACT_INT_LARGE($1, $2, $3, $4 )
dnl EXTRACT_INT_LARGE(type, index, arg_type, data_type)
define(`EXTRACT_INT_LARGE', `
instruct extract$1_index_ge$2($3 dst, vReg src, immI idx, vReg tmp) %{
predicate(n->in(2)->get_int() >= $2);
match(Set dst (Extract$1 src idx));
effect(TEMP tmp);
format %{ "extract$1_index_ge$2 $dst, $src, $idx\t# index >=$2. KILL $tmp" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
__ sve_extract_integral($dst$$Register, $4, $src$$FloatRegister,
(int)($idx$$constant), $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// BOOLEAN
instruct extractUB_ireg(iRegINoSp dst, vReg src, iRegI idx, vReg tmp) %{
match(Set dst (ExtractUB src idx));
effect(TEMP tmp);
format %{ "extractUB_ireg $dst, $src, $idx\t# variable index. KILL $tmp" %}
ins_encode %{
// Input "src" is a vector of boolean represented as
// bytes with 0x00/0x01 as element values.
// "idx" is expected to be in range.
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
__ mov($tmp$$FloatRegister, __ B, 0, $idx$$Register);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ tbl($tmp$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src$$FloatRegister, 1, $tmp$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_tbl($tmp$$FloatRegister, __ B, $src$$FloatRegister, $tmp$$FloatRegister);
}
__ smov($dst$$Register, $tmp$$FloatRegister, __ B, 0);
%}
ins_pipe(pipe_slow);
%}
EXTRACT_INT_SMALL(UB, 16, iRegINoSp, smov, B)
EXTRACT_INT_LARGE(UB, 16, iRegINoSp, T_BYTE)
// BYTE
EXTRACT_INT_SMALL(B, 16, iRegINoSp, smov, B)
EXTRACT_INT_LARGE(B, 16, iRegINoSp, T_BYTE)
// SHORT
EXTRACT_INT_SMALL(S, 8, iRegINoSp, smov, H)
EXTRACT_INT_LARGE(S, 8, iRegINoSp, T_SHORT)
// INT
EXTRACT_INT_SMALL(I, 4, iRegINoSp, umov, S)
EXTRACT_INT_LARGE(I, 4, iRegINoSp, T_INT)
// LONG
EXTRACT_INT_SMALL(L, 2, iRegLNoSp, umov, D)
EXTRACT_INT_LARGE(L, 2, iRegLNoSp, T_LONG)
dnl
dnl EXTRACT_FP($1, $2, $3, $4, $5 )
dnl EXTRACT_FP(type, insn, index, size, shift)
define(`EXTRACT_FP', `
instruct extract$1(vReg$1 dst, vReg src, immI idx) %{
match(Set dst (Extract$1 src idx));
effect(TEMP_DEF dst);
format %{ "extract$1 $dst, $src, $idx" %}
ins_encode %{
int index = (int)$idx$$constant;
if (index == 0) {
__ $2($dst$$FloatRegister, $src$$FloatRegister);
} else if (index < $3) {
__ ins($dst$$FloatRegister, __ $4, $src$$FloatRegister, 0, index);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
__ sve_ext($dst$$FloatRegister, $dst$$FloatRegister, index << $5);
}
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// FLOAT
EXTRACT_FP(F, fmovs, 4, S, 2)
// DOUBLE
EXTRACT_FP(D, fmovd, 2, D, 3)
// ------------------------------ Vector mask load/store -----------------------
// vector load mask
instruct vloadmask_neon(vReg dst, vReg src) %{
predicate(UseSVE == 0 &&
(Matcher::vector_length_in_bytes(n) == 8 ||
Matcher::vector_length_in_bytes(n) == 16));
match(Set dst (VectorLoadMask src ));
format %{ "vloadmask_neon $dst, $src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
if (bt == T_BYTE) {
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
__ negr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src$$FloatRegister);
} else {
__ uxtl($dst$$FloatRegister, __ T8H, $src$$FloatRegister, __ T8B);
if (type2aelembytes(bt) >= 4) {
__ uxtl($dst$$FloatRegister, __ T4S, $dst$$FloatRegister, __ T4H);
}
if (type2aelembytes(bt) == 8) {
__ uxtl($dst$$FloatRegister, __ T2D, $dst$$FloatRegister, __ T2S);
}
__ negr($dst$$FloatRegister, get_arrangement(this), $dst$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}
instruct vloadmaskB_sve(pReg dst, vReg src, rFlagsReg cr) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst (VectorLoadMask src));
effect(KILL cr);
format %{ "vloadmaskB_sve $dst, $src\t# KILL cr" %}
ins_encode %{
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ B,
ptrue, $src$$FloatRegister, 0);
%}
ins_pipe(pipe_slow);
%}
instruct vloadmask_extend_sve(pReg dst, vReg src, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) != T_BYTE);
match(Set dst (VectorLoadMask src));
effect(TEMP tmp, KILL cr);
format %{ "vloadmask_extend_sve $dst, $src\t# KILL $tmp, cr" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_vector_extend($tmp$$FloatRegister, size, $src$$FloatRegister, __ B);
__ sve_cmp(Assembler::NE, $dst$$PRegister, size, ptrue, $tmp$$FloatRegister, 0);
%}
ins_pipe(pipe_slow);
%}
instruct vloadmaskB_masked(pReg dst, vReg src, pRegGov pg, rFlagsReg cr) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst (VectorLoadMask src pg));
effect(KILL cr);
format %{ "vloadmaskB_masked $dst, $pg, $src\t# KILL cr" %}
ins_encode %{
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ B,
$pg$$PRegister, $src$$FloatRegister, 0);
%}
ins_pipe(pipe_slow);
%}
instruct vloadmask_extend_masked(pReg dst, vReg src, pRegGov pg, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) != T_BYTE);
match(Set dst (VectorLoadMask src pg));
effect(TEMP tmp, KILL cr);
format %{ "vloadmask_extend_masked $dst, $pg, $src\t# KILL $tmp, cr" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_vector_extend($tmp$$FloatRegister, size, $src$$FloatRegister, __ B);
__ sve_cmp(Assembler::NE, $dst$$PRegister, size,
$pg$$PRegister, $tmp$$FloatRegister, 0);
%}
ins_pipe(pipe_slow);
%}
// vector store mask - neon
instruct vstoremaskB_neon(vReg dst, vReg src, immI_1 size) %{
predicate(UseSVE == 0);
match(Set dst (VectorStoreMask src size));
format %{ "vstoremaskB_neon $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == 8 || length_in_bytes == 16, "must be");
__ negr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct vstoremask_narrow_neon(vReg dst, vReg src, immI_gt_1 size) %{
predicate(UseSVE == 0);
match(Set dst (VectorStoreMask src size));
format %{ "vstoremask_narrow_neon $dst, $src" %}
ins_encode %{
int esize = (int)$size$$constant;
if (esize == 2) {
__ xtn($dst$$FloatRegister, __ T8B, $src$$FloatRegister, __ T8H);
} else if (esize == 4) {
__ xtn($dst$$FloatRegister, __ T4H, $src$$FloatRegister, __ T4S);
__ xtn($dst$$FloatRegister, __ T8B, $dst$$FloatRegister, __ T8H);
} else {
assert(esize == 8, "must be");
__ xtn($dst$$FloatRegister, __ T2S, $src$$FloatRegister, __ T2D);
__ xtn($dst$$FloatRegister, __ T4H, $dst$$FloatRegister, __ T4S);
__ xtn($dst$$FloatRegister, __ T8B, $dst$$FloatRegister, __ T8H);
}
__ negr($dst$$FloatRegister, __ T8B, $dst$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// vector store mask - sve
instruct vstoremaskB_sve(vReg dst, pReg src, immI_1 size) %{
predicate(UseSVE > 0);
match(Set dst (VectorStoreMask src size));
format %{ "vstoremaskB_sve $dst, $src" %}
ins_encode %{
__ sve_cpy($dst$$FloatRegister, __ B, $src$$PRegister, 1, false);
%}
ins_pipe(pipe_slow);
%}
instruct vstoremask_narrow_sve(vReg dst, pReg src, immI_gt_1 size, vReg tmp) %{
predicate(UseSVE > 0);
match(Set dst (VectorStoreMask src size));
effect(TEMP_DEF dst, TEMP tmp);
format %{ "vstoremask_narrow_sve $dst, $src\t# KILL $tmp" %}
ins_encode %{
Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant((int)$size$$constant);
__ sve_cpy($dst$$FloatRegister, size, $src$$PRegister, 1, false);
__ sve_vector_narrow($dst$$FloatRegister, __ B,
$dst$$FloatRegister, size, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// Combined rules for vector mask load when the vector element type is not T_BYTE
// VectorLoadMask+LoadVector, and the VectorLoadMask is unpredicated.
instruct vloadmask_loadV(pReg dst, indirect mem, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
match(Set dst (VectorLoadMask (LoadVector mem)));
effect(TEMP tmp, KILL cr);
format %{ "vloadmask_loadV $dst, $mem\t# KILL $tmp, cr" %}
ins_encode %{
// Load mask values which are boolean type, and extend them to the
// defined vector element type. Convert the vector to predicate.
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
ptrue, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
ptrue, $tmp$$FloatRegister, 0);
%}
ins_pipe(pipe_slow);
%}
// VectorLoadMask+LoadVector, and the VectorLoadMask is predicated.
instruct vloadmask_loadV_masked(pReg dst, indirect mem, pRegGov pg,
vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
match(Set dst (VectorLoadMask (LoadVector mem) pg));
effect(TEMP tmp, KILL cr);
format %{ "vloadmask_loadV_masked $dst, $pg, $mem\t# KILL $tmp, cr" %}
ins_encode %{
// Load valid mask values which are boolean type, and extend them to the
// defined vector element type. Convert the vector to predicate.
BasicType bt = Matcher::vector_element_basic_type(this);
loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
$pg$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $tmp$$FloatRegister, 0);
%}
ins_pipe(pipe_slow);
%}
// VectorLoadMask+LoadVectorMasked, and the VectorLoadMask is unpredicated.
instruct vloadmask_loadVMasked(pReg dst, vmemA mem, pRegGov pg, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
match(Set dst (VectorLoadMask (LoadVectorMasked mem pg)));
effect(TEMP tmp, KILL cr);
format %{ "vloadmask_loadVMasked $dst, $mem\t# KILL $tmp, cr" %}
ins_encode %{
// Load mask values which are boolean type, and extend them to the
// defined vector element type. Convert the vector to predicate.
//
// Note that we cannot use "pg" here, since it is the predicate used
// for the vector load with boolean type. But the predicate used in
// the extending "sve_ld1b" is based on the final extended vector type,
// which is the full-sized predicate (ptrue) used in VectorLoadMask.
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
ptrue, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
ptrue, $tmp$$FloatRegister, 0);
%}
ins_pipe(pipe_slow);
%}
// VectorLoadMask+LoadVectorMasked, and the VectorLoadMask is predicated.
instruct vloadmask_loadVMasked_masked(pReg dst, vmemA mem, pRegGov pg1, pRegGov pg2,
vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
match(Set dst (VectorLoadMask (LoadVectorMasked mem pg1) pg2));
effect(TEMP tmp, KILL cr);
format %{ "vloadmask_loadVMasked_masked $dst, $pg2, $mem\t# KILL $tmp, cr" %}
ins_encode %{
// Load valid mask values which are boolean type, and extend them to the
// defined vector element type. Convert the vector to predicate.
//
// Note that we cannot use "pg1" here, since it is the predicate used
// for the vector load with boolean type. But the predicate used in
// the extending "sve_ld1b" is based on the final extended vector type,
// which is the "pg2" used in VectorLoadMask.
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
$pg2$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
$pg2$$PRegister, $tmp$$FloatRegister, 0);
%}
ins_pipe(pipe_slow);
%}
// Combined rules for vector mask store when the vector element type is not T_BYTE
// StoreVector+VectorStoreMask, and the vector size of "src" is equal to the MaxVectorSize.
instruct storeV_vstoremask(indirect mem, pReg src, immI_gt_1 esize, vReg tmp) %{
predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
match(Set mem (StoreVector mem (VectorStoreMask src esize)));
effect(TEMP tmp);
format %{ "storeV_vstoremask $mem, $src\t# KILL $tmp" %}
ins_encode %{
// Convert the valid src predicate to vector, and store the vector elements
// as boolean values.
BasicType bt = Matcher::vector_element_basic_type(this, $src);
assert(type2aelembytes(bt) == (int)$esize$$constant, "unsupported type");
Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
__ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
ptrue, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
ins_pipe(pipe_slow);
%}
// StoreVector+VectorStoreMask, and the vector size of "src" is less than the MaxVectorSize.
instruct storeV_vstoremask_masked(indirect mem, pReg src, immI_gt_1 esize,
vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
match(Set mem (StoreVector mem (VectorStoreMask src esize)));
effect(TEMP tmp, TEMP pgtmp, KILL cr);
format %{ "storeV_vstoremask_masked $mem, $src\t# KILL $tmp, $pgtmp, cr" %}
ins_encode %{
// Convert the valid src predicate to vector, and store the vector elements
// as boolean values.
BasicType bt = Matcher::vector_element_basic_type(this, $src);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
__ sve_gen_mask_imm($pgtmp$$PRegister, bt, Matcher::vector_length(this, $src));
loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
$pgtmp$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
ins_pipe(pipe_slow);
%}
// StoreVectorMasked+VectorStoreMask, and the vector size of "src" is equal to the MaxVectorSize.
instruct storeVMasked_vstoremask(vmemA mem, pReg src, pRegGov pg, immI_gt_1 esize, vReg tmp) %{
predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
match(Set mem (StoreVectorMasked mem (Binary (VectorStoreMask src esize) pg)));
effect(TEMP tmp);
format %{ "storeVMasked_vstoremask $mem, $src\t# KILL $tmp" %}
ins_encode %{
// Convert the valid src predicate to vector, and store the vector elements
// as boolean values.
//
// Note that we cannot use "pg" here, since it is the predicate used
// for the vector store with boolean type. But the predicate used in
// the narrowing "sve_st1b" is based on the "src" vector type, which
// is the full-sized predicate (ptrue) here.
BasicType bt = Matcher::vector_element_basic_type(this, $src);
assert(type2aelembytes(bt) == (int)$esize$$constant, "unsupported type.");
Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
__ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
ptrue, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
ins_pipe(pipe_slow);
%}
// StoreVectorMasked+VectorStoreMask, and the vector size of "src" is less than the MaxVectorSize.
instruct storeVMasked_vstoremask_masked(vmemA mem, pReg src, pRegGov pg, immI_gt_1 esize,
vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
match(Set mem (StoreVectorMasked mem (Binary (VectorStoreMask src esize) pg)));
effect(TEMP tmp, TEMP pgtmp, KILL cr);
format %{ "storeVMasked_vstoremask_masked $mem, $src\t# KILL $tmp, $pgtmp, cr" %}
ins_encode %{
// Convert the valid src predicate to vector, and store the vector elements
// as boolean values.
//
// Note that we cannot use "pg" here, since it is the predicate used for the
// vector store with boolean type. But the predicate used in the narrowing
// "sve_st1b" is based on the "src" vector type, which needed to be generated
// here.
BasicType bt = Matcher::vector_element_basic_type(this, $src);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
__ sve_gen_mask_imm($pgtmp$$PRegister, bt, Matcher::vector_length(this, $src));
loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
$pgtmp$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Vector mask basic OPs ------------------------
dnl
dnl VMASK_BITWISE_OP($1, $2, $3 )
dnl VMASK_BITWISE_OP(type, op_name, insn)
define(`VMASK_BITWISE_OP', `
instruct vmask_$1(pReg pd, pReg pn, pReg pm) %{
predicate(UseSVE > 0);
match(Set pd ($2 pn pm));
format %{ "vmask_$1 $pd, $pn, $pm" %}
ins_encode %{
__ $3($pd$$PRegister, ptrue, $pn$$PRegister, $pm$$PRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VMASK_AND_NOT($1 )
dnl VMASK_AND_NOT(type)
define(`VMASK_AND_NOT', `
instruct vmask_and_not$1(pReg pd, pReg pn, pReg pm, imm$1_M1 m1) %{
predicate(UseSVE > 0);
match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
format %{ "vmask_and_not$1 $pd, $pn, $pm" %}
ins_encode %{
__ sve_bic($pd$$PRegister, ptrue, $pn$$PRegister, $pm$$PRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// vector mask logical ops: and/or/xor/and_not
VMASK_BITWISE_OP(and, AndVMask, sve_and)
VMASK_BITWISE_OP(or, OrVMask, sve_orr)
VMASK_BITWISE_OP(xor, XorVMask, sve_eor)
VMASK_AND_NOT(I)
VMASK_AND_NOT(L)
// vector mask compare
instruct vmaskcmp_neon(vReg dst, vReg src1, vReg src2, immI cond) %{
predicate(UseSVE == 0 &&
(Matcher::vector_length_in_bytes(n) == 8 ||
Matcher::vector_length_in_bytes(n) == 16));
match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
format %{ "vmaskcmp_neon $dst, $src1, $src2, $cond" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
__ neon_compare($dst$$FloatRegister, bt, $src1$$FloatRegister,
$src2$$FloatRegister, condition, /* isQ */ length_in_bytes == 16);
%}
ins_pipe(pipe_slow);
%}
dnl VMASKCMP_ZERO_NEON($1, $2 )
dnl VMASKCMP_ZERO_NEON(type, basic_type)
define(`VMASKCMP_ZERO_NEON', `
instruct vmaskcmp_zero$1_neon(vReg dst, vReg src, imm`$1'0 zero, immI_cmp_cond cond) %{
predicate(UseSVE == 0);
match(Set dst (VectorMaskCmp (Binary src (Replicate zero)) cond));
format %{ "vmaskcmp_zero$1_neon $dst, $src, #0, $cond" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
__ neon_compare_zero($dst$$FloatRegister, ifelse($1, I, Matcher::vector_element_basic_type(this), $2),
$src$$FloatRegister,
condition, /* isQ */ length_in_bytes == 16);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
VMASKCMP_ZERO_NEON(I)
VMASKCMP_ZERO_NEON(L, T_LONG)
VMASKCMP_ZERO_NEON(F, T_FLOAT)
VMASKCMP_ZERO_NEON(D, T_DOUBLE)
instruct vmaskcmp_sve(pReg dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
effect(KILL cr);
format %{ "vmaskcmp_sve $dst, $src1, $src2, $cond\t# KILL cr" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_compare($dst$$PRegister, bt, ptrue, $src1$$FloatRegister,
$src2$$FloatRegister, condition);
%}
ins_pipe(pipe_slow);
%}
dnl
dnl VMASKCMP_SVE_IMM($1 , $2 , $3 , $4 )
dnl VMASKCMP_SVE_IMM(element_size, element_type, type_imm, type_condition)
define(`VMASKCMP_SVE_IMM', `
instruct vmask$3_imm$1_sve(pReg dst, vReg src, $2 imm, immI_$3_cond cond, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (VectorMaskCmp (Binary src (Replicate imm)) cond));
effect(KILL cr);
format %{ "vmask$3_imm$1_sve $dst, $src, $imm, $cond\t# KILL cr" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_cmp(condition, $dst$$PRegister, get_reg_variant(in(operand_index($src))),
ptrue, $src$$FloatRegister, (int)$imm$$constant);
%}
ins_pipe(pipe_slow);
%}')dnl
VMASKCMP_SVE_IMM(I, immI5, cmp)
VMASKCMP_SVE_IMM(I, immIU7, cmpU)
VMASKCMP_SVE_IMM(L, immL5, cmp)
VMASKCMP_SVE_IMM(L, immLU7, cmpU)
instruct vmaskcmp_masked(pReg dst, vReg src1, vReg src2, immI cond,
pRegGov pg, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg)));
effect(KILL cr);
format %{ "vmaskcmp_masked $dst, $pg, $src1, $src2, $cond\t# KILL cr" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_compare($dst$$PRegister, bt, $pg$$PRegister, $src1$$FloatRegister,
$src2$$FloatRegister, condition);
%}
ins_pipe(pipe_slow);
%}
// vector mask cast
instruct vmaskcast_same_esize_neon(vReg dst_src) %{
predicate(UseSVE == 0 &&
Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)) &&
(Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16));
match(Set dst_src (VectorMaskCast dst_src));
ins_cost(0);
format %{ "vmaskcast_same_esize_neon $dst_src\t# do nothing" %}
ins_encode(/* empty encoding */);
ins_pipe(pipe_class_empty);
%}
instruct vmaskcast_extend_neon(vReg dst, vReg src) %{
predicate(UseSVE == 0 &&
Matcher::vector_length_in_bytes(n) > Matcher::vector_length_in_bytes(n->in(1)));
match(Set dst (VectorMaskCast src));
format %{ "vmaskcast_extend_neon $dst, $src" %}
ins_encode %{
BasicType dst_bt = Matcher::vector_element_basic_type(this);
if (is_floating_point_type(dst_bt)) {
dst_bt = (dst_bt == T_FLOAT) ? T_INT : T_LONG;
}
uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
BasicType src_bt = Matcher::vector_element_basic_type(this, $src);
if (is_floating_point_type(src_bt)) {
src_bt = (src_bt == T_FLOAT) ? T_INT : T_LONG;
}
__ neon_vector_extend($dst$$FloatRegister, dst_bt, length_in_bytes_dst,
$src$$FloatRegister, src_bt);
%}
ins_pipe(pipe_slow);
%}
instruct vmaskcast_narrow_neon(vReg dst, vReg src) %{
predicate(UseSVE == 0 &&
Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)));
match(Set dst (VectorMaskCast src));
format %{ "vmaskcast_narrow_neon $dst, $src" %}
ins_encode %{
BasicType dst_bt = Matcher::vector_element_basic_type(this);
if (is_floating_point_type(dst_bt)) {
dst_bt = (dst_bt == T_FLOAT) ? T_INT : T_LONG;
}
BasicType src_bt = Matcher::vector_element_basic_type(this, $src);
if (is_floating_point_type(src_bt)) {
src_bt = (src_bt == T_FLOAT) ? T_INT : T_LONG;
}
uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
__ neon_vector_narrow($dst$$FloatRegister, dst_bt,
$src$$FloatRegister, src_bt, length_in_bytes_src);
%}
ins_pipe(pipe_slow);
%}
instruct vmaskcast_same_esize_sve(pReg dst_src) %{
predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
match(Set dst_src (VectorMaskCast dst_src));
ins_cost(0);
format %{ "vmaskcast_same_esize_sve $dst_src\t# do nothing" %}
ins_encode(/* empty encoding */);
ins_pipe(pipe_class_empty);
%}
instruct vmaskcast_extend_sve(pReg dst, pReg src) %{
predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n) > Matcher::vector_length_in_bytes(n->in(1)));
match(Set dst (VectorMaskCast src));
format %{ "vmaskcast_extend_sve $dst, $src" %}
ins_encode %{
uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
assert(length_in_bytes_dst == 2 * length_in_bytes_src ||
length_in_bytes_dst == 4 * length_in_bytes_src ||
length_in_bytes_dst == 8 * length_in_bytes_src, "invalid vector length");
__ sve_vmaskcast_extend($dst$$PRegister, $src$$PRegister,
length_in_bytes_dst, length_in_bytes_src);
%}
ins_pipe(pipe_slow);
%}
instruct vmaskcast_narrow_sve(pReg dst, pReg src, pReg ptmp) %{
predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)));
match(Set dst (VectorMaskCast src));
effect(TEMP_DEF dst, TEMP ptmp);
format %{ "vmaskcast_narrow_sve $dst, $src\t# KILL $ptmp" %}
ins_encode %{
uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
assert(length_in_bytes_dst * 2 == length_in_bytes_src ||
length_in_bytes_dst * 4 == length_in_bytes_src ||
length_in_bytes_dst * 8 == length_in_bytes_src, "invalid vector length");
__ sve_vmaskcast_narrow($dst$$PRegister, $src$$PRegister, $ptmp$$PRegister,
length_in_bytes_dst, length_in_bytes_src);
%}
ins_pipe(pipe_slow);
%}
// vector mask reinterpret
instruct vmask_reinterpret_same_esize(pReg dst_src) %{
predicate(UseSVE > 0 &&
Matcher::vector_length(n) == Matcher::vector_length(n->in(1)) &&
Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
match(Set dst_src (VectorReinterpret dst_src));
ins_cost(0);
format %{ "vmask_reinterpret_same_esize $dst_src\t# do nothing" %}
ins_encode(/* empty encoding */);
ins_pipe(pipe_class_empty);
%}
instruct vmask_reinterpret_diff_esize(pReg dst, pReg src, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
match(Set dst (VectorReinterpret src));
effect(TEMP tmp, KILL cr);
format %{ "vmask_reinterpret_diff_esize $dst, $src\t# KILL $tmp, cr" %}
ins_encode %{
BasicType from_bt = Matcher::vector_element_basic_type(this, $src);
Assembler::SIMD_RegVariant from_size = __ elemType_to_regVariant(from_bt);
BasicType to_bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
__ sve_cpy($tmp$$FloatRegister, from_size, $src$$PRegister, -1, false);
__ sve_cmp(Assembler::EQ, $dst$$PRegister, to_size, ptrue, $tmp$$FloatRegister, -1);
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Vector mask reductions -----------------------
// true count
instruct vmask_truecount_neon(iRegINoSp dst, vReg src, vReg tmp) %{
predicate(UseSVE == 0);
match(Set dst (VectorMaskTrueCount src));
effect(TEMP tmp);
format %{ "vmask_truecount_neon $dst, $src\t# KILL $tmp" %}
ins_encode %{
// Input "src" is a vector of boolean represented as bytes with
// 0x00/0x01 as element values.
BasicType bt = Matcher::vector_element_basic_type(this, $src);
assert(bt == T_BOOLEAN, "unsupported type");
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
__ addv($tmp$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src$$FloatRegister);
__ umov($dst$$Register, $tmp$$FloatRegister, __ B, 0);
%}
ins_pipe(pipe_slow);
%}
instruct vmask_truecount_sve(iRegINoSp dst, pReg src) %{
predicate(UseSVE > 0);
match(Set dst (VectorMaskTrueCount src));
format %{ "vmask_truecount_sve $dst, $src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
__ sve_cntp($dst$$Register, __ elemType_to_regVariant(bt),
ptrue, $src$$PRegister);
%}
ins_pipe(pipe_slow);
%}
// Combined rule for VectorMaskTrueCount (VectorStoreMask) when the vector element type is not T_BYTE.
instruct vstoremask_truecount_neon(iRegINoSp dst, vReg src, immI_gt_1 size, vReg vtmp) %{
match(Set dst (VectorMaskTrueCount (VectorStoreMask src size)));
effect(TEMP vtmp);
format %{ "vstoremask_truecount_neon $dst, $src\t# KILL $vtmp" %}
ins_encode %{
// Input "src" is a vector mask represented as lanes with
// 0/-1 as element values.
uint esize = (uint)$size$$constant;
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
Assembler::SIMD_Arrangement arrangement = Assembler::esize2arrangement(esize,
/* isQ */ length_in_bytes == 16);
if (arrangement == __ T2D || arrangement == __ T2S) {
__ addpv($vtmp$$FloatRegister, arrangement, $src$$FloatRegister, $src$$FloatRegister);
} else {
__ addv($vtmp$$FloatRegister, arrangement, $src$$FloatRegister);
}
__ smov($dst$$Register, $vtmp$$FloatRegister, __ B, 0);
__ neg($dst$$Register, $dst$$Register);
%}
ins_pipe(pipe_slow);
%}
// first true
instruct vmask_firsttrue_neon(iRegINoSp dst, vReg src) %{
predicate(UseSVE == 0);
match(Set dst (VectorMaskFirstTrue src));
format %{ "vmask_firsttrue_neon $dst, $src" %}
ins_encode %{
// Returns the index of the first active lane of the
// vector mask, or VLENGTH if no lane is active.
//
// Input "src" is a vector of boolean represented as
// bytes with 0x00/0x01 as element values.
//
// Computed by reversing the bits and counting the leading
// zero bytes.
BasicType bt = Matcher::vector_element_basic_type(this, $src);
assert(bt == T_BOOLEAN, "unsupported type");
uint vlength = Matcher::vector_length(this, $src);
if (vlength <= 8) {
__ fmovd($dst$$Register, $src$$FloatRegister);
if (vlength == 2 || vlength == 4) {
// Special handling for 2B or 4B cases:
// Vector mask is moved to a 64-bit general register, but only the low 16/32 bits are
// significant for 2B/4B cases. We initialize the 16th/32nd bit as bit 1, so as to generate
// the expected result (i.e. VLENGTH) for the case that all lanes are zero.
__ orr($dst$$Register, $dst$$Register, vlength == 2 ? 0x10000 : 0x100000000);
}
__ rbit($dst$$Register, $dst$$Register);
__ clz($dst$$Register, $dst$$Register);
__ lsrw($dst$$Register, $dst$$Register, 3);
} else {
assert(vlength == 16, "must be");
Label FIRST_TRUE_INDEX;
// Try to compute the result from lower 64 bits.
__ fmovd($dst$$Register, $src$$FloatRegister);
__ movw(rscratch1, zr);
__ cbnz($dst$$Register, FIRST_TRUE_INDEX);
// Compute the result from the higher 64 bits.
__ fmovhid($dst$$Register, $src$$FloatRegister);
__ movw(rscratch1, 8);
// Reverse the bits and count the leading zero bytes.
__ bind(FIRST_TRUE_INDEX);
__ rbit($dst$$Register, $dst$$Register);
__ clz($dst$$Register, $dst$$Register);
__ addw($dst$$Register, rscratch1, $dst$$Register, Assembler::LSR, 3);
}
%}
ins_pipe(pipe_slow);
%}
// Return the index of the first mask lane that is set, or vector length if none of
// them are set.
instruct vmask_firsttrue_sve(iRegINoSp dst, pReg src, pReg ptmp) %{
predicate(UseSVE > 0);
match(Set dst (VectorMaskFirstTrue src));
effect(TEMP ptmp);
format %{ "vmask_firsttrue_sve $dst, $src\t# KILL $ptmp" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_brkb($ptmp$$PRegister, ptrue, $src$$PRegister, false);
__ sve_cntp($dst$$Register, __ elemType_to_regVariant(bt), ptrue, $ptmp$$PRegister);
%}
ins_pipe(pipe_slow);
%}
instruct vmask_firsttrue_masked(iRegINoSp dst, pReg src, pReg pg, pReg ptmp) %{
predicate(UseSVE > 0);
match(Set dst (VectorMaskFirstTrue src pg));
effect(TEMP ptmp);
format %{ "vmask_firsttrue_masked $dst, $pg, $src\t# KILL $ptmp" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
__ sve_brkb($ptmp$$PRegister, $pg$$PRegister, $src$$PRegister, false);
__ sve_cntp($dst$$Register, __ elemType_to_regVariant(bt), ptrue, $ptmp$$PRegister);
%}
ins_pipe(pipe_slow);
%}
// last true
instruct vmask_lasttrue_neon(iRegINoSp dst, vReg src) %{
predicate(UseSVE == 0);
match(Set dst (VectorMaskLastTrue src));
format %{ "vmask_lasttrue_neon $dst, $src" %}
ins_encode %{
// Returns the index of the last active lane of the
// vector mask, or -1 if no lane is active.
//
// Input "src" is a vector of boolean represented as
// bytes with 0x00/0x01 as element values.
BasicType bt = Matcher::vector_element_basic_type(this, $src);
assert(bt == T_BOOLEAN, "unsupported type");
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
if (length_in_bytes <= 8) {
// Computed by counting the leading zero bytes and
// subtracting it by 7 (VLENGTH - 1).
__ fmovd($dst$$Register, $src$$FloatRegister);
__ clz($dst$$Register, $dst$$Register);
__ movw(rscratch1, 7);
__ subw($dst$$Register, rscratch1, $dst$$Register, Assembler::LSR, 3);
} else {
assert(length_in_bytes == 16, "must be");
Label LAST_TRUE_INDEX;
// Try to compute the result from higher 64 bits.
__ fmovhid($dst$$Register, $src$$FloatRegister);
__ movw(rscratch1, 16 - 1);
__ cbnz($dst$$Register, LAST_TRUE_INDEX);
// Compute the result from the lower 64 bits.
__ fmovd($dst$$Register, $src$$FloatRegister);
__ movw(rscratch1, 8 - 1);
// Count the leading zero bytes and subtract it by 15 (VLENGTH - 1).
__ bind(LAST_TRUE_INDEX);
__ clz($dst$$Register, $dst$$Register);
__ subw($dst$$Register, rscratch1, $dst$$Register, Assembler::LSR, 3);
}
%}
ins_pipe(pipe_slow);
%}
instruct vmask_lasttrue_sve(iRegINoSp dst, pReg src, pReg ptmp) %{
predicate(UseSVE > 0);
match(Set dst (VectorMaskLastTrue src));
effect(TEMP ptmp);
format %{ "vmask_lasttrue_sve $dst, $src\t# KILL $ptmp" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
__ sve_vmask_lasttrue($dst$$Register, bt, $src$$PRegister, $ptmp$$PRegister);
%}
ins_pipe(pipe_slow);
%}
// tolong
instruct vmask_tolong_neon(iRegLNoSp dst, vReg src) %{
predicate(UseSVE == 0);
match(Set dst (VectorMaskToLong src));
format %{ "vmask_tolong_neon $dst, $src" %}
ins_encode %{
// Input "src" is a vector of boolean represented as
// bytes with 0x00/0x01 as element values.
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
if (length_in_bytes <= 8) {
__ fmovd($dst$$Register, $src$$FloatRegister);
__ bytemask_compress($dst$$Register);
} else {
assert(length_in_bytes == 16, "must be");
__ umov($dst$$Register, $src$$FloatRegister, __ D, 0);
__ umov(rscratch1, $src$$FloatRegister, __ D, 1);
__ bytemask_compress($dst$$Register);
__ bytemask_compress(rscratch1);
__ orr($dst$$Register, $dst$$Register, rscratch1, Assembler::LSL, 8);
}
%}
ins_pipe(pipe_slow);
%}
instruct vmask_tolong_sve(iRegLNoSp dst, pReg src, vReg tmp1, vReg tmp2) %{
predicate(UseSVE > 0);
match(Set dst (VectorMaskToLong src));
effect(TEMP tmp1, TEMP tmp2);
format %{ "vmask_tolong_sve $dst, $src\t# KILL $tmp1, $tmp2" %}
ins_encode %{
__ sve_vmask_tolong($dst$$Register, $src$$PRegister,
Matcher::vector_element_basic_type(this, $src),
Matcher::vector_length(this, $src),
$tmp1$$FloatRegister, $tmp2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// fromlong
instruct vmask_fromlong(pReg dst, iRegL src, vReg tmp1, vReg tmp2) %{
match(Set dst (VectorLongToMask src));
effect(TEMP tmp1, TEMP tmp2);
format %{ "vmask_fromlong $dst, $src\t# vector (sve2). KILL $tmp1, $tmp2" %}
ins_encode %{
__ sve_vmask_fromlong($dst$$PRegister, $src$$Register,
Matcher::vector_element_basic_type(this),
Matcher::vector_length(this),
$tmp1$$FloatRegister, $tmp2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Vector mask generation -----------------------
// maskAll
dnl
dnl VMASKALL_IMM($1, $2 )
dnl VMASKALL_IMM(type, var_type)
define(`VMASKALL_IMM', `
instruct vmaskAll_imm$1(pReg dst, imm$1 src, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (MaskAll src));
effect(KILL cr);
format %{ "vmaskAll_imm$1 $dst, $src\t# KILL cr" %}
ins_encode %{
$2 con = ($2)$src$$constant;
if (con == 0) {
__ sve_pfalse($dst$$PRegister);
} else {
assert(con == -1, "invalid constant value for mask");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_gen_mask_imm($dst$$PRegister, bt, Matcher::vector_length(this));
}
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VMASKALL($1, $2 )
dnl VMASKALL(type, arg_type)
define(`VMASKALL', `
instruct vmaskAll$1(pReg dst, $2 src, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (MaskAll src));
effect(TEMP tmp, KILL cr);
format %{ "vmaskAll$1 $dst, $src\t# KILL $tmp, cr" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_dup($tmp$$FloatRegister, size, $src$$Register);
__ sve_cmp(Assembler::NE, $dst$$PRegister, size, ptrue, $tmp$$FloatRegister, 0);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl VMASKALL_PREDICATE($1, $2 )
dnl VMASKALL_PREDICATE(type, arg_type)
define(`VMASKALL_PREDICATE', `
instruct vmaskAll$1_masked(pReg dst, $2 src, pRegGov pg, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (MaskAll src pg));
effect(TEMP tmp, KILL cr);
format %{ "vmaskAll$1_masked $dst, $pg, $src\t# KILL $tmp, cr" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_dup($tmp$$FloatRegister, size, $src$$Register);
__ sve_cmp(Assembler::NE, $dst$$PRegister, size,
$pg$$PRegister, $tmp$$FloatRegister, 0);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
VMASKALL_IMM(I, int)
VMASKALL(I, iRegIorL2I)
VMASKALL_PREDICATE(I, iRegIorL2I)
VMASKALL_IMM(L, long)
VMASKALL(L, iRegL)
VMASKALL_PREDICATE(L, iRegL)
// vetcor mask generation
instruct vmask_gen_I(pReg pd, iRegIorL2I src, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set pd (VectorMaskGen (ConvI2L src)));
effect(KILL cr);
format %{ "vmask_gen_I $pd, $src\t# KILL cr" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_whileltw($pd$$PRegister, __ elemType_to_regVariant(bt), zr, $src$$Register);
%}
ins_pipe(pipe_class_default);
%}
instruct vmask_gen_L(pReg pd, iRegL src, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set pd (VectorMaskGen src));
effect(KILL cr);
format %{ "vmask_gen_L $pd, $src\t# KILL cr" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_whilelt($pd$$PRegister, __ elemType_to_regVariant(bt), zr, $src$$Register);
%}
ins_pipe(pipe_slow);
%}
instruct vmask_gen_imm(pReg pd, immL con, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set pd (VectorMaskGen con));
effect(KILL cr);
format %{ "vmask_gen_imm $pd, $con\t# KILL cr" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_gen_mask_imm($pd$$PRegister, bt, (uint)($con$$constant));
%}
ins_pipe(pipe_slow);
%}
instruct vmask_gen_sub(pReg pd, iRegL src1, iRegL src2, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set pd (VectorMaskGen (SubL src1 src2)));
effect(KILL cr);
format %{ "vmask_gen_sub $pd, $src2, $src1\t# KILL cr" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_whilelt($pd$$PRegister, __ elemType_to_regVariant(bt), $src2$$Register, $src1$$Register);
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Popcount vector ------------------------------
// vector popcount - INT
instruct vpopcountI(vReg dst, vReg src) %{
match(Set dst (PopCountVI src));
format %{ "vpopcountI $dst, $src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (bt == T_BYTE) {
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ cnt($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_cnt($dst$$FloatRegister, __ B, ptrue, $src$$FloatRegister);
}
} else {
assert(bt == T_SHORT || bt == T_INT, "unsupported");
if (UseSVE == 0) {
assert(length_in_bytes == 8 || length_in_bytes == 16, "unsupported");
__ cnt($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src$$FloatRegister);
__ uaddlp($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$dst$$FloatRegister);
if (bt == T_INT) {
__ uaddlp($dst$$FloatRegister, length_in_bytes == 16 ? __ T8H : __ T4H,
$dst$$FloatRegister);
}
} else {
__ sve_cnt($dst$$FloatRegister, __ elemType_to_regVariant(bt),
ptrue, $src$$FloatRegister);
}
}
%}
ins_pipe(pipe_slow);
%}
// vector popcount - LONG
instruct vpopcountL(vReg dst, vReg src) %{
match(Set dst (PopCountVL src));
format %{ "vpopcountL $dst, $src" %}
ins_encode %{
if (UseSVE == 0) {
__ cnt($dst$$FloatRegister, __ T16B, $src$$FloatRegister);
__ uaddlp($dst$$FloatRegister, __ T16B, $dst$$FloatRegister);
__ uaddlp($dst$$FloatRegister, __ T8H, $dst$$FloatRegister);
__ uaddlp($dst$$FloatRegister, __ T4S, $dst$$FloatRegister);
} else {
__ sve_cnt($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}
// vector popcount - predicated
UNARY_OP_PREDICATE(vpopcountI, PopCountVI, sve_cnt)
instruct vpopcountL_masked(vReg dst_src, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src (PopCountVL dst_src pg));
format %{ "vpopcountL_masked $dst_src, $pg, $dst_src" %}
ins_encode %{
__ sve_cnt($dst_src$$FloatRegister, __ D,
$pg$$PRegister, $dst_src$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Vector blend ---------------------------------
instruct vblend_neon(vReg dst, vReg src1, vReg src2) %{
predicate(UseSVE == 0);
match(Set dst (VectorBlend (Binary src1 src2) dst));
format %{ "vblend_neon $dst, $src1, $src2" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == 8 || length_in_bytes == 16, "must be");
__ bsl($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src2$$FloatRegister, $src1$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct vblend_sve(vReg dst, vReg src1, vReg src2, pReg pg) %{
predicate(UseSVE > 0);
match(Set dst (VectorBlend (Binary src1 src2) pg));
format %{ "vblend_sve $dst, $pg, $src1, $src2" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_sel($dst$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $src2$$FloatRegister, $src1$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Vector round ---------------------------------
// vector Math.round
instruct vround_le128b(vReg dst, vReg src, vReg tmp1, vReg tmp2,
vReg tmp3, rFlagsReg cr) %{
predicate(Matcher::vector_length_in_bytes(n) <= 16);
match(Set dst (RoundVF src));
match(Set dst (RoundVD src));
effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
format %{ "vround_le128b $dst, $src\t# vector <= 128 bits. KILL $tmp1, $tmp2, $tmp3, cr" %}
ins_encode %{
__ vector_round_neon($dst$$FloatRegister, $src$$FloatRegister,
$tmp1$$FloatRegister, $tmp2$$FloatRegister,
$tmp3$$FloatRegister, get_arrangement(this));
%}
ins_pipe(pipe_slow);
%}
instruct vround_gt128b(vReg dst, vReg src, vReg tmp1, vReg tmp2,
pRegGov pgtmp, rFlagsReg cr) %{
predicate(Matcher::vector_length_in_bytes(n) > 16);
match(Set dst (RoundVF src));
match(Set dst (RoundVD src));
effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP pgtmp, KILL cr);
format %{ "vround_gt128b $dst, $src\t# vector > 128 bits. KILL $tmp1, $tmp2, $pgtmp, cr" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this);
__ vector_round_sve($dst$$FloatRegister, $src$$FloatRegister,
$tmp1$$FloatRegister, $tmp2$$FloatRegister,
$pgtmp$$PRegister, __ elemType_to_regVariant(bt));
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ RoundDouble ----------------------------------
// vector Math.rint, floor, ceil
instruct vroundD(vReg dst, vReg src, immI rmode) %{
predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
match(Set dst (RoundDoubleModeV src rmode));
format %{ "vroundD $dst, $src, $rmode" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
switch ($rmode$$constant) {
case RoundDoubleModeNode::rmode_rint:
__ frintn($dst$$FloatRegister, __ T2D, $src$$FloatRegister);
break;
case RoundDoubleModeNode::rmode_floor:
__ frintm($dst$$FloatRegister, __ T2D, $src$$FloatRegister);
break;
case RoundDoubleModeNode::rmode_ceil:
__ frintp($dst$$FloatRegister, __ T2D, $src$$FloatRegister);
break;
default:
assert(false, "unsupported");
ShouldNotReachHere();
}
} else {
assert(UseSVE > 0, "must be sve");
switch ($rmode$$constant) {
case RoundDoubleModeNode::rmode_rint:
__ sve_frintn($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
break;
case RoundDoubleModeNode::rmode_floor:
__ sve_frintm($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
break;
case RoundDoubleModeNode::rmode_ceil:
__ sve_frintp($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
break;
default:
assert(false, "unsupported");
ShouldNotReachHere();
}
}
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ VectorTest -----------------------------------
// anytrue
instruct vtest_anytrue_neon(rFlagsReg cr, vReg src1, vReg src2, vReg tmp) %{
predicate(UseSVE == 0 &&
static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
match(Set cr (VectorTest src1 src2));
effect(TEMP tmp);
format %{ "vtest_anytrue_neon $src1\t# KILL $tmp" %}
ins_encode %{
// No need to use src2.
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src1);
assert(length_in_bytes == 8 || length_in_bytes == 16, "must be");
__ addv($tmp$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B, $src1$$FloatRegister);
__ umov(rscratch1, $tmp$$FloatRegister, __ B, 0);
__ cmpw(rscratch1, zr);
%}
ins_pipe(pipe_slow);
%}
instruct vtest_anytrue_sve(rFlagsReg cr, pReg src1, pReg src2) %{
predicate(UseSVE > 0 &&
static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
match(Set cr (VectorTest src1 src2));
format %{ "vtest_anytrue_sve $src1" %}
ins_encode %{
// "src2" is not used for sve.
__ sve_ptest(ptrue, $src1$$PRegister);
%}
ins_pipe(pipe_slow);
%}
// alltrue
instruct vtest_alltrue_neon(rFlagsReg cr, vReg src1, vReg src2, vReg tmp) %{
predicate(UseSVE == 0 &&
static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
match(Set cr (VectorTest src1 src2));
effect(TEMP tmp);
format %{ "vtest_alltrue_neon $src1\t# KILL $tmp" %}
ins_encode %{
// No need to use src2.
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src1);
assert(length_in_bytes == 8 || length_in_bytes == 16, "must be");
__ uminv($tmp$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B, $src1$$FloatRegister);
__ umov(rscratch1, $tmp$$FloatRegister, __ B, 0);
__ cmpw(rscratch1, 0xff);
%}
ins_pipe(pipe_slow);
%}
instruct vtest_alltrue_sve(rFlagsReg cr, pReg src1, pReg src2, pReg ptmp) %{
predicate(UseSVE > 0 &&
static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
match(Set cr (VectorTest src1 src2));
effect(TEMP ptmp);
format %{ "vtest_alltrue_sve $src1, $src2\t# KILL $ptmp" %}
ins_encode %{
__ sve_eors($ptmp$$PRegister, ptrue, $src1$$PRegister, $src2$$PRegister);
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Vector rearrange -----------------------------
instruct rearrange_HSD_neon(vReg dst, vReg src, vReg shuffle, vReg tmp) %{
predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n) != T_BYTE);
match(Set dst (VectorRearrange src shuffle));
effect(TEMP_DEF dst, TEMP tmp);
format %{ "rearrange_HSD_neon $dst, $src, $shuffle\t# vector (4H/8H/2S/4S/2D). KILL $tmp" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == 8 || length_in_bytes == 16, "must be");
__ neon_rearrange_hsd($dst$$FloatRegister, $src$$FloatRegister,
$shuffle$$FloatRegister, $tmp$$FloatRegister,
bt, length_in_bytes == 16);
%}
ins_pipe(pipe_slow);
%}
instruct rearrange(vReg dst, vReg src, vReg shuffle) %{
predicate(UseSVE > 0 || Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst (VectorRearrange src shuffle));
format %{ "rearrange $dst, $src, $shuffle" %}
ins_encode %{
BasicType bt_dst = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (bt_dst == T_BYTE && VM_Version::use_neon_for_vector(length_in_bytes)) {
__ tbl($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src$$FloatRegister, 1, $shuffle$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
BasicType bt_src = Matcher::vector_element_basic_type(this, $src);
__ sve_tbl($dst$$FloatRegister, __ elemType_to_regVariant(bt_src),
$src$$FloatRegister, $shuffle$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Vector Load Gather ---------------------------
instruct gather_loadS(vReg dst, indirect mem, vReg idx) %{
predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n)) == 4);
match(Set dst (LoadVectorGather mem idx));
format %{ "gather_loadS $dst, $mem, $idx\t# vector (sve)" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_ld1w_gather($dst$$FloatRegister, ptrue,
as_Register($mem$$base), $idx$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct gather_loadD(vReg dst, indirect mem, vReg idx, vReg tmp) %{
predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n)) == 8);
match(Set dst (LoadVectorGather mem idx));
effect(TEMP tmp);
format %{ "gather_loadD $dst, $mem, $idx\t# vector (sve). KILL $tmp" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_uunpklo($tmp$$FloatRegister, __ D, $idx$$FloatRegister);
__ sve_ld1d_gather($dst$$FloatRegister, ptrue, as_Register($mem$$base),
$tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct gather_loadS_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n)) == 4);
match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
format %{ "gather_loadS_masked $dst, $pg, $mem, $idx" %}
ins_encode %{
__ sve_ld1w_gather($dst$$FloatRegister, $pg$$PRegister,
as_Register($mem$$base), $idx$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct gather_loadD_masked(vReg dst, indirect mem, vReg idx, pRegGov pg, vReg tmp) %{
predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n)) == 8);
match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
effect(TEMP tmp);
format %{ "gather_loadD_masked $dst, $pg, $mem, $idx\t# KILL $tmp" %}
ins_encode %{
__ sve_uunpklo($tmp$$FloatRegister, __ D, $idx$$FloatRegister);
__ sve_ld1d_gather($dst$$FloatRegister, $pg$$PRegister,
as_Register($mem$$base), $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Vector Store Scatter -------------------------
instruct scatter_storeS(indirect mem, vReg src, vReg idx) %{
predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 4);
match(Set mem (StoreVectorScatter mem (Binary src idx)));
format %{ "scatter_storeS $mem, $idx, $src\t# vector (sve)" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_st1w_scatter($src$$FloatRegister, ptrue,
as_Register($mem$$base), $idx$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct scatter_storeD(indirect mem, vReg src, vReg idx, vReg tmp) %{
predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 8);
match(Set mem (StoreVectorScatter mem (Binary src idx)));
effect(TEMP tmp);
format %{ "scatter_storeD $mem, $idx, $src\t# vector (sve). KILL $tmp" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_uunpklo($tmp$$FloatRegister, __ D, $idx$$FloatRegister);
__ sve_st1d_scatter($src$$FloatRegister, ptrue,
as_Register($mem$$base), $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct scatter_storeS_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 4);
match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
format %{ "scatter_storeS_masked $mem, $pg, $idx, $src" %}
ins_encode %{
__ sve_st1w_scatter($src$$FloatRegister, $pg$$PRegister,
as_Register($mem$$base), $idx$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
instruct scatter_storeD_masked(indirect mem, vReg src, vReg idx, pRegGov pg, vReg tmp) %{
predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 8);
match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
effect(TEMP tmp);
format %{ "scatter_storeD_masked $mem, $pg, $idx, $src\t# KILL $tmp" %}
ins_encode %{
__ sve_uunpklo($tmp$$FloatRegister, __ D, $idx$$FloatRegister);
__ sve_st1d_scatter($src$$FloatRegister, $pg$$PRegister,
as_Register($mem$$base), $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ CountLeadingZerosV ---------------------------
instruct vcountLeadingZeros(vReg dst, vReg src) %{
match(Set dst (CountLeadingZerosV src));
format %{ "vcountLeadingZeros $dst, $src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (bt == T_LONG) {
if (UseSVE == 0) {
__ umov(rscratch1, $src$$FloatRegister, __ D, 0);
__ clz(rscratch1, rscratch1);
__ mov($dst$$FloatRegister, __ D, 0, rscratch1);
__ umov(rscratch1, $src$$FloatRegister, __ D, 1);
__ clz(rscratch1, rscratch1);
__ mov($dst$$FloatRegister, __ D, 1, rscratch1);
} else {
__ sve_clz($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
}
} else {
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ clz($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_clz($dst$$FloatRegister, __ elemType_to_regVariant(bt),
ptrue, $src$$FloatRegister);
}
}
%}
ins_pipe(pipe_slow);
%}
// The dst and src should use the same register to make sure the
// inactive lanes in dst save the same elements as src.
UNARY_OP_PREDICATE(vcountLeadingZeros, CountLeadingZerosV, sve_clz)
// ------------------------------ CountTrailingZerosV --------------------------
instruct vcountTrailingZeros(vReg dst, vReg src) %{
match(Set dst (CountTrailingZerosV src));
format %{ "vcountTrailingZeros $dst, $src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (bt == T_BYTE) {
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ rbit($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src$$FloatRegister);
__ clz($dst$$FloatRegister, get_arrangement(this), $dst$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_rbit($dst$$FloatRegister, __ B, ptrue, $src$$FloatRegister);
__ sve_clz($dst$$FloatRegister, __ B, ptrue, $dst$$FloatRegister);
}
} else {
assert(bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported type");
if (UseSVE == 0) {
assert(length_in_bytes == 8 || length_in_bytes == 16, "unsupported");
__ neon_reverse_bits($dst$$FloatRegister, $src$$FloatRegister,
bt, /* isQ */ length_in_bytes == 16);
if (bt != T_LONG) {
__ clz($dst$$FloatRegister, get_arrangement(this), $dst$$FloatRegister);
} else {
__ umov(rscratch1, $dst$$FloatRegister, __ D, 0);
__ clz(rscratch1, rscratch1);
__ mov($dst$$FloatRegister, __ D, 0, rscratch1);
__ umov(rscratch1, $dst$$FloatRegister, __ D, 1);
__ clz(rscratch1, rscratch1);
__ mov($dst$$FloatRegister, __ D, 1, rscratch1);
}
} else {
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_rbit($dst$$FloatRegister, size, ptrue, $src$$FloatRegister);
__ sve_clz($dst$$FloatRegister, size, ptrue, $dst$$FloatRegister);
}
}
%}
ins_pipe(pipe_slow);
%}
// The dst and src should use the same register to make sure the
// inactive lanes in dst save the same elements as src.
instruct vcountTrailingZeros_masked(vReg dst_src, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src (CountTrailingZerosV dst_src pg));
format %{ "vcountTrailingZeros_masked $dst_src, $pg, $dst_src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_rbit($dst_src$$FloatRegister, size,
$pg$$PRegister, $dst_src$$FloatRegister);
__ sve_clz($dst_src$$FloatRegister, size,
$pg$$PRegister, $dst_src$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ ReverseV -------------------------------------
instruct vreverse(vReg dst, vReg src) %{
match(Set dst (ReverseV src));
format %{ "vreverse $dst, $src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (bt == T_BYTE) {
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ rbit($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_rbit($dst$$FloatRegister, __ B, ptrue, $src$$FloatRegister);
}
} else {
assert(bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported type");
if (UseSVE == 0) {
assert(length_in_bytes == 8 || length_in_bytes == 16, "unsupported");
__ neon_reverse_bits($dst$$FloatRegister, $src$$FloatRegister,
bt, /* isQ */ length_in_bytes == 16);
} else {
__ sve_rbit($dst$$FloatRegister, __ elemType_to_regVariant(bt),
ptrue, $src$$FloatRegister);
}
}
%}
ins_pipe(pipe_slow);
%}
// The dst and src should use the same register to make sure the
// inactive lanes in dst save the same elements as src.
UNARY_OP_PREDICATE(vreverse, ReverseV, sve_rbit)
// ------------------------------ ReverseBytesV --------------------------------
instruct vreverseBytes(vReg dst, vReg src) %{
match(Set dst (ReverseBytesV src));
format %{ "vreverseBytes $dst, $src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
assert(length_in_bytes == 8 || length_in_bytes == 16, "unsupported");
if (bt == T_BYTE) {
if ($dst$$FloatRegister != $src$$FloatRegister) {
__ orr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src$$FloatRegister, $src$$FloatRegister);
}
} else {
__ neon_reverse_bytes($dst$$FloatRegister, $src$$FloatRegister,
bt, /* isQ */ length_in_bytes == 16);
}
} else {
assert(UseSVE > 0, "must be sve");
if (bt == T_BYTE) {
if ($dst$$FloatRegister != $src$$FloatRegister) {
__ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
}
} else {
__ sve_revb($dst$$FloatRegister, __ elemType_to_regVariant(bt),
ptrue, $src$$FloatRegister);
}
}
%}
ins_pipe(pipe_slow);
%}
// The dst and src should use the same register to make sure the
// inactive lanes in dst save the same elements as src.
instruct vreverseBytes_masked(vReg dst_src, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src (ReverseBytesV dst_src pg));
format %{ "vreverseBytes_masked $dst_src, $pg, $dst_src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
if (bt == T_BYTE) {
// do nothing
} else {
__ sve_revb($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $dst_src$$FloatRegister);
}
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Populate Index to a Vector -------------------
instruct populateindex(vReg dst, iRegIorL2I src1, immI src2) %{
predicate(UseSVE > 0);
match(Set dst (PopulateIndex src1 src2));
format %{ "populateindex $dst, $src1, $src2\t # populate index (sve)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_index($dst$$FloatRegister, __ elemType_to_regVariant(bt),
$src1$$Register, (int)($src2$$constant));
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Compress/Expand Operations -------------------
instruct mcompress(pReg dst, pReg pg, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (CompressM pg));
effect(KILL cr);
format %{ "mcompress $dst, $pg\t# KILL cr" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_cntp(rscratch1, size, ptrue, $pg$$PRegister);
__ sve_whilelt(as_PRegister($dst$$reg), size, zr, rscratch1);
%}
ins_pipe(pipe_slow);
%}
instruct vcompress(vReg dst, vReg src, pRegGov pg) %{
predicate(UseSVE > 0 &&
!is_subword_type(Matcher::vector_element_basic_type(n)));
match(Set dst (CompressV src pg));
format %{ "vcompress $dst, $src, $pg" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_compact($dst$$FloatRegister, __ elemType_to_regVariant(bt),
$src$$FloatRegister, $pg$$PRegister);
%}
ins_pipe(pipe_slow);
%}
instruct vcompressB(vReg dst, vReg src, pReg pg, vReg tmp1, vReg tmp2,
vReg tmp3, vReg tmp4, pReg ptmp, pRegGov pgtmp) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ptmp, TEMP pgtmp);
match(Set dst (CompressV src pg));
format %{ "vcompressB $dst, $src, $pg\t# KILL $tmp1, $tmp2, $tmp3, tmp4, $ptmp, $pgtmp" %}
ins_encode %{
__ sve_compress_byte($dst$$FloatRegister, $src$$FloatRegister, $pg$$PRegister,
$tmp1$$FloatRegister,$tmp2$$FloatRegister,
$tmp3$$FloatRegister,$tmp4$$FloatRegister,
$ptmp$$PRegister, $pgtmp$$PRegister);
%}
ins_pipe(pipe_slow);
%}
instruct vcompressS(vReg dst, vReg src, pReg pg,
vReg tmp1, vReg tmp2, pRegGov pgtmp) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_SHORT);
effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP pgtmp);
match(Set dst (CompressV src pg));
format %{ "vcompressS $dst, $src, $pg\t# KILL $tmp1, $tmp2, $pgtmp" %}
ins_encode %{
__ sve_compress_short($dst$$FloatRegister, $src$$FloatRegister, $pg$$PRegister,
$tmp1$$FloatRegister,$tmp2$$FloatRegister, $pgtmp$$PRegister);
%}
ins_pipe(pipe_slow);
%}
instruct vexpand(vReg dst, vReg src, pRegGov pg) %{
match(Set dst (ExpandV src pg));
effect(TEMP_DEF dst);
format %{ "vexpand $dst, $pg, $src" %}
ins_encode %{
// Example input: src = 1 2 3 4 5 6 7 8
// pg = 1 0 0 1 1 0 1 1
// Expected result: dst = 4 0 0 5 6 0 7 8
// The basic idea is to use TBL which can shuffle the elements in the given
// vector flexibly. HISTCNT + SUB is used to generate the second source input
// for TBL whose value is used to select the indexed element from src vector.
BasicType bt = Matcher::vector_element_basic_type(this);
assert(UseSVE == 2 && !is_subword_type(bt), "unsupported");
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
// dst = 0 0 0 0 0 0 0 0
__ sve_dup($dst$$FloatRegister, size, 0);
// dst = 5 0 0 4 3 0 2 1
__ sve_histcnt($dst$$FloatRegister, size, $pg$$PRegister,
$dst$$FloatRegister, $dst$$FloatRegister);
// dst = 4 -1 -1 3 2 -1 1 0
__ sve_sub($dst$$FloatRegister, size, 1);
// dst = 4 0 0 5 6 0 7 8
__ sve_tbl($dst$$FloatRegister, size, $src$$FloatRegister, $dst$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Vector signum --------------------------------
// Vector Math.signum
instruct vsignum_le128b(vReg dst, vReg src, vReg zero, vReg one) %{
predicate(Matcher::vector_length_in_bytes(n) <= 16);
match(Set dst (SignumVF src (Binary zero one)));
match(Set dst (SignumVD src (Binary zero one)));
effect(TEMP_DEF dst);
format %{ "vsignum_le128b $dst, $src\t# vector <= 128 bits" %}
ins_encode %{
__ vector_signum_neon($dst$$FloatRegister, $src$$FloatRegister, $zero$$FloatRegister,
$one$$FloatRegister, get_arrangement(this));
%}
ins_pipe(pipe_slow);
%}
instruct vsignum_gt128b(vReg dst, vReg src, vReg zero, vReg one, vReg tmp, pRegGov pgtmp) %{
predicate(Matcher::vector_length_in_bytes(n) > 16);
match(Set dst (SignumVF src (Binary zero one)));
match(Set dst (SignumVD src (Binary zero one)));
effect(TEMP_DEF dst, TEMP tmp, TEMP pgtmp);
format %{ "vsignum_gt128b $dst, $src\t# vector > 128 bits. KILL $tmp, $pgtmp" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
BasicType bt = Matcher::vector_element_basic_type(this);
__ vector_signum_sve($dst$$FloatRegister, $src$$FloatRegister, $zero$$FloatRegister,
$one$$FloatRegister, $tmp$$FloatRegister, $pgtmp$$PRegister,
__ elemType_to_regVariant(bt));
%}
ins_pipe(pipe_slow);
%}
dnl
dnl BITPERM($1, $2, $3 )
dnl BITPERM(insn_name, op_name, insn)
define(`BITPERM', `
instruct $1(vReg dst, vReg src1, vReg src2) %{
match(Set dst ($2 src1 src2));
format %{ "$1 $dst, $src1, $src2\t# vector (sve)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ $3($dst$$FloatRegister, size,
$src1$$FloatRegister, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// ---------------------------------- CompressBitsV --------------------------------
BITPERM(vcompressBits, CompressBitsV, sve_bext)
// ----------------------------------- ExpandBitsV ---------------------------------
BITPERM(vexpandBits, ExpandBitsV, sve_bdep)