| /* |
| * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved. |
| * Copyright (c) 2017, 2024 SAP SE. All rights reserved. |
| * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| * |
| * This code is free software; you can redistribute it and/or modify it |
| * under the terms of the GNU General Public License version 2 only, as |
| * published by the Free Software Foundation. |
| * |
| * This code is distributed in the hope that it will be useful, but WITHOUT |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| * version 2 for more details (a copy is included in the LICENSE file that |
| * accompanied this code). |
| * |
| * You should have received a copy of the GNU General Public License version |
| * 2 along with this work; if not, write to the Free Software Foundation, |
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| * |
| * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| * or visit www.oracle.com if you need additional information or have any |
| * questions. |
| * |
| */ |
| |
| #include "asm/assembler.hpp" |
| #include "asm/assembler.inline.hpp" |
| #include "opto/c2_MacroAssembler.hpp" |
| #include "opto/intrinsicnode.hpp" |
| #include "runtime/stubRoutines.hpp" |
| |
| #define BLOCK_COMMENT(str) block_comment(str) |
| #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") |
| |
| void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Register temp1, Register temp2) { |
| compiler_fast_lock_lightweight_object(obj, box, temp1, temp2); |
| } |
| |
| |
| void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register box, Register temp1, Register temp2) { |
| compiler_fast_unlock_lightweight_object(obj, box, temp1, temp2); |
| } |
| |
| void C2_MacroAssembler::load_narrow_klass_compact_c2(Register dst, Address src) { |
| // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract |
| // obj-start, so that we can load from the object's mark-word instead. |
| z_lg(dst, src.plus_disp(-oopDesc::klass_offset_in_bytes())); |
| z_srlg(dst, dst, markWord::klass_shift); // TODO: could be z_sra |
| } |
| |
| //------------------------------------------------------ |
| // Special String Intrinsics. Implementation |
| //------------------------------------------------------ |
| |
| // Intrinsics for CompactStrings |
| |
| // Compress char[] to byte[]. |
| // Restores: src, dst |
| // Uses: cnt |
| // Kills: tmp, Z_R0, Z_R1. |
| // Early clobber: result. |
| // Note: |
| // cnt is signed int. Do not rely on high word! |
| // counts # characters, not bytes. |
| // |
| // The result indicates success or failure of the operation. |
| // General compress operation (cut off high order byte which must be all zeroes). |
| // = len - all characters have been successfully compressed. |
| // = 0 - compress failed. At least one character was found with a non-zero high order byte. |
| // This is the failure return value which exactly corresponds to the Java implementation. |
| // 0 <= result < len - compress failed. That many characters were compressed successfully |
| // before the first non-compressable character was found. This is the |
| // current, but not fully compatible, implementation. See below. |
| // Encode to ISO or 7-bit ASCII array. |
| // = len - all characters have been encoded successfully. |
| // < len - encode failed. That many characters were encoded successfully. |
| // When used as an index into the character array, the return value addresses the |
| // first not encodeable character. |
| // |
| // If precise is true, the processing stops exactly at the point where a failure is detected. |
| // More characters than indicated by the return value may have been read from the src array. |
| // Exactly the number of characters indicated by the return value have been written to dst. |
| // If precise is false, a few characters more than indicated by the return value may have been |
| // written to the dst array. In any failure case, The result value indexes the first invalid character. |
| unsigned int C2_MacroAssembler::string_compress(Register result, Register Rsrc, Register Rdst, Register Rcnt, |
| Register tmp, bool precise, bool toASCII, VectorRegister Vtmp1, VectorRegister Vtmp2, |
| VectorRegister Vmask, VectorRegister Vzero, VectorRegister Vsrc_first, VectorRegister v21, |
| VectorRegister v22, VectorRegister Vsrc_last) { |
| assert_different_registers(Z_R0, Z_R1, result, Rsrc, Rdst, Rcnt, tmp); |
| |
| unsigned short char_mask = 0xff00; // all selected bits must be '0' for a char to be valid |
| unsigned int mask_ix_l = 0; // leftmost one bit pos in mask |
| unsigned int mask_ix_r = 7; // rightmost one bit pos in mask |
| if (precise) { |
| if (toASCII) { |
| BLOCK_COMMENT("encode_ascii_array {"); |
| char_mask = 0xff80; |
| mask_ix_r = 8; // rightmost one bit pos in mask. ASCII only uses codes 0..127 |
| } else { |
| BLOCK_COMMENT("encode_iso_array {"); |
| } |
| } else { |
| BLOCK_COMMENT("string_compress {"); |
| assert(!toASCII, "Can't compress strings to 7-bit ASCII"); |
| } |
| int block_start = offset(); |
| |
| Register Rix = tmp; |
| Register Rmask = result; // holds incompatibility check mask until result value is stored. |
| Label ScalarShortcut, AllDone; |
| |
| z_iilf(Rmask, (unsigned int)char_mask<<16 | (unsigned int)char_mask); |
| z_iihf(Rmask, (unsigned int)char_mask<<16 | (unsigned int)char_mask); |
| |
| #if 0 // Sacrifice shortcuts for code compactness |
| { |
| //---< shortcuts for short strings (very frequent) >--- |
| // Strings with 4 and 8 characters were found to occur very frequently. |
| // Therefore, we handle them right away with minimal overhead. |
| Label skipShortcut, skip4Shortcut, skip8Shortcut; |
| Register Rout = Z_R0; |
| z_chi(Rcnt, 4); |
| z_brne(skip4Shortcut); // 4 characters are very frequent |
| z_lg(Z_R0, 0, Rsrc); // Treat exactly 4 characters specially. |
| if (VM_Version::has_DistinctOpnds()) { |
| Rout = Z_R0; |
| z_ngrk(Rix, Z_R0, Rmask); |
| } else { |
| Rout = Rix; |
| z_lgr(Rix, Z_R0); |
| z_ngr(Z_R0, Rmask); |
| } |
| z_brnz(skipShortcut); |
| z_stcmh(Rout, 5, 0, Rdst); |
| z_stcm(Rout, 5, 2, Rdst); |
| z_lgfr(result, Rcnt); |
| z_bru(AllDone); |
| bind(skip4Shortcut); |
| |
| z_chi(Rcnt, 8); |
| z_brne(skip8Shortcut); // There's more to do... |
| z_lmg(Z_R0, Z_R1, 0, Rsrc); // Treat exactly 8 characters specially. |
| if (VM_Version::has_DistinctOpnds()) { |
| Rout = Z_R0; |
| z_ogrk(Rix, Z_R0, Z_R1); |
| z_ngr(Rix, Rmask); |
| } else { |
| Rout = Rix; |
| z_lgr(Rix, Z_R0); |
| z_ogr(Z_R0, Z_R1); |
| z_ngr(Z_R0, Rmask); |
| } |
| z_brnz(skipShortcut); |
| z_stcmh(Rout, 5, 0, Rdst); |
| z_stcm(Rout, 5, 2, Rdst); |
| z_stcmh(Z_R1, 5, 4, Rdst); |
| z_stcm(Z_R1, 5, 6, Rdst); |
| z_lgfr(result, Rcnt); |
| z_bru(AllDone); |
| |
| bind(skip8Shortcut); |
| clear_reg(Z_R0, true, false); // #characters already processed (none). Precond for scalar loop. |
| z_brl(ScalarShortcut); // Just a few characters |
| |
| bind(skipShortcut); |
| } |
| #endif |
| clear_reg(Z_R0); // make sure register is properly initialized. |
| |
| if (VM_Version::has_VectorFacility()) { |
| const int min_vcnt = 32; // Minimum #characters required to use vector instructions. |
| // Otherwise just do nothing in vector mode. |
| // Must correspond to # vector registers used by implementation, |
| // and must be a power of 2. |
| const int log_min_vcnt = exact_log2(min_vcnt); |
| Label VectorLoop, VectorDone, VectorBreak; |
| |
| assert((Vsrc_last->encoding() - Vsrc_first->encoding() + 1) == min_vcnt/8, "logic error"); |
| assert(VM_Version::has_DistinctOpnds(), "Assumption when has_VectorFacility()"); |
| z_srak(Rix, Rcnt, log_min_vcnt); // # vector loop iterations |
| z_brz(VectorDone); // not enough data for vector loop |
| |
| z_vzero(Vzero); // all zeroes |
| z_vgmh(Vmask, mask_ix_l, mask_ix_r); // generate 0xff00/0xff80 mask for all 2-byte elements |
| z_sllg(Z_R0, Rix, log_min_vcnt); // remember #chars that will be processed by vector loop |
| |
| bind(VectorLoop); |
| z_vlm(Vsrc_first, Vsrc_last, 0, Rsrc); |
| add2reg(Rsrc, min_vcnt*2); |
| |
| //---< check for incompatible character >--- |
| z_vo(Vtmp1, Vsrc_first, v21); |
| z_vo(Vtmp2, v22, Vsrc_last); |
| z_vo(Vtmp1, Vtmp1, Vtmp2); |
| z_vn(Vtmp1, Vtmp1, Vmask); |
| z_vceqhs(Vtmp1, Vtmp1, Vzero); // all bits selected by mask must be zero for successful compress. |
| z_bvnt(VectorBreak); // break vector loop if not all vector elements compare eq -> incompatible character found. |
| // re-process data from current iteration in break handler. |
| |
| //---< pack & store characters >--- |
| z_vpkh(Vtmp1, Vsrc_first, v21); // pack (src1, src2) -> tmp1 |
| z_vpkh(Vtmp2, v22, Vsrc_last); // pack (src3, src4) -> tmp2 |
| z_vstm(Vtmp1, Vtmp2, 0, Rdst); // store packed string |
| add2reg(Rdst, min_vcnt); |
| |
| z_brct(Rix, VectorLoop); |
| |
| z_bru(VectorDone); |
| |
| bind(VectorBreak); |
| add2reg(Rsrc, -min_vcnt*2); // Fix Rsrc. Rsrc was already updated, but Rdst and Rix are not. |
| z_sll(Rix, log_min_vcnt); // # chars processed so far in VectorLoop, excl. current iteration. |
| z_sr(Z_R0, Rix); // correct # chars processed in total. |
| |
| bind(VectorDone); |
| } |
| |
| { |
| const int min_cnt = 8; // Minimum #characters required to use unrolled loop. |
| // Otherwise just do nothing in unrolled loop. |
| // Must correspond to # registers used by implementation, |
| // and must be a power of 2. |
| const int log_min_cnt = exact_log2(min_cnt); |
| Label UnrolledLoop, UnrolledDone, UnrolledBreak; |
| |
| if (VM_Version::has_DistinctOpnds()) { |
| z_srk(Rix, Rcnt, Z_R0); // remaining # chars to compress in unrolled loop |
| } else { |
| z_lr(Rix, Rcnt); |
| z_sr(Rix, Z_R0); |
| } |
| z_sra(Rix, log_min_cnt); // unrolled loop count |
| z_brz(UnrolledDone); |
| |
| bind(UnrolledLoop); |
| z_lmg(Z_R0, Z_R1, 0, Rsrc); |
| if (precise) { |
| z_ogr(Z_R1, Z_R0); // check all 8 chars for incompatibility |
| z_ngr(Z_R1, Rmask); |
| z_brnz(UnrolledBreak); |
| |
| z_lg(Z_R1, 8, Rsrc); // reload destroyed register |
| z_stcmh(Z_R0, 5, 0, Rdst); |
| z_stcm(Z_R0, 5, 2, Rdst); |
| } else { |
| z_stcmh(Z_R0, 5, 0, Rdst); |
| z_stcm(Z_R0, 5, 2, Rdst); |
| |
| z_ogr(Z_R0, Z_R1); |
| z_ngr(Z_R0, Rmask); |
| z_brnz(UnrolledBreak); |
| } |
| z_stcmh(Z_R1, 5, 4, Rdst); |
| z_stcm(Z_R1, 5, 6, Rdst); |
| |
| add2reg(Rsrc, min_cnt*2); |
| add2reg(Rdst, min_cnt); |
| z_brct(Rix, UnrolledLoop); |
| |
| z_lgfr(Z_R0, Rcnt); // # chars processed in total after unrolled loop. |
| z_nilf(Z_R0, ~(min_cnt-1)); |
| z_tmll(Rcnt, min_cnt-1); |
| z_brnaz(ScalarShortcut); // if all bits zero, there is nothing left to do for scalar loop. |
| // Rix == 0 in all cases. |
| z_sllg(Z_R1, Rcnt, 1); // # src bytes already processed. Only lower 32 bits are valid! |
| // Z_R1 contents must be treated as unsigned operand! For huge strings, |
| // (Rcnt >= 2**30), the value may spill into the sign bit by sllg. |
| z_lgfr(result, Rcnt); // all characters processed. |
| z_slgfr(Rdst, Rcnt); // restore ptr |
| z_slgfr(Rsrc, Z_R1); // restore ptr, double the element count for Rsrc restore |
| z_bru(AllDone); |
| |
| bind(UnrolledBreak); |
| z_lgfr(Z_R0, Rcnt); // # chars processed in total after unrolled loop |
| z_nilf(Z_R0, ~(min_cnt-1)); |
| z_sll(Rix, log_min_cnt); // # chars not yet processed in UnrolledLoop (due to break), broken iteration not included. |
| z_sr(Z_R0, Rix); // fix # chars processed OK so far. |
| if (!precise) { |
| // Because we don't need to be precise, we just return the # of characters which have been written. |
| // The first illegal character is in the index range [result-min_cnt/2, result+min_cnt/2). |
| z_lgfr(result, Z_R0); |
| z_sllg(Z_R1, Z_R0, 1); // # src bytes already processed. Only lower 32 bits are valid! |
| // Z_R1 contents must be treated as unsigned operand! For huge strings, |
| // (Rcnt >= 2**30), the value may spill into the sign bit by sllg. |
| z_aghi(result, min_cnt/2); // min_cnt/2 characters have already been written |
| // but ptrs were not updated yet. |
| z_slgfr(Rdst, Z_R0); // restore ptr |
| z_slgfr(Rsrc, Z_R1); // restore ptr, double the element count for Rsrc restore |
| z_bru(AllDone); |
| } |
| bind(UnrolledDone); |
| } |
| |
| { |
| Label ScalarLoop, ScalarDone, ScalarBreak; |
| |
| bind(ScalarShortcut); |
| z_ltgfr(result, Rcnt); |
| z_brz(AllDone); |
| |
| #if 0 // Sacrifice shortcuts for code compactness |
| { |
| //---< Special treatment for very short strings (one or two characters) >--- |
| // For these strings, we are sure that the above code was skipped. |
| // Thus, no registers were modified, register restore is not required. |
| Label ScalarDoit, Scalar2Char; |
| z_chi(Rcnt, 2); |
| z_brh(ScalarDoit); |
| z_llh(Z_R1, 0, Z_R0, Rsrc); |
| z_bre(Scalar2Char); |
| z_tmll(Z_R1, char_mask); |
| z_lghi(result, 0); // cnt == 1, first char invalid, no chars successfully processed |
| z_brnaz(AllDone); |
| z_stc(Z_R1, 0, Z_R0, Rdst); |
| z_lghi(result, 1); |
| z_bru(AllDone); |
| |
| bind(Scalar2Char); |
| z_llh(Z_R0, 2, Z_R0, Rsrc); |
| z_tmll(Z_R1, char_mask); |
| z_lghi(result, 0); // cnt == 2, first char invalid, no chars successfully processed |
| z_brnaz(AllDone); |
| z_stc(Z_R1, 0, Z_R0, Rdst); |
| z_tmll(Z_R0, char_mask); |
| z_lghi(result, 1); // cnt == 2, second char invalid, one char successfully processed |
| z_brnaz(AllDone); |
| z_stc(Z_R0, 1, Z_R0, Rdst); |
| z_lghi(result, 2); |
| z_bru(AllDone); |
| |
| bind(ScalarDoit); |
| } |
| #endif |
| |
| if (VM_Version::has_DistinctOpnds()) { |
| z_srk(Rix, Rcnt, Z_R0); // remaining # chars to compress in scalar loop |
| } else { |
| z_lr(Rix, Rcnt); |
| z_sr(Rix, Z_R0); |
| } |
| z_lgfr(result, Rcnt); // # processed characters (if all encodes ok). |
| z_brz(ScalarDone); // anything left to do? (uses CC from Rix calculation) |
| |
| bind(ScalarLoop); |
| z_llh(Z_R1, 0, Z_R0, Rsrc); |
| z_tmll(Z_R1, char_mask); |
| z_brnaz(ScalarBreak); |
| z_stc(Z_R1, 0, Z_R0, Rdst); |
| add2reg(Rsrc, 2); |
| add2reg(Rdst, 1); |
| z_brct(Rix, ScalarLoop); |
| |
| z_bru(ScalarDone); |
| |
| bind(ScalarBreak); |
| z_sr(result, Rix); |
| |
| bind(ScalarDone); |
| z_sgfr(Rdst, result); // restore ptr |
| z_sgfr(Rsrc, result); // restore ptr, double the element count for Rsrc restore |
| z_sgfr(Rsrc, result); |
| } |
| bind(AllDone); |
| |
| if (precise) { |
| if (toASCII) { |
| BLOCK_COMMENT("} encode_ascii_array"); |
| } else { |
| BLOCK_COMMENT("} encode_iso_array"); |
| } |
| } else { |
| BLOCK_COMMENT("} string_compress"); |
| } |
| return offset() - block_start; |
| } |
| |
| // Inflate byte[] to char[]. |
| unsigned int C2_MacroAssembler::string_inflate_trot(Register src, Register dst, Register cnt, Register tmp) { |
| int block_start = offset(); |
| |
| BLOCK_COMMENT("string_inflate {"); |
| |
| Register stop_char = Z_R0; |
| Register table = Z_R1; |
| Register src_addr = tmp; |
| |
| assert_different_registers(Z_R0, Z_R1, tmp, src, dst, cnt); |
| assert(dst->encoding()%2 == 0, "must be even reg"); |
| assert(cnt->encoding()%2 == 1, "must be odd reg"); |
| assert(cnt->encoding() - dst->encoding() == 1, "must be even/odd pair"); |
| |
| StubRoutines::zarch::generate_load_trot_table_addr(this, table); // kills Z_R0 (if ASSERT) |
| clear_reg(stop_char); // Stop character. Not used here, but initialized to have a defined value. |
| lgr_if_needed(src_addr, src); |
| z_llgfr(cnt, cnt); // # src characters, must be a positive simm32. |
| |
| translate_ot(dst, src_addr, /* mask = */ 0x0001); |
| |
| BLOCK_COMMENT("} string_inflate"); |
| |
| return offset() - block_start; |
| } |
| |
| // Inflate byte[] to char[]. |
| // Restores: src, dst |
| // Uses: cnt |
| // Kills: tmp, Z_R0, Z_R1. |
| // Note: |
| // cnt is signed int. Do not rely on high word! |
| // counts # characters, not bytes. |
| unsigned int C2_MacroAssembler::string_inflate(Register src, Register dst, Register cnt, Register tmp, |
| VectorRegister v20, VectorRegister v21, VectorRegister v22, |
| VectorRegister v23, VectorRegister v24, VectorRegister v25) { |
| assert_different_registers(Z_R0, Z_R1, src, dst, cnt, tmp); |
| |
| BLOCK_COMMENT("string_inflate {"); |
| int block_start = offset(); |
| |
| Register Rcnt = cnt; // # characters (src: bytes, dst: char (2-byte)), remaining after current loop. |
| Register Rix = tmp; // loop index |
| Register Rsrc = src; // addr(src array) |
| Register Rdst = dst; // addr(dst array) |
| Label ScalarShortcut, AllDone; |
| |
| #if 0 // Sacrifice shortcuts for code compactness |
| { |
| //---< shortcuts for short strings (very frequent) >--- |
| Label skipShortcut, skip4Shortcut; |
| z_ltr(Rcnt, Rcnt); // absolutely nothing to do for strings of len == 0. |
| z_brz(AllDone); |
| clear_reg(Z_R0); // make sure registers are properly initialized. |
| clear_reg(Z_R1); |
| z_chi(Rcnt, 4); |
| z_brne(skip4Shortcut); // 4 characters are very frequent |
| z_icm(Z_R0, 5, 0, Rsrc); // Treat exactly 4 characters specially. |
| z_icm(Z_R1, 5, 2, Rsrc); |
| z_stm(Z_R0, Z_R1, 0, Rdst); |
| z_bru(AllDone); |
| bind(skip4Shortcut); |
| |
| z_chi(Rcnt, 8); |
| z_brh(skipShortcut); // There's a lot to do... |
| z_lgfr(Z_R0, Rcnt); // remaining #characters (<= 8). Precond for scalar loop. |
| // This does not destroy the "register cleared" state of Z_R0. |
| z_brl(ScalarShortcut); // Just a few characters |
| z_icmh(Z_R0, 5, 0, Rsrc); // Treat exactly 8 characters specially. |
| z_icmh(Z_R1, 5, 4, Rsrc); |
| z_icm(Z_R0, 5, 2, Rsrc); |
| z_icm(Z_R1, 5, 6, Rsrc); |
| z_stmg(Z_R0, Z_R1, 0, Rdst); |
| z_bru(AllDone); |
| bind(skipShortcut); |
| } |
| #endif |
| clear_reg(Z_R0); // make sure register is properly initialized. |
| |
| if (VM_Version::has_VectorFacility()) { |
| const int min_vcnt = 32; // Minimum #characters required to use vector instructions. |
| // Otherwise just do nothing in vector mode. |
| // Must be multiple of vector register length (16 bytes = 128 bits). |
| const int log_min_vcnt = exact_log2(min_vcnt); |
| Label VectorLoop, VectorDone; |
| |
| assert(VM_Version::has_DistinctOpnds(), "Assumption when has_VectorFacility()"); |
| z_srak(Rix, Rcnt, log_min_vcnt); // calculate # vector loop iterations |
| z_brz(VectorDone); // skip if none |
| |
| z_sllg(Z_R0, Rix, log_min_vcnt); // remember #chars that will be processed by vector loop |
| |
| bind(VectorLoop); |
| z_vlm(v20, v21, 0, Rsrc); // get next 32 characters (single-byte) |
| add2reg(Rsrc, min_vcnt); |
| |
| z_vuplhb(v22, v20); // V2 <- (expand) V0(high) |
| z_vupllb(v23, v20); // V3 <- (expand) V0(low) |
| z_vuplhb(v24, v21); // V4 <- (expand) V1(high) |
| z_vupllb(v25, v21); // V5 <- (expand) V1(low) |
| z_vstm(v22, v25, 0, Rdst); // store next 32 bytes |
| add2reg(Rdst, min_vcnt*2); |
| |
| z_brct(Rix, VectorLoop); |
| |
| bind(VectorDone); |
| } |
| |
| const int min_cnt = 8; // Minimum #characters required to use unrolled scalar loop. |
| // Otherwise just do nothing in unrolled scalar mode. |
| // Must be multiple of 8. |
| { |
| const int log_min_cnt = exact_log2(min_cnt); |
| Label UnrolledLoop, UnrolledDone; |
| |
| |
| if (VM_Version::has_DistinctOpnds()) { |
| z_srk(Rix, Rcnt, Z_R0); // remaining # chars to process in unrolled loop |
| } else { |
| z_lr(Rix, Rcnt); |
| z_sr(Rix, Z_R0); |
| } |
| z_sra(Rix, log_min_cnt); // unrolled loop count |
| z_brz(UnrolledDone); |
| |
| clear_reg(Z_R0); |
| clear_reg(Z_R1); |
| |
| bind(UnrolledLoop); |
| z_icmh(Z_R0, 5, 0, Rsrc); |
| z_icmh(Z_R1, 5, 4, Rsrc); |
| z_icm(Z_R0, 5, 2, Rsrc); |
| z_icm(Z_R1, 5, 6, Rsrc); |
| add2reg(Rsrc, min_cnt); |
| |
| z_stmg(Z_R0, Z_R1, 0, Rdst); |
| |
| add2reg(Rdst, min_cnt*2); |
| z_brct(Rix, UnrolledLoop); |
| |
| bind(UnrolledDone); |
| z_lgfr(Z_R0, Rcnt); // # chars left over after unrolled loop. |
| z_nilf(Z_R0, min_cnt-1); |
| z_brnz(ScalarShortcut); // if zero, there is nothing left to do for scalar loop. |
| // Rix == 0 in all cases. |
| z_sgfr(Z_R0, Rcnt); // negative # characters the ptrs have been advanced previously. |
| z_agr(Rdst, Z_R0); // restore ptr, double the element count for Rdst restore. |
| z_agr(Rdst, Z_R0); |
| z_agr(Rsrc, Z_R0); // restore ptr. |
| z_bru(AllDone); |
| } |
| |
| { |
| bind(ScalarShortcut); |
| // Z_R0 must contain remaining # characters as 64-bit signed int here. |
| // register contents is preserved over scalar processing (for register fixup). |
| |
| #if 0 // Sacrifice shortcuts for code compactness |
| { |
| Label ScalarDefault; |
| z_chi(Rcnt, 2); |
| z_brh(ScalarDefault); |
| z_llc(Z_R0, 0, Z_R0, Rsrc); // 6 bytes |
| z_sth(Z_R0, 0, Z_R0, Rdst); // 4 bytes |
| z_brl(AllDone); |
| z_llc(Z_R0, 1, Z_R0, Rsrc); // 6 bytes |
| z_sth(Z_R0, 2, Z_R0, Rdst); // 4 bytes |
| z_bru(AllDone); |
| bind(ScalarDefault); |
| } |
| #endif |
| |
| Label CodeTable; |
| // Some comments on Rix calculation: |
| // - Rcnt is small, therefore no bits shifted out of low word (sll(g) instructions). |
| // - high word of both Rix and Rcnt may contain garbage |
| // - the final lngfr takes care of that garbage, extending the sign to high word |
| z_sllg(Rix, Z_R0, 2); // calculate 10*Rix = (4*Rix + Rix)*2 |
| z_ar(Rix, Z_R0); |
| z_larl(Z_R1, CodeTable); |
| z_sll(Rix, 1); |
| z_lngfr(Rix, Rix); // ix range: [0..7], after inversion & mult: [-(7*12)..(0*12)]. |
| z_bc(Assembler::bcondAlways, 0, Rix, Z_R1); |
| |
| z_llc(Z_R1, 6, Z_R0, Rsrc); // 6 bytes |
| z_sth(Z_R1, 12, Z_R0, Rdst); // 4 bytes |
| |
| z_llc(Z_R1, 5, Z_R0, Rsrc); |
| z_sth(Z_R1, 10, Z_R0, Rdst); |
| |
| z_llc(Z_R1, 4, Z_R0, Rsrc); |
| z_sth(Z_R1, 8, Z_R0, Rdst); |
| |
| z_llc(Z_R1, 3, Z_R0, Rsrc); |
| z_sth(Z_R1, 6, Z_R0, Rdst); |
| |
| z_llc(Z_R1, 2, Z_R0, Rsrc); |
| z_sth(Z_R1, 4, Z_R0, Rdst); |
| |
| z_llc(Z_R1, 1, Z_R0, Rsrc); |
| z_sth(Z_R1, 2, Z_R0, Rdst); |
| |
| z_llc(Z_R1, 0, Z_R0, Rsrc); |
| z_sth(Z_R1, 0, Z_R0, Rdst); |
| bind(CodeTable); |
| |
| z_chi(Rcnt, 8); // no fixup for small strings. Rdst, Rsrc were not modified. |
| z_brl(AllDone); |
| |
| z_sgfr(Z_R0, Rcnt); // # characters the ptrs have been advanced previously. |
| z_agr(Rdst, Z_R0); // restore ptr, double the element count for Rdst restore. |
| z_agr(Rdst, Z_R0); |
| z_agr(Rsrc, Z_R0); // restore ptr. |
| } |
| bind(AllDone); |
| |
| BLOCK_COMMENT("} string_inflate"); |
| return offset() - block_start; |
| } |
| |
| // Inflate byte[] to char[], length known at compile time. |
| // Restores: src, dst |
| // Kills: tmp, Z_R0, Z_R1. |
| // Note: |
| // len is signed int. Counts # characters, not bytes. |
| unsigned int C2_MacroAssembler::string_inflate_const(Register src, Register dst, Register tmp, int len , |
| VectorRegister v20, VectorRegister v21, VectorRegister v22, |
| VectorRegister v23, VectorRegister v24, VectorRegister v25) { |
| assert_different_registers(Z_R0, Z_R1, src, dst, tmp); |
| |
| BLOCK_COMMENT("string_inflate_const {"); |
| int block_start = offset(); |
| |
| Register Rix = tmp; // loop index |
| Register Rsrc = src; // addr(src array) |
| Register Rdst = dst; // addr(dst array) |
| Label ScalarShortcut, AllDone; |
| int nprocessed = 0; |
| int src_off = 0; // compensate for saved (optimized away) ptr advancement. |
| int dst_off = 0; // compensate for saved (optimized away) ptr advancement. |
| bool restore_inputs = false; |
| bool workreg_clear = false; |
| |
| if ((len >= 32) && VM_Version::has_VectorFacility()) { |
| const int min_vcnt = 32; // Minimum #characters required to use vector instructions. |
| // Otherwise just do nothing in vector mode. |
| // Must be multiple of vector register length (16 bytes = 128 bits). |
| const int log_min_vcnt = exact_log2(min_vcnt); |
| const int iterations = (len - nprocessed) >> log_min_vcnt; |
| nprocessed += iterations << log_min_vcnt; |
| Label VectorLoop; |
| |
| if (iterations == 1) { |
| z_vlm(v20, v21, 0+src_off, Rsrc); // get next 32 characters (single-byte) |
| z_vuplhb(v22, v20); // V2 <- (expand) V0(high) |
| z_vupllb(v23, v20); // V3 <- (expand) V0(low) |
| z_vuplhb(v24, v21); // V4 <- (expand) V1(high) |
| z_vupllb(v25, v21); // V5 <- (expand) V1(low) |
| z_vstm(v22, v25, 0+dst_off, Rdst); // store next 32 bytes |
| |
| src_off += min_vcnt; |
| dst_off += min_vcnt*2; |
| } else { |
| restore_inputs = true; |
| |
| z_lgfi(Rix, len>>log_min_vcnt); |
| bind(VectorLoop); |
| z_vlm(v20, v21, 0, Rsrc); // get next 32 characters (single-byte) |
| add2reg(Rsrc, min_vcnt); |
| |
| z_vuplhb(v22, v20); // V2 <- (expand) V0(high) |
| z_vupllb(v23, v20); // V3 <- (expand) V0(low) |
| z_vuplhb(v24, v21); // V4 <- (expand) V1(high) |
| z_vupllb(v25, v21); // V5 <- (expand) V1(low) |
| z_vstm(v22, v25, 0, Rdst); // store next 32 bytes |
| add2reg(Rdst, min_vcnt*2); |
| |
| z_brct(Rix, VectorLoop); |
| } |
| } |
| |
| if (((len-nprocessed) >= 16) && VM_Version::has_VectorFacility()) { |
| const int min_vcnt = 16; // Minimum #characters required to use vector instructions. |
| // Otherwise just do nothing in vector mode. |
| // Must be multiple of vector register length (16 bytes = 128 bits). |
| const int log_min_vcnt = exact_log2(min_vcnt); |
| const int iterations = (len - nprocessed) >> log_min_vcnt; |
| nprocessed += iterations << log_min_vcnt; |
| assert(iterations == 1, "must be!"); |
| |
| z_vl(v20, 0+src_off, Z_R0, Rsrc); // get next 16 characters (single-byte) |
| z_vuplhb(v22, v20); // V2 <- (expand) V0(high) |
| z_vupllb(v23, v20); // V3 <- (expand) V0(low) |
| z_vstm(v22, v23, 0+dst_off, Rdst); // store next 32 bytes |
| |
| src_off += min_vcnt; |
| dst_off += min_vcnt*2; |
| } |
| |
| if ((len-nprocessed) > 8) { |
| const int min_cnt = 8; // Minimum #characters required to use unrolled scalar loop. |
| // Otherwise just do nothing in unrolled scalar mode. |
| // Must be multiple of 8. |
| const int log_min_cnt = exact_log2(min_cnt); |
| const int iterations = (len - nprocessed) >> log_min_cnt; |
| nprocessed += iterations << log_min_cnt; |
| |
| //---< avoid loop overhead/ptr increment for small # iterations >--- |
| if (iterations <= 2) { |
| clear_reg(Z_R0); |
| clear_reg(Z_R1); |
| workreg_clear = true; |
| |
| z_icmh(Z_R0, 5, 0+src_off, Rsrc); |
| z_icmh(Z_R1, 5, 4+src_off, Rsrc); |
| z_icm(Z_R0, 5, 2+src_off, Rsrc); |
| z_icm(Z_R1, 5, 6+src_off, Rsrc); |
| z_stmg(Z_R0, Z_R1, 0+dst_off, Rdst); |
| |
| src_off += min_cnt; |
| dst_off += min_cnt*2; |
| } |
| |
| if (iterations == 2) { |
| z_icmh(Z_R0, 5, 0+src_off, Rsrc); |
| z_icmh(Z_R1, 5, 4+src_off, Rsrc); |
| z_icm(Z_R0, 5, 2+src_off, Rsrc); |
| z_icm(Z_R1, 5, 6+src_off, Rsrc); |
| z_stmg(Z_R0, Z_R1, 0+dst_off, Rdst); |
| |
| src_off += min_cnt; |
| dst_off += min_cnt*2; |
| } |
| |
| if (iterations > 2) { |
| Label UnrolledLoop; |
| restore_inputs = true; |
| |
| clear_reg(Z_R0); |
| clear_reg(Z_R1); |
| workreg_clear = true; |
| |
| z_lgfi(Rix, iterations); |
| bind(UnrolledLoop); |
| z_icmh(Z_R0, 5, 0, Rsrc); |
| z_icmh(Z_R1, 5, 4, Rsrc); |
| z_icm(Z_R0, 5, 2, Rsrc); |
| z_icm(Z_R1, 5, 6, Rsrc); |
| add2reg(Rsrc, min_cnt); |
| |
| z_stmg(Z_R0, Z_R1, 0, Rdst); |
| add2reg(Rdst, min_cnt*2); |
| |
| z_brct(Rix, UnrolledLoop); |
| } |
| } |
| |
| if ((len-nprocessed) > 0) { |
| switch (len-nprocessed) { |
| case 8: |
| if (!workreg_clear) { |
| clear_reg(Z_R0); |
| clear_reg(Z_R1); |
| } |
| z_icmh(Z_R0, 5, 0+src_off, Rsrc); |
| z_icmh(Z_R1, 5, 4+src_off, Rsrc); |
| z_icm(Z_R0, 5, 2+src_off, Rsrc); |
| z_icm(Z_R1, 5, 6+src_off, Rsrc); |
| z_stmg(Z_R0, Z_R1, 0+dst_off, Rdst); |
| break; |
| case 7: |
| if (!workreg_clear) { |
| clear_reg(Z_R0); |
| clear_reg(Z_R1); |
| } |
| clear_reg(Rix); |
| z_icm(Z_R0, 5, 0+src_off, Rsrc); |
| z_icm(Z_R1, 5, 2+src_off, Rsrc); |
| z_icm(Rix, 5, 4+src_off, Rsrc); |
| z_stm(Z_R0, Z_R1, 0+dst_off, Rdst); |
| z_llc(Z_R0, 6+src_off, Z_R0, Rsrc); |
| z_st(Rix, 8+dst_off, Z_R0, Rdst); |
| z_sth(Z_R0, 12+dst_off, Z_R0, Rdst); |
| break; |
| case 6: |
| if (!workreg_clear) { |
| clear_reg(Z_R0); |
| clear_reg(Z_R1); |
| } |
| clear_reg(Rix); |
| z_icm(Z_R0, 5, 0+src_off, Rsrc); |
| z_icm(Z_R1, 5, 2+src_off, Rsrc); |
| z_icm(Rix, 5, 4+src_off, Rsrc); |
| z_stm(Z_R0, Z_R1, 0+dst_off, Rdst); |
| z_st(Rix, 8+dst_off, Z_R0, Rdst); |
| break; |
| case 5: |
| if (!workreg_clear) { |
| clear_reg(Z_R0); |
| clear_reg(Z_R1); |
| } |
| z_icm(Z_R0, 5, 0+src_off, Rsrc); |
| z_icm(Z_R1, 5, 2+src_off, Rsrc); |
| z_llc(Rix, 4+src_off, Z_R0, Rsrc); |
| z_stm(Z_R0, Z_R1, 0+dst_off, Rdst); |
| z_sth(Rix, 8+dst_off, Z_R0, Rdst); |
| break; |
| case 4: |
| if (!workreg_clear) { |
| clear_reg(Z_R0); |
| clear_reg(Z_R1); |
| } |
| z_icm(Z_R0, 5, 0+src_off, Rsrc); |
| z_icm(Z_R1, 5, 2+src_off, Rsrc); |
| z_stm(Z_R0, Z_R1, 0+dst_off, Rdst); |
| break; |
| case 3: |
| if (!workreg_clear) { |
| clear_reg(Z_R0); |
| } |
| z_llc(Z_R1, 2+src_off, Z_R0, Rsrc); |
| z_icm(Z_R0, 5, 0+src_off, Rsrc); |
| z_sth(Z_R1, 4+dst_off, Z_R0, Rdst); |
| z_st(Z_R0, 0+dst_off, Rdst); |
| break; |
| case 2: |
| z_llc(Z_R0, 0+src_off, Z_R0, Rsrc); |
| z_llc(Z_R1, 1+src_off, Z_R0, Rsrc); |
| z_sth(Z_R0, 0+dst_off, Z_R0, Rdst); |
| z_sth(Z_R1, 2+dst_off, Z_R0, Rdst); |
| break; |
| case 1: |
| z_llc(Z_R0, 0+src_off, Z_R0, Rsrc); |
| z_sth(Z_R0, 0+dst_off, Z_R0, Rdst); |
| break; |
| default: |
| guarantee(false, "Impossible"); |
| break; |
| } |
| src_off += len-nprocessed; |
| dst_off += (len-nprocessed)*2; |
| nprocessed = len; |
| } |
| |
| //---< restore modified input registers >--- |
| if ((nprocessed > 0) && restore_inputs) { |
| z_agfi(Rsrc, -(nprocessed-src_off)); |
| if (nprocessed < 1000000000) { // avoid int overflow |
| z_agfi(Rdst, -(nprocessed*2-dst_off)); |
| } else { |
| z_agfi(Rdst, -(nprocessed-dst_off)); |
| z_agfi(Rdst, -nprocessed); |
| } |
| } |
| |
| BLOCK_COMMENT("} string_inflate_const"); |
| return offset() - block_start; |
| } |
| |
| // Returns the number of non-negative bytes (aka US-ASCII characters) found |
| // before the first negative byte is encountered. |
| unsigned int C2_MacroAssembler::count_positives(Register result, Register src, Register cnt, Register tmp) { |
| const unsigned int block_start = offset(); |
| const unsigned int byte_mask = 0x80; |
| const unsigned int twobyte_mask = byte_mask<<8 | byte_mask; |
| const unsigned int unroll_factor = 16; |
| const unsigned int log_unroll_factor = exact_log2(unroll_factor); |
| Register pos = src; // current position in src array, restored at end |
| Register ctr = result; // loop counter, result value |
| Register mask = tmp; // holds the sign detection mask |
| Label unrolledLoop, unrolledDone, byteLoop, allDone; |
| |
| assert_different_registers(result, src, cnt, tmp); |
| |
| BLOCK_COMMENT("count_positives {"); |
| |
| lgr_if_needed(pos, src); // current position in src array |
| z_srak(ctr, cnt, log_unroll_factor); // # iterations of unrolled loop |
| z_brnh(unrolledDone); // array too short for unrolled loop |
| |
| z_iilf(mask, twobyte_mask<<16 | twobyte_mask); |
| z_iihf(mask, twobyte_mask<<16 | twobyte_mask); |
| |
| bind(unrolledLoop); |
| z_lmg(Z_R0, Z_R1, 0, pos); |
| z_ogr(Z_R0, Z_R1); |
| z_ngr(Z_R0, mask); |
| z_brne(unrolledDone); // There is a negative byte somewhere. |
| // ctr and pos are not updated yet -> |
| // delegate finding correct pos to byteLoop. |
| add2reg(pos, unroll_factor); |
| z_brct(ctr, unrolledLoop); |
| |
| // Once we arrive here, we have to examine at most (unroll_factor - 1) bytes more. |
| // We then either have reached the end of the array or we hit a negative byte. |
| bind(unrolledDone); |
| z_sll(ctr, log_unroll_factor); // calculate # bytes not processed by unrolled loop |
| // > 0 only if a negative byte was found |
| z_lr(Z_R0, cnt); // calculate remainder bytes |
| z_nilf(Z_R0, unroll_factor - 1); |
| z_ar(ctr, Z_R0); // remaining bytes |
| z_brnh(allDone); // shortcut if nothing left to do |
| |
| bind(byteLoop); |
| z_cli(0, pos, byte_mask); // unsigned comparison! byte@pos must be smaller that byte_mask |
| z_brnl(allDone); // negative byte found. |
| |
| add2reg(pos, 1); |
| z_brct(ctr, byteLoop); |
| |
| bind(allDone); |
| |
| z_srk(ctr, cnt, ctr); // # bytes actually processed (= cnt or index of first negative byte) |
| z_sgfr(pos, ctr); // restore src |
| z_lgfr(result, ctr); // unnecessary. Only there to be sure the high word has a defined state. |
| |
| BLOCK_COMMENT("} count_positives"); |
| |
| return offset() - block_start; |
| } |
| |
| // kill: cnt1, cnt2, odd_reg, even_reg; early clobber: result |
| unsigned int C2_MacroAssembler::string_compare(Register str1, Register str2, |
| Register cnt1, Register cnt2, |
| Register odd_reg, Register even_reg, Register result, int ae) { |
| int block_start = offset(); |
| |
| assert_different_registers(str1, cnt1, cnt2, odd_reg, even_reg, result); |
| assert_different_registers(str2, cnt1, cnt2, odd_reg, even_reg, result); |
| |
| // If strings are equal up to min length, return the length difference. |
| const Register diff = result, // Pre-set result with length difference. |
| min = cnt1, // min number of bytes |
| tmp = cnt2; |
| |
| // Note: Making use of the fact that compareTo(a, b) == -compareTo(b, a) |
| // we interchange str1 and str2 in the UL case and negate the result. |
| // Like this, str1 is always latin1 encoded, except for the UU case. |
| // In addition, we need 0 (or sign which is 0) extend when using 64 bit register. |
| const bool used_as_LU = (ae == StrIntrinsicNode::LU || ae == StrIntrinsicNode::UL); |
| |
| BLOCK_COMMENT("string_compare {"); |
| |
| if (used_as_LU) { |
| z_srl(cnt2, 1); |
| } |
| |
| // See if the lengths are different, and calculate min in cnt1. |
| // Save diff in case we need it for a tie-breaker. |
| |
| // diff = cnt1 - cnt2 |
| if (VM_Version::has_DistinctOpnds()) { |
| z_srk(diff, cnt1, cnt2); |
| } else { |
| z_lr(diff, cnt1); |
| z_sr(diff, cnt2); |
| } |
| if (str1 != str2) { |
| if (VM_Version::has_LoadStoreConditional()) { |
| z_locr(min, cnt2, Assembler::bcondHigh); |
| } else { |
| Label Lskip; |
| z_brl(Lskip); // min ok if cnt1 < cnt2 |
| z_lr(min, cnt2); // min = cnt2 |
| bind(Lskip); |
| } |
| } |
| |
| if (ae == StrIntrinsicNode::UU) { |
| z_sra(diff, 1); |
| } |
| if (str1 != str2) { |
| Label Ldone; |
| if (used_as_LU) { |
| // Loop which searches the first difference character by character. |
| Label Lloop; |
| const Register ind1 = Z_R1, |
| ind2 = min; |
| int stride1 = 1, stride2 = 2; // See comment above. |
| |
| // ind1: index, even_reg: index increment, odd_reg: index limit |
| z_llilf(ind1, (unsigned int)(-stride1)); |
| z_lhi(even_reg, stride1); |
| add2reg(odd_reg, -stride1, min); |
| clear_reg(ind2); // kills min |
| |
| bind(Lloop); |
| z_brxh(ind1, even_reg, Ldone); |
| z_llc(tmp, Address(str1, ind1)); |
| z_llh(Z_R0, Address(str2, ind2)); |
| z_ahi(ind2, stride2); |
| z_sr(tmp, Z_R0); |
| z_bre(Lloop); |
| |
| z_lr(result, tmp); |
| |
| } else { |
| // Use clcle in fast loop (only for same encoding). |
| z_lgr(Z_R0, str1); |
| z_lgr(even_reg, str2); |
| z_llgfr(Z_R1, min); |
| z_llgfr(odd_reg, min); |
| |
| if (ae == StrIntrinsicNode::LL) { |
| compare_long_ext(Z_R0, even_reg, 0); |
| } else { |
| compare_long_uni(Z_R0, even_reg, 0); |
| } |
| z_bre(Ldone); |
| z_lgr(Z_R1, Z_R0); |
| if (ae == StrIntrinsicNode::LL) { |
| z_llc(Z_R0, Address(even_reg)); |
| z_llc(result, Address(Z_R1)); |
| } else { |
| z_llh(Z_R0, Address(even_reg)); |
| z_llh(result, Address(Z_R1)); |
| } |
| z_sr(result, Z_R0); |
| } |
| |
| // Otherwise, return the difference between the first mismatched chars. |
| bind(Ldone); |
| } |
| |
| if (ae == StrIntrinsicNode::UL) { |
| z_lcr(result, result); // Negate result (see note above). |
| } |
| |
| BLOCK_COMMENT("} string_compare"); |
| |
| return offset() - block_start; |
| } |
| |
| unsigned int C2_MacroAssembler::array_equals(bool is_array_equ, Register ary1, Register ary2, Register limit, |
| Register odd_reg, Register even_reg, Register result, bool is_byte) { |
| int block_start = offset(); |
| |
| BLOCK_COMMENT("array_equals {"); |
| |
| assert_different_registers(ary1, limit, odd_reg, even_reg); |
| assert_different_registers(ary2, limit, odd_reg, even_reg); |
| |
| Label Ldone, Ldone_true, Ldone_false, Lclcle, CLC_template; |
| int base_offset = 0; |
| |
| if (ary1 != ary2) { |
| if (is_array_equ) { |
| base_offset = arrayOopDesc::base_offset_in_bytes(is_byte ? T_BYTE : T_CHAR); |
| |
| // Return true if the same array. |
| compareU64_and_branch(ary1, ary2, Assembler::bcondEqual, Ldone_true); |
| |
| // Return false if one of them is null. |
| compareU64_and_branch(ary1, (intptr_t)0, Assembler::bcondEqual, Ldone_false); |
| compareU64_and_branch(ary2, (intptr_t)0, Assembler::bcondEqual, Ldone_false); |
| |
| // Load the lengths of arrays. |
| z_llgf(odd_reg, Address(ary1, arrayOopDesc::length_offset_in_bytes())); |
| |
| // Return false if the two arrays are not equal length. |
| z_c(odd_reg, Address(ary2, arrayOopDesc::length_offset_in_bytes())); |
| z_brne(Ldone_false); |
| |
| // string len in bytes (right operand) |
| if (!is_byte) { |
| z_chi(odd_reg, 128); |
| z_sll(odd_reg, 1); // preserves flags |
| z_brh(Lclcle); |
| } else { |
| compareU32_and_branch(odd_reg, (intptr_t)256, Assembler::bcondHigh, Lclcle); |
| } |
| } else { |
| z_llgfr(odd_reg, limit); // Need to zero-extend prior to using the value. |
| compareU32_and_branch(limit, (intptr_t)256, Assembler::bcondHigh, Lclcle); |
| } |
| |
| |
| // Use clc instruction for up to 256 bytes. |
| { |
| Register str1_reg = ary1, |
| str2_reg = ary2; |
| if (is_array_equ) { |
| str1_reg = Z_R1; |
| str2_reg = even_reg; |
| add2reg(str1_reg, base_offset, ary1); // string addr (left operand) |
| add2reg(str2_reg, base_offset, ary2); // string addr (right operand) |
| } |
| z_ahi(odd_reg, -1); // Clc uses decremented limit. Also compare result to 0. |
| z_brl(Ldone_true); |
| // Note: We could jump to the template if equal. |
| |
| assert(VM_Version::has_ExecuteExtensions(), "unsupported hardware"); |
| z_exrl(odd_reg, CLC_template); |
| z_bre(Ldone_true); |
| // fall through |
| |
| bind(Ldone_false); |
| clear_reg(result); |
| z_bru(Ldone); |
| |
| bind(CLC_template); |
| z_clc(0, 0, str1_reg, 0, str2_reg); |
| } |
| |
| // Use clcle instruction. |
| { |
| bind(Lclcle); |
| add2reg(even_reg, base_offset, ary2); // string addr (right operand) |
| add2reg(Z_R0, base_offset, ary1); // string addr (left operand) |
| |
| z_lgr(Z_R1, odd_reg); // string len in bytes (left operand) |
| if (is_byte) { |
| compare_long_ext(Z_R0, even_reg, 0); |
| } else { |
| compare_long_uni(Z_R0, even_reg, 0); |
| } |
| z_lghi(result, 0); // Preserve flags. |
| z_brne(Ldone); |
| } |
| } |
| // fall through |
| |
| bind(Ldone_true); |
| z_lghi(result, 1); // All characters are equal. |
| bind(Ldone); |
| |
| BLOCK_COMMENT("} array_equals"); |
| |
| return offset() - block_start; |
| } |
| |
| // kill: haycnt, needlecnt, odd_reg, even_reg; early clobber: result |
| unsigned int C2_MacroAssembler::string_indexof(Register result, Register haystack, Register haycnt, |
| Register needle, Register needlecnt, int needlecntval, |
| Register odd_reg, Register even_reg, int ae) { |
| int block_start = offset(); |
| |
| // Ensure 0<needlecnt<=haycnt in ideal graph as prerequisite! |
| assert(ae != StrIntrinsicNode::LU, "Invalid encoding"); |
| const int h_csize = (ae == StrIntrinsicNode::LL) ? 1 : 2; |
| const int n_csize = (ae == StrIntrinsicNode::UU) ? 2 : 1; |
| Label L_needle1, L_Found, L_NotFound; |
| |
| BLOCK_COMMENT("string_indexof {"); |
| |
| if (needle == haystack) { |
| z_lhi(result, 0); |
| } else { |
| |
| // Load first character of needle (R0 used by search_string instructions). |
| if (n_csize == 2) { z_llgh(Z_R0, Address(needle)); } else { z_llgc(Z_R0, Address(needle)); } |
| |
| // Compute last haystack addr to use if no match gets found. |
| if (needlecnt != noreg) { // variable needlecnt |
| z_ahi(needlecnt, -1); // Remaining characters after first one. |
| z_sr(haycnt, needlecnt); // Compute index succeeding last element to compare. |
| if (n_csize == 2) { z_sll(needlecnt, 1); } // In bytes. |
| } else { // constant needlecnt |
| assert((needlecntval & 0x7fff) == needlecntval, "must be positive simm16 immediate"); |
| // Compute index succeeding last element to compare. |
| if (needlecntval != 1) { z_ahi(haycnt, 1 - needlecntval); } |
| } |
| |
| z_llgfr(haycnt, haycnt); // Clear high half. |
| z_lgr(result, haystack); // Final result will be computed from needle start pointer. |
| if (h_csize == 2) { z_sll(haycnt, 1); } // Scale to number of bytes. |
| z_agr(haycnt, haystack); // Point to address succeeding last element (haystack+scale*(haycnt-needlecnt+1)). |
| |
| if (h_csize != n_csize) { |
| assert(ae == StrIntrinsicNode::UL, "Invalid encoding"); |
| |
| if (needlecnt != noreg || needlecntval != 1) { |
| if (needlecnt != noreg) { |
| compare32_and_branch(needlecnt, (intptr_t)0, Assembler::bcondEqual, L_needle1); |
| } |
| |
| // Main Loop: UL version (now we have at least 2 characters). |
| Label L_OuterLoop, L_InnerLoop, L_Skip; |
| bind(L_OuterLoop); // Search for 1st 2 characters. |
| z_lgr(Z_R1, haycnt); |
| MacroAssembler::search_string_uni(Z_R1, result); |
| z_brc(Assembler::bcondNotFound, L_NotFound); |
| z_lgr(result, Z_R1); |
| |
| z_lghi(Z_R1, n_csize); |
| z_lghi(even_reg, h_csize); |
| bind(L_InnerLoop); |
| z_llgc(odd_reg, Address(needle, Z_R1)); |
| z_ch(odd_reg, Address(result, even_reg)); |
| z_brne(L_Skip); |
| if (needlecnt != noreg) { z_cr(Z_R1, needlecnt); } else { z_chi(Z_R1, needlecntval - 1); } |
| z_brnl(L_Found); |
| z_aghi(Z_R1, n_csize); |
| z_aghi(even_reg, h_csize); |
| z_bru(L_InnerLoop); |
| |
| bind(L_Skip); |
| z_aghi(result, h_csize); // This is the new address we want to use for comparing. |
| z_bru(L_OuterLoop); |
| } |
| |
| } else { |
| const intptr_t needle_bytes = (n_csize == 2) ? ((needlecntval - 1) << 1) : (needlecntval - 1); |
| Label L_clcle; |
| |
| if (needlecnt != noreg || (needlecntval != 1 && needle_bytes <= 256)) { |
| if (needlecnt != noreg) { |
| compare32_and_branch(needlecnt, 256, Assembler::bcondHigh, L_clcle); |
| z_ahi(needlecnt, -1); // remaining bytes -1 (for CLC) |
| z_brl(L_needle1); |
| } |
| |
| // Main Loop: clc version (now we have at least 2 characters). |
| Label L_OuterLoop, CLC_template; |
| bind(L_OuterLoop); // Search for 1st 2 characters. |
| z_lgr(Z_R1, haycnt); |
| if (h_csize == 1) { |
| MacroAssembler::search_string(Z_R1, result); |
| } else { |
| MacroAssembler::search_string_uni(Z_R1, result); |
| } |
| z_brc(Assembler::bcondNotFound, L_NotFound); |
| z_lgr(result, Z_R1); |
| |
| if (needlecnt != noreg) { |
| assert(VM_Version::has_ExecuteExtensions(), "unsupported hardware"); |
| z_exrl(needlecnt, CLC_template); |
| } else { |
| z_clc(h_csize, needle_bytes -1, Z_R1, n_csize, needle); |
| } |
| z_bre(L_Found); |
| z_aghi(result, h_csize); // This is the new address we want to use for comparing. |
| z_bru(L_OuterLoop); |
| |
| if (needlecnt != noreg) { |
| bind(CLC_template); |
| z_clc(h_csize, 0, Z_R1, n_csize, needle); |
| } |
| } |
| |
| if (needlecnt != noreg || needle_bytes > 256) { |
| bind(L_clcle); |
| |
| // Main Loop: clcle version (now we have at least 256 bytes). |
| Label L_OuterLoop, CLC_template; |
| bind(L_OuterLoop); // Search for 1st 2 characters. |
| z_lgr(Z_R1, haycnt); |
| if (h_csize == 1) { |
| MacroAssembler::search_string(Z_R1, result); |
| } else { |
| MacroAssembler::search_string_uni(Z_R1, result); |
| } |
| z_brc(Assembler::bcondNotFound, L_NotFound); |
| |
| add2reg(Z_R0, n_csize, needle); |
| add2reg(even_reg, h_csize, Z_R1); |
| z_lgr(result, Z_R1); |
| if (needlecnt != noreg) { |
| z_llgfr(Z_R1, needlecnt); // needle len in bytes (left operand) |
| z_llgfr(odd_reg, needlecnt); |
| } else { |
| load_const_optimized(Z_R1, needle_bytes); |
| if (Immediate::is_simm16(needle_bytes)) { z_lghi(odd_reg, needle_bytes); } else { z_lgr(odd_reg, Z_R1); } |
| } |
| if (h_csize == 1) { |
| compare_long_ext(Z_R0, even_reg, 0); |
| } else { |
| compare_long_uni(Z_R0, even_reg, 0); |
| } |
| z_bre(L_Found); |
| |
| if (n_csize == 2) { z_llgh(Z_R0, Address(needle)); } else { z_llgc(Z_R0, Address(needle)); } // Reload. |
| z_aghi(result, h_csize); // This is the new address we want to use for comparing. |
| z_bru(L_OuterLoop); |
| } |
| } |
| |
| if (needlecnt != noreg || needlecntval == 1) { |
| bind(L_needle1); |
| |
| // Single needle character version. |
| if (h_csize == 1) { |
| MacroAssembler::search_string(haycnt, result); |
| } else { |
| MacroAssembler::search_string_uni(haycnt, result); |
| } |
| z_lgr(result, haycnt); |
| z_brc(Assembler::bcondFound, L_Found); |
| } |
| |
| bind(L_NotFound); |
| add2reg(result, -1, haystack); // Return -1. |
| |
| bind(L_Found); // Return index (or -1 in fallthrough case). |
| z_sgr(result, haystack); |
| if (h_csize == 2) { z_srag(result, result, exact_log2(sizeof(jchar))); } |
| } |
| BLOCK_COMMENT("} string_indexof"); |
| |
| return offset() - block_start; |
| } |
| |
| // early clobber: result |
| unsigned int C2_MacroAssembler::string_indexof_char(Register result, Register haystack, Register haycnt, |
| Register needle, jchar needleChar, Register odd_reg, Register even_reg, bool is_byte) { |
| int block_start = offset(); |
| |
| BLOCK_COMMENT("string_indexof_char {"); |
| |
| if (needle == haystack) { |
| z_lhi(result, 0); |
| } else { |
| |
| Label Ldone; |
| |
| z_llgfr(odd_reg, haycnt); // Preset loop ctr/searchrange end. |
| if (needle == noreg) { |
| load_const_optimized(Z_R0, (unsigned long)needleChar); |
| } else { |
| if (is_byte) { |
| z_llgcr(Z_R0, needle); // First (and only) needle char. |
| } else { |
| z_llghr(Z_R0, needle); // First (and only) needle char. |
| } |
| } |
| |
| if (!is_byte) { |
| z_agr(odd_reg, odd_reg); // Calc #bytes to be processed with SRSTU. |
| } |
| |
| z_lgr(even_reg, haystack); // haystack addr |
| z_agr(odd_reg, haystack); // First char after range end. |
| z_lghi(result, -1); |
| |
| if (is_byte) { |
| MacroAssembler::search_string(odd_reg, even_reg); |
| } else { |
| MacroAssembler::search_string_uni(odd_reg, even_reg); |
| } |
| z_brc(Assembler::bcondNotFound, Ldone); |
| if (is_byte) { |
| if (VM_Version::has_DistinctOpnds()) { |
| z_sgrk(result, odd_reg, haystack); |
| } else { |
| z_sgr(odd_reg, haystack); |
| z_lgr(result, odd_reg); |
| } |
| } else { |
| z_slgr(odd_reg, haystack); |
| z_srlg(result, odd_reg, exact_log2(sizeof(jchar))); |
| } |
| |
| bind(Ldone); |
| } |
| BLOCK_COMMENT("} string_indexof_char"); |
| |
| return offset() - block_start; |
| } |
| |
