| /* |
| * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved. |
| * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| * |
| * This code is free software; you can redistribute it and/or modify it |
| * under the terms of the GNU General Public License version 2 only, as |
| * published by the Free Software Foundation. |
| * |
| * This code is distributed in the hope that it will be useful, but WITHOUT |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| * version 2 for more details (a copy is included in the LICENSE file that |
| * accompanied this code). |
| * |
| * You should have received a copy of the GNU General Public License version |
| * 2 along with this work; if not, write to the Free Software Foundation, |
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| * |
| * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| * or visit www.oracle.com if you need additional information or have any |
| * questions. |
| * |
| */ |
| |
| #include "asm/assembler.hpp" |
| #include "asm/assembler.inline.hpp" |
| #include "opto/c2_MacroAssembler.hpp" |
| #include "opto/intrinsicnode.hpp" |
| #include "runtime/vm_version.hpp" |
| |
| #ifdef PRODUCT |
| #define BLOCK_COMMENT(str) // nothing |
| #else |
| #define BLOCK_COMMENT(str) block_comment(str) |
| #endif |
| #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") |
| |
| |
| void C2_MacroAssembler::fast_lock_lightweight(ConditionRegister flag, Register obj, Register box, |
| Register tmp1, Register tmp2, Register tmp3) { |
| compiler_fast_lock_lightweight_object(flag, obj, box, tmp1, tmp2, tmp3); |
| } |
| |
| void C2_MacroAssembler::fast_unlock_lightweight(ConditionRegister flag, Register obj, Register box, |
| Register tmp1, Register tmp2, Register tmp3) { |
| compiler_fast_unlock_lightweight_object(flag, obj, box, tmp1, tmp2, tmp3); |
| } |
| |
| void C2_MacroAssembler::load_narrow_klass_compact_c2(Register dst, Register obj, int disp) { |
| // Note: Don't clobber obj anywhere in that method! |
| |
| // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract |
| // obj-start, so that we can load from the object's mark-word instead. |
| ld(dst, disp - oopDesc::klass_offset_in_bytes(), obj); |
| srdi(dst, dst, markWord::klass_shift); |
| } |
| |
| // Intrinsics for CompactStrings |
| |
| // Compress char[] to byte[] by compressing 16 bytes at once. |
| void C2_MacroAssembler::string_compress_16(Register src, Register dst, Register cnt, |
| Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, |
| Label& Lfailure, bool ascii) { |
| |
| const Register tmp0 = R0; |
| const int byte_mask = ascii ? 0x7F : 0xFF; |
| assert_different_registers(src, dst, cnt, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5); |
| Label Lloop, Lslow; |
| |
| // Check if cnt >= 8 (= 16 bytes) |
| lis(tmp1, byte_mask); // tmp1 = 0x00FF00FF00FF00FF (non ascii case) |
| srwi_(tmp2, cnt, 3); |
| beq(CR0, Lslow); |
| ori(tmp1, tmp1, byte_mask); |
| rldimi(tmp1, tmp1, 32, 0); |
| mtctr(tmp2); |
| |
| // 2x unrolled loop |
| bind(Lloop); |
| ld(tmp2, 0, src); // _0_1_2_3 (Big Endian) |
| ld(tmp4, 8, src); // _4_5_6_7 |
| |
| orr(tmp0, tmp2, tmp4); |
| rldicl(tmp3, tmp2, 6*8, 64-24); // _____1_2 |
| rldimi(tmp2, tmp2, 2*8, 2*8); // _0_2_3_3 |
| rldicl(tmp5, tmp4, 6*8, 64-24); // _____5_6 |
| rldimi(tmp4, tmp4, 2*8, 2*8); // _4_6_7_7 |
| |
| andc_(tmp0, tmp0, tmp1); |
| bne(CR0, Lfailure); // Not latin1/ascii. |
| addi(src, src, 16); |
| |
| rlwimi(tmp3, tmp2, 0*8, 24, 31);// _____1_3 |
| srdi(tmp2, tmp2, 3*8); // ____0_2_ |
| rlwimi(tmp5, tmp4, 0*8, 24, 31);// _____5_7 |
| srdi(tmp4, tmp4, 3*8); // ____4_6_ |
| |
| orr(tmp2, tmp2, tmp3); // ____0123 |
| orr(tmp4, tmp4, tmp5); // ____4567 |
| |
| stw(tmp2, 0, dst); |
| stw(tmp4, 4, dst); |
| addi(dst, dst, 8); |
| bdnz(Lloop); |
| |
| bind(Lslow); // Fallback to slow version |
| } |
| |
| // Compress char[] to byte[]. cnt must be positive int. |
| void C2_MacroAssembler::string_compress(Register src, Register dst, Register cnt, Register tmp, |
| Label& Lfailure, bool ascii) { |
| const int byte_mask = ascii ? 0x7F : 0xFF; |
| Label Lloop; |
| mtctr(cnt); |
| |
| bind(Lloop); |
| lhz(tmp, 0, src); |
| cmplwi(CR0, tmp, byte_mask); |
| bgt(CR0, Lfailure); // Not latin1/ascii. |
| addi(src, src, 2); |
| stb(tmp, 0, dst); |
| addi(dst, dst, 1); |
| bdnz(Lloop); |
| } |
| |
| void C2_MacroAssembler::encode_iso_array(Register src, Register dst, Register len, |
| Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, |
| Register result, bool ascii) { |
| Label Lslow, Lfailure1, Lfailure2, Ldone; |
| |
| string_compress_16(src, dst, len, tmp1, tmp2, tmp3, tmp4, tmp5, Lfailure1, ascii); |
| rldicl_(result, len, 0, 64-3); // Remaining characters. |
| beq(CR0, Ldone); |
| bind(Lslow); |
| string_compress(src, dst, result, tmp2, Lfailure2, ascii); |
| li(result, 0); |
| b(Ldone); |
| |
| bind(Lfailure1); |
| mr(result, len); |
| mfctr(tmp1); |
| rldimi_(result, tmp1, 3, 0); // Remaining characters. |
| beq(CR0, Ldone); |
| b(Lslow); |
| |
| bind(Lfailure2); |
| mfctr(result); // Remaining characters. |
| |
| bind(Ldone); |
| subf(result, result, len); |
| } |
| |
| // Inflate byte[] to char[] by inflating 16 bytes at once. |
| void C2_MacroAssembler::string_inflate_16(Register src, Register dst, Register cnt, |
| Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) { |
| const Register tmp0 = R0; |
| assert_different_registers(src, dst, cnt, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5); |
| Label Lloop, Lslow; |
| |
| // Check if cnt >= 8 |
| srwi_(tmp2, cnt, 3); |
| beq(CR0, Lslow); |
| lis(tmp1, 0xFF); // tmp1 = 0x00FF00FF |
| ori(tmp1, tmp1, 0xFF); |
| mtctr(tmp2); |
| |
| // 2x unrolled loop |
| bind(Lloop); |
| lwz(tmp2, 0, src); // ____0123 (Big Endian) |
| lwz(tmp4, 4, src); // ____4567 |
| addi(src, src, 8); |
| |
| rldicl(tmp3, tmp2, 7*8, 64-8); // _______2 |
| rlwimi(tmp2, tmp2, 3*8, 16, 23);// ____0113 |
| rldicl(tmp5, tmp4, 7*8, 64-8); // _______6 |
| rlwimi(tmp4, tmp4, 3*8, 16, 23);// ____4557 |
| |
| andc(tmp0, tmp2, tmp1); // ____0_1_ |
| rlwimi(tmp2, tmp3, 2*8, 0, 23); // _____2_3 |
| andc(tmp3, tmp4, tmp1); // ____4_5_ |
| rlwimi(tmp4, tmp5, 2*8, 0, 23); // _____6_7 |
| |
| rldimi(tmp2, tmp0, 3*8, 0*8); // _0_1_2_3 |
| rldimi(tmp4, tmp3, 3*8, 0*8); // _4_5_6_7 |
| |
| std(tmp2, 0, dst); |
| std(tmp4, 8, dst); |
| addi(dst, dst, 16); |
| bdnz(Lloop); |
| |
| bind(Lslow); // Fallback to slow version |
| } |
| |
| // Inflate byte[] to char[]. cnt must be positive int. |
| void C2_MacroAssembler::string_inflate(Register src, Register dst, Register cnt, Register tmp) { |
| Label Lloop; |
| mtctr(cnt); |
| |
| bind(Lloop); |
| lbz(tmp, 0, src); |
| addi(src, src, 1); |
| sth(tmp, 0, dst); |
| addi(dst, dst, 2); |
| bdnz(Lloop); |
| } |
| |
| void C2_MacroAssembler::string_compare(Register str1, Register str2, |
| Register cnt1, Register cnt2, |
| Register tmp1, Register result, int ae) { |
| const Register tmp0 = R0, |
| diff = tmp1; |
| |
| assert_different_registers(str1, str2, cnt1, cnt2, tmp0, tmp1, result); |
| Label Ldone, Lslow, Lloop, Lreturn_diff; |
| |
| // Note: Making use of the fact that compareTo(a, b) == -compareTo(b, a) |
| // we interchange str1 and str2 in the UL case and negate the result. |
| // Like this, str1 is always latin1 encoded, except for the UU case. |
| // In addition, we need 0 (or sign which is 0) extend. |
| |
| if (ae == StrIntrinsicNode::UU) { |
| srwi(cnt1, cnt1, 1); |
| } else { |
| clrldi(cnt1, cnt1, 32); |
| } |
| |
| if (ae != StrIntrinsicNode::LL) { |
| srwi(cnt2, cnt2, 1); |
| } else { |
| clrldi(cnt2, cnt2, 32); |
| } |
| |
| // See if the lengths are different, and calculate min in cnt1. |
| // Save diff in case we need it for a tie-breaker. |
| subf_(diff, cnt2, cnt1); // diff = cnt1 - cnt2 |
| // if (diff > 0) { cnt1 = cnt2; } |
| isel(cnt1, CR0, Assembler::greater, /*invert*/ false, cnt2); |
| |
| // Rename registers |
| Register chr1 = result; |
| Register chr2 = tmp0; |
| |
| // Compare multiple characters in fast loop (only implemented for same encoding). |
| int stride1 = 8, stride2 = 8; |
| if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { |
| int log2_chars_per_iter = (ae == StrIntrinsicNode::LL) ? 3 : 2; |
| Label Lfastloop, Lskipfast; |
| |
| srwi_(tmp0, cnt1, log2_chars_per_iter); |
| beq(CR0, Lskipfast); |
| rldicl(cnt2, cnt1, 0, 64 - log2_chars_per_iter); // Remaining characters. |
| li(cnt1, 1 << log2_chars_per_iter); // Initialize for failure case: Rescan characters from current iteration. |
| mtctr(tmp0); |
| |
| bind(Lfastloop); |
| ld(chr1, 0, str1); |
| ld(chr2, 0, str2); |
| cmpd(CR0, chr1, chr2); |
| bne(CR0, Lslow); |
| addi(str1, str1, stride1); |
| addi(str2, str2, stride2); |
| bdnz(Lfastloop); |
| mr(cnt1, cnt2); // Remaining characters. |
| bind(Lskipfast); |
| } |
| |
| // Loop which searches the first difference character by character. |
| cmpwi(CR0, cnt1, 0); |
| beq(CR0, Lreturn_diff); |
| bind(Lslow); |
| mtctr(cnt1); |
| |
| switch (ae) { |
| case StrIntrinsicNode::LL: stride1 = 1; stride2 = 1; break; |
| case StrIntrinsicNode::UL: // fallthru (see comment above) |
| case StrIntrinsicNode::LU: stride1 = 1; stride2 = 2; break; |
| case StrIntrinsicNode::UU: stride1 = 2; stride2 = 2; break; |
| default: ShouldNotReachHere(); break; |
| } |
| |
| bind(Lloop); |
| if (stride1 == 1) { lbz(chr1, 0, str1); } else { lhz(chr1, 0, str1); } |
| if (stride2 == 1) { lbz(chr2, 0, str2); } else { lhz(chr2, 0, str2); } |
| subf_(result, chr2, chr1); // result = chr1 - chr2 |
| bne(CR0, Ldone); |
| addi(str1, str1, stride1); |
| addi(str2, str2, stride2); |
| bdnz(Lloop); |
| |
| // If strings are equal up to min length, return the length difference. |
| bind(Lreturn_diff); |
| mr(result, diff); |
| |
| // Otherwise, return the difference between the first mismatched chars. |
| bind(Ldone); |
| if (ae == StrIntrinsicNode::UL) { |
| neg(result, result); // Negate result (see note above). |
| } |
| } |
| |
| void C2_MacroAssembler::array_equals(bool is_array_equ, Register ary1, Register ary2, |
| Register limit, Register tmp1, Register result, bool is_byte) { |
| const Register tmp0 = R0; |
| assert_different_registers(ary1, ary2, limit, tmp0, tmp1, result); |
| Label Ldone, Lskiploop, Lloop, Lfastloop, Lskipfast; |
| bool limit_needs_shift = false; |
| |
| if (is_array_equ) { |
| const int length_offset = arrayOopDesc::length_offset_in_bytes(); |
| const int base_offset = arrayOopDesc::base_offset_in_bytes(is_byte ? T_BYTE : T_CHAR); |
| |
| // Return true if the same array. |
| cmpd(CR0, ary1, ary2); |
| beq(CR0, Lskiploop); |
| |
| // Return false if one of them is null. |
| cmpdi(CR0, ary1, 0); |
| cmpdi(CR1, ary2, 0); |
| li(result, 0); |
| cror(CR0, Assembler::equal, CR1, Assembler::equal); |
| beq(CR0, Ldone); |
| |
| // Load the lengths of arrays. |
| lwz(limit, length_offset, ary1); |
| lwz(tmp0, length_offset, ary2); |
| |
| // Return false if the two arrays are not equal length. |
| cmpw(CR0, limit, tmp0); |
| bne(CR0, Ldone); |
| |
| // Load array addresses. |
| addi(ary1, ary1, base_offset); |
| addi(ary2, ary2, base_offset); |
| } else { |
| limit_needs_shift = !is_byte; |
| li(result, 0); // Assume not equal. |
| } |
| |
| // Rename registers |
| Register chr1 = tmp0; |
| Register chr2 = tmp1; |
| |
| // Compare 8 bytes per iteration in fast loop. |
| const int log2_chars_per_iter = is_byte ? 3 : 2; |
| |
| srwi_(tmp0, limit, log2_chars_per_iter + (limit_needs_shift ? 1 : 0)); |
| beq(CR0, Lskipfast); |
| mtctr(tmp0); |
| |
| bind(Lfastloop); |
| ld(chr1, 0, ary1); |
| ld(chr2, 0, ary2); |
| addi(ary1, ary1, 8); |
| addi(ary2, ary2, 8); |
| cmpd(CR0, chr1, chr2); |
| bne(CR0, Ldone); |
| bdnz(Lfastloop); |
| |
| bind(Lskipfast); |
| rldicl_(limit, limit, limit_needs_shift ? 64 - 1 : 0, 64 - log2_chars_per_iter); // Remaining characters. |
| beq(CR0, Lskiploop); |
| mtctr(limit); |
| |
| // Character by character. |
| bind(Lloop); |
| if (is_byte) { |
| lbz(chr1, 0, ary1); |
| lbz(chr2, 0, ary2); |
| addi(ary1, ary1, 1); |
| addi(ary2, ary2, 1); |
| } else { |
| lhz(chr1, 0, ary1); |
| lhz(chr2, 0, ary2); |
| addi(ary1, ary1, 2); |
| addi(ary2, ary2, 2); |
| } |
| cmpw(CR0, chr1, chr2); |
| bne(CR0, Ldone); |
| bdnz(Lloop); |
| |
| bind(Lskiploop); |
| li(result, 1); // All characters are equal. |
| bind(Ldone); |
| } |
| |
| void C2_MacroAssembler::string_indexof(Register result, Register haystack, Register haycnt, |
| Register needle, ciTypeArray* needle_values, Register needlecnt, int needlecntval, |
| Register tmp1, Register tmp2, Register tmp3, Register tmp4, int ae) { |
| |
| // Ensure 0<needlecnt<=haycnt in ideal graph as prerequisite! |
| Label L_TooShort, L_Found, L_NotFound, L_End; |
| Register last_addr = haycnt, // Kill haycnt at the beginning. |
| addr = tmp1, |
| n_start = tmp2, |
| ch1 = tmp3, |
| ch2 = R0; |
| |
| assert(ae != StrIntrinsicNode::LU, "Invalid encoding"); |
| const int h_csize = (ae == StrIntrinsicNode::LL) ? 1 : 2; |
| const int n_csize = (ae == StrIntrinsicNode::UU) ? 2 : 1; |
| |
| // ************************************************************************************************** |
| // Prepare for main loop: optimized for needle count >=2, bail out otherwise. |
| // ************************************************************************************************** |
| |
| // Compute last haystack addr to use if no match gets found. |
| clrldi(haycnt, haycnt, 32); // Ensure positive int is valid as 64 bit value. |
| addi(addr, haystack, -h_csize); // Accesses use pre-increment. |
| if (needlecntval == 0) { // variable needlecnt |
| cmpwi(CR6, needlecnt, 2); |
| clrldi(needlecnt, needlecnt, 32); // Ensure positive int is valid as 64 bit value. |
| blt(CR6, L_TooShort); // Variable needlecnt: handle short needle separately. |
| } |
| |
| if (n_csize == 2) { lwz(n_start, 0, needle); } else { lhz(n_start, 0, needle); } // Load first 2 characters of needle. |
| |
| if (needlecntval == 0) { // variable needlecnt |
| subf(ch1, needlecnt, haycnt); // Last character index to compare is haycnt-needlecnt. |
| addi(needlecnt, needlecnt, -2); // Rest of needle. |
| } else { // constant needlecnt |
| guarantee(needlecntval != 1, "IndexOf with single-character needle must be handled separately"); |
| assert((needlecntval & 0x7fff) == needlecntval, "wrong immediate"); |
| addi(ch1, haycnt, -needlecntval); // Last character index to compare is haycnt-needlecnt. |
| if (needlecntval > 3) { li(needlecnt, needlecntval - 2); } // Rest of needle. |
| } |
| |
| if (h_csize == 2) { slwi(ch1, ch1, 1); } // Scale to number of bytes. |
| |
| if (ae ==StrIntrinsicNode::UL) { |
| srwi(tmp4, n_start, 1*8); // ___0 |
| rlwimi(n_start, tmp4, 2*8, 0, 23); // _0_1 |
| } |
| |
| add(last_addr, haystack, ch1); // Point to last address to compare (haystack+2*(haycnt-needlecnt)). |
| |
| // Main Loop (now we have at least 2 characters). |
| Label L_OuterLoop, L_InnerLoop, L_FinalCheck, L_Comp1, L_Comp2; |
| bind(L_OuterLoop); // Search for 1st 2 characters. |
| Register addr_diff = tmp4; |
| subf(addr_diff, addr, last_addr); // Difference between already checked address and last address to check. |
| addi(addr, addr, h_csize); // This is the new address we want to use for comparing. |
| srdi_(ch2, addr_diff, h_csize); |
| beq(CR0, L_FinalCheck); // 2 characters left? |
| mtctr(ch2); // num of characters / 2 |
| bind(L_InnerLoop); // Main work horse (2x unrolled search loop) |
| if (h_csize == 2) { // Load 2 characters of haystack (ignore alignment). |
| lwz(ch1, 0, addr); |
| lwz(ch2, 2, addr); |
| } else { |
| lhz(ch1, 0, addr); |
| lhz(ch2, 1, addr); |
| } |
| cmpw(CR0, ch1, n_start); // Compare 2 characters (1 would be sufficient but try to reduce branches to CompLoop). |
| cmpw(CR1, ch2, n_start); |
| beq(CR0, L_Comp1); // Did we find the needle start? |
| beq(CR1, L_Comp2); |
| addi(addr, addr, 2 * h_csize); |
| bdnz(L_InnerLoop); |
| bind(L_FinalCheck); |
| andi_(addr_diff, addr_diff, h_csize); // Remaining characters not covered by InnerLoop: (num of characters) & 1. |
| beq(CR0, L_NotFound); |
| if (h_csize == 2) { lwz(ch1, 0, addr); } else { lhz(ch1, 0, addr); } // One position left at which we have to compare. |
| cmpw(CR1, ch1, n_start); |
| beq(CR1, L_Comp1); |
| bind(L_NotFound); |
| li(result, -1); // not found |
| b(L_End); |
| |
| // ************************************************************************************************** |
| // Special Case: unfortunately, the variable needle case can be called with needlecnt<2 |
| // ************************************************************************************************** |
| if (needlecntval == 0) { // We have to handle these cases separately. |
| Label L_OneCharLoop; |
| bind(L_TooShort); |
| mtctr(haycnt); |
| if (n_csize == 2) { lhz(n_start, 0, needle); } else { lbz(n_start, 0, needle); } // First character of needle |
| bind(L_OneCharLoop); |
| if (h_csize == 2) { lhzu(ch1, 2, addr); } else { lbzu(ch1, 1, addr); } |
| cmpw(CR1, ch1, n_start); |
| beq(CR1, L_Found); // Did we find the one character needle? |
| bdnz(L_OneCharLoop); |
| li(result, -1); // Not found. |
| b(L_End); |
| } |
| |
| // ************************************************************************************************** |
| // Regular Case Part II: compare rest of needle (first 2 characters have been compared already) |
| // ************************************************************************************************** |
| |
| // Compare the rest |
| bind(L_Comp2); |
| addi(addr, addr, h_csize); // First comparison has failed, 2nd one hit. |
| bind(L_Comp1); // Addr points to possible needle start. |
| if (needlecntval != 2) { // Const needlecnt==2? |
| if (needlecntval != 3) { |
| if (needlecntval == 0) { beq(CR6, L_Found); } // Variable needlecnt==2? |
| Register n_ind = tmp4, |
| h_ind = n_ind; |
| li(n_ind, 2 * n_csize); // First 2 characters are already compared, use index 2. |
| mtctr(needlecnt); // Decremented by 2, still > 0. |
| Label L_CompLoop; |
| bind(L_CompLoop); |
| if (ae ==StrIntrinsicNode::UL) { |
| h_ind = ch1; |
| sldi(h_ind, n_ind, 1); |
| } |
| if (n_csize == 2) { lhzx(ch2, needle, n_ind); } else { lbzx(ch2, needle, n_ind); } |
| if (h_csize == 2) { lhzx(ch1, addr, h_ind); } else { lbzx(ch1, addr, h_ind); } |
| cmpw(CR1, ch1, ch2); |
| bne(CR1, L_OuterLoop); |
| addi(n_ind, n_ind, n_csize); |
| bdnz(L_CompLoop); |
| } else { // No loop required if there's only one needle character left. |
| if (n_csize == 2) { lhz(ch2, 2 * 2, needle); } else { lbz(ch2, 2 * 1, needle); } |
| if (h_csize == 2) { lhz(ch1, 2 * 2, addr); } else { lbz(ch1, 2 * 1, addr); } |
| cmpw(CR1, ch1, ch2); |
| bne(CR1, L_OuterLoop); |
| } |
| } |
| // Return index ... |
| bind(L_Found); |
| subf(result, haystack, addr); // relative to haystack, ... |
| if (h_csize == 2) { srdi(result, result, 1); } // in characters. |
| bind(L_End); |
| } // string_indexof |
| |
| void C2_MacroAssembler::string_indexof_char(Register result, Register haystack, Register haycnt, |
| Register needle, jchar needleChar, Register tmp1, Register tmp2, bool is_byte) { |
| assert_different_registers(haystack, haycnt, needle, tmp1, tmp2); |
| |
| Label L_InnerLoop, L_FinalCheck, L_Found1, L_Found2, L_NotFound, L_End; |
| Register addr = tmp1, |
| ch1 = tmp2, |
| ch2 = R0; |
| |
| const int h_csize = is_byte ? 1 : 2; |
| |
| //4: |
| srwi_(tmp2, haycnt, 1); // Shift right by exact_log2(UNROLL_FACTOR). |
| mr(addr, haystack); |
| beq(CR0, L_FinalCheck); |
| mtctr(tmp2); // Move to count register. |
| //8: |
| bind(L_InnerLoop); // Main work horse (2x unrolled search loop). |
| if (!is_byte) { |
| lhz(ch1, 0, addr); |
| lhz(ch2, 2, addr); |
| } else { |
| lbz(ch1, 0, addr); |
| lbz(ch2, 1, addr); |
| } |
| (needle != R0) ? cmpw(CR0, ch1, needle) : cmplwi(CR0, ch1, (unsigned int)needleChar); |
| (needle != R0) ? cmpw(CR1, ch2, needle) : cmplwi(CR1, ch2, (unsigned int)needleChar); |
| beq(CR0, L_Found1); // Did we find the needle? |
| beq(CR1, L_Found2); |
| addi(addr, addr, 2 * h_csize); |
| bdnz(L_InnerLoop); |
| //16: |
| bind(L_FinalCheck); |
| andi_(R0, haycnt, 1); |
| beq(CR0, L_NotFound); |
| if (!is_byte) { lhz(ch1, 0, addr); } else { lbz(ch1, 0, addr); } // One position left at which we have to compare. |
| (needle != R0) ? cmpw(CR1, ch1, needle) : cmplwi(CR1, ch1, (unsigned int)needleChar); |
| beq(CR1, L_Found1); |
| //21: |
| bind(L_NotFound); |
| li(result, -1); // Not found. |
| b(L_End); |
| |
| bind(L_Found2); |
| addi(addr, addr, h_csize); |
| //24: |
| bind(L_Found1); // Return index ... |
| subf(result, haystack, addr); // relative to haystack, ... |
| if (!is_byte) { srdi(result, result, 1); } // in characters. |
| bind(L_End); |
| } // string_indexof_char |
| |
| |
| void C2_MacroAssembler::count_positives(Register src, Register cnt, Register result, |
| Register tmp1, Register tmp2) { |
| const Register tmp0 = R0; |
| assert_different_registers(src, result, cnt, tmp0, tmp1, tmp2); |
| Label Lfastloop, Lslow, Lloop, Ldone; |
| |
| // Check if cnt >= 8 (= 16 bytes) |
| lis(tmp1, (int)(short)0x8080); // tmp1 = 0x8080808080808080 |
| srwi_(tmp2, cnt, 4); |
| mr(result, src); // Use result reg to point to the current position. |
| beq(CR0, Lslow); |
| ori(tmp1, tmp1, 0x8080); |
| rldimi(tmp1, tmp1, 32, 0); |
| mtctr(tmp2); |
| |
| // 2x unrolled loop |
| bind(Lfastloop); |
| ld(tmp2, 0, result); |
| ld(tmp0, 8, result); |
| |
| orr(tmp0, tmp2, tmp0); |
| |
| and_(tmp0, tmp0, tmp1); |
| bne(CR0, Lslow); // Found negative byte. |
| addi(result, result, 16); |
| bdnz(Lfastloop); |
| |
| bind(Lslow); // Fallback to slow version. |
| subf(tmp0, src, result); // Bytes known positive. |
| subf_(tmp0, tmp0, cnt); // Remaining Bytes. |
| beq(CR0, Ldone); |
| mtctr(tmp0); |
| bind(Lloop); |
| lbz(tmp0, 0, result); |
| andi_(tmp0, tmp0, 0x80); |
| bne(CR0, Ldone); // Found negative byte. |
| addi(result, result, 1); |
| bdnz(Lloop); |
| |
| bind(Ldone); |
| subf(result, src, result); // Result is offset from src. |
| } |
| |
| void C2_MacroAssembler::reduceI(int opcode, Register dst, Register iSrc, VectorRegister vSrc, |
| VectorRegister vTmp1, VectorRegister vTmp2) { |
| |
| auto fn_vec_op = [this](int opcode, const VectorRegister &dst, const VectorRegister &a, const VectorRegister &b) { |
| switch(opcode) { |
| case Op_AddReductionVI: vadduwm(dst, a, b); break; |
| case Op_MulReductionVI: vmuluwm(dst, a , b); break; |
| case Op_AndReductionV: vand(dst, a, b); break; |
| case Op_OrReductionV: vor(dst, a, b); break; |
| case Op_XorReductionV: vxor(dst, a, b); break; |
| case Op_MinReductionV: vminsw(dst, a, b); break; |
| case Op_MaxReductionV: vmaxsw(dst, a, b); break; |
| default: assert(false, "wrong opcode"); |
| } |
| }; |
| |
| auto fn_scalar_op = [this](int opcode, const Register &dst, const Register &a, const Register &b) { |
| switch (opcode) { |
| case Op_AddReductionVI: add(dst, a, b); break; |
| case Op_MulReductionVI: mullw(dst, a, b); break; |
| case Op_AndReductionV: andr(dst, a, b); break; |
| case Op_OrReductionV: orr(dst, a, b); break; |
| case Op_XorReductionV: xorr(dst, a, b); break; |
| case Op_MinReductionV: |
| cmpw(CR0, a, b); |
| isel(dst, CR0, Assembler::less, /*invert*/false, a, b); |
| break; |
| case Op_MaxReductionV: |
| cmpw(CR0, a, b); |
| isel(dst, CR0, Assembler::greater, /*invert*/false, a, b); |
| break; |
| default: assert(false, "wrong opcode"); |
| } |
| }; |
| |
| // vSrc = [i0,i1,i2,i3] |
| vsldoi(vTmp1, vSrc, vSrc, 8); // vTmp1 <- [i2,i3,i0,i1] |
| fn_vec_op(opcode, vTmp2, vSrc, vTmp1); // vTmp2 <- [op(i0,i2), op(i1,i3), op(i2,i0), op(i3,i1)] |
| vsldoi(vTmp1, vTmp2, vTmp2, 4); // vTmp1 <- [op(i1,i3), op(i2,i0), op(i3,i1), op(i0,i2)] |
| fn_vec_op(opcode, vTmp1, vTmp1, vTmp2); // vTmp1 <- [op(i0,i1,i2,i3), op(i0,i1,i2,i3), op(i0,i1,i2,i3), op(i0,i1,i2,i3)] |
| mfvsrwz(R0, vTmp1.to_vsr()); // R0 <- op(i0,i1,i2,i3) |
| fn_scalar_op(opcode, dst, iSrc, R0); // dst <- op(iSrc, R0) |
| } |
| |
