| /* strstr with unaligned loads |
| Copyright (C) 2009-2014 Free Software Foundation, Inc. |
| This file is part of the GNU C Library. |
| |
| The GNU C Library is free software; you can redistribute it and/or |
| modify it under the terms of the GNU Lesser General Public |
| License as published by the Free Software Foundation; either |
| version 2.1 of the License, or (at your option) any later version. |
| |
| The GNU C Library is distributed in the hope that it will be useful, |
| but WITHOUT ANY WARRANTY; without even the implied warranty of |
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| Lesser General Public License for more details. |
| |
| You should have received a copy of the GNU Lesser General Public |
| License along with the GNU C Library; if not, see |
| <http://www.gnu.org/licenses/>. */ |
| |
| #include <sysdep.h> |
| |
| ENTRY(__strstr_sse2_unaligned) |
| movzbl (%rsi), %eax |
| testb %al, %al |
| je L(empty) |
| movzbl 1(%rsi), %edx |
| testb %dl, %dl |
| je L(strchr) |
| movd %eax, %xmm1 |
| movd %edx, %xmm2 |
| movq %rdi, %rax |
| andl $4095, %eax |
| punpcklbw %xmm1, %xmm1 |
| cmpq $4031, %rax |
| punpcklbw %xmm2, %xmm2 |
| punpcklwd %xmm1, %xmm1 |
| punpcklwd %xmm2, %xmm2 |
| pshufd $0, %xmm1, %xmm1 |
| pshufd $0, %xmm2, %xmm2 |
| ja L(cross_page) |
| movdqu (%rdi), %xmm3 |
| pxor %xmm5, %xmm5 |
| movdqu 1(%rdi), %xmm4 |
| movdqa %xmm3, %xmm6 |
| pcmpeqb %xmm1, %xmm3 |
| pcmpeqb %xmm2, %xmm4 |
| movdqu 16(%rdi), %xmm0 |
| pcmpeqb %xmm5, %xmm6 |
| pminub %xmm4, %xmm3 |
| movdqa %xmm3, %xmm4 |
| movdqu 17(%rdi), %xmm3 |
| pcmpeqb %xmm0, %xmm5 |
| pcmpeqb %xmm2, %xmm3 |
| por %xmm6, %xmm4 |
| pcmpeqb %xmm1, %xmm0 |
| pminub %xmm3, %xmm0 |
| por %xmm5, %xmm0 |
| pmovmskb %xmm4, %r8d |
| pmovmskb %xmm0, %eax |
| salq $16, %rax |
| orq %rax, %r8 |
| je L(next_32_bytes) |
| L(next_pair_index): |
| bsf %r8, %rax |
| addq %rdi, %rax |
| cmpb $0, (%rax) |
| je L(zero1) |
| movzbl 2(%rsi), %edx |
| testb %dl, %dl |
| je L(found1) |
| cmpb 2(%rax), %dl |
| jne L(next_pair) |
| xorl %edx, %edx |
| jmp L(pair_loop_start) |
| |
| .p2align 4 |
| L(strchr): |
| movzbl %al, %esi |
| jmp __strchr_sse2 |
| |
| .p2align 4 |
| L(pair_loop): |
| addq $1, %rdx |
| cmpb 2(%rax,%rdx), %cl |
| jne L(next_pair) |
| L(pair_loop_start): |
| movzbl 3(%rsi,%rdx), %ecx |
| testb %cl, %cl |
| jne L(pair_loop) |
| L(found1): |
| ret |
| L(zero1): |
| xorl %eax, %eax |
| ret |
| |
| .p2align 4 |
| L(next_pair): |
| leaq -1(%r8), %rax |
| andq %rax, %r8 |
| jne L(next_pair_index) |
| |
| .p2align 4 |
| L(next_32_bytes): |
| movdqu 32(%rdi), %xmm3 |
| pxor %xmm5, %xmm5 |
| movdqu 33(%rdi), %xmm4 |
| movdqa %xmm3, %xmm6 |
| pcmpeqb %xmm1, %xmm3 |
| pcmpeqb %xmm2, %xmm4 |
| movdqu 48(%rdi), %xmm0 |
| pcmpeqb %xmm5, %xmm6 |
| pminub %xmm4, %xmm3 |
| movdqa %xmm3, %xmm4 |
| movdqu 49(%rdi), %xmm3 |
| pcmpeqb %xmm0, %xmm5 |
| pcmpeqb %xmm2, %xmm3 |
| por %xmm6, %xmm4 |
| pcmpeqb %xmm1, %xmm0 |
| pminub %xmm3, %xmm0 |
| por %xmm5, %xmm0 |
| pmovmskb %xmm4, %eax |
| salq $32, %rax |
| pmovmskb %xmm0, %r8d |
| salq $48, %r8 |
| orq %rax, %r8 |
| je L(loop_header) |
| L(next_pair2_index): |
| bsfq %r8, %rax |
| addq %rdi, %rax |
| cmpb $0, (%rax) |
| je L(zero2) |
| movzbl 2(%rsi), %edx |
| testb %dl, %dl |
| je L(found2) |
| cmpb 2(%rax), %dl |
| jne L(next_pair2) |
| xorl %edx, %edx |
| jmp L(pair_loop2_start) |
| |
| .p2align 4 |
| L(pair_loop2): |
| addq $1, %rdx |
| cmpb 2(%rax,%rdx), %cl |
| jne L(next_pair2) |
| L(pair_loop2_start): |
| movzbl 3(%rsi,%rdx), %ecx |
| testb %cl, %cl |
| jne L(pair_loop2) |
| L(found2): |
| ret |
| L(zero2): |
| xorl %eax, %eax |
| ret |
| L(empty): |
| mov %rdi, %rax |
| ret |
| |
| .p2align 4 |
| L(next_pair2): |
| leaq -1(%r8), %rax |
| andq %rax, %r8 |
| jne L(next_pair2_index) |
| L(loop_header): |
| movq $-512, %r11 |
| movq %rdi, %r9 |
| |
| pxor %xmm7, %xmm7 |
| andq $-64, %rdi |
| |
| .p2align 4 |
| L(loop): |
| movdqa 64(%rdi), %xmm3 |
| movdqu 63(%rdi), %xmm6 |
| movdqa %xmm3, %xmm0 |
| pxor %xmm2, %xmm3 |
| pxor %xmm1, %xmm6 |
| movdqa 80(%rdi), %xmm10 |
| por %xmm3, %xmm6 |
| pminub %xmm10, %xmm0 |
| movdqu 79(%rdi), %xmm3 |
| pxor %xmm2, %xmm10 |
| pxor %xmm1, %xmm3 |
| movdqa 96(%rdi), %xmm9 |
| por %xmm10, %xmm3 |
| pminub %xmm9, %xmm0 |
| pxor %xmm2, %xmm9 |
| movdqa 112(%rdi), %xmm8 |
| addq $64, %rdi |
| pminub %xmm6, %xmm3 |
| movdqu 31(%rdi), %xmm4 |
| pminub %xmm8, %xmm0 |
| pxor %xmm2, %xmm8 |
| pxor %xmm1, %xmm4 |
| por %xmm9, %xmm4 |
| pminub %xmm4, %xmm3 |
| movdqu 47(%rdi), %xmm5 |
| pxor %xmm1, %xmm5 |
| por %xmm8, %xmm5 |
| pminub %xmm5, %xmm3 |
| pminub %xmm3, %xmm0 |
| pcmpeqb %xmm7, %xmm0 |
| pmovmskb %xmm0, %eax |
| testl %eax, %eax |
| je L(loop) |
| pminub (%rdi), %xmm6 |
| pminub 32(%rdi),%xmm4 |
| pminub 48(%rdi),%xmm5 |
| pcmpeqb %xmm7, %xmm6 |
| pcmpeqb %xmm7, %xmm5 |
| pmovmskb %xmm6, %edx |
| movdqa 16(%rdi), %xmm8 |
| pcmpeqb %xmm7, %xmm4 |
| movdqu 15(%rdi), %xmm0 |
| pmovmskb %xmm5, %r8d |
| movdqa %xmm8, %xmm3 |
| pmovmskb %xmm4, %ecx |
| pcmpeqb %xmm1,%xmm0 |
| pcmpeqb %xmm2,%xmm3 |
| salq $32, %rcx |
| pcmpeqb %xmm7,%xmm8 |
| salq $48, %r8 |
| pminub %xmm0,%xmm3 |
| orq %rcx, %rdx |
| por %xmm3,%xmm8 |
| orq %rdx, %r8 |
| pmovmskb %xmm8, %eax |
| salq $16, %rax |
| orq %rax, %r8 |
| je L(loop) |
| L(next_pair_index3): |
| bsfq %r8, %rcx |
| addq %rdi, %rcx |
| cmpb $0, (%rcx) |
| je L(zero) |
| xorl %eax, %eax |
| movzbl 2(%rsi), %edx |
| testb %dl, %dl |
| je L(success3) |
| cmpb 1(%rcx), %dl |
| jne L(next_pair3) |
| jmp L(pair_loop_start3) |
| |
| .p2align 4 |
| L(pair_loop3): |
| addq $1, %rax |
| cmpb 1(%rcx,%rax), %dl |
| jne L(next_pair3) |
| L(pair_loop_start3): |
| movzbl 3(%rsi,%rax), %edx |
| testb %dl, %dl |
| jne L(pair_loop3) |
| L(success3): |
| lea -1(%rcx), %rax |
| ret |
| |
| .p2align 4 |
| L(next_pair3): |
| addq %rax, %r11 |
| movq %rdi, %rax |
| subq %r9, %rax |
| cmpq %r11, %rax |
| jl L(switch_strstr) |
| leaq -1(%r8), %rax |
| andq %rax, %r8 |
| jne L(next_pair_index3) |
| jmp L(loop) |
| |
| .p2align 4 |
| L(switch_strstr): |
| movq %rdi, %rdi |
| jmp __strstr_sse2 |
| |
| .p2align 4 |
| L(cross_page): |
| |
| movq %rdi, %rax |
| pxor %xmm0, %xmm0 |
| andq $-64, %rax |
| movdqa (%rax), %xmm3 |
| movdqu -1(%rax), %xmm4 |
| movdqa %xmm3, %xmm8 |
| movdqa 16(%rax), %xmm5 |
| pcmpeqb %xmm1, %xmm4 |
| pcmpeqb %xmm0, %xmm8 |
| pcmpeqb %xmm2, %xmm3 |
| movdqa %xmm5, %xmm7 |
| pminub %xmm4, %xmm3 |
| movdqu 15(%rax), %xmm4 |
| pcmpeqb %xmm0, %xmm7 |
| por %xmm3, %xmm8 |
| movdqa %xmm5, %xmm3 |
| movdqa 32(%rax), %xmm5 |
| pcmpeqb %xmm1, %xmm4 |
| pcmpeqb %xmm2, %xmm3 |
| movdqa %xmm5, %xmm6 |
| pmovmskb %xmm8, %ecx |
| pminub %xmm4, %xmm3 |
| movdqu 31(%rax), %xmm4 |
| por %xmm3, %xmm7 |
| movdqa %xmm5, %xmm3 |
| pcmpeqb %xmm0, %xmm6 |
| movdqa 48(%rax), %xmm5 |
| pcmpeqb %xmm1, %xmm4 |
| pmovmskb %xmm7, %r8d |
| pcmpeqb %xmm2, %xmm3 |
| pcmpeqb %xmm5, %xmm0 |
| pminub %xmm4, %xmm3 |
| movdqu 47(%rax), %xmm4 |
| por %xmm3, %xmm6 |
| movdqa %xmm5, %xmm3 |
| salq $16, %r8 |
| pcmpeqb %xmm1, %xmm4 |
| pcmpeqb %xmm2, %xmm3 |
| pmovmskb %xmm6, %r10d |
| pminub %xmm4, %xmm3 |
| por %xmm3, %xmm0 |
| salq $32, %r10 |
| orq %r10, %r8 |
| orq %rcx, %r8 |
| movl %edi, %ecx |
| pmovmskb %xmm0, %edx |
| subl %eax, %ecx |
| salq $48, %rdx |
| orq %rdx, %r8 |
| shrq %cl, %r8 |
| je L(loop_header) |
| L(next_pair_index4): |
| bsfq %r8, %rax |
| addq %rdi, %rax |
| cmpb $0, (%rax) |
| je L(zero) |
| |
| cmpq %rax,%rdi |
| je L(next_pair4) |
| |
| movzbl 2(%rsi), %edx |
| testb %dl, %dl |
| je L(found3) |
| cmpb 1(%rax), %dl |
| jne L(next_pair4) |
| xorl %edx, %edx |
| jmp L(pair_loop_start4) |
| |
| .p2align 4 |
| L(pair_loop4): |
| addq $1, %rdx |
| cmpb 1(%rax,%rdx), %cl |
| jne L(next_pair4) |
| L(pair_loop_start4): |
| movzbl 3(%rsi,%rdx), %ecx |
| testb %cl, %cl |
| jne L(pair_loop4) |
| L(found3): |
| subq $1, %rax |
| ret |
| |
| .p2align 4 |
| L(next_pair4): |
| leaq -1(%r8), %rax |
| andq %rax, %r8 |
| jne L(next_pair_index4) |
| jmp L(loop_header) |
| |
| .p2align 4 |
| L(found): |
| rep |
| ret |
| |
| .p2align 4 |
| L(zero): |
| xorl %eax, %eax |
| ret |
| |
| |
| END(__strstr_sse2_unaligned) |
