| /* memcpy optimized with SSE2 unaligned memory access instructions. |
| Copyright (C) 2014-2018 Free Software Foundation, Inc. |
| This file is part of the GNU C Library. |
| |
| The GNU C Library is free software; you can redistribute it and/or |
| modify it under the terms of the GNU Lesser General Public |
| License as published by the Free Software Foundation; either |
| version 2.1 of the License, or (at your option) any later version. |
| |
| The GNU C Library is distributed in the hope that it will be useful, |
| but WITHOUT ANY WARRANTY; without even the implied warranty of |
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| Lesser General Public License for more details. |
| |
| You should have received a copy of the GNU Lesser General Public |
| License along with the GNU C Library; if not, see |
| <http://www.gnu.org/licenses/>. */ |
| |
| #if IS_IN (libc) \ |
| && (defined SHARED \ |
| || defined USE_AS_MEMMOVE \ |
| || !defined USE_MULTIARCH) |
| |
| # include <sysdep.h> |
| # include "asm-syntax.h" |
| |
| # ifndef MEMCPY |
| # define MEMCPY __memcpy_sse2_unaligned |
| # define MEMCPY_CHK __memcpy_chk_sse2_unaligned |
| # endif |
| |
| # ifdef USE_AS_BCOPY |
| # define SRC PARMS |
| # define DEST SRC+4 |
| # define LEN DEST+4 |
| # else |
| # define DEST PARMS |
| # define SRC DEST+4 |
| # define LEN SRC+4 |
| # endif |
| |
| # define CFI_PUSH(REG) \ |
| cfi_adjust_cfa_offset (4); \ |
| cfi_rel_offset (REG, 0) |
| |
| # define CFI_POP(REG) \ |
| cfi_adjust_cfa_offset (-4); \ |
| cfi_restore (REG) |
| |
| # define PUSH(REG) pushl REG; CFI_PUSH (REG) |
| # define POP(REG) popl REG; CFI_POP (REG) |
| |
| # define PARMS 8 /* Preserve EBX. */ |
| # define ENTRANCE PUSH (%ebx); |
| # define RETURN_END POP (%ebx); ret |
| # define RETURN RETURN_END; CFI_PUSH (%ebx) |
| |
| .section .text.sse2,"ax",@progbits |
| # if !defined USE_AS_BCOPY && defined SHARED |
| ENTRY (MEMCPY_CHK) |
| movl 12(%esp), %eax |
| cmpl %eax, 16(%esp) |
| jb HIDDEN_JUMPTARGET (__chk_fail) |
| END (MEMCPY_CHK) |
| # endif |
| |
| ENTRY (MEMCPY) |
| ENTRANCE |
| movl LEN(%esp), %ecx |
| movl SRC(%esp), %eax |
| movl DEST(%esp), %edx |
| cmp %edx, %eax |
| |
| # ifdef USE_AS_MEMMOVE |
| ja L(check_forward) |
| |
| L(mm_len_0_or_more_backward): |
| /* Now do checks for lengths. We do [0..16], [16..32], [32..64], [64..128] |
| separately. */ |
| cmp $16, %ecx |
| jbe L(mm_len_0_16_bytes_backward) |
| |
| cmpl $32, %ecx |
| ja L(mm_len_32_or_more_backward) |
| |
| /* Copy [0..32] and return. */ |
| movdqu (%eax), %xmm0 |
| movdqu -16(%eax, %ecx), %xmm1 |
| movdqu %xmm0, (%edx) |
| movdqu %xmm1, -16(%edx, %ecx) |
| jmp L(return) |
| |
| L(mm_len_32_or_more_backward): |
| cmpl $64, %ecx |
| ja L(mm_len_64_or_more_backward) |
| |
| /* Copy [0..64] and return. */ |
| movdqu (%eax), %xmm0 |
| movdqu 16(%eax), %xmm1 |
| movdqu -16(%eax, %ecx), %xmm2 |
| movdqu -32(%eax, %ecx), %xmm3 |
| movdqu %xmm0, (%edx) |
| movdqu %xmm1, 16(%edx) |
| movdqu %xmm2, -16(%edx, %ecx) |
| movdqu %xmm3, -32(%edx, %ecx) |
| jmp L(return) |
| |
| L(mm_len_64_or_more_backward): |
| cmpl $128, %ecx |
| ja L(mm_len_128_or_more_backward) |
| |
| /* Copy [0..128] and return. */ |
| movdqu (%eax), %xmm0 |
| movdqu 16(%eax), %xmm1 |
| movdqu 32(%eax), %xmm2 |
| movdqu 48(%eax), %xmm3 |
| movdqu -64(%eax, %ecx), %xmm4 |
| movdqu -48(%eax, %ecx), %xmm5 |
| movdqu -32(%eax, %ecx), %xmm6 |
| movdqu -16(%eax, %ecx), %xmm7 |
| movdqu %xmm0, (%edx) |
| movdqu %xmm1, 16(%edx) |
| movdqu %xmm2, 32(%edx) |
| movdqu %xmm3, 48(%edx) |
| movdqu %xmm4, -64(%edx, %ecx) |
| movdqu %xmm5, -48(%edx, %ecx) |
| movdqu %xmm6, -32(%edx, %ecx) |
| movdqu %xmm7, -16(%edx, %ecx) |
| jmp L(return) |
| |
| L(mm_len_128_or_more_backward): |
| add %ecx, %eax |
| cmp %edx, %eax |
| movl SRC(%esp), %eax |
| jbe L(forward) |
| PUSH (%esi) |
| PUSH (%edi) |
| PUSH (%ebx) |
| |
| /* Aligning the address of destination. */ |
| movdqu (%eax), %xmm4 |
| movdqu 16(%eax), %xmm5 |
| movdqu 32(%eax), %xmm6 |
| movdqu 48(%eax), %xmm7 |
| leal (%edx, %ecx), %esi |
| movdqu -16(%eax, %ecx), %xmm0 |
| subl $16, %esp |
| movdqu %xmm0, (%esp) |
| mov %ecx, %edi |
| movl %esi, %ecx |
| andl $-16, %ecx |
| leal (%ecx), %ebx |
| subl %edx, %ebx |
| leal (%eax, %ebx), %eax |
| shrl $6, %ebx |
| |
| # ifdef SHARED_CACHE_SIZE_HALF |
| cmp $SHARED_CACHE_SIZE_HALF, %edi |
| # else |
| # ifdef PIC |
| PUSH (%ebx) |
| SETUP_PIC_REG (bx) |
| add $_GLOBAL_OFFSET_TABLE_, %ebx |
| cmp __x86_shared_cache_size_half@GOTOFF(%ebx), %edi |
| POP (%ebx) |
| # else |
| cmp __x86_shared_cache_size_half, %edi |
| # endif |
| # endif |
| jae L(mm_large_page_loop_backward) |
| |
| .p2align 4 |
| L(mm_main_loop_backward): |
| |
| prefetcht0 -128(%eax) |
| |
| movdqu -64(%eax), %xmm0 |
| movdqu -48(%eax), %xmm1 |
| movdqu -32(%eax), %xmm2 |
| movdqu -16(%eax), %xmm3 |
| movaps %xmm0, -64(%ecx) |
| subl $64, %eax |
| movaps %xmm1, -48(%ecx) |
| movaps %xmm2, -32(%ecx) |
| movaps %xmm3, -16(%ecx) |
| subl $64, %ecx |
| sub $1, %ebx |
| jnz L(mm_main_loop_backward) |
| movdqu (%esp), %xmm0 |
| addl $16, %esp |
| movdqu %xmm0, -16(%esi) |
| movdqu %xmm4, (%edx) |
| movdqu %xmm5, 16(%edx) |
| movdqu %xmm6, 32(%edx) |
| movdqu %xmm7, 48(%edx) |
| POP (%ebx) |
| jmp L(mm_return_pop_all) |
| |
| /* Copy [0..16] and return. */ |
| L(mm_len_0_16_bytes_backward): |
| testb $24, %cl |
| jnz L(mm_len_9_16_bytes_backward) |
| testb $4, %cl |
| .p2align 4,,5 |
| jnz L(mm_len_5_8_bytes_backward) |
| testl %ecx, %ecx |
| .p2align 4,,2 |
| je L(return) |
| testb $2, %cl |
| .p2align 4,,1 |
| jne L(mm_len_3_4_bytes_backward) |
| movzbl -1(%eax,%ecx), %ebx |
| movzbl (%eax), %eax |
| movb %bl, -1(%edx,%ecx) |
| movb %al, (%edx) |
| jmp L(return) |
| |
| L(mm_len_3_4_bytes_backward): |
| movzwl -2(%eax,%ecx), %ebx |
| movzwl (%eax), %eax |
| movw %bx, -2(%edx,%ecx) |
| movw %ax, (%edx) |
| jmp L(return) |
| |
| L(mm_len_9_16_bytes_backward): |
| PUSH (%esi) |
| movl -4(%eax,%ecx), %ebx |
| movl -8(%eax,%ecx), %esi |
| movl %ebx, -4(%edx,%ecx) |
| movl %esi, -8(%edx,%ecx) |
| subl $8, %ecx |
| POP (%esi) |
| jmp L(mm_len_0_16_bytes_backward) |
| |
| L(mm_len_5_8_bytes_backward): |
| movl (%eax), %ebx |
| movl -4(%eax,%ecx), %eax |
| movl %ebx, (%edx) |
| movl %eax, -4(%edx,%ecx) |
| jmp L(return) |
| |
| /* Big length copy backward part. */ |
| .p2align 4 |
| L(mm_large_page_loop_backward): |
| movdqu -64(%eax), %xmm0 |
| movdqu -48(%eax), %xmm1 |
| movdqu -32(%eax), %xmm2 |
| movdqu -16(%eax), %xmm3 |
| movntdq %xmm0, -64(%ecx) |
| subl $64, %eax |
| movntdq %xmm1, -48(%ecx) |
| movntdq %xmm2, -32(%ecx) |
| movntdq %xmm3, -16(%ecx) |
| subl $64, %ecx |
| sub $1, %ebx |
| jnz L(mm_large_page_loop_backward) |
| sfence |
| movdqu (%esp), %xmm0 |
| addl $16, %esp |
| movdqu %xmm0, -16(%esi) |
| movdqu %xmm4, (%edx) |
| movdqu %xmm5, 16(%edx) |
| movdqu %xmm6, 32(%edx) |
| movdqu %xmm7, 48(%edx) |
| POP (%ebx) |
| jmp L(mm_return_pop_all) |
| |
| L(check_forward): |
| add %edx, %ecx |
| cmp %eax, %ecx |
| movl LEN(%esp), %ecx |
| jbe L(forward) |
| |
| /* Now do checks for lengths. We do [0..16], [0..32], [0..64], [0..128] |
| separately. */ |
| cmp $16, %ecx |
| jbe L(mm_len_0_16_bytes_forward) |
| |
| cmpl $32, %ecx |
| ja L(mm_len_32_or_more_forward) |
| |
| /* Copy [0..32] and return. */ |
| movdqu (%eax), %xmm0 |
| movdqu -16(%eax, %ecx), %xmm1 |
| movdqu %xmm0, (%edx) |
| movdqu %xmm1, -16(%edx, %ecx) |
| jmp L(return) |
| |
| L(mm_len_32_or_more_forward): |
| cmpl $64, %ecx |
| ja L(mm_len_64_or_more_forward) |
| |
| /* Copy [0..64] and return. */ |
| movdqu (%eax), %xmm0 |
| movdqu 16(%eax), %xmm1 |
| movdqu -16(%eax, %ecx), %xmm2 |
| movdqu -32(%eax, %ecx), %xmm3 |
| movdqu %xmm0, (%edx) |
| movdqu %xmm1, 16(%edx) |
| movdqu %xmm2, -16(%edx, %ecx) |
| movdqu %xmm3, -32(%edx, %ecx) |
| jmp L(return) |
| |
| L(mm_len_64_or_more_forward): |
| cmpl $128, %ecx |
| ja L(mm_len_128_or_more_forward) |
| |
| /* Copy [0..128] and return. */ |
| movdqu (%eax), %xmm0 |
| movdqu 16(%eax), %xmm1 |
| movdqu 32(%eax), %xmm2 |
| movdqu 48(%eax), %xmm3 |
| movdqu -64(%eax, %ecx), %xmm4 |
| movdqu -48(%eax, %ecx), %xmm5 |
| movdqu -32(%eax, %ecx), %xmm6 |
| movdqu -16(%eax, %ecx), %xmm7 |
| movdqu %xmm0, (%edx) |
| movdqu %xmm1, 16(%edx) |
| movdqu %xmm2, 32(%edx) |
| movdqu %xmm3, 48(%edx) |
| movdqu %xmm4, -64(%edx, %ecx) |
| movdqu %xmm5, -48(%edx, %ecx) |
| movdqu %xmm6, -32(%edx, %ecx) |
| movdqu %xmm7, -16(%edx, %ecx) |
| jmp L(return) |
| |
| L(mm_len_128_or_more_forward): |
| PUSH (%esi) |
| PUSH (%edi) |
| PUSH (%ebx) |
| |
| /* Aligning the address of destination. */ |
| movdqu -16(%eax, %ecx), %xmm4 |
| movdqu -32(%eax, %ecx), %xmm5 |
| movdqu -48(%eax, %ecx), %xmm6 |
| movdqu -64(%eax, %ecx), %xmm7 |
| leal (%edx, %ecx), %esi |
| movdqu (%eax), %xmm0 |
| subl $16, %esp |
| movdqu %xmm0, (%esp) |
| mov %ecx, %edi |
| leal 16(%edx), %ecx |
| andl $-16, %ecx |
| movl %ecx, %ebx |
| subl %edx, %ebx |
| addl %ebx, %eax |
| movl %esi, %ebx |
| subl %ecx, %ebx |
| shrl $6, %ebx |
| |
| # ifdef SHARED_CACHE_SIZE_HALF |
| cmp $SHARED_CACHE_SIZE_HALF, %edi |
| # else |
| # ifdef PIC |
| PUSH (%ebx) |
| SETUP_PIC_REG(bx) |
| add $_GLOBAL_OFFSET_TABLE_, %ebx |
| cmp __x86_shared_cache_size_half@GOTOFF(%ebx), %edi |
| POP (%ebx) |
| # else |
| cmp __x86_shared_cache_size_half, %edi |
| # endif |
| # endif |
| jae L(mm_large_page_loop_forward) |
| |
| .p2align 4 |
| L(mm_main_loop_forward): |
| |
| prefetcht0 128(%eax) |
| |
| movdqu (%eax), %xmm0 |
| movdqu 16(%eax), %xmm1 |
| movdqu 32(%eax), %xmm2 |
| movdqu 48(%eax), %xmm3 |
| movdqa %xmm0, (%ecx) |
| addl $64, %eax |
| movaps %xmm1, 16(%ecx) |
| movaps %xmm2, 32(%ecx) |
| movaps %xmm3, 48(%ecx) |
| addl $64, %ecx |
| sub $1, %ebx |
| jnz L(mm_main_loop_forward) |
| movdqu (%esp), %xmm0 |
| addl $16, %esp |
| movdqu %xmm0, (%edx) |
| movdqu %xmm4, -16(%esi) |
| movdqu %xmm5, -32(%esi) |
| movdqu %xmm6, -48(%esi) |
| movdqu %xmm7, -64(%esi) |
| POP (%ebx) |
| jmp L(mm_return_pop_all) |
| |
| L(mm_len_0_16_bytes_forward): |
| testb $24, %cl |
| jne L(mm_len_9_16_bytes_forward) |
| testb $4, %cl |
| .p2align 4,,5 |
| jne L(mm_len_5_8_bytes_forward) |
| testl %ecx, %ecx |
| .p2align 4,,2 |
| je L(return) |
| testb $2, %cl |
| .p2align 4,,1 |
| jne L(mm_len_2_4_bytes_forward) |
| movzbl -1(%eax,%ecx), %ebx |
| movzbl (%eax), %eax |
| movb %bl, -1(%edx,%ecx) |
| movb %al, (%edx) |
| jmp L(return) |
| |
| L(mm_len_2_4_bytes_forward): |
| movzwl -2(%eax,%ecx), %ebx |
| movzwl (%eax), %eax |
| movw %bx, -2(%edx,%ecx) |
| movw %ax, (%edx) |
| jmp L(return) |
| |
| L(mm_len_5_8_bytes_forward): |
| movl (%eax), %ebx |
| movl -4(%eax,%ecx), %eax |
| movl %ebx, (%edx) |
| movl %eax, -4(%edx,%ecx) |
| jmp L(return) |
| |
| L(mm_len_9_16_bytes_forward): |
| movq (%eax), %xmm0 |
| movq -8(%eax, %ecx), %xmm1 |
| movq %xmm0, (%edx) |
| movq %xmm1, -8(%edx, %ecx) |
| jmp L(return) |
| |
| L(mm_return_pop_all): |
| movl %edx, %eax |
| POP (%edi) |
| POP (%esi) |
| RETURN |
| |
| /* Big length copy forward part. */ |
| .p2align 4 |
| L(mm_large_page_loop_forward): |
| movdqu (%eax), %xmm0 |
| movdqu 16(%eax), %xmm1 |
| movdqu 32(%eax), %xmm2 |
| movdqu 48(%eax), %xmm3 |
| movntdq %xmm0, (%ecx) |
| addl $64, %eax |
| movntdq %xmm1, 16(%ecx) |
| movntdq %xmm2, 32(%ecx) |
| movntdq %xmm3, 48(%ecx) |
| addl $64, %ecx |
| sub $1, %ebx |
| jnz L(mm_large_page_loop_forward) |
| sfence |
| movdqu (%esp), %xmm0 |
| addl $16, %esp |
| movdqu %xmm0, (%edx) |
| movdqu %xmm4, -16(%esi) |
| movdqu %xmm5, -32(%esi) |
| movdqu %xmm6, -48(%esi) |
| movdqu %xmm7, -64(%esi) |
| POP (%ebx) |
| jmp L(mm_return_pop_all) |
| # endif |
| |
| L(forward): |
| cmp $16, %ecx |
| jbe L(len_0_16_bytes) |
| |
| # ifdef SHARED_CACHE_SIZE_HALF |
| cmp $SHARED_CACHE_SIZE_HALF, %ecx |
| # else |
| # ifdef PIC |
| SETUP_PIC_REG(bx) |
| add $_GLOBAL_OFFSET_TABLE_, %ebx |
| cmp __x86_shared_cache_size_half@GOTOFF(%ebx), %ecx |
| # else |
| cmp __x86_shared_cache_size_half, %ecx |
| # endif |
| # endif |
| jae L(large_page) |
| |
| movdqu (%eax), %xmm0 |
| movdqu -16(%eax, %ecx), %xmm1 |
| cmpl $32, %ecx |
| movdqu %xmm0, (%edx) |
| movdqu %xmm1, -16(%edx, %ecx) |
| jbe L(return) |
| |
| movdqu 16(%eax), %xmm0 |
| movdqu -32(%eax, %ecx), %xmm1 |
| cmpl $64, %ecx |
| movdqu %xmm0, 16(%edx) |
| movdqu %xmm1, -32(%edx, %ecx) |
| jbe L(return) |
| |
| movdqu 32(%eax), %xmm0 |
| movdqu 48(%eax), %xmm1 |
| movdqu -48(%eax, %ecx), %xmm2 |
| movdqu -64(%eax, %ecx), %xmm3 |
| cmpl $128, %ecx |
| movdqu %xmm0, 32(%edx) |
| movdqu %xmm1, 48(%edx) |
| movdqu %xmm2, -48(%edx, %ecx) |
| movdqu %xmm3, -64(%edx, %ecx) |
| jbe L(return) |
| |
| /* Now the main loop: we align the address of the destination. */ |
| leal 64(%edx), %ebx |
| andl $-64, %ebx |
| |
| addl %edx, %ecx |
| andl $-64, %ecx |
| |
| subl %edx, %eax |
| |
| /* We should stop two iterations before the termination |
| (in order not to misprefetch). */ |
| subl $64, %ecx |
| cmpl %ebx, %ecx |
| je L(main_loop_just_one_iteration) |
| |
| subl $64, %ecx |
| cmpl %ebx, %ecx |
| je L(main_loop_last_two_iterations) |
| |
| .p2align 4 |
| L(main_loop_cache): |
| |
| prefetcht0 128(%ebx, %eax) |
| |
| movdqu (%ebx, %eax), %xmm0 |
| movdqu 16(%ebx, %eax), %xmm1 |
| movdqu 32(%ebx, %eax), %xmm2 |
| movdqu 48(%ebx, %eax), %xmm3 |
| movdqa %xmm0, (%ebx) |
| movaps %xmm1, 16(%ebx) |
| movaps %xmm2, 32(%ebx) |
| movaps %xmm3, 48(%ebx) |
| lea 64(%ebx), %ebx |
| cmpl %ebx, %ecx |
| jne L(main_loop_cache) |
| |
| L(main_loop_last_two_iterations): |
| movdqu (%ebx, %eax), %xmm0 |
| movdqu 16(%ebx, %eax), %xmm1 |
| movdqu 32(%ebx, %eax), %xmm2 |
| movdqu 48(%ebx, %eax), %xmm3 |
| movdqu 64(%ebx, %eax), %xmm4 |
| movdqu 80(%ebx, %eax), %xmm5 |
| movdqu 96(%ebx, %eax), %xmm6 |
| movdqu 112(%ebx, %eax), %xmm7 |
| movdqa %xmm0, (%ebx) |
| movaps %xmm1, 16(%ebx) |
| movaps %xmm2, 32(%ebx) |
| movaps %xmm3, 48(%ebx) |
| movaps %xmm4, 64(%ebx) |
| movaps %xmm5, 80(%ebx) |
| movaps %xmm6, 96(%ebx) |
| movaps %xmm7, 112(%ebx) |
| jmp L(return) |
| |
| L(main_loop_just_one_iteration): |
| movdqu (%ebx, %eax), %xmm0 |
| movdqu 16(%ebx, %eax), %xmm1 |
| movdqu 32(%ebx, %eax), %xmm2 |
| movdqu 48(%ebx, %eax), %xmm3 |
| movdqa %xmm0, (%ebx) |
| movaps %xmm1, 16(%ebx) |
| movaps %xmm2, 32(%ebx) |
| movaps %xmm3, 48(%ebx) |
| jmp L(return) |
| |
| L(large_page): |
| movdqu (%eax), %xmm0 |
| movdqu 16(%eax), %xmm1 |
| movdqu 32(%eax), %xmm2 |
| movdqu 48(%eax), %xmm3 |
| movdqu -64(%eax, %ecx), %xmm4 |
| movdqu -48(%eax, %ecx), %xmm5 |
| movdqu -32(%eax, %ecx), %xmm6 |
| movdqu -16(%eax, %ecx), %xmm7 |
| movdqu %xmm0, (%edx) |
| movdqu %xmm1, 16(%edx) |
| movdqu %xmm2, 32(%edx) |
| movdqu %xmm3, 48(%edx) |
| movdqu %xmm4, -64(%edx, %ecx) |
| movdqu %xmm5, -48(%edx, %ecx) |
| movdqu %xmm6, -32(%edx, %ecx) |
| movdqu %xmm7, -16(%edx, %ecx) |
| |
| movdqu 64(%eax), %xmm0 |
| movdqu 80(%eax), %xmm1 |
| movdqu 96(%eax), %xmm2 |
| movdqu 112(%eax), %xmm3 |
| movdqu -128(%eax, %ecx), %xmm4 |
| movdqu -112(%eax, %ecx), %xmm5 |
| movdqu -96(%eax, %ecx), %xmm6 |
| movdqu -80(%eax, %ecx), %xmm7 |
| movdqu %xmm0, 64(%edx) |
| movdqu %xmm1, 80(%edx) |
| movdqu %xmm2, 96(%edx) |
| movdqu %xmm3, 112(%edx) |
| movdqu %xmm4, -128(%edx, %ecx) |
| movdqu %xmm5, -112(%edx, %ecx) |
| movdqu %xmm6, -96(%edx, %ecx) |
| movdqu %xmm7, -80(%edx, %ecx) |
| |
| /* Now the main loop with non temporal stores. We align |
| the address of the destination. */ |
| leal 128(%edx), %ebx |
| andl $-128, %ebx |
| |
| addl %edx, %ecx |
| andl $-128, %ecx |
| |
| subl %edx, %eax |
| |
| .p2align 4 |
| L(main_loop_large_page): |
| movdqu (%ebx, %eax), %xmm0 |
| movdqu 16(%ebx, %eax), %xmm1 |
| movdqu 32(%ebx, %eax), %xmm2 |
| movdqu 48(%ebx, %eax), %xmm3 |
| movdqu 64(%ebx, %eax), %xmm4 |
| movdqu 80(%ebx, %eax), %xmm5 |
| movdqu 96(%ebx, %eax), %xmm6 |
| movdqu 112(%ebx, %eax), %xmm7 |
| movntdq %xmm0, (%ebx) |
| movntdq %xmm1, 16(%ebx) |
| movntdq %xmm2, 32(%ebx) |
| movntdq %xmm3, 48(%ebx) |
| movntdq %xmm4, 64(%ebx) |
| movntdq %xmm5, 80(%ebx) |
| movntdq %xmm6, 96(%ebx) |
| movntdq %xmm7, 112(%ebx) |
| lea 128(%ebx), %ebx |
| cmpl %ebx, %ecx |
| jne L(main_loop_large_page) |
| sfence |
| jmp L(return) |
| |
| L(len_0_16_bytes): |
| testb $24, %cl |
| jne L(len_9_16_bytes) |
| testb $4, %cl |
| .p2align 4,,5 |
| jne L(len_5_8_bytes) |
| testl %ecx, %ecx |
| .p2align 4,,2 |
| je L(return) |
| movzbl (%eax), %ebx |
| testb $2, %cl |
| movb %bl, (%edx) |
| je L(return) |
| movzwl -2(%eax,%ecx), %ebx |
| movw %bx, -2(%edx,%ecx) |
| jmp L(return) |
| |
| L(len_9_16_bytes): |
| movq (%eax), %xmm0 |
| movq -8(%eax, %ecx), %xmm1 |
| movq %xmm0, (%edx) |
| movq %xmm1, -8(%edx, %ecx) |
| jmp L(return) |
| |
| L(len_5_8_bytes): |
| movl (%eax), %ebx |
| movl %ebx, (%edx) |
| movl -4(%eax,%ecx), %ebx |
| movl %ebx, -4(%edx,%ecx) |
| |
| L(return): |
| movl %edx, %eax |
| # if !defined USE_AS_BCOPY && defined USE_AS_MEMPCPY |
| movl LEN(%esp), %ecx |
| add %ecx, %eax |
| # endif |
| RETURN |
| |
| END (MEMCPY) |
| #endif |
