| /* Optimized memset implementation for PowerPC32/POWER7. |
| Copyright (C) 2010-2018 Free Software Foundation, Inc. |
| Contributed by Luis Machado <luisgpm@br.ibm.com>. |
| This file is part of the GNU C Library. |
| |
| The GNU C Library is free software; you can redistribute it and/or |
| modify it under the terms of the GNU Lesser General Public |
| License as published by the Free Software Foundation; either |
| version 2.1 of the License, or (at your option) any later version. |
| |
| The GNU C Library is distributed in the hope that it will be useful, |
| but WITHOUT ANY WARRANTY; without even the implied warranty of |
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| Lesser General Public License for more details. |
| |
| You should have received a copy of the GNU Lesser General Public |
| License along with the GNU C Library; if not, see |
| <http://www.gnu.org/licenses/>. */ |
| |
| #include <sysdep.h> |
| |
| /* void * [r3] memset (void *s [r3], int c [r4], size_t n [r5])); |
| Returns 's'. */ |
| |
| .machine power7 |
| EALIGN (memset, 5, 0) |
| CALL_MCOUNT |
| |
| .align 4 |
| L(_memset): |
| cmplwi cr7,5,31 |
| cmplwi cr6,5,8 |
| mr 10,3 /* Save original argument for later. */ |
| mr 7,1 /* Save original r1 for later. */ |
| cfi_offset(31,-8) |
| |
| /* Replicate byte to word. */ |
| insrwi 4,4,8,16 |
| insrwi 4,4,16,0 |
| |
| ble cr6,L(small) /* If length <= 8, use short copy code. */ |
| |
| neg 0,3 |
| ble cr7,L(medium) /* If length < 32, use medium copy code. */ |
| |
| /* Save our word twice to create a doubleword that we will later |
| copy to a FPR. */ |
| stwu 1,-32(1) |
| andi. 11,10,7 /* Check alignment of DST. */ |
| mr 12,5 |
| stw 4,24(1) |
| stw 4,28(1) |
| beq L(big_aligned) |
| |
| clrlwi 0,0,29 |
| mtocrf 0x01,0 |
| subf 5,0,5 |
| |
| /* Get DST aligned to 8 bytes. */ |
| 1: bf 31,2f |
| |
| stb 4,0(10) |
| addi 10,10,1 |
| 2: bf 30,4f |
| |
| sth 4,0(10) |
| addi 10,10,2 |
| 4: bf 29,L(big_aligned) |
| |
| stw 4,0(10) |
| addi 10,10,4 |
| |
| .align 4 |
| L(big_aligned): |
| cmplwi cr5,5,255 |
| li 0,32 |
| cmplwi cr1,5,160 |
| dcbtst 0,10 |
| cmplwi cr6,4,0 |
| srwi 9,5,3 /* Number of full doublewords remaining. */ |
| crand 27,26,21 |
| mtocrf 0x01,9 |
| bt 27,L(huge) |
| |
| /* From this point on, we'll copy 32+ bytes and the value |
| isn't 0 (so we can't use dcbz). */ |
| |
| srwi 8,5,5 |
| clrlwi 11,5,29 |
| cmplwi cr6,11,0 |
| cmplwi cr1,9,4 |
| mtctr 8 |
| |
| /* Copy 1~3 doublewords so the main loop starts |
| at a multiple of 32 bytes. */ |
| |
| bf 30,1f |
| |
| stw 4,0(10) |
| stw 4,4(10) |
| stw 4,8(10) |
| stw 4,12(10) |
| addi 10,10,16 |
| bf 31,L(big_loop) |
| |
| stw 4,0(10) |
| stw 4,4(10) |
| addi 10,10,8 |
| mr 12,10 |
| blt cr1,L(tail_bytes) |
| |
| b L(big_loop) |
| |
| .align 4 |
| 1: /* Copy 1 doubleword. */ |
| bf 31,L(big_loop) |
| |
| stw 4,0(10) |
| stw 4,4(10) |
| addi 10,10,8 |
| |
| /* First use a 32-bytes loop with stw's to try and avoid the LHS due |
| to the lfd we will do next. Also, ping-pong through r10 and r12 |
| to avoid AGEN delays. */ |
| .align 4 |
| L(big_loop): |
| addi 12,10,32 |
| stw 4,0(10) |
| stw 4,4(10) |
| stw 4,8(10) |
| stw 4,12(10) |
| stw 4,16(10) |
| stw 4,20(10) |
| stw 4,24(10) |
| stw 4,28(10) |
| bdz L(tail_bytes) |
| |
| addi 10,10,64 |
| stw 4,0(12) |
| stw 4,4(12) |
| stw 4,8(12) |
| stw 4,12(12) |
| stw 4,16(12) |
| stw 4,20(12) |
| stw 4,24(12) |
| stw 4,28(12) |
| bdnz L(big_loop_fast_setup) |
| |
| mr 12,10 |
| b L(tail_bytes) |
| |
| /* Now that we're probably past the LHS window, use the VSX to |
| speed up the loop. */ |
| L(big_loop_fast_setup): |
| li 11,24 |
| li 6,16 |
| lxvdsx 4,1,11 |
| |
| .align 4 |
| L(big_loop_fast): |
| addi 12,10,32 |
| stxvd2x 4,0,10 |
| stxvd2x 4,10,6 |
| bdz L(tail_bytes) |
| |
| addi 10,10,64 |
| stxvd2x 4,0,12 |
| stxvd2x 4,12,6 |
| bdnz L(big_loop_fast) |
| |
| mr 12,10 |
| |
| .align 4 |
| L(tail_bytes): |
| |
| /* Check for tail bytes. */ |
| mr 1,7 /* Restore r1. */ |
| beqlr cr6 |
| |
| clrlwi 0,5,29 |
| mtocrf 0x01,0 |
| |
| /* At this point we have a tail of 0-7 bytes and we know that the |
| destination is doubleword-aligned. */ |
| 4: /* Copy 4 bytes. */ |
| bf 29,2f |
| |
| stw 4,0(12) |
| addi 12,12,4 |
| 2: /* Copy 2 bytes. */ |
| bf 30,1f |
| |
| sth 4,0(12) |
| addi 12,12,2 |
| 1: /* Copy 1 byte. */ |
| bflr 31 |
| |
| stb 4,0(12) |
| blr |
| |
| |
| /* Special case when value is 0 and we have a long length to deal |
| with. Use dcbz to zero out 128-bytes at a time. Before using |
| dcbz though, we need to get the destination 128-bytes aligned. */ |
| .align 4 |
| L(huge): |
| lfd 4,24(1) |
| andi. 11,10,127 |
| neg 0,10 |
| beq L(huge_aligned) |
| |
| clrlwi 0,0,25 |
| subf 5,0,5 |
| srwi 0,0,3 |
| mtocrf 0x01,0 |
| |
| /* Get DST aligned to 128 bytes. */ |
| 8: bf 28,4f |
| |
| stfd 4,0(10) |
| stfd 4,8(10) |
| stfd 4,16(10) |
| stfd 4,24(10) |
| stfd 4,32(10) |
| stfd 4,40(10) |
| stfd 4,48(10) |
| stfd 4,56(10) |
| addi 10,10,64 |
| .align 4 |
| 4: bf 29,2f |
| |
| stfd 4,0(10) |
| stfd 4,8(10) |
| stfd 4,16(10) |
| stfd 4,24(10) |
| addi 10,10,32 |
| .align 4 |
| 2: bf 30,1f |
| |
| stfd 4,0(10) |
| stfd 4,8(10) |
| addi 10,10,16 |
| .align 4 |
| 1: bf 31,L(huge_aligned) |
| |
| stfd 4,0(10) |
| addi 10,10,8 |
| |
| L(huge_aligned): |
| srwi 8,5,7 |
| clrlwi 11,5,25 |
| cmplwi cr6,11,0 |
| mtctr 8 |
| |
| /* Copies 128-bytes at a time. */ |
| .align 4 |
| L(huge_loop): |
| dcbz 0,10 |
| addi 10,10,128 |
| bdnz L(huge_loop) |
| |
| /* We have a tail of 0~127 bytes to handle. */ |
| mr 1,7 /* Restore r1. */ |
| beqlr cr6 |
| |
| subf 9,3,10 |
| subf 5,9,12 |
| srwi 8,5,3 |
| cmplwi cr6,8,0 |
| mtocrf 0x01,8 |
| |
| /* We have a tail o 1~127 bytes. Copy up to 15 doublewords for |
| speed. We'll handle the resulting tail bytes later. */ |
| beq cr6,L(tail) |
| |
| 8: bf 28,4f |
| |
| stfd 4,0(10) |
| stfd 4,8(10) |
| stfd 4,16(10) |
| stfd 4,24(10) |
| stfd 4,32(10) |
| stfd 4,40(10) |
| stfd 4,48(10) |
| stfd 4,56(10) |
| addi 10,10,64 |
| .align 4 |
| 4: bf 29,2f |
| |
| stfd 4,0(10) |
| stfd 4,8(10) |
| stfd 4,16(10) |
| stfd 4,24(10) |
| addi 10,10,32 |
| .align 4 |
| 2: bf 30,1f |
| |
| stfd 4,0(10) |
| stfd 4,8(10) |
| addi 10,10,16 |
| .align 4 |
| 1: bf 31,L(tail) |
| |
| stfd 4,0(10) |
| addi 10,10,8 |
| |
| /* Handle the rest of the tail bytes here. */ |
| L(tail): |
| mtocrf 0x01,5 |
| |
| .align 4 |
| 4: bf 29,2f |
| |
| stw 4,0(10) |
| addi 10,10,4 |
| .align 4 |
| 2: bf 30,1f |
| |
| sth 4,0(10) |
| addi 10,10,2 |
| .align 4 |
| 1: bflr 31 |
| |
| stb 4,0(10) |
| blr |
| |
| |
| /* Expanded tree to copy tail bytes without increments. */ |
| .align 4 |
| L(copy_tail): |
| bf 29,L(FXX) |
| |
| stw 4,0(10) |
| bf 30,L(TFX) |
| |
| sth 4,4(10) |
| bflr 31 |
| |
| stb 4,6(10) |
| blr |
| |
| .align 4 |
| L(FXX): bf 30,L(FFX) |
| |
| sth 4,0(10) |
| bflr 31 |
| |
| stb 4,2(10) |
| blr |
| |
| .align 4 |
| L(TFX): bflr 31 |
| |
| stb 4,4(10) |
| blr |
| |
| .align 4 |
| L(FFX): bflr 31 |
| |
| stb 4,0(10) |
| blr |
| |
| /* Handle copies of 9~31 bytes. */ |
| .align 4 |
| L(medium): |
| /* At least 9 bytes to go. */ |
| andi. 11,10,3 |
| clrlwi 0,0,30 |
| beq L(medium_aligned) |
| |
| /* Force 4-bytes alignment for DST. */ |
| mtocrf 0x01,0 |
| subf 5,0,5 |
| 1: /* Copy 1 byte. */ |
| bf 31,2f |
| |
| stb 4,0(10) |
| addi 10,10,1 |
| 2: /* Copy 2 bytes. */ |
| bf 30,L(medium_aligned) |
| |
| sth 4,0(10) |
| addi 10,10,2 |
| |
| .align 4 |
| L(medium_aligned): |
| /* At least 6 bytes to go, and DST is word-aligned. */ |
| cmplwi cr1,5,16 |
| mtocrf 0x01,5 |
| blt cr1,8f |
| |
| /* Copy 16 bytes. */ |
| stw 4,0(10) |
| stw 4,4(10) |
| stw 4,8(10) |
| stw 4,12(10) |
| addi 10,10,16 |
| 8: /* Copy 8 bytes. */ |
| bf 28,4f |
| |
| stw 4,0(10) |
| stw 4,4(10) |
| addi 10,10,8 |
| 4: /* Copy 4 bytes. */ |
| bf 29,2f |
| |
| stw 4,0(10) |
| addi 10,10,4 |
| 2: /* Copy 2-3 bytes. */ |
| bf 30,1f |
| |
| sth 4,0(10) |
| addi 10,10,2 |
| 1: /* Copy 1 byte. */ |
| bflr 31 |
| |
| stb 4,0(10) |
| blr |
| |
| /* Handles copies of 0~8 bytes. */ |
| .align 4 |
| L(small): |
| mtocrf 0x01,5 |
| bne cr6,L(copy_tail) |
| |
| stw 4,0(10) |
| stw 4,4(10) |
| blr |
| |
| END (memset) |
| libc_hidden_builtin_def (memset) |
