| /* Optimized memset implementation for PowerPC64/POWER7. |
| Copyright (C) 2010-2018 Free Software Foundation, Inc. |
| Contributed by Luis Machado <luisgpm@br.ibm.com>. |
| This file is part of the GNU C Library. |
| |
| The GNU C Library is free software; you can redistribute it and/or |
| modify it under the terms of the GNU Lesser General Public |
| License as published by the Free Software Foundation; either |
| version 2.1 of the License, or (at your option) any later version. |
| |
| The GNU C Library is distributed in the hope that it will be useful, |
| but WITHOUT ANY WARRANTY; without even the implied warranty of |
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| Lesser General Public License for more details. |
| |
| You should have received a copy of the GNU Lesser General Public |
| License along with the GNU C Library; if not, see |
| <http://www.gnu.org/licenses/>. */ |
| |
| #include <sysdep.h> |
| |
| /* void * [r3] memset (void *s [r3], int c [r4], size_t n [r5])); |
| Returns 's'. */ |
| |
| #ifndef MEMSET |
| # define MEMSET memset |
| #endif |
| .machine power7 |
| ENTRY_TOCLESS (MEMSET, 5) |
| CALL_MCOUNT 3 |
| |
| L(_memset): |
| cmpldi cr7,5,31 |
| cmpldi cr6,5,8 |
| mr 10,3 |
| |
| /* Replicate byte to word. */ |
| insrdi 4,4,8,48 |
| insrdi 4,4,16,32 |
| ble cr6,L(small) /* If length <= 8, use short copy code. */ |
| |
| neg 0,3 |
| ble cr7,L(medium) /* If length < 32, use medium copy code. */ |
| |
| andi. 11,10,7 /* Check alignment of SRC. */ |
| insrdi 4,4,32,0 /* Replicate word to double word. */ |
| |
| mr 12,5 |
| beq L(big_aligned) |
| |
| clrldi 0,0,61 |
| mtocrf 0x01,0 |
| subf 5,0,5 |
| |
| /* Get DST aligned to 8 bytes. */ |
| 1: bf 31,2f |
| |
| stb 4,0(10) |
| addi 10,10,1 |
| 2: bf 30,4f |
| |
| sth 4,0(10) |
| addi 10,10,2 |
| 4: bf 29,L(big_aligned) |
| |
| stw 4,0(10) |
| addi 10,10,4 |
| |
| .align 4 |
| L(big_aligned): |
| |
| cmpldi cr5,5,255 |
| li 0,32 |
| dcbtst 0,10 |
| cmpldi cr6,4,0 |
| srdi 9,5,3 /* Number of full doublewords remaining. */ |
| crand 27,26,21 |
| mtocrf 0x01,9 |
| bt 27,L(huge) |
| |
| /* From this point on, we'll copy 32+ bytes and the value |
| isn't 0 (so we can't use dcbz). */ |
| |
| srdi 8,5,5 |
| clrldi 11,5,61 |
| cmpldi cr6,11,0 |
| cmpldi cr1,9,4 |
| mtctr 8 |
| |
| /* Copy 1~3 doublewords so the main loop starts |
| at a multiple of 32 bytes. */ |
| |
| bf 30,1f |
| |
| std 4,0(10) |
| std 4,8(10) |
| addi 10,10,16 |
| bf 31,L(big_loop) |
| |
| std 4,0(10) |
| addi 10,10,8 |
| mr 12,10 |
| blt cr1,L(tail_bytes) |
| b L(big_loop) |
| |
| .align 4 |
| 1: /* Copy 1 doubleword. */ |
| bf 31,L(big_loop) |
| |
| std 4,0(10) |
| addi 10,10,8 |
| |
| /* Main aligned copy loop. Copies 32-bytes at a time and |
| ping-pong through r10 and r12 to avoid AGEN delays. */ |
| .align 4 |
| L(big_loop): |
| addi 12,10,32 |
| std 4,0(10) |
| std 4,8(10) |
| std 4,16(10) |
| std 4,24(10) |
| bdz L(tail_bytes) |
| |
| addi 10,10,64 |
| std 4,0(12) |
| std 4,8(12) |
| std 4,16(12) |
| std 4,24(12) |
| bdnz L(big_loop) |
| |
| mr 12,10 |
| b L(tail_bytes) |
| |
| .align 4 |
| L(tail_bytes): |
| |
| /* Check for tail bytes. */ |
| beqlr cr6 |
| |
| clrldi 0,5,61 |
| mtocrf 0x01,0 |
| |
| /* At this point we have a tail of 0-7 bytes and we know that the |
| destination is doubleword-aligned. */ |
| 4: /* Copy 4 bytes. */ |
| bf 29,2f |
| |
| stw 4,0(12) |
| addi 12,12,4 |
| 2: /* Copy 2 bytes. */ |
| bf 30,1f |
| |
| sth 4,0(12) |
| addi 12,12,2 |
| 1: /* Copy 1 byte. */ |
| bflr 31 |
| |
| stb 4,0(12) |
| blr |
| |
| /* Special case when value is 0 and we have a long length to deal |
| with. Use dcbz to zero out 128-bytes at a time. Before using |
| dcbz though, we need to get the destination 128-bytes aligned. */ |
| .align 4 |
| L(huge): |
| andi. 11,10,127 |
| neg 0,10 |
| beq L(huge_aligned) |
| |
| clrldi 0,0,57 |
| subf 5,0,5 |
| srdi 0,0,3 |
| mtocrf 0x01,0 |
| |
| /* Get DST aligned to 128 bytes. */ |
| 8: bf 28,4f |
| |
| std 4,0(10) |
| std 4,8(10) |
| std 4,16(10) |
| std 4,24(10) |
| std 4,32(10) |
| std 4,40(10) |
| std 4,48(10) |
| std 4,56(10) |
| addi 10,10,64 |
| .align 4 |
| 4: bf 29,2f |
| |
| std 4,0(10) |
| std 4,8(10) |
| std 4,16(10) |
| std 4,24(10) |
| addi 10,10,32 |
| .align 4 |
| 2: bf 30,1f |
| |
| std 4,0(10) |
| std 4,8(10) |
| addi 10,10,16 |
| .align 4 |
| 1: bf 31,L(huge_aligned) |
| |
| std 4,0(10) |
| addi 10,10,8 |
| |
| |
| L(huge_aligned): |
| srdi 8,5,7 |
| clrldi 11,5,57 |
| cmpldi cr6,11,0 |
| mtctr 8 |
| |
| .align 4 |
| L(huge_loop): |
| dcbz 0,10 |
| addi 10,10,128 |
| bdnz L(huge_loop) |
| |
| /* Check how many bytes are still left. */ |
| beqlr cr6 |
| |
| subf 9,3,10 |
| subf 5,9,12 |
| srdi 8,5,3 |
| cmpldi cr6,8,0 |
| mtocrf 0x01,8 |
| |
| /* We have a tail o 1~127 bytes. Copy up to 15 doublewords for |
| speed. We'll handle the resulting tail bytes later. */ |
| beq cr6,L(tail) |
| |
| 8: bf 28,4f |
| |
| std 4,0(10) |
| std 4,8(10) |
| std 4,16(10) |
| std 4,24(10) |
| std 4,32(10) |
| std 4,40(10) |
| std 4,48(10) |
| std 4,56(10) |
| addi 10,10,64 |
| .align 4 |
| 4: bf 29,2f |
| |
| std 4,0(10) |
| std 4,8(10) |
| std 4,16(10) |
| std 4,24(10) |
| addi 10,10,32 |
| .align 4 |
| 2: bf 30,1f |
| |
| std 4,0(10) |
| std 4,8(10) |
| addi 10,10,16 |
| .align 4 |
| 1: bf 31,L(tail) |
| |
| std 4,0(10) |
| addi 10,10,8 |
| |
| /* Handle the rest of the tail bytes here. */ |
| L(tail): |
| mtocrf 0x01,5 |
| |
| .align 4 |
| 4: bf 29,2f |
| |
| stw 4,0(10) |
| addi 10,10,4 |
| .align 4 |
| 2: bf 30,1f |
| |
| sth 4,0(10) |
| addi 10,10,2 |
| .align 4 |
| 1: bflr 31 |
| |
| stb 4,0(10) |
| blr |
| |
| /* Expanded tree to copy tail bytes without increments. */ |
| .align 4 |
| L(copy_tail): |
| bf 29,L(FXX) |
| |
| stw 4,0(10) |
| bf 30,L(TFX) |
| |
| sth 4,4(10) |
| bflr 31 |
| |
| stb 4,6(10) |
| blr |
| |
| .align 4 |
| L(FXX): bf 30,L(FFX) |
| |
| sth 4,0(10) |
| bflr 31 |
| |
| stb 4,2(10) |
| blr |
| |
| .align 4 |
| L(TFX): bflr 31 |
| |
| stb 4,4(10) |
| blr |
| |
| .align 4 |
| L(FFX): bflr 31 |
| |
| stb 4,0(10) |
| blr |
| |
| /* Handle copies of 9~31 bytes. */ |
| .align 4 |
| L(medium): |
| /* At least 9 bytes to go. */ |
| andi. 11,10,3 |
| clrldi 0,0,62 |
| beq L(medium_aligned) |
| |
| /* Force 4-bytes alignment for DST. */ |
| mtocrf 0x01,0 |
| subf 5,0,5 |
| 1: /* Copy 1 byte. */ |
| bf 31,2f |
| |
| stb 4,0(10) |
| addi 10,10,1 |
| 2: /* Copy 2 bytes. */ |
| bf 30,L(medium_aligned) |
| |
| sth 4,0(10) |
| addi 10,10,2 |
| |
| .align 4 |
| L(medium_aligned): |
| /* At least 6 bytes to go, and DST is word-aligned. */ |
| cmpldi cr1,5,16 |
| mtocrf 0x01,5 |
| blt cr1,8f |
| |
| /* Copy 16 bytes. */ |
| stw 4,0(10) |
| stw 4,4(10) |
| stw 4,8(10) |
| stw 4,12(10) |
| addi 10,10,16 |
| 8: /* Copy 8 bytes. */ |
| bf 28,4f |
| |
| stw 4,0(10) |
| stw 4,4(10) |
| addi 10,10,8 |
| 4: /* Copy 4 bytes. */ |
| bf 29,2f |
| |
| stw 4,0(10) |
| addi 10,10,4 |
| 2: /* Copy 2-3 bytes. */ |
| bf 30,1f |
| |
| sth 4,0(10) |
| addi 10,10,2 |
| 1: /* Copy 1 byte. */ |
| bflr 31 |
| |
| stb 4,0(10) |
| blr |
| |
| /* Handles copies of 0~8 bytes. */ |
| .align 4 |
| L(small): |
| mtocrf 0x01,5 |
| bne cr6,L(copy_tail) |
| |
| stw 4,0(10) |
| stw 4,4(10) |
| blr |
| |
| END_GEN_TB (MEMSET,TB_TOCLESS) |
| libc_hidden_builtin_def (memset) |
| |
| /* Copied from bzero.S to prevent the linker from inserting a stub |
| between bzero and memset. */ |
| ENTRY_TOCLESS (__bzero) |
| CALL_MCOUNT 3 |
| mr r5,r4 |
| li r4,0 |
| b L(_memset) |
| END (__bzero) |
| #ifndef __bzero |
| weak_alias (__bzero, bzero) |
| #endif |
