| /* Copyright (C) 2011-2014 Free Software Foundation, Inc. |
| This file is part of the GNU C Library. |
| Contributed by Chris Metcalf <cmetcalf@tilera.com>, 2011. |
| |
| The GNU C Library is free software; you can redistribute it and/or |
| modify it under the terms of the GNU Lesser General Public |
| License as published by the Free Software Foundation; either |
| version 2.1 of the License, or (at your option) any later version. |
| |
| The GNU C Library is distributed in the hope that it will be useful, |
| but WITHOUT ANY WARRANTY; without even the implied warranty of |
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| Lesser General Public License for more details. |
| |
| You should have received a copy of the GNU Lesser General Public |
| License along with the GNU C Library. If not, see |
| <http://www.gnu.org/licenses/>. */ |
| |
| #include <string.h> |
| #include <stdint.h> |
| #include <stdlib.h> |
| #include <memcopy.h> |
| #include <arch/chip.h> |
| |
| /* How many cache lines ahead should we prefetch? */ |
| #define PREFETCH_LINES_AHEAD 3 |
| |
| void * |
| __memcpy (void *__restrict dstv, const void *__restrict srcv, size_t n) |
| { |
| char *__restrict dst1 = (char *) dstv; |
| const char *__restrict src1 = (const char *) srcv; |
| const char *__restrict src1_end; |
| const char *__restrict prefetch; |
| op_t *__restrict dst8; /* 8-byte pointer to destination memory. */ |
| op_t final; /* Final bytes to write to trailing word, if any */ |
| long i; |
| |
| if (n < 16) |
| { |
| for (; n; n--) |
| *dst1++ = *src1++; |
| return dstv; |
| } |
| |
| /* Locate the end of source memory we will copy. Don't prefetch |
| past this. */ |
| src1_end = src1 + n - 1; |
| |
| /* Prefetch ahead a few cache lines, but not past the end. */ |
| prefetch = src1; |
| for (i = 0; i < PREFETCH_LINES_AHEAD; i++) |
| { |
| __insn_prefetch (prefetch); |
| prefetch += CHIP_L2_LINE_SIZE (); |
| prefetch = (prefetch < src1_end) ? prefetch : src1; |
| } |
| |
| /* Copy bytes until dst is word-aligned. */ |
| for (; (uintptr_t) dst1 & (sizeof (op_t) - 1); n--) |
| *dst1++ = *src1++; |
| |
| /* 8-byte pointer to destination memory. */ |
| dst8 = (op_t *) dst1; |
| |
| if (__builtin_expect ((uintptr_t) src1 & (sizeof (op_t) - 1), 0)) |
| { |
| /* Misaligned copy. Use glibc's _wordcopy_fwd_dest_aligned, but |
| inline it to avoid prologue/epilogue. TODO: Consider |
| prefetching and using wh64 as well. */ |
| void * srci; |
| op_t a0, a1, a2, a3; |
| long int dstp = (long int) dst1; |
| long int srcp = (long int) src1; |
| long int len = n / OPSIZ; |
| |
| /* Save the initial source pointer so we know the number of |
| bytes to shift for merging two unaligned results. */ |
| srci = (void *) srcp; |
| |
| /* Make SRCP aligned by rounding it down to the beginning of the |
| `op_t' it points in the middle of. */ |
| srcp &= -OPSIZ; |
| |
| switch (len % 4) |
| { |
| case 2: |
| a1 = ((op_t *) srcp)[0]; |
| a2 = ((op_t *) srcp)[1]; |
| len += 2; |
| srcp += 2 * OPSIZ; |
| goto do1; |
| case 3: |
| a0 = ((op_t *) srcp)[0]; |
| a1 = ((op_t *) srcp)[1]; |
| len += 1; |
| srcp += 2 * OPSIZ; |
| goto do2; |
| case 0: |
| if (OP_T_THRES <= 3 * OPSIZ && len == 0) |
| return dstv; |
| a3 = ((op_t *) srcp)[0]; |
| a0 = ((op_t *) srcp)[1]; |
| len += 0; |
| srcp += 2 * OPSIZ; |
| goto do3; |
| case 1: |
| a2 = ((op_t *) srcp)[0]; |
| a3 = ((op_t *) srcp)[1]; |
| srcp += 2 * OPSIZ; |
| len -= 1; |
| if (OP_T_THRES <= 3 * OPSIZ && len == 0) |
| goto do0; |
| goto do4; /* No-op. */ |
| } |
| |
| do |
| { |
| do4: |
| a0 = ((op_t *) srcp)[0]; |
| a2 = __insn_dblalign (a2, a3, srci); |
| ((op_t *) dstp)[0] = a2; |
| srcp += OPSIZ; |
| dstp += OPSIZ; |
| do3: |
| a1 = ((op_t *) srcp)[0]; |
| a3 = __insn_dblalign (a3, a0, srci); |
| ((op_t *) dstp)[0] = a3; |
| srcp += OPSIZ; |
| dstp += OPSIZ; |
| do2: |
| a2 = ((op_t *) srcp)[0]; |
| a0 = __insn_dblalign (a0, a1, srci); |
| ((op_t *) dstp)[0] = a0; |
| srcp += OPSIZ; |
| dstp += OPSIZ; |
| do1: |
| a3 = ((op_t *) srcp)[0]; |
| a1 = __insn_dblalign (a1, a2, srci); |
| ((op_t *) dstp)[0] = a1; |
| srcp += OPSIZ; |
| dstp += OPSIZ; |
| len -= 4; |
| } |
| while (len != 0); |
| |
| /* This is the right position for do0. Please don't move |
| it into the loop. */ |
| do0: |
| ((op_t *) dstp)[0] = __insn_dblalign (a2, a3, srci); |
| |
| n = n % OPSIZ; |
| if (n == 0) |
| return dstv; |
| |
| a0 = ((const char *) srcp <= src1_end) ? ((op_t *) srcp)[0] : 0; |
| |
| final = __insn_dblalign (a3, a0, srci); |
| dst8 = (op_t *)(dstp + OPSIZ); |
| } |
| else |
| { |
| /* Aligned copy. */ |
| |
| const op_t *__restrict src8 = (const op_t *) src1; |
| |
| /* src8 and dst8 are both word-aligned. */ |
| if (n >= CHIP_L2_LINE_SIZE ()) |
| { |
| /* Copy until 'dst' is cache-line-aligned. */ |
| for (; (uintptr_t) dst8 & (CHIP_L2_LINE_SIZE () - 1); |
| n -= sizeof (op_t)) |
| *dst8++ = *src8++; |
| |
| for (; n >= CHIP_L2_LINE_SIZE ();) |
| { |
| op_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; |
| |
| /* Prefetch and advance to next line to prefetch, but |
| don't go past the end. */ |
| __insn_prefetch (prefetch); |
| prefetch += CHIP_L2_LINE_SIZE (); |
| prefetch = (prefetch < src1_end) ? prefetch : |
| (const char *) src8; |
| |
| /* Do all the loads before wh64. This is necessary if |
| [src8, src8+7] and [dst8, dst8+7] share the same |
| cache line and dst8 <= src8, as can be the case when |
| called from memmove, or with code tested on x86 whose |
| memcpy always works with forward copies. */ |
| tmp0 = *src8++; |
| tmp1 = *src8++; |
| tmp2 = *src8++; |
| tmp3 = *src8++; |
| tmp4 = *src8++; |
| tmp5 = *src8++; |
| tmp6 = *src8++; |
| tmp7 = *src8++; |
| |
| __insn_wh64 (dst8); |
| |
| *dst8++ = tmp0; |
| *dst8++ = tmp1; |
| *dst8++ = tmp2; |
| *dst8++ = tmp3; |
| *dst8++ = tmp4; |
| *dst8++ = tmp5; |
| *dst8++ = tmp6; |
| *dst8++ = tmp7; |
| |
| n -= 64; |
| } |
| #if CHIP_L2_LINE_SIZE() != 64 |
| # error "Fix code that assumes particular L2 cache line size." |
| #endif |
| } |
| |
| for (; n >= sizeof (op_t); n -= sizeof (op_t)) |
| *dst8++ = *src8++; |
| |
| if (__builtin_expect (n == 0, 1)) |
| return dstv; |
| |
| final = *src8; |
| } |
| |
| /* n != 0 if we get here. Write out any trailing bytes. */ |
| dst1 = (char *) dst8; |
| #ifndef __BIG_ENDIAN__ |
| if (n & 4) |
| { |
| *(uint32_t *) dst1 = final; |
| dst1 += 4; |
| final >>= 32; |
| n &= 3; |
| } |
| if (n & 2) |
| { |
| *(uint16_t *) dst1 = final; |
| dst1 += 2; |
| final >>= 16; |
| n &= 1; |
| } |
| if (n) |
| *(uint8_t *) dst1 = final; |
| #else |
| if (n & 4) |
| { |
| *(uint32_t *) dst1 = final >> 32; |
| dst1 += 4; |
| } |
| else |
| { |
| final >>= 32; |
| } |
| if (n & 2) |
| { |
| *(uint16_t *) dst1 = final >> 16; |
| dst1 += 2; |
| } |
| else |
| { |
| final >>= 16; |
| } |
| if (n & 1) |
| *(uint8_t *) dst1 = final >> 8; |
| #endif |
| |
| return dstv; |
| } |
| weak_alias (__memcpy, memcpy) |
| libc_hidden_builtin_def (memcpy) |
