/* Copyright (C) 2011-2014 Free Software Foundation, Inc.
   This file is part of the GNU C Library.
   Contributed by Chris Metcalf <cmetcalf@tilera.com>, 2011.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library.  If not, see
   <http://www.gnu.org/licenses/>.  */

#include <arch/chip.h>
#include <sysdep.h>

	.text
ENTRY (__memcpy)
	FEEDBACK_ENTER(__memcpy)

	/* r0 is the dest, r1 is the source, r2 is the size. */

	/* Save aside original dest so we can return it at the end. */
	{ sw sp, lr; move r23, r0; or r4, r0, r1 }
	cfi_offset (lr, 0)

	/* Check for an empty size. */
	{ bz r2, .Ldone; andi r4, r4, 3 }

	/* Check for an unaligned source or dest. */
	{ bnz r4, .Lcopy_unaligned_maybe_many; addli r4, r2, -256 }

.Lcheck_aligned_copy_size:
	/* If we are copying < 256 bytes, branch to simple case. */
	{ blzt r4, .Lcopy_8_check; slti_u r8, r2, 8 }

	/* Copying >= 256 bytes, so jump to complex prefetching loop. */
	{ andi r6, r1, 63; j .Lcopy_many }

/* Aligned 4 byte at a time copy loop.  */

.Lcopy_8_loop:
	/* Copy two words at a time to hide load latency. */
	{ lw r3, r1; addi r1, r1, 4; slti_u r8, r2, 16 }
	{ lw r4, r1; addi r1, r1, 4 }
	{ sw r0, r3; addi r0, r0, 4; addi r2, r2, -4 }
	{ sw r0, r4; addi r0, r0, 4; addi r2, r2, -4 }
.Lcopy_8_check:
	{ bzt r8, .Lcopy_8_loop; slti_u r4, r2, 4 }

	/* Copy odd leftover word, if any. */
	{ bnzt r4, .Lcheck_odd_stragglers }
	{ lw r3, r1; addi r1, r1, 4 }
	{ sw r0, r3; addi r0, r0, 4; addi r2, r2, -4 }

.Lcheck_odd_stragglers:
	{ bnz r2, .Lcopy_unaligned_few }

.Ldone:
	{ move r0, r23; jrp lr }

/* Prefetching multiple cache line copy handler (for large transfers).  */

	/* Copy words until r1 is cache-line-aligned. */
.Lalign_loop:
	{ lw r3, r1; addi r1, r1, 4 }
	{ andi r6, r1, 63 }
	{ sw r0, r3; addi r0, r0, 4; addi r2, r2, -4 }
.Lcopy_many:
	{ bnzt r6, .Lalign_loop; addi r9, r0, 63 }

	{ addi r3, r1, 60; andi r9, r9, -64 }

	/* No need to prefetch dst, we'll just do the wh64
           right before we copy a line.  */
	{ lw r5, r3; addi r3, r3, 64; movei r4, 1 }
	/* Intentionally stall for a few cycles to leave L2 cache alone. */
	{ bnzt zero, .; move r27, lr }
	{ lw r6, r3; addi r3, r3, 64 }
	/* Intentionally stall for a few cycles to leave L2 cache alone. */
	{ bnzt zero, . }
	{ lw r7, r3; addi r3, r3, 64 }
	/* Intentionally stall for a few cycles to leave L2 cache alone. */
	{ bz zero, .Lbig_loop2 }

	/* On entry to this loop:
	   - r0 points to the start of dst line 0
	   - r1 points to start of src line 0
	   - r2 >= (256 - 60), only the first time the loop trips.
	   - r3 contains r1 + 128 + 60    [pointer to end of source line 2]
	     This is our prefetch address. When we get near the end
	     rather than prefetching off the end this is changed to point
	     to some "safe" recently loaded address.
	   - r5 contains *(r1 + 60)       [i.e. last word of source line 0]
	   - r6 contains *(r1 + 64 + 60)  [i.e. last word of source line 1]
	   - r9 contains ((r0 + 63) & -64)
	       [start of next dst cache line.]  */

.Lbig_loop:
	{ jal .Lcopy_line2; add r15, r1, r2 }

.Lbig_loop2:
	/* Copy line 0, first stalling until r5 is ready. */
	{ move r12, r5; lw r16, r1 }
	{ bz r4, .Lcopy_8_check; slti_u r8, r2, 8 }
	/* Prefetch several lines ahead. */
	{ lw r5, r3; addi r3, r3, 64 }
	{ jal .Lcopy_line }

	/* Copy line 1, first stalling until r6 is ready. */
	{ move r12, r6; lw r16, r1 }
	{ bz r4, .Lcopy_8_check; slti_u r8, r2, 8 }
	/* Prefetch several lines ahead. */
	{ lw r6, r3; addi r3, r3, 64 }
	{ jal .Lcopy_line }

	/* Copy line 2, first stalling until r7 is ready. */
	{ move r12, r7; lw r16, r1 }
	{ bz r4, .Lcopy_8_check; slti_u r8, r2, 8 }
	/* Prefetch several lines ahead. */
	{ lw r7, r3; addi r3, r3, 64 }
	/* Use up a caches-busy cycle by jumping back to the top of the
	   loop. Might as well get it out of the way now.  */
	{ j .Lbig_loop }


	/* On entry:
	   - r0 points to the destination line.
	   - r1 points to the source line.
	   - r3 is the next prefetch address.
	   - r9 holds the last address used for wh64.
	   - r12 = WORD_15
	   - r16 = WORD_0.
	   - r17 == r1 + 16.
	   - r27 holds saved lr to restore.

	   On exit:
	   - r0 is incremented by 64.
	   - r1 is incremented by 64, unless that would point to a word
	     beyond the end of the source array, in which case it is redirected
	     to point to an arbitrary word already in the cache.
	   - r2 is decremented by 64.
	   - r3 is unchanged, unless it points to a word beyond the
	     end of the source array, in which case it is redirected
	     to point to an arbitrary word already in the cache.
	     Redirecting is OK since if we are that close to the end
	     of the array we will not come back to this subroutine
	     and use the contents of the prefetched address.
	   - r4 is nonzero iff r2 >= 64.
	   - r9 is incremented by 64, unless it points beyond the
	     end of the last full destination cache line, in which
	     case it is redirected to a "safe address" that can be
	     clobbered (sp - 64)
	   - lr contains the value in r27.  */

/* r26 unused */

.Lcopy_line:
	/* TODO: when r3 goes past the end, we would like to redirect it
	   to prefetch the last partial cache line (if any) just once, for the
	   benefit of the final cleanup loop. But we don't want to
	   prefetch that line more than once, or subsequent prefetches
	   will go into the RTF. But then .Lbig_loop should unconditionally
	   branch to top of loop to execute final prefetch, and its
	   nop should become a conditional branch.  */

	/* We need two non-memory cycles here to cover the resources
	   used by the loads initiated by the caller.  */
	{ add r15, r1, r2 }
.Lcopy_line2:
	{ slt_u r13, r3, r15; addi r17, r1, 16 }

	/* NOTE: this will stall for one cycle as L1 is busy. */

	/* Fill second L1D line. */
	{ lw r17, r17; addi r1, r1, 48; mvz r3, r13, r1 } /* r17 = WORD_4 */

	/* Prepare destination line for writing. */
	{ wh64 r9; addi r9, r9, 64 }

	/* Load seven words that are L1D hits to cover wh64 L2 usage. */

	/* Load the three remaining words from the last L1D line, which
	   we know has already filled the L1D.  */
	{ lw r4, r1;  addi r1, r1, 4;   addi r20, r1, 16 }   /* r4 = WORD_12 */
	{ lw r8, r1;  addi r1, r1, 4;   slt_u r13, r20, r15 }/* r8 = WORD_13 */
	{ lw r11, r1; addi r1, r1, -52; mvz r20, r13, r1 }  /* r11 = WORD_14 */

	/* Load the three remaining words from the first L1D line, first
	   stalling until it has filled by "looking at" r16.  */
	{ lw r13, r1; addi r1, r1, 4; move zero, r16 }   /* r13 = WORD_1 */
	{ lw r14, r1; addi r1, r1, 4 }                   /* r14 = WORD_2 */
	{ lw r15, r1; addi r1, r1, 8; addi r10, r0, 60 } /* r15 = WORD_3 */

	/* Load second word from the second L1D line, first
	   stalling until it has filled by "looking at" r17.  */
	{ lw r19, r1; addi r1, r1, 4; move zero, r17 }  /* r19 = WORD_5 */

	/* Store last word to the destination line, potentially dirtying it
	   for the first time, which keeps the L2 busy for two cycles.  */
	{ sw r10, r12 }                                 /* store(WORD_15) */

	/* Use two L1D hits to cover the sw L2 access above. */
	{ lw r10, r1; addi r1, r1, 4 }                  /* r10 = WORD_6 */
	{ lw r12, r1; addi r1, r1, 4 }                  /* r12 = WORD_7 */

	/* Fill third L1D line. */
	{ lw r18, r1; addi r1, r1, 4 }                  /* r18 = WORD_8 */

	/* Store first L1D line. */
	{ sw r0, r16; addi r0, r0, 4; add r16, r0, r2 } /* store(WORD_0) */
	{ sw r0, r13; addi r0, r0, 4; andi r16, r16, -64 } /* store(WORD_1) */
	{ sw r0, r14; addi r0, r0, 4; slt_u r16, r9, r16 } /* store(WORD_2) */
	{ sw r0, r15; addi r0, r0, 4; addi r13, sp, -64 } /* store(WORD_3) */

	/* Store second L1D line. */
	{ sw r0, r17; addi r0, r0, 4; mvz r9, r16, r13 }/* store(WORD_4) */
	{ sw r0, r19; addi r0, r0, 4 }                  /* store(WORD_5) */
	{ sw r0, r10; addi r0, r0, 4 }                  /* store(WORD_6) */
	{ sw r0, r12; addi r0, r0, 4 }                  /* store(WORD_7) */

	{ lw r13, r1; addi r1, r1, 4; move zero, r18 }  /* r13 = WORD_9 */
	{ lw r14, r1; addi r1, r1, 4 }                  /* r14 = WORD_10 */
	{ lw r15, r1; move r1, r20   }                  /* r15 = WORD_11 */

	/* Store third L1D line. */
	{ sw r0, r18; addi r0, r0, 4 }                  /* store(WORD_8) */
	{ sw r0, r13; addi r0, r0, 4 }                  /* store(WORD_9) */
	{ sw r0, r14; addi r0, r0, 4 }                  /* store(WORD_10) */
	{ sw r0, r15; addi r0, r0, 4 }                  /* store(WORD_11) */

	/* Store rest of fourth L1D line. */
	{ sw r0, r4;  addi r0, r0, 4 }                  /* store(WORD_12) */
	{
	sw r0, r8                                       /* store(WORD_13) */
	addi r0, r0, 4
	/* Will r2 be > 64 after we subtract 64 below? */
	shri r4, r2, 7
	}
	{
	sw r0, r11                                      /* store(WORD_14) */
	addi r0, r0, 8
	/* Record 64 bytes successfully copied. */
	addi r2, r2, -64
	}

	{ jrp lr; move lr, r27 }

	/* Convey to the backtrace library that the stack frame is
	   size zero, and the real return address is on the stack
	   rather than in 'lr'.  */
	{ info 8 }

	.align 64
.Lcopy_unaligned_maybe_many:
	/* Skip the setup overhead if we aren't copying many bytes. */
	{ slti_u r8, r2, 20; sub r4, zero, r0 }
	{ bnzt r8, .Lcopy_unaligned_few; andi r4, r4, 3 }
	{ bz r4, .Ldest_is_word_aligned; add r18, r1, r2 }

/* Unaligned 4 byte at a time copy handler.  */

	/* Copy single bytes until r0 == 0 mod 4, so we can store words. */
.Lalign_dest_loop:
	{ lb_u r3, r1; addi r1, r1, 1; addi r4, r4, -1 }
	{ sb r0, r3;   addi r0, r0, 1; addi r2, r2, -1 }
	{ bnzt r4, .Lalign_dest_loop; andi r3, r1, 3 }

	/* If source and dest are now *both* aligned, do an aligned copy. */
	{ bz r3, .Lcheck_aligned_copy_size; addli r4, r2, -256 }

.Ldest_is_word_aligned:

	{ andi r8, r0, 63; lwadd_na r6, r1, 4}
	{ slti_u r9, r2, 64; bz r8, .Ldest_is_L2_line_aligned }

	/* This copies unaligned words until either there are fewer
	   than 4 bytes left to copy, or until the destination pointer
	   is cache-aligned, whichever comes first.

	   On entry:
	   - r0 is the next store address.
	   - r1 points 4 bytes past the load address corresponding to r0.
	   - r2 >= 4
	   - r6 is the next aligned word loaded.  */
.Lcopy_unaligned_src_words:
	{ lwadd_na r7, r1, 4; slti_u r8, r2, 4 + 4 }
	/* stall */
	{ dword_align r6, r7, r1; slti_u r9, r2, 64 + 4 }
	{ swadd r0, r6, 4; addi r2, r2, -4 }
	{ bnz r8, .Lcleanup_unaligned_words; andi r8, r0, 63 }
	{ bnzt r8, .Lcopy_unaligned_src_words; move r6, r7 }

	/* On entry:
	   - r0 is the next store address.
	   - r1 points 4 bytes past the load address corresponding to r0.
	   - r2 >= 4 (# of bytes left to store).
	   - r6 is the next aligned src word value.
	   - r9 = (r2 < 64U).
	   - r18 points one byte past the end of source memory.  */
.Ldest_is_L2_line_aligned:

	{
	/* Not a full cache line remains. */
	bnz r9, .Lcleanup_unaligned_words
	move r7, r6
	}

	/* r2 >= 64 */

	/* Kick off two prefetches, but don't go past the end. */
	{ addi r3, r1, 63 - 4; addi r8, r1, 64 + 63 - 4 }
	{ prefetch r3; move r3, r8; slt_u r8, r8, r18 }
	{ mvz r3, r8, r1; addi r8, r3, 64 }
	{ prefetch r3; move r3, r8; slt_u r8, r8, r18 }
	{ mvz r3, r8, r1; movei r17, 0 }

.Lcopy_unaligned_line:
	/* Prefetch another line. */
	{ prefetch r3; addi r15, r1, 60; addi r3, r3, 64 }
	/* Fire off a load of the last word we are about to copy. */
	{ lw_na r15, r15; slt_u r8, r3, r18 }

	{ mvz r3, r8, r1; wh64 r0 }

	/* This loop runs twice.

	   On entry:
	   - r17 is even before the first iteration, and odd before
	     the second.  It is incremented inside the loop.  Encountering
	     an even value at the end of the loop makes it stop.  */
.Lcopy_half_an_unaligned_line:
	{
	/* Stall until the last byte is ready. In the steady state this
	   guarantees all words to load below will be in the L2 cache, which
	   avoids shunting the loads to the RTF.  */
	move zero, r15
	lwadd_na r7, r1, 16
	}
	{ lwadd_na r11, r1, 12 }
	{ lwadd_na r14, r1, -24 }
	{ lwadd_na r8, r1, 4 }
	{ lwadd_na r9, r1, 4 }
	{
	lwadd_na r10, r1, 8
	/* r16 = (r2 < 64), after we subtract 32 from r2 below. */
	slti_u r16, r2, 64 + 32
	}
	{ lwadd_na r12, r1, 4; addi r17, r17, 1 }
	{ lwadd_na r13, r1, 8; dword_align r6, r7, r1 }
	{ swadd r0, r6,  4; dword_align r7,  r8,  r1 }
	{ swadd r0, r7,  4; dword_align r8,  r9,  r1 }
	{ swadd r0, r8,  4; dword_align r9,  r10, r1 }
	{ swadd r0, r9,  4; dword_align r10, r11, r1 }
	{ swadd r0, r10, 4; dword_align r11, r12, r1 }
	{ swadd r0, r11, 4; dword_align r12, r13, r1 }
	{ swadd r0, r12, 4; dword_align r13, r14, r1 }
	{ swadd r0, r13, 4; addi r2, r2, -32 }
	{ move r6, r14; bbst r17, .Lcopy_half_an_unaligned_line }

	{ bzt r16, .Lcopy_unaligned_line; move r7, r6 }

	/* On entry:
	   - r0 is the next store address.
	   - r1 points 4 bytes past the load address corresponding to r0.
	   - r2 >= 0 (# of bytes left to store).
	   - r7 is the next aligned src word value.  */
.Lcleanup_unaligned_words:
	/* Handle any trailing bytes. */
	{ bz r2, .Lcopy_unaligned_done; slti_u r8, r2, 4 }
	{ bzt r8, .Lcopy_unaligned_src_words; move r6, r7 }

	/* Move r1 back to the point where it corresponds to r0. */
	{ addi r1, r1, -4 }

	/* Fall through */

/* 1 byte at a time copy handler.  */

.Lcopy_unaligned_few:
	{ lb_u r3, r1; addi r1, r1, 1 }
	{ sb r0, r3;   addi r0, r0, 1; addi r2, r2, -1 }
	{ bnzt r2, .Lcopy_unaligned_few }

.Lcopy_unaligned_done:

	{ move r0, r23; jrp lr }

END (__memcpy)

weak_alias (__memcpy, memcpy)
libc_hidden_builtin_def (memcpy)