arch/mips/lib/cache_init.S - u-boot - Git at Google

 /* SPDX-License-Identifier: GPL-2.0+ */
 /*
  *  Cache-handling routined for MIPS CPUs
  *
  *  Copyright (c) 2003	Wolfgang Denk <wd@denx.de>
  */

 #include <asm-offsets.h>
 #include <config.h>
 #include <asm/asm.h>
 #include <asm/regdef.h>
 #include <asm/mipsregs.h>
 #include <asm/addrspace.h>
 #include <asm/cacheops.h>
 #include <asm/cm.h>

 #ifndef CONFIG_SYS_MIPS_CACHE_MODE
 #define CONFIG_SYS_MIPS_CACHE_MODE CONF_CM_CACHABLE_NONCOHERENT
 #endif

 #define INDEX_BASE	CKSEG0

 	.macro	f_fill64 dst, offset, val
 	LONG_S	\val, (\offset +  0 * LONGSIZE)(\dst)
 	LONG_S	\val, (\offset +  1 * LONGSIZE)(\dst)
 	LONG_S	\val, (\offset +  2 * LONGSIZE)(\dst)
 	LONG_S	\val, (\offset +  3 * LONGSIZE)(\dst)
 	LONG_S	\val, (\offset +  4 * LONGSIZE)(\dst)
 	LONG_S	\val, (\offset +  5 * LONGSIZE)(\dst)
 	LONG_S	\val, (\offset +  6 * LONGSIZE)(\dst)
 	LONG_S	\val, (\offset +  7 * LONGSIZE)(\dst)
 #if LONGSIZE == 4
 	LONG_S	\val, (\offset +  8 * LONGSIZE)(\dst)
 	LONG_S	\val, (\offset +  9 * LONGSIZE)(\dst)
 	LONG_S	\val, (\offset + 10 * LONGSIZE)(\dst)
 	LONG_S	\val, (\offset + 11 * LONGSIZE)(\dst)
 	LONG_S	\val, (\offset + 12 * LONGSIZE)(\dst)
 	LONG_S	\val, (\offset + 13 * LONGSIZE)(\dst)
 	LONG_S	\val, (\offset + 14 * LONGSIZE)(\dst)
 	LONG_S	\val, (\offset + 15 * LONGSIZE)(\dst)
 #endif
 	.endm

 	.macro cache_loop	curr, end, line_sz, op
 10:	cache		\op, 0(\curr)
 	PTR_ADDU	\curr, \curr, \line_sz
 	bne		\curr, \end, 10b
 	.endm

 	.macro	l1_info		sz, line_sz, off
 	.set	push
 	.set	noat

 	mfc0	$1, CP0_CONFIG, 1

 	/* detect line size */
 	srl	\line_sz, $1, \off + MIPS_CONF1_DL_SHF - MIPS_CONF1_DA_SHF
 	andi	\line_sz, \line_sz, (MIPS_CONF1_DL >> MIPS_CONF1_DL_SHF)
 	move	\sz, zero
 	beqz	\line_sz, 10f
 	li	\sz, 2
 	sllv	\line_sz, \sz, \line_sz

 	/* detect associativity */
 	srl	\sz, $1, \off + MIPS_CONF1_DA_SHF - MIPS_CONF1_DA_SHF
 	andi	\sz, \sz, (MIPS_CONF1_DA >> MIPS_CONF1_DA_SHF)
 	addiu	\sz, \sz, 1

 	/* sz *= line_sz */
 	mul	\sz, \sz, \line_sz

 	/* detect log32(sets) */
 	srl	$1, $1, \off + MIPS_CONF1_DS_SHF - MIPS_CONF1_DA_SHF
 	andi	$1, $1, (MIPS_CONF1_DS >> MIPS_CONF1_DS_SHF)
 	addiu	$1, $1, 1
 	andi	$1, $1, 0x7

 	/* sz <<= log32(sets) */
 	sllv	\sz, \sz, $1

 	/* sz *= 32 */
 	li	$1, 32
 	mul	\sz, \sz, $1
 10:
 	.set	pop
 	.endm
 /*
  * mips_cache_reset - low level initialisation of the primary caches
  *
  * This routine initialises the primary caches to ensure that they have good
  * parity.  It must be called by the ROM before any cached locations are used
  * to prevent the possibility of data with bad parity being written to memory.
  *
  * To initialise the instruction cache it is essential that a source of data
  * with good parity is available. This routine will initialise an area of
  * memory starting at location zero to be used as a source of parity.
  *
  * Note that this function does not follow the standard calling convention &
  * may clobber typically callee-saved registers.
  *
  * RETURNS: N/A
  *
  */
 #define R_RETURN	s0
 #define R_IC_SIZE	s1
 #define R_IC_LINE	s2
 #define R_DC_SIZE	s3
 #define R_DC_LINE	s4
 #define R_L2_SIZE	s5
 #define R_L2_LINE	s6
 #define R_L2_BYPASSED	s7
 #define R_L2_L2C	t8
 LEAF(mips_cache_reset)
 	move	R_RETURN, ra

 #ifdef CONFIG_MIPS_L2_CACHE
 	/*
 	 * For there to be an L2 present, Config2 must be present. If it isn't
 	 * then we proceed knowing there's no L2 cache.
 	 */
 	move	R_L2_SIZE, zero
 	move	R_L2_LINE, zero
 	move	R_L2_BYPASSED, zero
 	move	R_L2_L2C, zero
 	mfc0	t0, CP0_CONFIG, 1
 	bgez	t0, l2_probe_done

 	/*
 	 * From MIPSr6 onwards the L2 cache configuration might not be reported
 	 * by Config2. The Config5.L2C bit indicates whether this is the case,
 	 * and if it is then we need knowledge of where else to look. For cores
 	 * from Imagination Technologies this is a CM GCR.
 	 */
 # if __mips_isa_rev >= 6
 	/* Check that Config5 exists */
 	mfc0	t0, CP0_CONFIG, 2
 	bgez	t0, l2_probe_cop0
 	mfc0	t0, CP0_CONFIG, 3
 	bgez	t0, l2_probe_cop0
 	mfc0	t0, CP0_CONFIG, 4
 	bgez	t0, l2_probe_cop0

 	/* Check Config5.L2C is set */
 	mfc0	t0, CP0_CONFIG, 5
 	and	R_L2_L2C, t0, MIPS_CONF5_L2C
 	beqz	R_L2_L2C, l2_probe_cop0

 	/* Config5.L2C is set */
 #  ifdef CONFIG_MIPS_CM
 	/* The CM will provide L2 configuration */
 	PTR_LI	t0, CKSEG1ADDR(CONFIG_MIPS_CM_BASE)
 	lw	t1, GCR_L2_CONFIG(t0)
 	bgez	t1, l2_probe_done

 	ext	R_L2_LINE, t1, \
 		GCR_L2_CONFIG_LINESZ_SHIFT, GCR_L2_CONFIG_LINESZ_BITS
 	beqz	R_L2_LINE, l2_probe_done
 	li	t2, 2
 	sllv	R_L2_LINE, t2, R_L2_LINE

 	ext	t2, t1, GCR_L2_CONFIG_ASSOC_SHIFT, GCR_L2_CONFIG_ASSOC_BITS
 	addiu	t2, t2, 1
 	mul	R_L2_SIZE, R_L2_LINE, t2

 	ext	t2, t1, GCR_L2_CONFIG_SETSZ_SHIFT, GCR_L2_CONFIG_SETSZ_BITS
 	sllv	R_L2_SIZE, R_L2_SIZE, t2
 	li	t2, 64
 	mul	R_L2_SIZE, R_L2_SIZE, t2

 	/* Bypass the L2 cache so that we can init the L1s early */
 	or	t1, t1, GCR_L2_CONFIG_BYPASS
 	sw	t1, GCR_L2_CONFIG(t0)
 	sync
 	li	R_L2_BYPASSED, 1

 	/* Zero the L2 tag registers */
 	sw	zero, GCR_L2_TAG_ADDR(t0)
 	sw	zero, GCR_L2_TAG_ADDR_UPPER(t0)
 	sw	zero, GCR_L2_TAG_STATE(t0)
 	sw	zero, GCR_L2_TAG_STATE_UPPER(t0)
 	sw	zero, GCR_L2_DATA(t0)
 	sw	zero, GCR_L2_DATA_UPPER(t0)
 	sync
 #  else
 	/* We don't know how to retrieve L2 configuration on this system */
 #  endif
 	b	l2_probe_done
 # endif

 	/*
 	 * For pre-r6 systems, or r6 systems with Config5.L2C==0, probe the L2
 	 * cache configuration from the cop0 Config2 register.
 	 */
 l2_probe_cop0:
 	mfc0	t0, CP0_CONFIG, 2

 	srl	R_L2_LINE, t0, MIPS_CONF2_SL_SHF
 	andi	R_L2_LINE, R_L2_LINE, MIPS_CONF2_SL >> MIPS_CONF2_SL_SHF
 	beqz	R_L2_LINE, l2_probe_done
 	li	t1, 2
 	sllv	R_L2_LINE, t1, R_L2_LINE

 	srl	t1, t0, MIPS_CONF2_SA_SHF
 	andi	t1, t1, MIPS_CONF2_SA >> MIPS_CONF2_SA_SHF
 	addiu	t1, t1, 1
 	mul	R_L2_SIZE, R_L2_LINE, t1

 	srl	t1, t0, MIPS_CONF2_SS_SHF
 	andi	t1, t1, MIPS_CONF2_SS >> MIPS_CONF2_SS_SHF
 	sllv	R_L2_SIZE, R_L2_SIZE, t1
 	li	t1, 64
 	mul	R_L2_SIZE, R_L2_SIZE, t1

 	/* Attempt to bypass the L2 so that we can init the L1s early */
 	or	t0, t0, MIPS_CONF2_L2B
 	mtc0	t0, CP0_CONFIG, 2
 	ehb
 	mfc0	t0, CP0_CONFIG, 2
 	and	R_L2_BYPASSED, t0, MIPS_CONF2_L2B

 	/* Zero the L2 tag registers */
 	mtc0	zero, CP0_TAGLO, 4
 	ehb
 l2_probe_done:
 #endif

 #ifndef CONFIG_SYS_CACHE_SIZE_AUTO
 	li	R_IC_SIZE, CONFIG_SYS_ICACHE_SIZE
 	li	R_IC_LINE, CONFIG_SYS_ICACHE_LINE_SIZE
 #else
 	l1_info	R_IC_SIZE, R_IC_LINE, MIPS_CONF1_IA_SHF
 #endif

 #ifndef CONFIG_SYS_CACHE_SIZE_AUTO
 	li	R_DC_SIZE, CONFIG_SYS_DCACHE_SIZE
 	li	R_DC_LINE, CONFIG_SYS_DCACHE_LINE_SIZE
 #else
 	l1_info	R_DC_SIZE, R_DC_LINE, MIPS_CONF1_DA_SHF
 #endif

 #ifdef CONFIG_SYS_MIPS_CACHE_INIT_RAM_LOAD

 	/* Determine the largest L1 cache size */
 #ifndef CONFIG_SYS_CACHE_SIZE_AUTO
 #if CONFIG_SYS_ICACHE_SIZE > CONFIG_SYS_DCACHE_SIZE
 	li	v0, CONFIG_SYS_ICACHE_SIZE
 #else
 	li	v0, CONFIG_SYS_DCACHE_SIZE
 #endif
 #else
 	move	v0, R_IC_SIZE
 	sltu	t1, R_IC_SIZE, R_DC_SIZE
 	movn	v0, R_DC_SIZE, t1
 #endif
 	/*
 	 * Now clear that much memory starting from zero.
 	 */
 	PTR_LI		a0, CKSEG1
 	PTR_ADDU	a1, a0, v0
 2:	PTR_ADDIU	a0, 64
 	f_fill64	a0, -64, zero
 	bne		a0, a1, 2b

 #endif /* CONFIG_SYS_MIPS_CACHE_INIT_RAM_LOAD */

 #ifdef CONFIG_MIPS_L2_CACHE
 	/*
 	 * If the L2 is bypassed, init the L1 first so that we can execute the
 	 * rest of the cache initialisation using the L1 instruction cache.
 	 */
 	bnez		R_L2_BYPASSED, l1_init

 l2_init:
 	PTR_LI		t0, INDEX_BASE
 	PTR_ADDU	t1, t0, R_L2_SIZE
 1:	cache		INDEX_STORE_TAG_SD, 0(t0)
 	PTR_ADDU	t0, t0, R_L2_LINE
 	bne		t0, t1, 1b

 	/*
 	 * If the L2 was bypassed then we already initialised the L1s before
 	 * the L2, so we are now done.
 	 */
 	bnez		R_L2_BYPASSED, l2_unbypass
 #endif

 	/*
 	 * The TagLo registers used depend upon the CPU implementation, but the
 	 * architecture requires that it is safe for software to write to both
 	 * TagLo selects 0 & 2 covering supported cases.
 	 */
 l1_init:
 	mtc0		zero, CP0_TAGLO
 	mtc0		zero, CP0_TAGLO, 2
 	ehb

 	/*
 	 * The caches are probably in an indeterminate state, so we force good
 	 * parity into them by doing an invalidate for each line. If
 	 * CONFIG_SYS_MIPS_CACHE_INIT_RAM_LOAD is set then we'll proceed to
 	 * perform a load/fill & a further invalidate for each line, assuming
 	 * that the bottom of RAM (having just been cleared) will generate good
 	 * parity for the cache.
 	 */

 	/*
 	 * Initialize the I-cache first,
 	 */
 	blez		R_IC_SIZE, 1f
 	PTR_LI		t0, INDEX_BASE
 	PTR_ADDU	t1, t0, R_IC_SIZE
 	/* clear tag to invalidate */
 	cache_loop	t0, t1, R_IC_LINE, INDEX_STORE_TAG_I
 #ifdef CONFIG_SYS_MIPS_CACHE_INIT_RAM_LOAD
 	/* fill once, so data field parity is correct */
 	PTR_LI		t0, INDEX_BASE
 	cache_loop	t0, t1, R_IC_LINE, FILL
 	/* invalidate again - prudent but not strictly neccessary */
 	PTR_LI		t0, INDEX_BASE
 	cache_loop	t0, t1, R_IC_LINE, INDEX_STORE_TAG_I
 #endif

 	/* Enable use of the I-cache by setting Config.K0 */
 	sync
 	mfc0		t0, CP0_CONFIG
 	li		t1, CONFIG_SYS_MIPS_CACHE_MODE
 #if __mips_isa_rev >= 2
 	ins		t0, t1, 0, 3
 #else
 	ori		t0, t0, CONF_CM_CMASK
 	xori		t0, t0, CONF_CM_CMASK
 	or		t0, t0, t1
 #endif
 	mtc0		t0, CP0_CONFIG

 	/*
 	 * then initialize D-cache.
 	 */
 1:	blez		R_DC_SIZE, 3f
 	PTR_LI		t0, INDEX_BASE
 	PTR_ADDU	t1, t0, R_DC_SIZE
 	/* clear all tags */
 	cache_loop	t0, t1, R_DC_LINE, INDEX_STORE_TAG_D
 #ifdef CONFIG_SYS_MIPS_CACHE_INIT_RAM_LOAD
 	/* load from each line (in cached space) */
 	PTR_LI		t0, INDEX_BASE
 2:	LONG_L		zero, 0(t0)
 	PTR_ADDU	t0, R_DC_LINE
 	bne		t0, t1, 2b
 	/* clear all tags */
 	PTR_LI		t0, INDEX_BASE
 	cache_loop	t0, t1, R_DC_LINE, INDEX_STORE_TAG_D
 #endif
 3:

 #ifdef CONFIG_MIPS_L2_CACHE
 	/* If the L2 isn't bypassed then we're done */
 	beqz		R_L2_BYPASSED, return

 	/* The L2 is bypassed - go initialise it */
 	b		l2_init

 l2_unbypass:
 # if __mips_isa_rev >= 6
 	beqz		R_L2_L2C, 1f

 	li		t0, CKSEG1ADDR(CONFIG_MIPS_CM_BASE)
 	lw		t1, GCR_L2_CONFIG(t0)
 	xor		t1, t1, GCR_L2_CONFIG_BYPASS
 	sw		t1, GCR_L2_CONFIG(t0)
 	sync
 	ehb
 	b		2f
 # endif
 1:	mfc0		t0, CP0_CONFIG, 2
 	xor		t0, t0, MIPS_CONF2_L2B
 	mtc0		t0, CP0_CONFIG, 2
 	ehb

 2:
 # ifdef CONFIG_MIPS_CM
 	/* Config3 must exist for a CM to be present */
 	mfc0		t0, CP0_CONFIG, 1
 	bgez		t0, 2f
 	mfc0		t0, CP0_CONFIG, 2
 	bgez		t0, 2f

 	/* Check Config3.CMGCR to determine CM presence */
 	mfc0		t0, CP0_CONFIG, 3
 	and		t0, t0, MIPS_CONF3_CMGCR
 	beqz		t0, 2f

 	/* Change Config.K0 to a coherent CCA */
 	mfc0		t0, CP0_CONFIG
 	li		t1, CONF_CM_CACHABLE_COW
 #if __mips_isa_rev >= 2
 	ins		t0, t1, 0, 3
 #else
 	ori		t0, t0, CONF_CM_CMASK
 	xori		t0, t0, CONF_CM_CMASK
 	or		t0, t0, t1
 #endif
 	mtc0		t0, CP0_CONFIG

 	/*
 	 * Join the coherent domain such that the caches of this core are kept
 	 * coherent with those of other cores.
 	 */
 	PTR_LI		t0, CKSEG1ADDR(CONFIG_MIPS_CM_BASE)
 	lw		t1, GCR_REV(t0)
 	li		t2, GCR_REV_CM3
 	li		t3, GCR_Cx_COHERENCE_EN
 	bge		t1, t2, 1f
 	li		t3, GCR_Cx_COHERENCE_DOM_EN
 1:	sw		t3, GCR_Cx_COHERENCE(t0)
 	ehb
 2:
 # endif
 #endif

 return:
 	/* Ensure all cache operations complete before returning */
 	sync
 	jr	ra
 	END(mips_cache_reset)

 /*
  * dcache_status - get cache status
  *
  * RETURNS: 0 - cache disabled; 1 - cache enabled
  *
  */
 LEAF(dcache_status)
 	mfc0	t0, CP0_CONFIG
 	li	t1, CONF_CM_UNCACHED
 	andi	t0, t0, CONF_CM_CMASK
 	move	v0, zero
 	beq	t0, t1, 2f
 	li	v0, 1
 2:	jr	ra
 	END(dcache_status)

 /*
  * dcache_disable - disable cache
  *
  * RETURNS: N/A
  *
  */
 LEAF(dcache_disable)
 	mfc0	t0, CP0_CONFIG
 	li	t1, -8
 	and	t0, t0, t1
 	ori	t0, t0, CONF_CM_UNCACHED
 	mtc0	t0, CP0_CONFIG
 	jr	ra
 	END(dcache_disable)

 /*
  * dcache_enable - enable cache
  *
  * RETURNS: N/A
  *
  */
 LEAF(dcache_enable)
 	mfc0	t0, CP0_CONFIG
 	ori	t0, CONF_CM_CMASK
 	xori	t0, CONF_CM_CMASK
 	ori	t0, CONFIG_SYS_MIPS_CACHE_MODE
 	mtc0	t0, CP0_CONFIG
 	jr	ra
 	END(dcache_enable)
	/* SPDX-License-Identifier: GPL-2.0+ */
	/*
	* Cache-handling routined for MIPS CPUs
	*
	* Copyright (c) 2003 Wolfgang Denk <wd@denx.de>
	*/

	#include <asm-offsets.h>
	#include <config.h>
	#include <asm/asm.h>
	#include <asm/regdef.h>
	#include <asm/mipsregs.h>
	#include <asm/addrspace.h>
	#include <asm/cacheops.h>
	#include <asm/cm.h>

	#ifndef CONFIG_SYS_MIPS_CACHE_MODE
	#define CONFIG_SYS_MIPS_CACHE_MODE CONF_CM_CACHABLE_NONCOHERENT
	#endif

	#define INDEX_BASE CKSEG0

	.macro f_fill64 dst, offset, val
	LONG_S \val, (\offset + 0 * LONGSIZE)(\dst)
	LONG_S \val, (\offset + 1 * LONGSIZE)(\dst)
	LONG_S \val, (\offset + 2 * LONGSIZE)(\dst)
	LONG_S \val, (\offset + 3 * LONGSIZE)(\dst)
	LONG_S \val, (\offset + 4 * LONGSIZE)(\dst)
	LONG_S \val, (\offset + 5 * LONGSIZE)(\dst)
	LONG_S \val, (\offset + 6 * LONGSIZE)(\dst)
	LONG_S \val, (\offset + 7 * LONGSIZE)(\dst)
	#if LONGSIZE == 4
	LONG_S \val, (\offset + 8 * LONGSIZE)(\dst)
	LONG_S \val, (\offset + 9 * LONGSIZE)(\dst)
	LONG_S \val, (\offset + 10 * LONGSIZE)(\dst)
	LONG_S \val, (\offset + 11 * LONGSIZE)(\dst)
	LONG_S \val, (\offset + 12 * LONGSIZE)(\dst)
	LONG_S \val, (\offset + 13 * LONGSIZE)(\dst)
	LONG_S \val, (\offset + 14 * LONGSIZE)(\dst)
	LONG_S \val, (\offset + 15 * LONGSIZE)(\dst)
	#endif
	.endm

	.macro cache_loop curr, end, line_sz, op
	10: cache \op, 0(\curr)
	PTR_ADDU \curr, \curr, \line_sz
	bne \curr, \end, 10b
	.endm

	.macro l1_info sz, line_sz, off
	.set push
	.set noat

	mfc0 $1, CP0_CONFIG, 1

	/* detect line size */
	srl \line_sz, $1, \off + MIPS_CONF1_DL_SHF - MIPS_CONF1_DA_SHF
	andi \line_sz, \line_sz, (MIPS_CONF1_DL >> MIPS_CONF1_DL_SHF)
	move \sz, zero
	beqz \line_sz, 10f
	li \sz, 2
	sllv \line_sz, \sz, \line_sz

	/* detect associativity */
	srl \sz, $1, \off + MIPS_CONF1_DA_SHF - MIPS_CONF1_DA_SHF
	andi \sz, \sz, (MIPS_CONF1_DA >> MIPS_CONF1_DA_SHF)
	addiu \sz, \sz, 1

	/* sz = line_sz /
	mul \sz, \sz, \line_sz

	/* detect log32(sets) */
	srl $1, $1, \off + MIPS_CONF1_DS_SHF - MIPS_CONF1_DA_SHF
	andi $1, $1, (MIPS_CONF1_DS >> MIPS_CONF1_DS_SHF)
	addiu $1, $1, 1
	andi $1, $1, 0x7

	/* sz <<= log32(sets) */
	sllv \sz, \sz, $1

	/* sz = 32 /
	li $1, 32
	mul \sz, \sz, $1
	10:
	.set pop
	.endm
	/*
	* mips_cache_reset - low level initialisation of the primary caches
	*
	* This routine initialises the primary caches to ensure that they have good
	* parity. It must be called by the ROM before any cached locations are used
	* to prevent the possibility of data with bad parity being written to memory.
	*
	* To initialise the instruction cache it is essential that a source of data
	* with good parity is available. This routine will initialise an area of
	* memory starting at location zero to be used as a source of parity.
	*
	* Note that this function does not follow the standard calling convention &
	* may clobber typically callee-saved registers.
	*
	* RETURNS: N/A
	*
	*/
	#define R_RETURN s0
	#define R_IC_SIZE s1
	#define R_IC_LINE s2
	#define R_DC_SIZE s3
	#define R_DC_LINE s4
	#define R_L2_SIZE s5
	#define R_L2_LINE s6
	#define R_L2_BYPASSED s7
	#define R_L2_L2C t8
	LEAF(mips_cache_reset)
	move R_RETURN, ra

	#ifdef CONFIG_MIPS_L2_CACHE
	/*
	* For there to be an L2 present, Config2 must be present. If it isn't
	* then we proceed knowing there's no L2 cache.
	*/
	move R_L2_SIZE, zero
	move R_L2_LINE, zero
	move R_L2_BYPASSED, zero
	move R_L2_L2C, zero
	mfc0 t0, CP0_CONFIG, 1
	bgez t0, l2_probe_done

	/*
	* From MIPSr6 onwards the L2 cache configuration might not be reported
	* by Config2. The Config5.L2C bit indicates whether this is the case,
	* and if it is then we need knowledge of where else to look. For cores
	* from Imagination Technologies this is a CM GCR.
	*/
	# if __mips_isa_rev >= 6
	/* Check that Config5 exists */
	mfc0 t0, CP0_CONFIG, 2
	bgez t0, l2_probe_cop0
	mfc0 t0, CP0_CONFIG, 3
	bgez t0, l2_probe_cop0
	mfc0 t0, CP0_CONFIG, 4
	bgez t0, l2_probe_cop0

	/* Check Config5.L2C is set */
	mfc0 t0, CP0_CONFIG, 5
	and R_L2_L2C, t0, MIPS_CONF5_L2C
	beqz R_L2_L2C, l2_probe_cop0

	/* Config5.L2C is set */
	# ifdef CONFIG_MIPS_CM
	/* The CM will provide L2 configuration */
	PTR_LI t0, CKSEG1ADDR(CONFIG_MIPS_CM_BASE)
	lw t1, GCR_L2_CONFIG(t0)
	bgez t1, l2_probe_done

	ext R_L2_LINE, t1, \
	GCR_L2_CONFIG_LINESZ_SHIFT, GCR_L2_CONFIG_LINESZ_BITS
	beqz R_L2_LINE, l2_probe_done
	li t2, 2
	sllv R_L2_LINE, t2, R_L2_LINE

	ext t2, t1, GCR_L2_CONFIG_ASSOC_SHIFT, GCR_L2_CONFIG_ASSOC_BITS
	addiu t2, t2, 1
	mul R_L2_SIZE, R_L2_LINE, t2

	ext t2, t1, GCR_L2_CONFIG_SETSZ_SHIFT, GCR_L2_CONFIG_SETSZ_BITS
	sllv R_L2_SIZE, R_L2_SIZE, t2
	li t2, 64
	mul R_L2_SIZE, R_L2_SIZE, t2

	/* Bypass the L2 cache so that we can init the L1s early */
	or t1, t1, GCR_L2_CONFIG_BYPASS
	sw t1, GCR_L2_CONFIG(t0)
	sync
	li R_L2_BYPASSED, 1

	/* Zero the L2 tag registers */
	sw zero, GCR_L2_TAG_ADDR(t0)
	sw zero, GCR_L2_TAG_ADDR_UPPER(t0)
	sw zero, GCR_L2_TAG_STATE(t0)
	sw zero, GCR_L2_TAG_STATE_UPPER(t0)
	sw zero, GCR_L2_DATA(t0)
	sw zero, GCR_L2_DATA_UPPER(t0)
	sync
	# else
	/* We don't know how to retrieve L2 configuration on this system */
	# endif
	b l2_probe_done
	# endif

	/*
	* For pre-r6 systems, or r6 systems with Config5.L2C==0, probe the L2
	* cache configuration from the cop0 Config2 register.
	*/
	l2_probe_cop0:
	mfc0 t0, CP0_CONFIG, 2

	srl R_L2_LINE, t0, MIPS_CONF2_SL_SHF
	andi R_L2_LINE, R_L2_LINE, MIPS_CONF2_SL >> MIPS_CONF2_SL_SHF
	beqz R_L2_LINE, l2_probe_done
	li t1, 2
	sllv R_L2_LINE, t1, R_L2_LINE

	srl t1, t0, MIPS_CONF2_SA_SHF
	andi t1, t1, MIPS_CONF2_SA >> MIPS_CONF2_SA_SHF
	addiu t1, t1, 1
	mul R_L2_SIZE, R_L2_LINE, t1

	srl t1, t0, MIPS_CONF2_SS_SHF
	andi t1, t1, MIPS_CONF2_SS >> MIPS_CONF2_SS_SHF
	sllv R_L2_SIZE, R_L2_SIZE, t1
	li t1, 64
	mul R_L2_SIZE, R_L2_SIZE, t1

	/* Attempt to bypass the L2 so that we can init the L1s early */
	or t0, t0, MIPS_CONF2_L2B
	mtc0 t0, CP0_CONFIG, 2
	ehb
	mfc0 t0, CP0_CONFIG, 2
	and R_L2_BYPASSED, t0, MIPS_CONF2_L2B

	/* Zero the L2 tag registers */
	mtc0 zero, CP0_TAGLO, 4
	ehb
	l2_probe_done:
	#endif

	#ifndef CONFIG_SYS_CACHE_SIZE_AUTO
	li R_IC_SIZE, CONFIG_SYS_ICACHE_SIZE
	li R_IC_LINE, CONFIG_SYS_ICACHE_LINE_SIZE
	#else
	l1_info R_IC_SIZE, R_IC_LINE, MIPS_CONF1_IA_SHF
	#endif

	#ifndef CONFIG_SYS_CACHE_SIZE_AUTO
	li R_DC_SIZE, CONFIG_SYS_DCACHE_SIZE
	li R_DC_LINE, CONFIG_SYS_DCACHE_LINE_SIZE
	#else
	l1_info R_DC_SIZE, R_DC_LINE, MIPS_CONF1_DA_SHF
	#endif

	#ifdef CONFIG_SYS_MIPS_CACHE_INIT_RAM_LOAD

	/* Determine the largest L1 cache size */
	#ifndef CONFIG_SYS_CACHE_SIZE_AUTO
	#if CONFIG_SYS_ICACHE_SIZE > CONFIG_SYS_DCACHE_SIZE
	li v0, CONFIG_SYS_ICACHE_SIZE
	#else
	li v0, CONFIG_SYS_DCACHE_SIZE
	#endif
	#else
	move v0, R_IC_SIZE
	sltu t1, R_IC_SIZE, R_DC_SIZE
	movn v0, R_DC_SIZE, t1
	#endif
	/*
	* Now clear that much memory starting from zero.
	*/
	PTR_LI a0, CKSEG1
	PTR_ADDU a1, a0, v0
	2: PTR_ADDIU a0, 64
	f_fill64 a0, -64, zero
	bne a0, a1, 2b

	#endif /* CONFIG_SYS_MIPS_CACHE_INIT_RAM_LOAD */

	#ifdef CONFIG_MIPS_L2_CACHE
	/*
	* If the L2 is bypassed, init the L1 first so that we can execute the
	* rest of the cache initialisation using the L1 instruction cache.
	*/
	bnez R_L2_BYPASSED, l1_init

	l2_init:
	PTR_LI t0, INDEX_BASE
	PTR_ADDU t1, t0, R_L2_SIZE
	1: cache INDEX_STORE_TAG_SD, 0(t0)
	PTR_ADDU t0, t0, R_L2_LINE
	bne t0, t1, 1b

	/*
	* If the L2 was bypassed then we already initialised the L1s before
	* the L2, so we are now done.
	*/
	bnez R_L2_BYPASSED, l2_unbypass
	#endif

	/*
	* The TagLo registers used depend upon the CPU implementation, but the
	* architecture requires that it is safe for software to write to both
	* TagLo selects 0 & 2 covering supported cases.
	*/
	l1_init:
	mtc0 zero, CP0_TAGLO
	mtc0 zero, CP0_TAGLO, 2
	ehb

	/*
	* The caches are probably in an indeterminate state, so we force good
	* parity into them by doing an invalidate for each line. If
	* CONFIG_SYS_MIPS_CACHE_INIT_RAM_LOAD is set then we'll proceed to
	* perform a load/fill & a further invalidate for each line, assuming
	* that the bottom of RAM (having just been cleared) will generate good
	* parity for the cache.
	*/

	/*
	* Initialize the I-cache first,
	*/
	blez R_IC_SIZE, 1f
	PTR_LI t0, INDEX_BASE
	PTR_ADDU t1, t0, R_IC_SIZE
	/* clear tag to invalidate */
	cache_loop t0, t1, R_IC_LINE, INDEX_STORE_TAG_I
	#ifdef CONFIG_SYS_MIPS_CACHE_INIT_RAM_LOAD
	/* fill once, so data field parity is correct */
	PTR_LI t0, INDEX_BASE
	cache_loop t0, t1, R_IC_LINE, FILL
	/* invalidate again - prudent but not strictly neccessary */
	PTR_LI t0, INDEX_BASE
	cache_loop t0, t1, R_IC_LINE, INDEX_STORE_TAG_I
	#endif

	/* Enable use of the I-cache by setting Config.K0 */
	sync
	mfc0 t0, CP0_CONFIG
	li t1, CONFIG_SYS_MIPS_CACHE_MODE
	#if __mips_isa_rev >= 2
	ins t0, t1, 0, 3
	#else
	ori t0, t0, CONF_CM_CMASK
	xori t0, t0, CONF_CM_CMASK
	or t0, t0, t1
	#endif
	mtc0 t0, CP0_CONFIG

	/*
	* then initialize D-cache.
	*/
	1: blez R_DC_SIZE, 3f
	PTR_LI t0, INDEX_BASE
	PTR_ADDU t1, t0, R_DC_SIZE
	/* clear all tags */
	cache_loop t0, t1, R_DC_LINE, INDEX_STORE_TAG_D
	#ifdef CONFIG_SYS_MIPS_CACHE_INIT_RAM_LOAD
	/* load from each line (in cached space) */
	PTR_LI t0, INDEX_BASE
	2: LONG_L zero, 0(t0)
	PTR_ADDU t0, R_DC_LINE
	bne t0, t1, 2b
	/* clear all tags */
	PTR_LI t0, INDEX_BASE
	cache_loop t0, t1, R_DC_LINE, INDEX_STORE_TAG_D
	#endif
	3:

	#ifdef CONFIG_MIPS_L2_CACHE
	/* If the L2 isn't bypassed then we're done */
	beqz R_L2_BYPASSED, return

	/* The L2 is bypassed - go initialise it */
	b l2_init

	l2_unbypass:
	# if __mips_isa_rev >= 6
	beqz R_L2_L2C, 1f

	li t0, CKSEG1ADDR(CONFIG_MIPS_CM_BASE)
	lw t1, GCR_L2_CONFIG(t0)
	xor t1, t1, GCR_L2_CONFIG_BYPASS
	sw t1, GCR_L2_CONFIG(t0)
	sync
	ehb
	b 2f
	# endif
	1: mfc0 t0, CP0_CONFIG, 2
	xor t0, t0, MIPS_CONF2_L2B
	mtc0 t0, CP0_CONFIG, 2
	ehb

	2:
	# ifdef CONFIG_MIPS_CM
	/* Config3 must exist for a CM to be present */
	mfc0 t0, CP0_CONFIG, 1
	bgez t0, 2f
	mfc0 t0, CP0_CONFIG, 2
	bgez t0, 2f

	/* Check Config3.CMGCR to determine CM presence */
	mfc0 t0, CP0_CONFIG, 3
	and t0, t0, MIPS_CONF3_CMGCR
	beqz t0, 2f

	/* Change Config.K0 to a coherent CCA */
	mfc0 t0, CP0_CONFIG
	li t1, CONF_CM_CACHABLE_COW
	#if __mips_isa_rev >= 2
	ins t0, t1, 0, 3
	#else
	ori t0, t0, CONF_CM_CMASK
	xori t0, t0, CONF_CM_CMASK
	or t0, t0, t1
	#endif
	mtc0 t0, CP0_CONFIG

	/*
	* Join the coherent domain such that the caches of this core are kept
	* coherent with those of other cores.
	*/
	PTR_LI t0, CKSEG1ADDR(CONFIG_MIPS_CM_BASE)
	lw t1, GCR_REV(t0)
	li t2, GCR_REV_CM3
	li t3, GCR_Cx_COHERENCE_EN
	bge t1, t2, 1f
	li t3, GCR_Cx_COHERENCE_DOM_EN
	1: sw t3, GCR_Cx_COHERENCE(t0)
	ehb
	2:
	# endif
	#endif

	return:
	/* Ensure all cache operations complete before returning */
	sync
	jr ra
	END(mips_cache_reset)

	/*
	* dcache_status - get cache status
	*
	* RETURNS: 0 - cache disabled; 1 - cache enabled
	*
	*/
	LEAF(dcache_status)
	mfc0 t0, CP0_CONFIG
	li t1, CONF_CM_UNCACHED
	andi t0, t0, CONF_CM_CMASK
	move v0, zero
	beq t0, t1, 2f
	li v0, 1
	2: jr ra
	END(dcache_status)

	/*
	* dcache_disable - disable cache
	*
	* RETURNS: N/A
	*
	*/
	LEAF(dcache_disable)
	mfc0 t0, CP0_CONFIG
	li t1, -8
	and t0, t0, t1
	ori t0, t0, CONF_CM_UNCACHED
	mtc0 t0, CP0_CONFIG
	jr ra
	END(dcache_disable)

	/*
	* dcache_enable - enable cache
	*
	* RETURNS: N/A
	*
	*/
	LEAF(dcache_enable)
	mfc0 t0, CP0_CONFIG
	ori t0, CONF_CM_CMASK
	xori t0, CONF_CM_CMASK
	ori t0, CONFIG_SYS_MIPS_CACHE_MODE
	mtc0 t0, CP0_CONFIG
	jr ra
	END(dcache_enable)