svn commit: r255367 - in head/sys: conf mips/mips
Jayachandran C.
jchandra at FreeBSD.org
Sat Sep 7 16:31:31 UTC 2013
Author: jchandra
Date: Sat Sep 7 16:31:30 2013
New Revision: 255367
URL: http://svnweb.freebsd.org/changeset/base/255367
Log:
Use a better version of memcpy/bcopy for mips kernel.
Use a variant of mips libc memcpy for kernel. This implementation uses
64-bit operations when compiled for 64-bit, and is significantly faster
in that case.
Submitted by: Tanmay Jagdale <tanmayj at broadcom.com>
Added:
head/sys/mips/mips/bcopy.S (contents, props changed)
Modified:
head/sys/conf/files.mips
head/sys/mips/mips/support.S
Modified: head/sys/conf/files.mips
==============================================================================
--- head/sys/conf/files.mips Sat Sep 7 16:16:57 2013 (r255366)
+++ head/sys/conf/files.mips Sat Sep 7 16:31:30 2013 (r255367)
@@ -38,6 +38,7 @@ mips/mips/stack_machdep.c optional ddb
mips/mips/stdatomic.c standard \
compile-with "${NORMAL_C:N-Wmissing-prototypes}"
mips/mips/support.S standard
+mips/mips/bcopy.S standard
mips/mips/swtch.S standard
mips/mips/sys_machdep.c standard
mips/mips/tlb.c standard
Added: head/sys/mips/mips/bcopy.S
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/sys/mips/mips/bcopy.S Sat Sep 7 16:31:30 2013 (r255367)
@@ -0,0 +1,286 @@
+/* $NetBSD: bcopy.S,v 1.3 2009/12/14 00:39:00 matt Exp $ */
+
+/*
+ * Mach Operating System
+ * Copyright (c) 1993 Carnegie Mellon University
+ * All Rights Reserved.
+ *
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ *
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ *
+ * Carnegie Mellon requests users of this software to return to
+ *
+ * Software Distribution Coordinator or Software.Distribution at CS.CMU.EDU
+ * School of Computer Science
+ * Carnegie Mellon University
+ * Pittsburgh PA 15213-3890
+ *
+ * any improvements or extensions that they make and grant Carnegie Mellon
+ * the rights to redistribute these changes.
+ */
+
+/*
+ * File: mips_bcopy.s
+ * Author: Chris Maeda
+ * Date: June 1993
+ *
+ * Fast copy routine. Derived from aligned_block_copy.
+ */
+
+
+#include <machine/asm.h>
+__FBSDID("$FreeBSD$");
+
+#include <machine/endian.h>
+
+#if defined(LIBC_SCCS) && !defined(lint)
+#if 0
+ ASMSTR("from: @(#)mips_bcopy.s 2.2 CMU 18/06/93")
+#else
+ ASMSTR("$NetBSD: bcopy.S,v 1.3 2009/12/14 00:39:00 matt Exp $")
+#endif
+#endif /* LIBC_SCCS and not lint */
+
+#ifdef __ABICALLS__
+ .abicalls
+#endif
+
+/*
+ * bcopy(caddr_t src, caddr_t dst, unsigned int len)
+ *
+ * a0 src address
+ * a1 dst address
+ * a2 length
+ */
+
+#define SRCREG a0
+#define DSTREG a1
+#define SIZEREG a2
+
+LEAF(memcpy)
+ .set noat
+ .set noreorder
+
+ move v0, a0
+ move a0, a1
+ move a1, v0
+
+ALEAF(bcopy)
+ALEAF(ovbcopy)
+ /*
+ * Make sure we can copy forwards.
+ */
+ sltu t0,SRCREG,DSTREG # t0 == SRCREG < DSTREG
+ bne t0,zero,6f # copy backwards
+
+ /*
+ * There are four alignment cases (with frequency)
+ * (Based on measurements taken with a DECstation 5000/200
+ * inside a Mach kernel.)
+ *
+ * aligned -> aligned (mostly)
+ * unaligned -> aligned (sometimes)
+ * aligned,unaligned -> unaligned (almost never)
+ *
+ * Note that we could add another case that checks if
+ * the destination and source are unaligned but the
+ * copy is alignable. eg if src and dest are both
+ * on a halfword boundary.
+ */
+ andi t1,DSTREG,(SZREG-1) # get last bits of dest
+ bne t1,zero,3f # dest unaligned
+ andi t0,SRCREG,(SZREG-1) # get last bits of src
+ bne t0,zero,5f
+
+ /*
+ * Forward aligned->aligned copy, 8 words at a time.
+ */
+98:
+ li AT,-(SZREG*8)
+ and t0,SIZEREG,AT # count truncated to multiples
+ PTR_ADDU a3,SRCREG,t0 # run fast loop up to this addr
+ sltu AT,SRCREG,a3 # any work to do?
+ beq AT,zero,2f
+ PTR_SUBU SIZEREG,t0
+
+ /*
+ * loop body
+ */
+1: # cp
+ REG_L t3,(0*SZREG)(SRCREG)
+ REG_L v1,(1*SZREG)(SRCREG)
+ REG_L t0,(2*SZREG)(SRCREG)
+ REG_L t1,(3*SZREG)(SRCREG)
+ PTR_ADDU SRCREG,SZREG*8
+ REG_S t3,(0*SZREG)(DSTREG)
+ REG_S v1,(1*SZREG)(DSTREG)
+ REG_S t0,(2*SZREG)(DSTREG)
+ REG_S t1,(3*SZREG)(DSTREG)
+ REG_L t1,(-1*SZREG)(SRCREG)
+ REG_L t0,(-2*SZREG)(SRCREG)
+ REG_L v1,(-3*SZREG)(SRCREG)
+ REG_L t3,(-4*SZREG)(SRCREG)
+ PTR_ADDU DSTREG,SZREG*8
+ REG_S t1,(-1*SZREG)(DSTREG)
+ REG_S t0,(-2*SZREG)(DSTREG)
+ REG_S v1,(-3*SZREG)(DSTREG)
+ bne SRCREG,a3,1b
+ REG_S t3,(-4*SZREG)(DSTREG)
+
+ /*
+ * Copy a word at a time, no loop unrolling.
+ */
+2: # wordcopy
+ andi t2,SIZEREG,(SZREG-1) # get byte count / SZREG
+ PTR_SUBU t2,SIZEREG,t2 # t2 = words to copy * SZREG
+ beq t2,zero,3f
+ PTR_ADDU t0,SRCREG,t2 # stop at t0
+ PTR_SUBU SIZEREG,SIZEREG,t2
+1:
+ REG_L t3,0(SRCREG)
+ PTR_ADDU SRCREG,SZREG
+ REG_S t3,0(DSTREG)
+ bne SRCREG,t0,1b
+ PTR_ADDU DSTREG,SZREG
+
+3: # bytecopy
+ beq SIZEREG,zero,4f # nothing left to do?
+ nop
+1:
+ lb t3,0(SRCREG)
+ PTR_ADDU SRCREG,1
+ sb t3,0(DSTREG)
+ PTR_SUBU SIZEREG,1
+ bgtz SIZEREG,1b
+ PTR_ADDU DSTREG,1
+
+4: # copydone
+ j ra
+ nop
+
+ /*
+ * Copy from unaligned source to aligned dest.
+ */
+5: # destaligned
+ andi t0,SIZEREG,(SZREG-1) # t0 = bytecount mod SZREG
+ PTR_SUBU a3,SIZEREG,t0 # number of words to transfer
+ beq a3,zero,3b
+ nop
+ move SIZEREG,t0 # this many to do after we are done
+ PTR_ADDU a3,SRCREG,a3 # stop point
+
+1:
+ REG_LHI t3,0(SRCREG)
+ REG_LLO t3,SZREG-1(SRCREG)
+ PTR_ADDI SRCREG,SZREG
+ REG_S t3,0(DSTREG)
+ bne SRCREG,a3,1b
+ PTR_ADDI DSTREG,SZREG
+
+ b 3b
+ nop
+
+6: # backcopy -- based on above
+ PTR_ADDU SRCREG,SIZEREG
+ PTR_ADDU DSTREG,SIZEREG
+ andi t1,DSTREG,SZREG-1 # get last 3 bits of dest
+ bne t1,zero,3f
+ andi t0,SRCREG,SZREG-1 # get last 3 bits of src
+ bne t0,zero,5f
+
+ /*
+ * Forward aligned->aligned copy, 8*4 bytes at a time.
+ */
+ li AT,(-8*SZREG)
+ and t0,SIZEREG,AT # count truncated to multiple of 32
+ beq t0,zero,2f # any work to do?
+ PTR_SUBU SIZEREG,t0
+ PTR_SUBU a3,SRCREG,t0
+
+ /*
+ * loop body
+ */
+1: # cp
+ REG_L t3,(-4*SZREG)(SRCREG)
+ REG_L v1,(-3*SZREG)(SRCREG)
+ REG_L t0,(-2*SZREG)(SRCREG)
+ REG_L t1,(-1*SZREG)(SRCREG)
+ PTR_SUBU SRCREG,8*SZREG
+ REG_S t3,(-4*SZREG)(DSTREG)
+ REG_S v1,(-3*SZREG)(DSTREG)
+ REG_S t0,(-2*SZREG)(DSTREG)
+ REG_S t1,(-1*SZREG)(DSTREG)
+ REG_L t1,(3*SZREG)(SRCREG)
+ REG_L t0,(2*SZREG)(SRCREG)
+ REG_L v1,(1*SZREG)(SRCREG)
+ REG_L t3,(0*SZREG)(SRCREG)
+ PTR_SUBU DSTREG,8*SZREG
+ REG_S t1,(3*SZREG)(DSTREG)
+ REG_S t0,(2*SZREG)(DSTREG)
+ REG_S v1,(1*SZREG)(DSTREG)
+ bne SRCREG,a3,1b
+ REG_S t3,(0*SZREG)(DSTREG)
+
+ /*
+ * Copy a word at a time, no loop unrolling.
+ */
+2: # wordcopy
+ andi t2,SIZEREG,SZREG-1 # get byte count / 4
+ PTR_SUBU t2,SIZEREG,t2 # t2 = number of words to copy
+ beq t2,zero,3f
+ PTR_SUBU t0,SRCREG,t2 # stop at t0
+ PTR_SUBU SIZEREG,SIZEREG,t2
+1:
+ REG_L t3,-SZREG(SRCREG)
+ PTR_SUBU SRCREG,SZREG
+ REG_S t3,-SZREG(DSTREG)
+ bne SRCREG,t0,1b
+ PTR_SUBU DSTREG,SZREG
+
+3: # bytecopy
+ beq SIZEREG,zero,4f # nothing left to do?
+ nop
+1:
+ lb t3,-1(SRCREG)
+ PTR_SUBU SRCREG,1
+ sb t3,-1(DSTREG)
+ PTR_SUBU SIZEREG,1
+ bgtz SIZEREG,1b
+ PTR_SUBU DSTREG,1
+
+4: # copydone
+ j ra
+ nop
+
+ /*
+ * Copy from unaligned source to aligned dest.
+ */
+5: # destaligned
+ andi t0,SIZEREG,SZREG-1 # t0 = bytecount mod 4
+ PTR_SUBU a3,SIZEREG,t0 # number of words to transfer
+ beq a3,zero,3b
+ nop
+ move SIZEREG,t0 # this many to do after we are done
+ PTR_SUBU a3,SRCREG,a3 # stop point
+
+1:
+ REG_LHI t3,-SZREG(SRCREG)
+ REG_LLO t3,-1(SRCREG)
+ PTR_SUBU SRCREG,SZREG
+ REG_S t3,-SZREG(DSTREG)
+ bne SRCREG,a3,1b
+ PTR_SUBU DSTREG,SZREG
+
+ b 3b
+ nop
+
+ .set reorder
+ .set at
+END(memcpy)
Modified: head/sys/mips/mips/support.S
==============================================================================
--- head/sys/mips/mips/support.S Sat Sep 7 16:16:57 2013 (r255366)
+++ head/sys/mips/mips/support.S Sat Sep 7 16:31:30 2013 (r255367)
@@ -507,98 +507,6 @@ LEAF(fswintrberr)
END(fswintrberr)
/*
- * memcpy(to, from, len)
- * {ov}bcopy(from, to, len)
- */
-LEAF(memcpy)
- .set noreorder
- move v0, a0 # swap from and to
- move a0, a1
- move a1, v0
-ALEAF(bcopy)
-ALEAF(ovbcopy)
- .set noreorder
- PTR_ADDU t0, a0, a2 # t0 = end of s1 region
- sltu t1, a1, t0
- sltu t2, a0, a1
- and t1, t1, t2 # t1 = true if from < to < (from+len)
- beq t1, zero, forward # non overlapping, do forward copy
- slt t2, a2, 12 # check for small copy
-
- ble a2, zero, 2f
- PTR_ADDU t1, a1, a2 # t1 = end of to region
-1:
- lb v1, -1(t0) # copy bytes backwards,
- PTR_SUBU t0, t0, 1 # doesnt happen often so do slow way
- PTR_SUBU t1, t1, 1
- bne t0, a0, 1b
- sb v1, 0(t1)
-2:
- j ra
- nop
-forward:
- bne t2, zero, smallcpy # do a small bcopy
- xor v1, a0, a1 # compare low two bits of addresses
- and v1, v1, 3
- PTR_SUBU a3, zero, a1 # compute # bytes to word align address
- beq v1, zero, aligned # addresses can be word aligned
- and a3, a3, 3
-
- beq a3, zero, 1f
- PTR_SUBU a2, a2, a3 # subtract from remaining count
- LWHI v1, 0(a0) # get next 4 bytes (unaligned)
- LWLO v1, 3(a0)
- PTR_ADDU a0, a0, a3
- SWHI v1, 0(a1) # store 1, 2, or 3 bytes to align a1
- PTR_ADDU a1, a1, a3
-1:
- and v1, a2, 3 # compute number of words left
- PTR_SUBU a3, a2, v1
- move a2, v1
- PTR_ADDU a3, a3, a0 # compute ending address
-2:
- LWHI v1, 0(a0) # copy words a0 unaligned, a1 aligned
- LWLO v1, 3(a0)
- PTR_ADDU a0, a0, 4
- sw v1, 0(a1)
- PTR_ADDU a1, a1, 4
- bne a0, a3, 2b
- nop # We have to do this mmu-bug.
- b smallcpy
- nop
-aligned:
- beq a3, zero, 1f
- PTR_SUBU a2, a2, a3 # subtract from remaining count
- LWHI v1, 0(a0) # copy 1, 2, or 3 bytes to align
- PTR_ADDU a0, a0, a3
- SWHI v1, 0(a1)
- PTR_ADDU a1, a1, a3
-1:
- and v1, a2, 3 # compute number of whole words left
- PTR_SUBU a3, a2, v1
- move a2, v1
- PTR_ADDU a3, a3, a0 # compute ending address
-2:
- lw v1, 0(a0) # copy words
- PTR_ADDU a0, a0, 4
- sw v1, 0(a1)
- bne a0, a3, 2b
- PTR_ADDU a1, a1, 4
-smallcpy:
- ble a2, zero, 2f
- PTR_ADDU a3, a2, a0 # compute ending address
-1:
- lbu v1, 0(a0) # copy bytes
- PTR_ADDU a0, a0, 1
- sb v1, 0(a1)
- bne a0, a3, 1b
- PTR_ADDU a1, a1, 1 # MMU BUG ? can not do -1(a1) at 0x80000000!!
-2:
- j ra
- nop
-END(memcpy)
-
-/*
* memset(void *s1, int c, int len)
* NetBSD: memset.S,v 1.3 2001/10/16 15:40:53 uch Exp
*/
More information about the svn-src-all
mailing list