svn commit: r341364 - in head: lib/libc/amd64/string sys/amd64/amd64

[Mateusz Guzik]

Author: mjg
Date: Sat Dec  1 14:20:32 2018
New Revision: 341364
URL: https://svnweb.freebsd.org/changeset/base/341364

Log:
  amd64: align target memmove buffer to 16 bytes before using rep movs
  
  See the review for sample test results.
  
  Reviewed by:	kib (kernel part)
  Sponsored by:	The FreeBSD Foundation
  Differential Revision:	https://reviews.freebsd.org/D18401

Modified:
  head/lib/libc/amd64/string/memmove.S
  head/sys/amd64/amd64/support.S

Modified: head/lib/libc/amd64/string/memmove.S
==============================================================================
--- head/lib/libc/amd64/string/memmove.S	Sat Dec  1 12:08:16 2018	(r341363)
+++ head/lib/libc/amd64/string/memmove.S	Sat Dec  1 14:20:32 2018	(r341364)
@@ -139,6 +139,8 @@ __FBSDID("$FreeBSD$");
 
 	ALIGN_TEXT
 1256:
+	testb	$15,%dil
+	jnz	100f
 .if \erms == 1
 	rep
 	movsb
@@ -146,6 +148,34 @@ __FBSDID("$FreeBSD$");
 	shrq	$3,%rcx                         /* copy by 64-bit words */
 	rep
 	movsq
+	movq	%rdx,%rcx
+	andl	$7,%ecx                         /* any bytes left? */
+	jne	100408b
+.endif
+	\end
+	ret
+100:
+	movq	(%rsi),%r8
+	movq	8(%rsi),%r9
+	movq	%rdi,%r10
+	movq	%rdi,%rcx
+	andq	$15,%rcx
+	leaq	-16(%rdx,%rcx),%rdx
+	neg	%rcx
+	leaq	16(%rdi,%rcx),%rdi
+	leaq	16(%rsi,%rcx),%rsi
+	movq	%rdx,%rcx
+.if \erms == 1
+	rep
+	movsb
+	movq	%r8,(%r10)
+	movq	%r9,8(%r10)
+.else
+	shrq	$3,%rcx                         /* copy by 64-bit words */
+	rep
+	movsq
+	movq	%r8,(%r10)
+	movq	%r9,8(%r10)
 	movq	%rdx,%rcx
 	andl	$7,%ecx                         /* any bytes left? */
 	jne	100408b

Modified: head/sys/amd64/amd64/support.S
==============================================================================
--- head/sys/amd64/amd64/support.S	Sat Dec  1 12:08:16 2018	(r341363)
+++ head/sys/amd64/amd64/support.S	Sat Dec  1 14:20:32 2018	(r341364)
@@ -302,6 +302,8 @@ END(memcmp)
 
 	ALIGN_TEXT
 1256:
+	testb	$15,%dil
+	jnz	100f
 .if \erms == 1
 	rep
 	movsb
@@ -309,6 +311,34 @@ END(memcmp)
 	shrq	$3,%rcx                         /* copy by 64-bit words */
 	rep
 	movsq
+	movq	%rdx,%rcx
+	andl	$7,%ecx                         /* any bytes left? */
+	jne	100408b
+.endif
+	\end
+	ret
+100:
+	movq	(%rsi),%r8
+	movq	8(%rsi),%r9
+	movq	%rdi,%r10
+	movq	%rdi,%rcx
+	andq	$15,%rcx
+	leaq	-16(%rdx,%rcx),%rdx
+	neg	%rcx
+	leaq	16(%rdi,%rcx),%rdi
+	leaq	16(%rsi,%rcx),%rsi
+	movq	%rdx,%rcx
+.if \erms == 1
+	rep
+	movsb
+	movq	%r8,(%r10)
+	movq	%r9,8(%r10)
+.else
+	shrq	$3,%rcx                         /* copy by 64-bit words */
+	rep
+	movsq
+	movq	%r8,(%r10)
+	movq	%r9,8(%r10)
 	movq	%rdx,%rcx
 	andl	$7,%ecx                         /* any bytes left? */
 	jne	100408b