svn commit: r338508 - head/sys/amd64/amd64

[Mateusz Guzik]

Author: mjg
Date: Thu Sep  6 19:42:40 2018
New Revision: 338508
URL: https://svnweb.freebsd.org/changeset/base/338508

Log:
  amd64: depessimize copyinstr_smap
  
  The stac/clac combo around each byte copy is causing a measurable
  slowdown in benchmarks. Do it only before and after all data is
  copied. While here reorder the code to avoid a forward branch in
  the common case.
  
  Note the copying loop (originating from copyinstr) is avoidably slow
  and will be fixed later.
  
  Reviewed by:	kib
  Approved by:	re (gjb)
  Differential Revision:	https://reviews.freebsd.org/D17063

Modified:
  head/sys/amd64/amd64/support.S

Modified: head/sys/amd64/amd64/support.S
==============================================================================
--- head/sys/amd64/amd64/support.S	Thu Sep  6 19:28:52 2018	(r338507)
+++ head/sys/amd64/amd64/support.S	Thu Sep  6 19:42:40 2018	(r338508)
@@ -914,6 +914,8 @@ ENTRY(copyinstr_smap)
 	subq	%rsi,%rax
 	jbe	cpystrflt
 
+	stac
+
 	/* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */
 	cmpq	%rdx,%rax
 	jae	1f
@@ -924,32 +926,20 @@ ENTRY(copyinstr_smap)
 
 2:
 	decq	%rdx
-	jz	copyinstr_toolong
+	jz	copyinstr_toolong_smap
 
-	stac
 	lodsb
 	stosb
-	clac
 	orb	%al,%al
 	jnz	2b
 
+	clac
+
 copyinstr_succ:
 	/* Success -- 0 byte reached */
 	decq	%rdx
 	xorl	%eax,%eax
-	jmp	cpystrflt_x
-copyinstr_toolong:
-	/* rdx is zero - return ENAMETOOLONG or EFAULT */
-	movq	$VM_MAXUSER_ADDRESS,%rax
-	cmpq	%rax,%rsi
-	jae	cpystrflt
-	movq	$ENAMETOOLONG,%rax
-	jmp	cpystrflt_x
 
-	/* Fault entry clears PSL.AC */
-cpystrflt:
-	movq	$EFAULT,%rax
-
 cpystrflt_x:
 	/* set *lencopied and return %eax */
 	movq	PCPU(CURPCB),%rcx
@@ -962,6 +952,21 @@ cpystrflt_x:
 1:
 	POP_FRAME_POINTER
 	ret
+	/* Fault entry clears PSL.AC */
+cpystrflt:
+	movq	$EFAULT,%rax
+	jmp	cpystrflt_x
+
+copyinstr_toolong_smap:
+	clac
+copyinstr_toolong:
+	/* rdx is zero - return ENAMETOOLONG or EFAULT */
+	movq	$VM_MAXUSER_ADDRESS,%rax
+	cmpq	%rax,%rsi
+	jae	cpystrflt
+	movq	$ENAMETOOLONG,%rax
+	jmp	cpystrflt_x
+
 END(copyinstr_smap)
 
 /*