git: f4fe1b19bce8 - stable/13 - i386 copyout/in_fast: handle page fault from KVA access

The branch stable/13 has been updated by kib:

URL: https://cgit.FreeBSD.org/src/commit/?id=f4fe1b19bce87552c1c32dad2c62fcf48fc327ac

commit f4fe1b19bce87552c1c32dad2c62fcf48fc327ac
Author:     Konstantin Belousov <kib@FreeBSD.org>
AuthorDate: 2022-09-08 20:15:32 +0000
Commit:     Konstantin Belousov <kib@FreeBSD.org>
CommitDate: 2022-09-21 09:29:02 +0000

    i386 copyout/in_fast: handle page fault from KVA access
    
    (cherry picked from commit 49587b8fb4d12166d30efb7111d32f4606a1e73b)
---
 sys/i386/i386/copyout_fast.s | 21 +++++++++------------
 sys/i386/i386/exception.s    | 25 +++++++++++++++----------
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/sys/i386/i386/copyout_fast.s b/sys/i386/i386/copyout_fast.s
index 70647fe7613d..4391f36b18e4 100644
--- a/sys/i386/i386/copyout_fast.s
+++ b/sys/i386/i386/copyout_fast.s
@@ -47,32 +47,30 @@ ENTRY(copyout_fast)
 	pushl	%ebx
 
 	movl	20(%ebp),%ebx		/* KCR3 */
-	movl	PCPU(CURPCB),%edx
-	movl	PCB_CR3(%edx),%edx	/* UCR3 */
 	/* bcopy(%esi = kaddr, %edi = PCPU(copyout_buf), %ecx = len) */
 	movl	16(%ebp),%ecx
 	movl	8(%ebp),%esi
+	movl	%esp,%eax
+	movl	$copyout_fault,%edx
+
 	cli
 	movl	PCPU(COPYOUT_BUF),%edi
-	rep; movsb
+pf_y1:	rep; movsb
 
 	movl	16(%ebp),%ecx		/* len */
 	movl	PCPU(COPYOUT_BUF),%esi	/* kaddr */
 	movl	12(%ebp),%edi		/* uaddr */
-
-	movl	%esp,%eax
 	movl	PCPU(TRAMPSTK),%esp
-
+	movl	PCPU(CURPCB),%edx
+	movl	PCB_CR3(%edx),%edx	/* UCR3 */
 	movl	%edx,%cr3
 	movl	$copyout_fault,%edx
-
 	/* bcopy(%esi = PCPU(copyout_buf), %edi = udaddr, %ecx = len) */
 pf_x1:	rep; movsb
 
 	movl	%ebx,%cr3
 	movl	%eax,%esp
 	sti
-
 	xorl	%eax,%eax
 	popl	%ebx
 	popl	%edi
@@ -93,10 +91,10 @@ ENTRY(copyin_fast)
 	movl	PCB_CR3(%eax),%edx	/* UCR3 */
 	movl	16(%ebp),%ecx		/* len */
 	movl	8(%ebp),%esi		/* udaddr */
+	movl	%esp,%eax
+
 	cli
 	movl	PCPU(COPYOUT_BUF),%edi	/* kaddr */
-
-	movl	%esp,%eax
 	movl	PCPU(TRAMPSTK),%esp
 	movl	%edx,%cr3
 	movl	$copyout_fault,%edx
@@ -110,10 +108,9 @@ pf_x2:	rep; movsb
 	movl	16(%ebp),%ecx
 	movl	12(%ebp),%edi
 	movl	PCPU(COPYOUT_BUF),%esi
-	rep; movsb
+pf_y2:	rep; movsb
 
 	sti
-
 	xorl	%eax,%eax
 	popl	%ebx
 	popl	%edi
diff --git a/sys/i386/i386/exception.s b/sys/i386/i386/exception.s
index 272ebb5a2460..8a2194038af6 100644
--- a/sys/i386/i386/exception.s
+++ b/sys/i386/i386/exception.s
@@ -133,15 +133,15 @@ IDTVEC(prot)
 	jmp	irettraps
 IDTVEC(page)
 	testl	$PSL_VM, TF_EFLAGS-TF_ERR(%esp)
-	jnz	4f
+	jnz	5f
 	testb	$SEL_RPL_MASK, TF_CS-TF_ERR(%esp)
-	jnz	4f
+	jnz	5f
 	cmpl	$PMAP_TRM_MIN_ADDRESS, TF_EIP-TF_ERR(%esp)
-	jb	4f
+	jb	5f
 	pushl	%eax
 	movl	TF_EIP-TF_ERR+4(%esp), %eax
 	addl	$1f, %eax
-	call	5f
+	call	6f
 1:	cmpl	$pf_x1, %eax
 	je	2f
 	cmpl	$pf_x2, %eax
@@ -157,16 +157,21 @@ IDTVEC(page)
 	cmpl	$pf_x7, %eax
 	je	2f
 	cmpl	$pf_x8, %eax
-	jne	3f
-2:	popl	%eax
-	movl	%ebx, %cr3
+	je	2f
+	cmpl	$pf_y1, %eax
+	je	3f
+	cmpl	$pf_y2, %eax
+	je	3f
+	jmp	4f
+2:	movl	%ebx, %cr3
+3:	popl	%eax
 	movl	%edx, TF_EIP-TF_ERR(%esp)
 	addl	$4, %esp
 	iret
-3:	popl	%eax
-4:	pushl	$T_PAGEFLT
+4:	popl	%eax
+5:	pushl	$T_PAGEFLT
 	jmp	alltraps
-5:	subl	(%esp), %eax
+6:	subl	(%esp), %eax
 	retl
 IDTVEC(rsvd_pti)
 IDTVEC(rsvd)