git: 49587b8fb4d1 - main - i386 copyout/in_fast: handle page fault from KVA access

The branch main has been updated by kib:

URL: https://cgit.FreeBSD.org/src/commit/?id=49587b8fb4d12166d30efb7111d32f4606a1e73b

commit 49587b8fb4d12166d30efb7111d32f4606a1e73b
Author:     Konstantin Belousov <kib@FreeBSD.org>
AuthorDate: 2022-09-08 20:15:32 +0000
Commit:     Konstantin Belousov <kib@FreeBSD.org>
CommitDate: 2022-09-14 15:46:32 +0000

    i386 copyout/in_fast: handle page fault from KVA access
    
    by delegating the work to the slow path.
    
    Some kernel memory, like pipe buffers, is pageable.  We must not enable
    interrupts, and consequently, preemption, while in critical section in
    the fast copyout path, because we use pcpu buffers.  If page fault
    occurs while copying from the pcpu copyout_buf to kernel memory, abort
    fast path and delegate work to the slow implementation.
    
    In collaboration with:  pho, tijl
    Sponsored by:   The FreeBSD Foundation
    MFC after:      1 week
---
 sys/i386/i386/copyout_fast.s | 21 +++++++++------------
 sys/i386/i386/exception.s    | 25 +++++++++++++++----------
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/sys/i386/i386/copyout_fast.s b/sys/i386/i386/copyout_fast.s
index 70647fe7613d..4391f36b18e4 100644
--- a/sys/i386/i386/copyout_fast.s
+++ b/sys/i386/i386/copyout_fast.s
@@ -47,32 +47,30 @@ ENTRY(copyout_fast)
 	pushl	%ebx
 
 	movl	20(%ebp),%ebx		/* KCR3 */
-	movl	PCPU(CURPCB),%edx
-	movl	PCB_CR3(%edx),%edx	/* UCR3 */
 	/* bcopy(%esi = kaddr, %edi = PCPU(copyout_buf), %ecx = len) */
 	movl	16(%ebp),%ecx
 	movl	8(%ebp),%esi
+	movl	%esp,%eax
+	movl	$copyout_fault,%edx
+
 	cli
 	movl	PCPU(COPYOUT_BUF),%edi
-	rep; movsb
+pf_y1:	rep; movsb
 
 	movl	16(%ebp),%ecx		/* len */
 	movl	PCPU(COPYOUT_BUF),%esi	/* kaddr */
 	movl	12(%ebp),%edi		/* uaddr */
-
-	movl	%esp,%eax
 	movl	PCPU(TRAMPSTK),%esp
-
+	movl	PCPU(CURPCB),%edx
+	movl	PCB_CR3(%edx),%edx	/* UCR3 */
 	movl	%edx,%cr3
 	movl	$copyout_fault,%edx
-
 	/* bcopy(%esi = PCPU(copyout_buf), %edi = udaddr, %ecx = len) */
 pf_x1:	rep; movsb
 
 	movl	%ebx,%cr3
 	movl	%eax,%esp
 	sti
-
 	xorl	%eax,%eax
 	popl	%ebx
 	popl	%edi
@@ -93,10 +91,10 @@ ENTRY(copyin_fast)
 	movl	PCB_CR3(%eax),%edx	/* UCR3 */
 	movl	16(%ebp),%ecx		/* len */
 	movl	8(%ebp),%esi		/* udaddr */
+	movl	%esp,%eax
+
 	cli
 	movl	PCPU(COPYOUT_BUF),%edi	/* kaddr */
-
-	movl	%esp,%eax
 	movl	PCPU(TRAMPSTK),%esp
 	movl	%edx,%cr3
 	movl	$copyout_fault,%edx
@@ -110,10 +108,9 @@ pf_x2:	rep; movsb
 	movl	16(%ebp),%ecx
 	movl	12(%ebp),%edi
 	movl	PCPU(COPYOUT_BUF),%esi
-	rep; movsb
+pf_y2:	rep; movsb
 
 	sti
-
 	xorl	%eax,%eax
 	popl	%ebx
 	popl	%edi
diff --git a/sys/i386/i386/exception.s b/sys/i386/i386/exception.s
index f8d61c884a21..e83d95e6afe1 100644
--- a/sys/i386/i386/exception.s
+++ b/sys/i386/i386/exception.s
@@ -130,15 +130,15 @@ IDTVEC(prot)
 	jmp	irettraps
 IDTVEC(page)
 	testl	$PSL_VM, TF_EFLAGS-TF_ERR(%esp)
-	jnz	4f
+	jnz	5f
 	testb	$SEL_RPL_MASK, TF_CS-TF_ERR(%esp)
-	jnz	4f
+	jnz	5f
 	cmpl	$PMAP_TRM_MIN_ADDRESS, TF_EIP-TF_ERR(%esp)
-	jb	4f
+	jb	5f
 	pushl	%eax
 	movl	TF_EIP-TF_ERR+4(%esp), %eax
 	addl	$1f, %eax
-	call	5f
+	call	6f
 1:	cmpl	$pf_x1, %eax
 	je	2f
 	cmpl	$pf_x2, %eax
@@ -154,16 +154,21 @@ IDTVEC(page)
 	cmpl	$pf_x7, %eax
 	je	2f
 	cmpl	$pf_x8, %eax
-	jne	3f
-2:	popl	%eax
-	movl	%ebx, %cr3
+	je	2f
+	cmpl	$pf_y1, %eax
+	je	3f
+	cmpl	$pf_y2, %eax
+	je	3f
+	jmp	4f
+2:	movl	%ebx, %cr3
+3:	popl	%eax
 	movl	%edx, TF_EIP-TF_ERR(%esp)
 	addl	$4, %esp
 	iret
-3:	popl	%eax
-4:	pushl	$T_PAGEFLT
+4:	popl	%eax
+5:	pushl	$T_PAGEFLT
 	jmp	alltraps
-5:	subl	(%esp), %eax
+6:	subl	(%esp), %eax
 	retl
 IDTVEC(rsvd_pti)
 IDTVEC(rsvd)