git: c4c2d50242c9 - stable/13 - vm_fault: Factor out per-object operations into vm_fault_object()

From: Mark Johnston <markj_at_FreeBSD.org>
Date: Wed, 08 Dec 2021 13:46:28 UTC
The branch stable/13 has been updated by markj:

URL: https://cgit.FreeBSD.org/src/commit/?id=c4c2d50242c9d3bc596d22174fbf6ea93c577872

commit c4c2d50242c9d3bc596d22174fbf6ea93c577872
Author:     Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2021-11-24 18:43:38 +0000
Commit:     Mark Johnston <markj@FreeBSD.org>
CommitDate: 2021-12-08 13:41:30 +0000

    vm_fault: Factor out per-object operations into vm_fault_object()
    
    No functional change intended.
    
    Obtained from:  jeff (object_concurrency patches)
    Reviewed by:    kib
    
    (cherry picked from commit d47d3a94bb2d873b1c3a5cdfe9cc00e4619f649b)
---
 sys/vm/vm_fault.c | 213 ++++++++++++++++++++++++++++++------------------------
 1 file changed, 118 insertions(+), 95 deletions(-)

diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c
index b05c2439699b..9b80b4188c40 100644
--- a/sys/vm/vm_fault.c
+++ b/sys/vm/vm_fault.c
@@ -166,6 +166,7 @@ enum fault_status {
 	FAULT_OUT_OF_BOUNDS,	/* Invalid address for pager. */
 	FAULT_HARD,		/* Performed I/O. */
 	FAULT_SOFT,		/* Found valid page. */
+	FAULT_PROTECTION_FAILURE, /* Invalid access. */
 };
 
 static void vm_fault_dontneed(const struct faultstate *fs, vm_offset_t vaddr,
@@ -1346,6 +1347,96 @@ vm_fault_busy_sleep(struct faultstate *fs)
 	vm_object_deallocate(fs->first_object);
 }
 
+/*
+ * Handle page lookup, populate, allocate, page-in for the current
+ * object.
+ *
+ * The object is locked on entry and will remain locked with a return
+ * code of FAULT_CONTINUE so that fault may follow the shadow chain.
+ * Otherwise, the object will be unlocked upon return.
+ */
+static enum fault_status
+vm_fault_object(struct faultstate *fs, int *behindp, int *aheadp)
+{
+	enum fault_status res;
+	bool dead;
+
+	/*
+	 * If the object is marked for imminent termination, we retry
+	 * here, since the collapse pass has raced with us.  Otherwise,
+	 * if we see terminally dead object, return fail.
+	 */
+	if ((fs->object->flags & OBJ_DEAD) != 0) {
+		dead = fs->object->type == OBJT_DEAD;
+		unlock_and_deallocate(fs);
+		if (dead)
+			return (FAULT_PROTECTION_FAILURE);
+		pause("vmf_de", 1);
+		return (FAULT_RESTART);
+	}
+
+	/*
+	 * See if the page is resident.
+	 */
+	fs->m = vm_page_lookup(fs->object, fs->pindex);
+	if (fs->m != NULL) {
+		if (!vm_page_tryxbusy(fs->m)) {
+			vm_fault_busy_sleep(fs);
+			return (FAULT_RESTART);
+		}
+
+		/*
+		 * The page is marked busy for other processes and the
+		 * pagedaemon.  If it is still completely valid we are
+		 * done.
+		 */
+		if (vm_page_all_valid(fs->m)) {
+			VM_OBJECT_WUNLOCK(fs->object);
+			return (FAULT_SOFT);
+		}
+	}
+	VM_OBJECT_ASSERT_WLOCKED(fs->object);
+
+	/*
+	 * Page is not resident.  If the pager might contain the page
+	 * or this is the beginning of the search, allocate a new
+	 * page.  (Default objects are zero-fill, so there is no real
+	 * pager for them.)
+	 */
+	if (fs->m == NULL && (fs->object->type != OBJT_DEFAULT ||
+	    fs->object == fs->first_object)) {
+		res = vm_fault_allocate(fs);
+		if (res != FAULT_CONTINUE)
+			return (res);
+	}
+
+	/*
+	 * Default objects have no pager so no exclusive busy exists
+	 * to protect this page in the chain.  Skip to the next
+	 * object without dropping the lock to preserve atomicity of
+	 * shadow faults.
+	 */
+	if (fs->object->type != OBJT_DEFAULT) {
+		/*
+		 * At this point, we have either allocated a new page
+		 * or found an existing page that is only partially
+		 * valid.
+		 *
+		 * We hold a reference on the current object and the
+		 * page is exclusive busied.  The exclusive busy
+		 * prevents simultaneous faults and collapses while
+		 * the object lock is dropped.
+		 */
+		VM_OBJECT_WUNLOCK(fs->object);
+		res = vm_fault_getpages(fs, behindp, aheadp);
+		if (res == FAULT_CONTINUE)
+			VM_OBJECT_WLOCK(fs->object);
+	} else {
+		res = FAULT_CONTINUE;
+	}
+	return (res);
+}
+
 int
 vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type,
     int fault_flags, vm_page_t *m_hold)
@@ -1353,7 +1444,7 @@ vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type,
 	struct faultstate fs;
 	int ahead, behind, faultcount, rv;
 	enum fault_status res;
-	bool dead, hardfault;
+	bool hardfault;
 
 	VM_CNT_INC(v_vm_faults);
 
@@ -1448,98 +1539,29 @@ RetryFault:
 	while (TRUE) {
 		KASSERT(fs.m == NULL,
 		    ("page still set %p at loop start", fs.m));
-		/*
-		 * If the object is marked for imminent termination,
-		 * we retry here, since the collapse pass has raced
-		 * with us.  Otherwise, if we see terminally dead
-		 * object, return fail.
-		 */
-		if ((fs.object->flags & OBJ_DEAD) != 0) {
-			dead = fs.object->type == OBJT_DEAD;
-			unlock_and_deallocate(&fs);
-			if (dead)
-				return (KERN_PROTECTION_FAILURE);
-			pause("vmf_de", 1);
-			goto RetryFault;
-		}
-
-		/*
-		 * See if page is resident
-		 */
-		fs.m = vm_page_lookup(fs.object, fs.pindex);
-		if (fs.m != NULL) {
-			if (vm_page_tryxbusy(fs.m) == 0) {
-				vm_fault_busy_sleep(&fs);
-				goto RetryFault;
-			}
-
-			/*
-			 * The page is marked busy for other processes and the
-			 * pagedaemon.  If it still is completely valid we
-			 * are done.
-			 */
-			if (vm_page_all_valid(fs.m)) {
-				VM_OBJECT_WUNLOCK(fs.object);
-				break; /* break to PAGE HAS BEEN FOUND. */
-			}
-		}
-		VM_OBJECT_ASSERT_WLOCKED(fs.object);
-
-		/*
-		 * Page is not resident.  If the pager might contain the page
-		 * or this is the beginning of the search, allocate a new
-		 * page.  (Default objects are zero-fill, so there is no real
-		 * pager for them.)
-		 */
-		if (fs.m == NULL && (fs.object->type != OBJT_DEFAULT ||
-		    fs.object == fs.first_object)) {
-			res = vm_fault_allocate(&fs);
-			switch (res) {
-			case FAULT_RESTART:
-				goto RetryFault;
-			case FAULT_SUCCESS:
-				return (KERN_SUCCESS);
-			case FAULT_FAILURE:
-				return (KERN_FAILURE);
-			case FAULT_OUT_OF_BOUNDS:
-				return (KERN_OUT_OF_BOUNDS);
-			case FAULT_CONTINUE:
-				break;
-			default:
-				panic("vm_fault: Unhandled status %d", res);
-			}
-		}
 
-		/*
-		 * Default objects have no pager so no exclusive busy exists
-		 * to protect this page in the chain.  Skip to the next
-		 * object without dropping the lock to preserve atomicity of
-		 * shadow faults.
-		 */
-		if (fs.object->type != OBJT_DEFAULT) {
-			/*
-			 * At this point, we have either allocated a new page
-			 * or found an existing page that is only partially
-			 * valid.
-			 *
-			 * We hold a reference on the current object and the
-			 * page is exclusive busied.  The exclusive busy
-			 * prevents simultaneous faults and collapses while
-			 * the object lock is dropped.
-		 	 */
-			VM_OBJECT_WUNLOCK(fs.object);
-
-			res = vm_fault_getpages(&fs, &behind, &ahead);
-			if (res == FAULT_SUCCESS) {
-				faultcount = behind + 1 + ahead;
-				hardfault = true;
-				break; /* break to PAGE HAS BEEN FOUND. */
-			}
-			if (res == FAULT_RESTART)
-				goto RetryFault;
-			if (res == FAULT_OUT_OF_BOUNDS)
-				return (KERN_OUT_OF_BOUNDS);
-			VM_OBJECT_WLOCK(fs.object);
+		res = vm_fault_object(&fs, &behind, &ahead);
+		switch (res) {
+		case FAULT_SOFT:
+			goto found;
+		case FAULT_HARD:
+			faultcount = behind + 1 + ahead;
+			hardfault = true;
+			goto found;
+		case FAULT_RESTART:
+			goto RetryFault;
+		case FAULT_SUCCESS:
+			return (KERN_SUCCESS);
+		case FAULT_FAILURE:
+			return (KERN_FAILURE);
+		case FAULT_OUT_OF_BOUNDS:
+			return (KERN_OUT_OF_BOUNDS);
+		case FAULT_PROTECTION_FAILURE:
+			return (KERN_PROTECTION_FAILURE);
+		case FAULT_CONTINUE:
+			break;
+		default:
+			panic("vm_fault: Unhandled status %d", res);
 		}
 
 		/*
@@ -1559,12 +1581,13 @@ RetryFault:
 		vm_fault_zerofill(&fs);
 		/* Don't try to prefault neighboring pages. */
 		faultcount = 1;
-		break;	/* break to PAGE HAS BEEN FOUND. */
+		break;
 	}
 
+found:
 	/*
-	 * PAGE HAS BEEN FOUND.  A valid page has been found and exclusively
-	 * busied.  The object lock must no longer be held.
+	 * A valid page has been found and exclusively busied.  The
+	 * object lock must no longer be held.
 	 */
 	vm_page_assert_xbusied(fs.m);
 	VM_OBJECT_ASSERT_UNLOCKED(fs.object);