svn commit: r312073 - stable/11/sys/vm
Konstantin Belousov
kib at FreeBSD.org
Fri Jan 13 13:45:35 UTC 2017
Author: kib
Date: Fri Jan 13 13:45:34 2017
New Revision: 312073
URL: https://svnweb.freebsd.org/changeset/base/312073
Log:
MFC r309710:
Add a new populate() pager method and extend device pager ops vector
with cdev_pg_populate() to provide device drivers access to it.
MFC r310849:
Fix two similar bugs in the populate vm_fault() code.
Modified:
stable/11/sys/vm/device_pager.c
stable/11/sys/vm/vm_fault.c
stable/11/sys/vm/vm_object.h
stable/11/sys/vm/vm_pager.h
Directory Properties:
stable/11/ (props changed)
Modified: stable/11/sys/vm/device_pager.c
==============================================================================
--- stable/11/sys/vm/device_pager.c Fri Jan 13 13:37:09 2017 (r312072)
+++ stable/11/sys/vm/device_pager.c Fri Jan 13 13:45:34 2017 (r312073)
@@ -63,6 +63,8 @@ static int dev_pager_getpages(vm_object_
static void dev_pager_putpages(vm_object_t, vm_page_t *, int, int, int *);
static boolean_t dev_pager_haspage(vm_object_t, vm_pindex_t, int *, int *);
static void dev_pager_free_page(vm_object_t object, vm_page_t m);
+static int dev_pager_populate(vm_object_t object, vm_pindex_t pidx,
+ int fault_type, vm_prot_t, vm_pindex_t *first, vm_pindex_t *last);
/* list of device pager objects */
static struct pagerlst dev_pager_object_list;
@@ -84,6 +86,7 @@ struct pagerops mgtdevicepagerops = {
.pgo_getpages = dev_pager_getpages,
.pgo_putpages = dev_pager_putpages,
.pgo_haspage = dev_pager_haspage,
+ .pgo_populate = dev_pager_populate,
};
static int old_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
@@ -127,6 +130,8 @@ cdev_pager_allocate(void *handle, enum o
if (tp != OBJT_DEVICE && tp != OBJT_MGTDEVICE)
return (NULL);
+ KASSERT(tp == OBJT_MGTDEVICE || ops->cdev_pg_populate == NULL,
+ ("populate on unmanaged device pager"));
/*
* Offset should be page aligned.
@@ -179,6 +184,8 @@ cdev_pager_allocate(void *handle, enum o
object->handle = handle;
TAILQ_INSERT_TAIL(&dev_pager_object_list, object,
pager_object_list);
+ if (ops->cdev_pg_populate != NULL)
+ vm_object_set_flag(object, OBJ_POPULATE);
}
} else {
if (pindex > object->size)
@@ -268,6 +275,8 @@ dev_pager_getpages(vm_object_t object, v
/* Since our haspage reports zero after/before, the count is 1. */
KASSERT(count == 1, ("%s: count %d", __func__, count));
VM_OBJECT_ASSERT_WLOCKED(object);
+ if (object->un_pager.devp.ops->cdev_pg_fault == NULL)
+ return (VM_PAGER_FAIL);
error = object->un_pager.devp.ops->cdev_pg_fault(object,
IDX_TO_OFF(ma[0]->pindex), PROT_READ, &ma[0]);
@@ -293,6 +302,18 @@ dev_pager_getpages(vm_object_t object, v
}
static int
+dev_pager_populate(vm_object_t object, vm_pindex_t pidx, int fault_type,
+ vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last)
+{
+
+ VM_OBJECT_ASSERT_WLOCKED(object);
+ if (object->un_pager.devp.ops->cdev_pg_populate == NULL)
+ return (VM_PAGER_FAIL);
+ return (object->un_pager.devp.ops->cdev_pg_populate(object, pidx,
+ fault_type, max_prot, first, last));
+}
+
+static int
old_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, int prot,
vm_page_t *mres)
{
Modified: stable/11/sys/vm/vm_fault.c
==============================================================================
--- stable/11/sys/vm/vm_fault.c Fri Jan 13 13:37:09 2017 (r312072)
+++ stable/11/sys/vm/vm_fault.c Fri Jan 13 13:45:34 2017 (r312073)
@@ -289,6 +289,157 @@ vm_fault_soft_fast(struct faultstate *fs
return (KERN_SUCCESS);
}
+static void
+vm_fault_restore_map_lock(struct faultstate *fs)
+{
+
+ VM_OBJECT_ASSERT_WLOCKED(fs->first_object);
+ MPASS(fs->first_object->paging_in_progress > 0);
+
+ if (!vm_map_trylock_read(fs->map)) {
+ VM_OBJECT_WUNLOCK(fs->first_object);
+ vm_map_lock_read(fs->map);
+ VM_OBJECT_WLOCK(fs->first_object);
+ }
+ fs->lookup_still_valid = true;
+}
+
+static void
+vm_fault_populate_check_page(vm_page_t m)
+{
+
+ /*
+ * Check each page to ensure that the pager is obeying the
+ * interface: the page must be installed in the object, fully
+ * valid, and exclusively busied.
+ */
+ MPASS(m != NULL);
+ MPASS(m->valid == VM_PAGE_BITS_ALL);
+ MPASS(vm_page_xbusied(m));
+}
+
+static void
+vm_fault_populate_cleanup(vm_object_t object, vm_pindex_t first,
+ vm_pindex_t last)
+{
+ vm_page_t m;
+ vm_pindex_t pidx;
+
+ VM_OBJECT_ASSERT_WLOCKED(object);
+ MPASS(first <= last);
+ for (pidx = first, m = vm_page_lookup(object, pidx);
+ pidx <= last; pidx++, m = vm_page_next(m)) {
+ vm_fault_populate_check_page(m);
+ vm_page_lock(m);
+ vm_page_deactivate(m);
+ vm_page_unlock(m);
+ vm_page_xunbusy(m);
+ }
+}
+
+static int
+vm_fault_populate(struct faultstate *fs, vm_offset_t vaddr, vm_prot_t prot,
+ int fault_type, int fault_flags, boolean_t wired, vm_page_t *m_hold)
+{
+ vm_page_t m;
+ vm_pindex_t map_first, map_last, pager_first, pager_last, pidx;
+ int rv;
+
+ MPASS(fs->object == fs->first_object);
+ VM_OBJECT_ASSERT_WLOCKED(fs->first_object);
+ MPASS(fs->first_object->paging_in_progress > 0);
+ MPASS(fs->first_object->backing_object == NULL);
+ MPASS(fs->lookup_still_valid);
+
+ pager_first = OFF_TO_IDX(fs->entry->offset);
+ pager_last = OFF_TO_IDX(fs->entry->offset + fs->entry->end -
+ fs->entry->start) - 1;
+ unlock_map(fs);
+ unlock_vp(fs);
+
+ /*
+ * Call the pager (driver) populate() method.
+ *
+ * There is no guarantee that the method will be called again
+ * if the current fault is for read, and a future fault is
+ * for write. Report the entry's maximum allowed protection
+ * to the driver.
+ */
+ rv = vm_pager_populate(fs->first_object, fs->first_pindex,
+ fault_type, fs->entry->max_protection, &pager_first, &pager_last);
+
+ VM_OBJECT_ASSERT_WLOCKED(fs->first_object);
+ if (rv == VM_PAGER_BAD) {
+ /*
+ * VM_PAGER_BAD is the backdoor for a pager to request
+ * normal fault handling.
+ */
+ vm_fault_restore_map_lock(fs);
+ if (fs->map->timestamp != fs->map_generation)
+ return (KERN_RESOURCE_SHORTAGE); /* RetryFault */
+ return (KERN_NOT_RECEIVER);
+ }
+ if (rv != VM_PAGER_OK)
+ return (KERN_FAILURE); /* AKA SIGSEGV */
+
+ /* Ensure that the driver is obeying the interface. */
+ MPASS(pager_first <= pager_last);
+ MPASS(fs->first_pindex <= pager_last);
+ MPASS(fs->first_pindex >= pager_first);
+ MPASS(pager_last < fs->first_object->size);
+
+ vm_fault_restore_map_lock(fs);
+ if (fs->map->timestamp != fs->map_generation) {
+ vm_fault_populate_cleanup(fs->first_object, pager_first,
+ pager_last);
+ return (KERN_RESOURCE_SHORTAGE); /* RetryFault */
+ }
+
+ /*
+ * The map is unchanged after our last unlock. Process the fault.
+ *
+ * The range [pager_first, pager_last] that is given to the
+ * pager is only a hint. The pager may populate any range
+ * within the object that includes the requested page index.
+ * In case the pager expanded the range, clip it to fit into
+ * the map entry.
+ */
+ map_first = MAX(OFF_TO_IDX(fs->entry->offset), pager_first);
+ if (map_first > pager_first)
+ vm_fault_populate_cleanup(fs->first_object, pager_first,
+ map_first - 1);
+ map_last = MIN(OFF_TO_IDX(fs->entry->end - fs->entry->start +
+ fs->entry->offset), pager_last);
+ if (map_last < pager_last)
+ vm_fault_populate_cleanup(fs->first_object, map_last + 1,
+ pager_last);
+
+ for (pidx = map_first, m = vm_page_lookup(fs->first_object, pidx);
+ pidx <= map_last; pidx++, m = vm_page_next(m)) {
+ vm_fault_populate_check_page(m);
+ vm_fault_dirty(fs->entry, m, prot, fault_type, fault_flags,
+ true);
+ VM_OBJECT_WUNLOCK(fs->first_object);
+ pmap_enter(fs->map->pmap, fs->entry->start + IDX_TO_OFF(pidx) -
+ fs->entry->offset, m, prot, fault_type | (wired ?
+ PMAP_ENTER_WIRED : 0), 0);
+ VM_OBJECT_WLOCK(fs->first_object);
+ if (pidx == fs->first_pindex)
+ vm_fault_fill_hold(m_hold, m);
+ vm_page_lock(m);
+ if ((fault_flags & VM_FAULT_WIRE) != 0) {
+ KASSERT(wired, ("VM_FAULT_WIRE && !wired"));
+ vm_page_wire(m);
+ } else {
+ vm_page_activate(m);
+ }
+ vm_page_unlock(m);
+ vm_page_xunbusy(m);
+ }
+ curthread->td_ru.ru_majflt++;
+ return (KERN_SUCCESS);
+}
+
/*
* vm_fault:
*
@@ -554,6 +705,30 @@ RetryFault:;
return (KERN_PROTECTION_FAILURE);
}
+ if (fs.object == fs.first_object &&
+ (fs.first_object->flags & OBJ_POPULATE) != 0 &&
+ fs.first_object->shadow_count == 0) {
+ rv = vm_fault_populate(&fs, vaddr, prot,
+ fault_type, fault_flags, wired, m_hold);
+ switch (rv) {
+ case KERN_SUCCESS:
+ case KERN_FAILURE:
+ unlock_and_deallocate(&fs);
+ return (rv);
+ case KERN_RESOURCE_SHORTAGE:
+ unlock_and_deallocate(&fs);
+ goto RetryFault;
+ case KERN_NOT_RECEIVER:
+ /*
+ * Pager's populate() method
+ * returned VM_PAGER_BAD.
+ */
+ break;
+ default:
+ panic("inconsistent return codes");
+ }
+ }
+
/*
* Allocate a new page for this object/offset pair.
*
Modified: stable/11/sys/vm/vm_object.h
==============================================================================
--- stable/11/sys/vm/vm_object.h Fri Jan 13 13:37:09 2017 (r312072)
+++ stable/11/sys/vm/vm_object.h Fri Jan 13 13:45:34 2017 (r312073)
@@ -182,6 +182,7 @@ struct vm_object {
*/
#define OBJ_FICTITIOUS 0x0001 /* (c) contains fictitious pages */
#define OBJ_UNMANAGED 0x0002 /* (c) contains unmanaged pages */
+#define OBJ_POPULATE 0x0004 /* pager implements populate() */
#define OBJ_DEAD 0x0008 /* dead objects (during rundown) */
#define OBJ_NOSPLIT 0x0010 /* dont split this object */
#define OBJ_UMTXDEAD 0x0020 /* umtx pshared was terminated */
Modified: stable/11/sys/vm/vm_pager.h
==============================================================================
--- stable/11/sys/vm/vm_pager.h Fri Jan 13 13:37:09 2017 (r312072)
+++ stable/11/sys/vm/vm_pager.h Fri Jan 13 13:45:34 2017 (r312073)
@@ -56,6 +56,8 @@ typedef int pgo_getpages_async_t(vm_obje
pgo_getpages_iodone_t, void *);
typedef void pgo_putpages_t(vm_object_t, vm_page_t *, int, int, int *);
typedef boolean_t pgo_haspage_t(vm_object_t, vm_pindex_t, int *, int *);
+typedef int pgo_populate_t(vm_object_t, vm_pindex_t, int, vm_prot_t,
+ vm_pindex_t *, vm_pindex_t *);
typedef void pgo_pageunswapped_t(vm_page_t);
struct pagerops {
@@ -66,6 +68,7 @@ struct pagerops {
pgo_getpages_async_t *pgo_getpages_async; /* Get page asyncly. */
pgo_putpages_t *pgo_putpages; /* Put (write) page. */
pgo_haspage_t *pgo_haspage; /* Query page. */
+ pgo_populate_t *pgo_populate; /* Bulk spec pagein. */
pgo_pageunswapped_t *pgo_pageunswapped;
};
@@ -151,6 +154,19 @@ vm_pager_has_page(
return (ret);
}
+static __inline int
+vm_pager_populate(vm_object_t object, vm_pindex_t pidx, int fault_type,
+ vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last)
+{
+
+ MPASS((object->flags & OBJ_POPULATE) != 0);
+ MPASS(pidx < object->size);
+ MPASS(object->paging_in_progress > 0);
+ return ((*pagertab[object->type]->pgo_populate)(object, pidx,
+ fault_type, max_prot, first, last));
+}
+
+
/*
* vm_pager_page_unswapped
*
@@ -176,6 +192,9 @@ vm_pager_page_unswapped(vm_page_t m)
struct cdev_pager_ops {
int (*cdev_pg_fault)(vm_object_t vm_obj, vm_ooffset_t offset,
int prot, vm_page_t *mres);
+ int (*cdev_pg_populate)(vm_object_t vm_obj, vm_pindex_t pidx,
+ int fault_type, vm_prot_t max_prot, vm_pindex_t *first,
+ vm_pindex_t *last);
int (*cdev_pg_ctor)(void *handle, vm_ooffset_t size, vm_prot_t prot,
vm_ooffset_t foff, struct ucred *cred, u_short *color);
void (*cdev_pg_dtor)(void *handle);
More information about the svn-src-all
mailing list