svn commit: r349846 - in head: share/man/man9 sys/amd64/amd64 sys/amd64/vmm sys/arm/arm sys/arm64/arm64 sys/cddl/contrib/opensolaris/uts/common/fs/zfs sys/compat/linuxkpi/common/src sys/contrib/vch...
Mark Johnston
markj at FreeBSD.org
Mon Jul 8 19:46:28 UTC 2019
Author: markj
Date: Mon Jul 8 19:46:20 2019
New Revision: 349846
URL: https://svnweb.freebsd.org/changeset/base/349846
Log:
Merge the vm_page hold and wire mechanisms.
The hold_count and wire_count fields of struct vm_page are separate
reference counters with similar semantics. The remaining essential
differences are that holds are not counted as a reference with respect
to LRU, and holds have an implicit free-on-last unhold semantic whereas
vm_page_unwire() callers must explicitly determine whether to free the
page once the last reference to the page is released.
This change removes the KPIs which directly manipulate hold_count.
Functions such as vm_fault_quick_hold_pages() now return wired pages
instead. Since r328977 the overhead of maintaining LRU for wired pages
is lower, and in many cases vm_fault_quick_hold_pages() callers would
swap holds for wirings on the returned pages anyway, so with this change
we remove a number of page lock acquisitions.
No functional change is intended. __FreeBSD_version is bumped.
Reviewed by: alc, kib
Discussed with: jeff
Discussed with: jhb, np (cxgbe)
Tested by: pho (previous version)
Sponsored by: Netflix
Differential Revision: https://reviews.freebsd.org/D19247
Deleted:
head/share/man/man9/vm_page_hold.9
Modified:
head/share/man/man9/Makefile
head/sys/amd64/amd64/pmap.c
head/sys/amd64/vmm/vmm.c
head/sys/arm/arm/pmap-v4.c
head/sys/arm/arm/pmap-v6.c
head/sys/arm64/arm64/pmap.c
head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
head/sys/compat/linuxkpi/common/src/linux_page.c
head/sys/contrib/vchiq/interface/vchiq_arm/vchiq_2835_arm.c
head/sys/dev/cxgbe/tom/t4_cpl_io.c
head/sys/dev/cxgbe/tom/t4_ddp.c
head/sys/dev/cxgbe/tom/t4_tom.h
head/sys/i386/i386/pmap.c
head/sys/kern/kern_exec.c
head/sys/kern/sys_process.c
head/sys/kern/uipc_shm.c
head/sys/mips/mips/pmap.c
head/sys/net/bpf_zerocopy.c
head/sys/powerpc/aim/mmu_oea.c
head/sys/powerpc/aim/mmu_oea64.c
head/sys/powerpc/booke/pmap.c
head/sys/riscv/riscv/pmap.c
head/sys/sparc64/sparc64/pmap.c
head/sys/sys/param.h
head/sys/vm/vm_fault.c
head/sys/vm/vm_glue.c
head/sys/vm/vm_object.c
head/sys/vm/vm_page.c
head/sys/vm/vm_page.h
head/sys/vm/vm_pageout.c
head/sys/vm/vm_swapout.c
Modified: head/share/man/man9/Makefile
==============================================================================
--- head/share/man/man9/Makefile Mon Jul 8 19:38:49 2019 (r349845)
+++ head/share/man/man9/Makefile Mon Jul 8 19:46:20 2019 (r349846)
@@ -381,7 +381,6 @@ MAN= accept_filter.9 \
vm_page_aflag.9 \
vm_page_free.9 \
vm_page_grab.9 \
- vm_page_hold.9 \
vm_page_insert.9 \
vm_page_lookup.9 \
vm_page_rename.9 \
@@ -2248,7 +2247,6 @@ MLINKS+=vm_page_aflag.9 vm_page_aflag_clear.9 \
MLINKS+=vm_page_free.9 vm_page_free_toq.9 \
vm_page_free.9 vm_page_free_zero.9 \
vm_page_free.9 vm_page_try_to_free.9
-MLINKS+=vm_page_hold.9 vm_page_unhold.9
MLINKS+=vm_page_insert.9 vm_page_remove.9
MLINKS+=vm_page_wire.9 vm_page_unwire.9
MLINKS+=VOP_ACCESS.9 VOP_ACCESSX.9
Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/amd64/amd64/pmap.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -3035,7 +3035,7 @@ retry:
}
}
if (m != NULL)
- vm_page_hold(m);
+ vm_page_wire(m);
}
PA_UNLOCK_COND(pa);
PMAP_UNLOCK(pmap);
Modified: head/sys/amd64/vmm/vmm.c
==============================================================================
--- head/sys/amd64/vmm/vmm.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/amd64/vmm/vmm.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -1003,7 +1003,7 @@ vm_gpa_release(void *cookie)
vm_page_t m = cookie;
vm_page_lock(m);
- vm_page_unhold(m);
+ vm_page_unwire(m, PQ_ACTIVE);
vm_page_unlock(m);
}
Modified: head/sys/arm/arm/pmap-v4.c
==============================================================================
--- head/sys/arm/arm/pmap-v4.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/arm/arm/pmap-v4.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -3438,9 +3438,8 @@ retry:
goto retry;
if (l1pd & L1_S_PROT_W || (prot & VM_PROT_WRITE) == 0) {
m = PHYS_TO_VM_PAGE(pa);
- vm_page_hold(m);
+ vm_page_wire(m);
}
-
} else {
/*
* Note that we can't rely on the validity of the L1
@@ -3470,7 +3469,7 @@ retry:
if (vm_page_pa_tryrelock(pmap, pa & PG_FRAME, &paddr))
goto retry;
m = PHYS_TO_VM_PAGE(pa);
- vm_page_hold(m);
+ vm_page_wire(m);
}
}
Modified: head/sys/arm/arm/pmap-v6.c
==============================================================================
--- head/sys/arm/arm/pmap-v6.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/arm/arm/pmap-v6.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -2002,7 +2002,7 @@ retry:
if (vm_page_pa_tryrelock(pmap, pa, &lockpa))
goto retry;
m = PHYS_TO_VM_PAGE(pa);
- vm_page_hold(m);
+ vm_page_wire(m);
}
} else if (pte1_is_link(pte1)) {
pte2p = pmap_pte2(pmap, va);
@@ -2014,7 +2014,7 @@ retry:
if (vm_page_pa_tryrelock(pmap, pa, &lockpa))
goto retry;
m = PHYS_TO_VM_PAGE(pa);
- vm_page_hold(m);
+ vm_page_wire(m);
}
}
PA_UNLOCK_COND(lockpa);
@@ -6710,10 +6710,9 @@ pmap_pid_dump(int pid)
pa = pte2_pa(pte2);
m = PHYS_TO_VM_PAGE(pa);
- printf("va: 0x%x, pa: 0x%x, h: %d, w:"
- " %d, f: 0x%x", va, pa,
- m->hold_count, m->wire_count,
- m->flags);
+ printf("va: 0x%x, pa: 0x%x, w: %d, "
+ "f: 0x%x", va, pa,
+ m->wire_count, m->flags);
npte2++;
index++;
if (index >= 2) {
@@ -6823,8 +6822,8 @@ dump_link(pmap_t pmap, uint32_t pte1_idx, boolean_t in
printf(" 0x%08X: 0x%08X, TEX%d, s:%d, g:%d, m:%p", va , pte2,
pte2_class(pte2), !!(pte2 & PTE2_S), !(pte2 & PTE2_NG), m);
if (m != NULL) {
- printf(" v:%d h:%d w:%d f:0x%04X\n", m->valid,
- m->hold_count, m->wire_count, m->flags);
+ printf(" v:%d w:%d f:0x%04X\n", m->valid,
+ m->wire_count, m->flags);
} else {
printf("\n");
}
@@ -6933,8 +6932,8 @@ dump_pt2tab(pmap_t pmap)
printf(" 0x%08X: 0x%08X, TEX%d, s:%d, m:%p", va, pte2,
pte2_class(pte2), !!(pte2 & PTE2_S), m);
if (m != NULL)
- printf(" , h: %d, w: %d, f: 0x%04X pidx: %lld",
- m->hold_count, m->wire_count, m->flags, m->pindex);
+ printf(" , w: %d, f: 0x%04X pidx: %lld",
+ m->wire_count, m->flags, m->pindex);
printf("\n");
}
}
Modified: head/sys/arm64/arm64/pmap.c
==============================================================================
--- head/sys/arm64/arm64/pmap.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/arm64/arm64/pmap.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -1100,7 +1100,7 @@ retry:
(tpte & ~ATTR_MASK) | off, &pa))
goto retry;
m = PHYS_TO_VM_PAGE((tpte & ~ATTR_MASK) | off);
- vm_page_hold(m);
+ vm_page_wire(m);
}
}
PA_UNLOCK_COND(pa);
Modified: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
==============================================================================
--- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -455,7 +455,7 @@ page_unbusy(vm_page_t pp)
}
static vm_page_t
-page_hold(vnode_t *vp, int64_t start)
+page_wire(vnode_t *vp, int64_t start)
{
vm_object_t obj;
vm_page_t pp;
@@ -482,9 +482,8 @@ page_hold(vnode_t *vp, int64_t start)
ASSERT3U(pp->valid, ==, VM_PAGE_BITS_ALL);
vm_page_lock(pp);
- vm_page_hold(pp);
+ vm_page_wire(pp);
vm_page_unlock(pp);
-
} else
pp = NULL;
break;
@@ -493,11 +492,11 @@ page_hold(vnode_t *vp, int64_t start)
}
static void
-page_unhold(vm_page_t pp)
+page_unwire(vm_page_t pp)
{
vm_page_lock(pp);
- vm_page_unhold(pp);
+ vm_page_unwire(pp, PQ_ACTIVE);
vm_page_unlock(pp);
}
@@ -647,7 +646,7 @@ mappedread(vnode_t *vp, int nbytes, uio_t *uio)
vm_page_t pp;
uint64_t bytes = MIN(PAGESIZE - off, len);
- if (pp = page_hold(vp, start)) {
+ if (pp = page_wire(vp, start)) {
struct sf_buf *sf;
caddr_t va;
@@ -660,7 +659,7 @@ mappedread(vnode_t *vp, int nbytes, uio_t *uio)
#endif
zfs_unmap_page(sf);
zfs_vmobject_wlock(obj);
- page_unhold(pp);
+ page_unwire(pp);
} else {
zfs_vmobject_wunlock(obj);
error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl),
Modified: head/sys/compat/linuxkpi/common/src/linux_page.c
==============================================================================
--- head/sys/compat/linuxkpi/common/src/linux_page.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/compat/linuxkpi/common/src/linux_page.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -198,23 +198,11 @@ linux_get_user_pages_internal(vm_map_t map, unsigned l
vm_prot_t prot;
size_t len;
int count;
- int i;
prot = write ? (VM_PROT_READ | VM_PROT_WRITE) : VM_PROT_READ;
len = ((size_t)nr_pages) << PAGE_SHIFT;
count = vm_fault_quick_hold_pages(map, start, len, prot, pages, nr_pages);
- if (count == -1)
- return (-EFAULT);
-
- for (i = 0; i != nr_pages; i++) {
- struct page *pg = pages[i];
-
- vm_page_lock(pg);
- vm_page_wire(pg);
- vm_page_unhold(pg);
- vm_page_unlock(pg);
- }
- return (nr_pages);
+ return (count == -1 ? -EFAULT : nr_pages);
}
int
@@ -243,11 +231,6 @@ __get_user_pages_fast(unsigned long start, int nr_page
*mp = pmap_extract_and_hold(map->pmap, va, prot);
if (*mp == NULL)
break;
-
- vm_page_lock(*mp);
- vm_page_wire(*mp);
- vm_page_unhold(*mp);
- vm_page_unlock(*mp);
if ((prot & VM_PROT_WRITE) != 0 &&
(*mp)->dirty != VM_PAGE_BITS_ALL) {
Modified: head/sys/contrib/vchiq/interface/vchiq_arm/vchiq_2835_arm.c
==============================================================================
--- head/sys/contrib/vchiq/interface/vchiq_arm/vchiq_2835_arm.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/contrib/vchiq/interface/vchiq_arm/vchiq_2835_arm.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -474,13 +474,6 @@ create_pagelist(char __user *buf, size_t count, unsign
return (-ENOMEM);
}
- for (i = 0; i < actual_pages; i++) {
- vm_page_lock(pages[i]);
- vm_page_wire(pages[i]);
- vm_page_unhold(pages[i]);
- vm_page_unlock(pages[i]);
- }
-
pagelist->length = count;
pagelist->type = type;
pagelist->offset = offset;
Modified: head/sys/dev/cxgbe/tom/t4_cpl_io.c
==============================================================================
--- head/sys/dev/cxgbe/tom/t4_cpl_io.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/dev/cxgbe/tom/t4_cpl_io.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -1944,7 +1944,7 @@ aiotx_free_pgs(struct mbuf *m)
for (int i = 0; i < ext_pgs->npgs; i++) {
pg = PHYS_TO_VM_PAGE(ext_pgs->pa[i]);
vm_page_change_lock(pg, &mtx);
- vm_page_unhold(pg);
+ vm_page_unwire(pg, PQ_ACTIVE);
}
if (mtx != NULL)
mtx_unlock(mtx);
Modified: head/sys/dev/cxgbe/tom/t4_ddp.c
==============================================================================
--- head/sys/dev/cxgbe/tom/t4_ddp.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/dev/cxgbe/tom/t4_ddp.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -112,15 +112,12 @@ free_pageset(struct tom_data *td, struct pageset *ps)
if (ps->prsv.prsv_nppods > 0)
t4_free_page_pods(&ps->prsv);
- if (ps->flags & PS_WIRED) {
- for (i = 0; i < ps->npages; i++) {
- p = ps->pages[i];
- vm_page_lock(p);
- vm_page_unwire(p, PQ_INACTIVE);
- vm_page_unlock(p);
- }
- } else
- vm_page_unhold_pages(ps->pages, ps->npages);
+ for (i = 0; i < ps->npages; i++) {
+ p = ps->pages[i];
+ vm_page_lock(p);
+ vm_page_unwire(p, PQ_INACTIVE);
+ vm_page_unlock(p);
+ }
mtx_lock(&ddp_orphan_pagesets_lock);
TAILQ_INSERT_TAIL(&ddp_orphan_pagesets, ps, link);
taskqueue_enqueue(taskqueue_thread, &ddp_orphan_task);
@@ -150,7 +147,7 @@ recycle_pageset(struct toepcb *toep, struct pageset *p
{
DDP_ASSERT_LOCKED(toep);
- if (!(toep->ddp.flags & DDP_DEAD) && ps->flags & PS_WIRED) {
+ if (!(toep->ddp.flags & DDP_DEAD)) {
KASSERT(toep->ddp.cached_count + toep->ddp.active_count <
nitems(toep->ddp.db), ("too many wired pagesets"));
TAILQ_INSERT_HEAD(&toep->ddp.cached_pagesets, ps, link);
@@ -1179,35 +1176,14 @@ t4_write_page_pods_for_buf(struct adapter *sc, struct
return (0);
}
-static void
-wire_pageset(struct pageset *ps)
-{
- vm_page_t p;
- int i;
-
- KASSERT(!(ps->flags & PS_WIRED), ("pageset already wired"));
-
- for (i = 0; i < ps->npages; i++) {
- p = ps->pages[i];
- vm_page_lock(p);
- vm_page_wire(p);
- vm_page_unhold(p);
- vm_page_unlock(p);
- }
- ps->flags |= PS_WIRED;
-}
-
/*
- * Prepare a pageset for DDP. This wires the pageset and sets up page
- * pods.
+ * Prepare a pageset for DDP. This sets up page pods.
*/
static int
prep_pageset(struct adapter *sc, struct toepcb *toep, struct pageset *ps)
{
struct tom_data *td = sc->tom_softc;
- if (!(ps->flags & PS_WIRED))
- wire_pageset(ps);
if (ps->prsv.prsv_nppods == 0 &&
!t4_alloc_page_pods_for_ps(&td->pr, ps)) {
return (0);
Modified: head/sys/dev/cxgbe/tom/t4_tom.h
==============================================================================
--- head/sys/dev/cxgbe/tom/t4_tom.h Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/dev/cxgbe/tom/t4_tom.h Mon Jul 8 19:46:20 2019 (r349846)
@@ -124,8 +124,7 @@ struct pageset {
TAILQ_HEAD(pagesetq, pageset);
-#define PS_WIRED 0x0001 /* Pages wired rather than held. */
-#define PS_PPODS_WRITTEN 0x0002 /* Page pods written to the card. */
+#define PS_PPODS_WRITTEN 0x0001 /* Page pods written to the card. */
struct ddp_buffer {
struct pageset *ps;
Modified: head/sys/i386/i386/pmap.c
==============================================================================
--- head/sys/i386/i386/pmap.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/i386/i386/pmap.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -1716,7 +1716,7 @@ retry:
}
}
if (m != NULL)
- vm_page_hold(m);
+ vm_page_wire(m);
}
PA_UNLOCK_COND(pa);
PMAP_UNLOCK(pmap);
Modified: head/sys/kern/kern_exec.c
==============================================================================
--- head/sys/kern/kern_exec.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/kern/kern_exec.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -1015,8 +1015,7 @@ exec_map_first_page(struct image_params *imgp)
vm_page_readahead_finish(ma[i]);
}
vm_page_lock(ma[0]);
- vm_page_hold(ma[0]);
- vm_page_activate(ma[0]);
+ vm_page_wire(ma[0]);
vm_page_unlock(ma[0]);
VM_OBJECT_WUNLOCK(object);
@@ -1036,7 +1035,7 @@ exec_unmap_first_page(struct image_params *imgp)
sf_buf_free(imgp->firstpage);
imgp->firstpage = NULL;
vm_page_lock(m);
- vm_page_unhold(m);
+ vm_page_unwire(m, PQ_ACTIVE);
vm_page_unlock(m);
}
}
Modified: head/sys/kern/sys_process.c
==============================================================================
--- head/sys/kern/sys_process.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/kern/sys_process.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -307,7 +307,8 @@ proc_rwmem(struct proc *p, struct uio *uio)
* Release the page.
*/
vm_page_lock(m);
- vm_page_unhold(m);
+ if (vm_page_unwire(m, PQ_ACTIVE) && m->object == NULL)
+ vm_page_free(m);
vm_page_unlock(m);
} while (error == 0 && uio->uio_resid > 0);
Modified: head/sys/kern/uipc_shm.c
==============================================================================
--- head/sys/kern/uipc_shm.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/kern/uipc_shm.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -207,11 +207,7 @@ uiomove_object_page(vm_object_t obj, size_t len, struc
vm_page_xunbusy(m);
}
vm_page_lock(m);
- vm_page_hold(m);
- if (vm_page_active(m))
- vm_page_reference(m);
- else
- vm_page_activate(m);
+ vm_page_wire(m);
vm_page_unlock(m);
VM_OBJECT_WUNLOCK(obj);
error = uiomove_fromphys(&m, offset, tlen, uio);
@@ -222,7 +218,7 @@ uiomove_object_page(vm_object_t obj, size_t len, struc
VM_OBJECT_WUNLOCK(obj);
}
vm_page_lock(m);
- vm_page_unhold(m);
+ vm_page_unwire(m, PQ_ACTIVE);
vm_page_unlock(m);
return (error);
Modified: head/sys/mips/mips/pmap.c
==============================================================================
--- head/sys/mips/mips/pmap.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/mips/mips/pmap.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -811,7 +811,7 @@ retry:
if (vm_page_pa_tryrelock(pmap, pte_pa, &pa))
goto retry;
m = PHYS_TO_VM_PAGE(pte_pa);
- vm_page_hold(m);
+ vm_page_wire(m);
}
}
PA_UNLOCK_COND(pa);
Modified: head/sys/net/bpf_zerocopy.c
==============================================================================
--- head/sys/net/bpf_zerocopy.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/net/bpf_zerocopy.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -166,10 +166,6 @@ zbuf_sfbuf_get(struct vm_map *map, vm_offset_t uaddr)
if (vm_fault_quick_hold_pages(map, uaddr, PAGE_SIZE, VM_PROT_READ |
VM_PROT_WRITE, &pp, 1) < 0)
return (NULL);
- vm_page_lock(pp);
- vm_page_wire(pp);
- vm_page_unhold(pp);
- vm_page_unlock(pp);
sf = sf_buf_alloc(pp, SFB_NOWAIT);
if (sf == NULL) {
zbuf_page_free(pp);
Modified: head/sys/powerpc/aim/mmu_oea.c
==============================================================================
--- head/sys/powerpc/aim/mmu_oea.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/powerpc/aim/mmu_oea.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -1275,7 +1275,7 @@ retry:
if (vm_page_pa_tryrelock(pmap, pvo->pvo_pte.pte.pte_lo & PTE_RPGN, &pa))
goto retry;
m = PHYS_TO_VM_PAGE(pvo->pvo_pte.pte.pte_lo & PTE_RPGN);
- vm_page_hold(m);
+ vm_page_wire(m);
}
PA_UNLOCK_COND(pa);
PMAP_UNLOCK(pmap);
Modified: head/sys/powerpc/aim/mmu_oea64.c
==============================================================================
--- head/sys/powerpc/aim/mmu_oea64.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/powerpc/aim/mmu_oea64.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -1595,7 +1595,7 @@ retry:
pvo->pvo_pte.pa & LPTE_RPGN, &pa))
goto retry;
m = PHYS_TO_VM_PAGE(pvo->pvo_pte.pa & LPTE_RPGN);
- vm_page_hold(m);
+ vm_page_wire(m);
}
PA_UNLOCK_COND(pa);
PMAP_UNLOCK(pmap);
Modified: head/sys/powerpc/booke/pmap.c
==============================================================================
--- head/sys/powerpc/booke/pmap.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/powerpc/booke/pmap.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -2951,7 +2951,7 @@ retry:
if (vm_page_pa_tryrelock(pmap, PTE_PA(pte), &pa))
goto retry;
m = PHYS_TO_VM_PAGE(PTE_PA(pte));
- vm_page_hold(m);
+ vm_page_wire(m);
}
}
Modified: head/sys/riscv/riscv/pmap.c
==============================================================================
--- head/sys/riscv/riscv/pmap.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/riscv/riscv/pmap.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -884,7 +884,7 @@ retry:
if (vm_page_pa_tryrelock(pmap, phys, &pa))
goto retry;
m = PHYS_TO_VM_PAGE(phys);
- vm_page_hold(m);
+ vm_page_wire(m);
}
}
PA_UNLOCK_COND(pa);
Modified: head/sys/sparc64/sparc64/pmap.c
==============================================================================
--- head/sys/sparc64/sparc64/pmap.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/sparc64/sparc64/pmap.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -859,7 +859,7 @@ retry:
m = PHYS_TO_VM_PAGE(TLB_DIRECT_TO_PHYS(va));
(void)vm_page_pa_tryrelock(pm, TLB_DIRECT_TO_PHYS(va),
&pa);
- vm_page_hold(m);
+ vm_page_wire(m);
} else {
tp = tsb_kvtotte(va);
if ((tp->tte_data & TD_V) == 0)
@@ -872,7 +872,7 @@ retry:
if (vm_page_pa_tryrelock(pm, TTE_GET_PA(tp), &pa))
goto retry;
m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp));
- vm_page_hold(m);
+ vm_page_wire(m);
}
PA_UNLOCK_COND(pa);
PMAP_UNLOCK(pm);
Modified: head/sys/sys/param.h
==============================================================================
--- head/sys/sys/param.h Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/sys/param.h Mon Jul 8 19:46:20 2019 (r349846)
@@ -60,7 +60,7 @@
* in the range 5 to 9.
*/
#undef __FreeBSD_version
-#define __FreeBSD_version 1300034 /* Master, propagated to newvers */
+#define __FreeBSD_version 1300035 /* Master, propagated to newvers */
/*
* __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,
Modified: head/sys/vm/vm_fault.c
==============================================================================
--- head/sys/vm/vm_fault.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/vm/vm_fault.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -84,6 +84,7 @@ __FBSDID("$FreeBSD$");
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/mman.h>
+#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/racct.h>
#include <sys/resourcevar.h>
@@ -257,7 +258,7 @@ vm_fault_fill_hold(vm_page_t *m_hold, vm_page_t m)
if (m_hold != NULL) {
*m_hold = m;
vm_page_lock(m);
- vm_page_hold(m);
+ vm_page_wire(m);
vm_page_unlock(m);
}
}
@@ -505,7 +506,7 @@ vm_fault_populate(struct faultstate *fs, vm_prot_t pro
vm_page_activate(&m[i]);
if (m_hold != NULL && m[i].pindex == fs->first_pindex) {
*m_hold = &m[i];
- vm_page_hold(&m[i]);
+ vm_page_wire(&m[i]);
}
vm_page_xunbusy_maybelocked(&m[i]);
}
@@ -563,6 +564,7 @@ vm_fault_hold(vm_map_t map, vm_offset_t vaddr, vm_prot
struct faultstate fs;
struct vnode *vp;
struct domainset *dset;
+ struct mtx *mtx;
vm_object_t next_object, retry_object;
vm_offset_t e_end, e_start;
vm_pindex_t retry_pindex;
@@ -1142,15 +1144,23 @@ readrest:
* We don't chase down the shadow chain
*/
fs.object == fs.first_object->backing_object) {
- vm_page_lock(fs.m);
- vm_page_dequeue(fs.m);
+ /*
+ * Keep the page wired to ensure that it is not
+ * freed by another thread, such as the page
+ * daemon, while it is disassociated from an
+ * object.
+ */
+ mtx = NULL;
+ vm_page_change_lock(fs.m, &mtx);
+ vm_page_wire(fs.m);
(void)vm_page_remove(fs.m);
- vm_page_unlock(fs.m);
- vm_page_lock(fs.first_m);
+ vm_page_change_lock(fs.first_m, &mtx);
vm_page_replace_checked(fs.m, fs.first_object,
fs.first_pindex, fs.first_m);
vm_page_free(fs.first_m);
- vm_page_unlock(fs.first_m);
+ vm_page_change_lock(fs.m, &mtx);
+ vm_page_unwire(fs.m, PQ_ACTIVE);
+ mtx_unlock(mtx);
vm_page_dirty(fs.m);
#if VM_NRESERVLEVEL > 0
/*
@@ -1327,7 +1337,7 @@ readrest:
vm_page_activate(fs.m);
if (m_hold != NULL) {
*m_hold = fs.m;
- vm_page_hold(fs.m);
+ vm_page_wire(fs.m);
}
vm_page_unlock(fs.m);
vm_page_xunbusy(fs.m);
@@ -1600,7 +1610,9 @@ error:
for (mp = ma; mp < ma + count; mp++)
if (*mp != NULL) {
vm_page_lock(*mp);
- vm_page_unhold(*mp);
+ if (vm_page_unwire(*mp, PQ_INACTIVE) &&
+ (*mp)->object == NULL)
+ vm_page_free(*mp);
vm_page_unlock(*mp);
}
return (-1);
Modified: head/sys/vm/vm_glue.c
==============================================================================
--- head/sys/vm/vm_glue.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/vm/vm_glue.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -223,12 +223,14 @@ vm_imgact_hold_page(vm_object_t object, vm_ooffset_t o
VM_OBJECT_WLOCK(object);
pindex = OFF_TO_IDX(offset);
- m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY);
+ m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY |
+ VM_ALLOC_WIRED);
if (m->valid != VM_PAGE_BITS_ALL) {
vm_page_xbusy(m);
rv = vm_pager_get_pages(object, &m, 1, NULL, NULL);
if (rv != VM_PAGER_OK) {
vm_page_lock(m);
+ vm_page_unwire(m, PQ_NONE);
vm_page_free(m);
vm_page_unlock(m);
m = NULL;
@@ -236,10 +238,6 @@ vm_imgact_hold_page(vm_object_t object, vm_ooffset_t o
}
vm_page_xunbusy(m);
}
- vm_page_lock(m);
- vm_page_hold(m);
- vm_page_activate(m);
- vm_page_unlock(m);
out:
VM_OBJECT_WUNLOCK(object);
return (m);
@@ -273,7 +271,7 @@ vm_imgact_unmap_page(struct sf_buf *sf)
sf_buf_free(sf);
sched_unpin();
vm_page_lock(m);
- vm_page_unhold(m);
+ vm_page_unwire(m, PQ_ACTIVE);
vm_page_unlock(m);
}
Modified: head/sys/vm/vm_object.c
==============================================================================
--- head/sys/vm/vm_object.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/vm/vm_object.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -1212,7 +1212,7 @@ next_page:
if (tm->valid != VM_PAGE_BITS_ALL)
goto next_pindex;
vm_page_lock(tm);
- if (vm_page_held(tm)) {
+ if (vm_page_wired(tm)) {
vm_page_unlock(tm);
goto next_pindex;
}
Modified: head/sys/vm/vm_page.c
==============================================================================
--- head/sys/vm/vm_page.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/vm/vm_page.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -431,8 +431,7 @@ sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS)
/*
* Initialize a dummy page for use in scans of the specified paging queue.
* In principle, this function only needs to set the flag PG_MARKER.
- * Nonetheless, it write busies and initializes the hold count to one as
- * safety precautions.
+ * Nonetheless, it write busies the page as a safety precaution.
*/
static void
vm_page_init_marker(vm_page_t marker, int queue, uint8_t aflags)
@@ -443,7 +442,6 @@ vm_page_init_marker(vm_page_t marker, int queue, uint8
marker->aflags = aflags;
marker->busy_lock = VPB_SINGLE_EXCLUSIVER;
marker->queue = queue;
- marker->hold_count = 1;
}
static void
@@ -513,7 +511,6 @@ vm_page_init_page(vm_page_t m, vm_paddr_t pa, int segi
m->object = NULL;
m->wire_count = 0;
m->busy_lock = VPB_UNBUSIED;
- m->hold_count = 0;
m->flags = m->aflags = 0;
m->phys_addr = pa;
m->queue = PQ_NONE;
@@ -1096,31 +1093,6 @@ vm_page_change_lock(vm_page_t m, struct mtx **mtx)
}
/*
- * Keep page from being freed by the page daemon
- * much of the same effect as wiring, except much lower
- * overhead and should be used only for *very* temporary
- * holding ("wiring").
- */
-void
-vm_page_hold(vm_page_t mem)
-{
-
- vm_page_lock_assert(mem, MA_OWNED);
- mem->hold_count++;
-}
-
-void
-vm_page_unhold(vm_page_t mem)
-{
-
- vm_page_lock_assert(mem, MA_OWNED);
- KASSERT(mem->hold_count >= 1, ("vm_page_unhold: hold count < 0!!!"));
- --mem->hold_count;
- if (mem->hold_count == 0 && (mem->flags & PG_UNHOLDFREE) != 0)
- vm_page_free_toq(mem);
-}
-
-/*
* vm_page_unhold_pages:
*
* Unhold each of the pages that is referenced by the given array.
@@ -1133,7 +1105,8 @@ vm_page_unhold_pages(vm_page_t *ma, int count)
mtx = NULL;
for (; count != 0; count--) {
vm_page_change_lock(*ma, &mtx);
- vm_page_unhold(*ma);
+ if (vm_page_unwire(*ma, PQ_ACTIVE) && (*ma)->object == NULL)
+ vm_page_free(*ma);
ma++;
}
if (mtx != NULL)
@@ -1595,7 +1568,7 @@ vm_page_replace(vm_page_t mnew, vm_object_t object, vm
VM_OBJECT_ASSERT_WLOCKED(object);
KASSERT(mnew->object == NULL,
("vm_page_replace: page %p already in object", mnew));
- KASSERT(mnew->queue == PQ_NONE,
+ KASSERT(mnew->queue == PQ_NONE || vm_page_wired(mnew),
("vm_page_replace: new page %p is on a paging queue", mnew));
/*
@@ -2143,7 +2116,7 @@ vm_page_alloc_check(vm_page_t m)
KASSERT(m->queue == PQ_NONE && (m->aflags & PGA_QUEUE_STATE_MASK) == 0,
("page %p has unexpected queue %d, flags %#x",
m, m->queue, (m->aflags & PGA_QUEUE_STATE_MASK)));
- KASSERT(!vm_page_held(m), ("page %p is held", m));
+ KASSERT(!vm_page_wired(m), ("page %p is wired", m));
KASSERT(!vm_page_busied(m), ("page %p is busy", m));
KASSERT(m->dirty == 0, ("page %p is dirty", m));
KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT,
@@ -2350,7 +2323,7 @@ vm_page_scan_contig(u_long npages, vm_page_t m_start,
vm_page_change_lock(m, &m_mtx);
m_inc = 1;
retry:
- if (vm_page_held(m))
+ if (vm_page_wired(m))
run_ext = 0;
#if VM_NRESERVLEVEL > 0
else if ((level = vm_reserv_level(m)) >= 0 &&
@@ -2378,13 +2351,11 @@ retry:
*/
VM_OBJECT_RUNLOCK(object);
goto retry;
- } else if (vm_page_held(m)) {
+ } else if (vm_page_wired(m)) {
run_ext = 0;
goto unlock;
}
}
- KASSERT((m->flags & PG_UNHOLDFREE) == 0,
- ("page %p is PG_UNHOLDFREE", m));
/* Don't care: PG_NODUMP, PG_ZERO. */
if (object->type != OBJT_DEFAULT &&
object->type != OBJT_SWAP &&
@@ -2520,7 +2491,7 @@ vm_page_reclaim_run(int req_class, int domain, u_long
*/
vm_page_change_lock(m, &m_mtx);
retry:
- if (vm_page_held(m))
+ if (vm_page_wired(m))
error = EBUSY;
else if ((object = m->object) != NULL) {
/*
@@ -2537,13 +2508,11 @@ retry:
*/
VM_OBJECT_WUNLOCK(object);
goto retry;
- } else if (vm_page_held(m)) {
+ } else if (vm_page_wired(m)) {
error = EBUSY;
goto unlock;
}
}
- KASSERT((m->flags & PG_UNHOLDFREE) == 0,
- ("page %p is PG_UNHOLDFREE", m));
/* Don't care: PG_NODUMP, PG_ZERO. */
if (object->type != OBJT_DEFAULT &&
object->type != OBJT_SWAP &&
@@ -3476,13 +3445,6 @@ vm_page_free_prep(vm_page_t m)
if (vm_page_wired(m) != 0)
panic("vm_page_free_prep: freeing wired page %p", m);
- if (m->hold_count != 0) {
- m->flags &= ~PG_ZERO;
- KASSERT((m->flags & PG_UNHOLDFREE) == 0,
- ("vm_page_free_prep: freeing PG_UNHOLDFREE page %p", m));
- m->flags |= PG_UNHOLDFREE;
- return (false);
- }
/*
* Restore the default memory attribute to the page.
@@ -3799,7 +3761,7 @@ vm_page_try_to_free(vm_page_t m)
vm_page_assert_locked(m);
VM_OBJECT_ASSERT_WLOCKED(m->object);
KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("page %p is unmanaged", m));
- if (m->dirty != 0 || vm_page_held(m) || vm_page_busied(m))
+ if (m->dirty != 0 || vm_page_wired(m) || vm_page_busied(m))
return (false);
if (m->object->ref_count != 0) {
pmap_remove_all(m);
@@ -4539,10 +4501,10 @@ DB_SHOW_COMMAND(pginfo, vm_page_print_pginfo)
else
m = (vm_page_t)addr;
db_printf(
- "page %p obj %p pidx 0x%jx phys 0x%jx q %d hold %d wire %d\n"
+ "page %p obj %p pidx 0x%jx phys 0x%jx q %d wire %d\n"
" af 0x%x of 0x%x f 0x%x act %d busy %x valid 0x%x dirty 0x%x\n",
m, m->object, (uintmax_t)m->pindex, (uintmax_t)m->phys_addr,
- m->queue, m->hold_count, m->wire_count, m->aflags, m->oflags,
+ m->queue, m->wire_count, m->aflags, m->oflags,
m->flags, m->act_count, m->busy_lock, m->valid, m->dirty);
}
#endif /* DDB */
Modified: head/sys/vm/vm_page.h
==============================================================================
--- head/sys/vm/vm_page.h Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/vm/vm_page.h Mon Jul 8 19:46:20 2019 (r349846)
@@ -204,15 +204,14 @@ struct vm_page {
struct md_page md; /* machine dependent stuff */
u_int wire_count; /* wired down maps refs (P) */
volatile u_int busy_lock; /* busy owners lock */
- uint16_t hold_count; /* page hold count (P) */
uint16_t flags; /* page PG_* flags (P) */
+ uint8_t order; /* index of the buddy queue (F) */
+ uint8_t pool; /* vm_phys freepool index (F) */
uint8_t aflags; /* access is atomic */
uint8_t oflags; /* page VPO_* flags (O) */
uint8_t queue; /* page queue index (Q) */
int8_t psind; /* pagesizes[] index (O) */
int8_t segind; /* vm_phys segment index (C) */
- uint8_t order; /* index of the buddy queue (F) */
- uint8_t pool; /* vm_phys freepool index (F) */
u_char act_count; /* page usage count (P) */
/* NOTE that these must support one bit per DEV_BSIZE in a page */
/* so, on normal X86 kernels, they must be at least 8 bits wide */
@@ -388,7 +387,6 @@ extern struct mtx_padalign pa_lock[];
#define PG_ZERO 0x0008 /* page is zeroed */
#define PG_MARKER 0x0010 /* special queue marker page */
#define PG_NODUMP 0x0080 /* don't include this page in a dump */
-#define PG_UNHOLDFREE 0x0100 /* delayed free of a held page */
/*
* Misc constants.
@@ -516,8 +514,6 @@ malloc2vm_flags(int malloc_flags)
void vm_page_busy_downgrade(vm_page_t m);
void vm_page_busy_sleep(vm_page_t m, const char *msg, bool nonshared);
void vm_page_flash(vm_page_t m);
-void vm_page_hold(vm_page_t mem);
-void vm_page_unhold(vm_page_t mem);
void vm_page_free(vm_page_t m);
void vm_page_free_zero(vm_page_t m);
@@ -816,17 +812,10 @@ vm_page_in_laundry(vm_page_t m)
}
/*
- * vm_page_held:
+ * vm_page_wired:
*
* Return true if a reference prevents the page from being reclaimable.
*/
-static inline bool
-vm_page_held(vm_page_t m)
-{
-
- return (m->hold_count > 0 || m->wire_count > 0);
-}
-
static inline bool
vm_page_wired(vm_page_t m)
{
Modified: head/sys/vm/vm_pageout.c
==============================================================================
--- head/sys/vm/vm_pageout.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/vm/vm_pageout.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -334,7 +334,7 @@ vm_pageout_cluster(vm_page_t m)
pindex = m->pindex;
vm_page_assert_unbusied(m);
- KASSERT(!vm_page_held(m), ("page %p is held", m));
+ KASSERT(!vm_page_wired(m), ("page %p is wired", m));
pmap_remove_write(m);
vm_page_unlock(m);
@@ -373,7 +373,7 @@ more:
break;
}
vm_page_lock(p);
- if (vm_page_held(p) || !vm_page_in_laundry(p)) {
+ if (vm_page_wired(p) || !vm_page_in_laundry(p)) {
vm_page_unlock(p);
ib = 0;
break;
@@ -399,7 +399,7 @@ more:
if (p->dirty == 0)
break;
vm_page_lock(p);
- if (vm_page_held(p) || !vm_page_in_laundry(p)) {
+ if (vm_page_wired(p) || !vm_page_in_laundry(p)) {
vm_page_unlock(p);
break;
}
@@ -651,7 +651,7 @@ vm_pageout_clean(vm_page_t m, int *numpagedout)
* The page may have been busied or referenced while the object
* and page locks were released.
*/
- if (vm_page_busied(m) || vm_page_held(m)) {
+ if (vm_page_busied(m) || vm_page_wired(m)) {
vm_page_unlock(m);
error = EBUSY;
goto unlock_all;
@@ -747,14 +747,10 @@ recheck:
}
/*
- * Held pages are essentially stuck in the queue.
- *
* Wired pages may not be freed. Complete their removal
* from the queue now to avoid needless revisits during
* future scans.
*/
- if (m->hold_count != 0)
- continue;
if (vm_page_wired(m)) {
vm_page_dequeue_deferred(m);
continue;
@@ -1419,18 +1415,10 @@ recheck:
goto reinsert;
/*
- * Held pages are essentially stuck in the queue. So,
- * they ought to be discounted from the inactive count.
- * See the description of addl_page_shortage above.
- *
* Wired pages may not be freed. Complete their removal
* from the queue now to avoid needless revisits during
* future scans.
*/
- if (m->hold_count != 0) {
- addl_page_shortage++;
- goto reinsert;
- }
if (vm_page_wired(m)) {
vm_page_dequeue_deferred(m);
continue;
Modified: head/sys/vm/vm_swapout.c
==============================================================================
--- head/sys/vm/vm_swapout.c Mon Jul 8 19:38:49 2019 (r349845)
+++ head/sys/vm/vm_swapout.c Mon Jul 8 19:46:20 2019 (r349846)
@@ -212,7 +212,7 @@ vm_swapout_object_deactivate_pages(pmap_t pmap, vm_obj
continue;
VM_CNT_INC(v_pdpages);
vm_page_lock(p);
- if (vm_page_held(p) ||
+ if (vm_page_wired(p) ||
!pmap_page_exists_quick(pmap, p)) {
vm_page_unlock(p);
continue;
More information about the svn-src-head
mailing list