svn commit: r354158 - in head/sys: fs/nfsclient fs/nfsserver fs/tmpfs kern ufs/ffs vm
Jeff Roberson
jeff at FreeBSD.org
Tue Oct 29 21:06:37 UTC 2019
Author: jeff
Date: Tue Oct 29 21:06:34 2019
New Revision: 354158
URL: https://svnweb.freebsd.org/changeset/base/354158
Log:
Replace OBJ_MIGHTBEDIRTY with a system using atomics. Remove the TMPFS_DIRTY
flag and use the same system.
This enables further fault locking improvements by allowing more faults to
proceed with a shared lock.
Reviewed by: kib
Tested by: pho
Differential Revision: https://reviews.freebsd.org/D22116
Modified:
head/sys/fs/nfsclient/nfs_clvnops.c
head/sys/fs/nfsserver/nfs_nfsdport.c
head/sys/fs/tmpfs/tmpfs_subr.c
head/sys/fs/tmpfs/tmpfs_vfsops.c
head/sys/fs/tmpfs/tmpfs_vnops.c
head/sys/kern/vfs_subr.c
head/sys/ufs/ffs/ffs_rawread.c
head/sys/vm/vm_fault.c
head/sys/vm/vm_object.c
head/sys/vm/vm_object.h
head/sys/vm/vm_page.c
Modified: head/sys/fs/nfsclient/nfs_clvnops.c
==============================================================================
--- head/sys/fs/nfsclient/nfs_clvnops.c Tue Oct 29 20:58:46 2019 (r354157)
+++ head/sys/fs/nfsclient/nfs_clvnops.c Tue Oct 29 21:06:34 2019 (r354158)
@@ -715,7 +715,7 @@ nfs_open(struct vop_open_args *ap)
*/
if (vp->v_writecount <= -1) {
if ((obj = vp->v_object) != NULL &&
- (obj->flags & OBJ_MIGHTBEDIRTY) != 0) {
+ vm_object_mightbedirty(obj)) {
VM_OBJECT_WLOCK(obj);
vm_object_page_clean(obj, 0, 0, OBJPC_SYNC);
VM_OBJECT_WUNLOCK(obj);
Modified: head/sys/fs/nfsserver/nfs_nfsdport.c
==============================================================================
--- head/sys/fs/nfsserver/nfs_nfsdport.c Tue Oct 29 20:58:46 2019 (r354157)
+++ head/sys/fs/nfsserver/nfs_nfsdport.c Tue Oct 29 21:06:34 2019 (r354158)
@@ -1498,8 +1498,7 @@ nfsvno_fsync(struct vnode *vp, u_int64_t off, int cnt,
/*
* Give up and do the whole thing
*/
- if (vp->v_object &&
- (vp->v_object->flags & OBJ_MIGHTBEDIRTY)) {
+ if (vp->v_object && vm_object_mightbedirty(vp->v_object)) {
VM_OBJECT_WLOCK(vp->v_object);
vm_object_page_clean(vp->v_object, 0, 0, OBJPC_SYNC);
VM_OBJECT_WUNLOCK(vp->v_object);
@@ -1529,8 +1528,7 @@ nfsvno_fsync(struct vnode *vp, u_int64_t off, int cnt,
}
lblkno = off / iosize;
- if (vp->v_object &&
- (vp->v_object->flags & OBJ_MIGHTBEDIRTY)) {
+ if (vp->v_object && vm_object_mightbedirty(vp->v_object)) {
VM_OBJECT_WLOCK(vp->v_object);
vm_object_page_clean(vp->v_object, off, off + cnt,
OBJPC_SYNC);
Modified: head/sys/fs/tmpfs/tmpfs_subr.c
==============================================================================
--- head/sys/fs/tmpfs/tmpfs_subr.c Tue Oct 29 20:58:46 2019 (r354157)
+++ head/sys/fs/tmpfs/tmpfs_subr.c Tue Oct 29 21:06:34 2019 (r354158)
@@ -1477,10 +1477,10 @@ tmpfs_check_mtime(struct vnode *vp)
KASSERT((obj->flags & (OBJ_TMPFS_NODE | OBJ_TMPFS)) ==
(OBJ_TMPFS_NODE | OBJ_TMPFS), ("non-tmpfs obj"));
/* unlocked read */
- if ((obj->flags & OBJ_TMPFS_DIRTY) != 0) {
+ if (obj->generation != obj->cleangeneration) {
VM_OBJECT_WLOCK(obj);
- if ((obj->flags & OBJ_TMPFS_DIRTY) != 0) {
- obj->flags &= ~OBJ_TMPFS_DIRTY;
+ if (obj->generation != obj->cleangeneration) {
+ obj->cleangeneration = obj->generation;
node = VP_TO_TMPFS_NODE(vp);
node->tn_status |= TMPFS_NODE_MODIFIED |
TMPFS_NODE_CHANGED;
Modified: head/sys/fs/tmpfs/tmpfs_vfsops.c
==============================================================================
--- head/sys/fs/tmpfs/tmpfs_vfsops.c Tue Oct 29 20:58:46 2019 (r354157)
+++ head/sys/fs/tmpfs/tmpfs_vfsops.c Tue Oct 29 21:06:34 2019 (r354158)
@@ -172,7 +172,7 @@ tmpfs_update_mtime(struct mount *mp, bool lazy)
* For non-lazy case, we must flush all pending
* metadata changes now.
*/
- if (!lazy || (obj->flags & OBJ_TMPFS_DIRTY) != 0) {
+ if (!lazy || obj->generation != obj->cleangeneration) {
if (vget(vp, LK_EXCLUSIVE | LK_INTERLOCK,
curthread) != 0)
continue;
Modified: head/sys/fs/tmpfs/tmpfs_vnops.c
==============================================================================
--- head/sys/fs/tmpfs/tmpfs_vnops.c Tue Oct 29 20:58:46 2019 (r354157)
+++ head/sys/fs/tmpfs/tmpfs_vnops.c Tue Oct 29 21:06:34 2019 (r354158)
@@ -1323,7 +1323,7 @@ tmpfs_need_inactive(struct vop_need_inactive_args *ap)
goto need;
if (vp->v_type == VREG) {
obj = vp->v_object;
- if ((obj->flags & OBJ_TMPFS_DIRTY) != 0)
+ if (obj->generation != obj->cleangeneration)
goto need;
}
return (0);
Modified: head/sys/kern/vfs_subr.c
==============================================================================
--- head/sys/kern/vfs_subr.c Tue Oct 29 20:58:46 2019 (r354157)
+++ head/sys/kern/vfs_subr.c Tue Oct 29 21:06:34 2019 (r354158)
@@ -3346,7 +3346,7 @@ vinactive(struct vnode *vp, struct thread *td)
* pending I/O and dirty pages in the object.
*/
if ((obj = vp->v_object) != NULL && (vp->v_vflag & VV_NOSYNC) == 0 &&
- (obj->flags & OBJ_MIGHTBEDIRTY) != 0) {
+ vm_object_mightbedirty(obj)) {
VM_OBJECT_WLOCK(obj);
vm_object_page_clean(obj, 0, 0, 0);
VM_OBJECT_WUNLOCK(obj);
@@ -4406,7 +4406,7 @@ vfs_msync(struct mount *mp, int flags)
MNT_VNODE_FOREACH_ACTIVE(vp, mp, mvp) {
obj = vp->v_object;
- if (obj != NULL && (obj->flags & OBJ_MIGHTBEDIRTY) != 0 &&
+ if (obj != NULL && vm_object_mightbedirty(obj) &&
(flags == MNT_WAIT || VOP_ISLOCKED(vp) == 0)) {
if (!vget(vp,
LK_EXCLUSIVE | LK_RETRY | LK_INTERLOCK,
@@ -4696,7 +4696,7 @@ vn_need_pageq_flush(struct vnode *vp)
MPASS(mtx_owned(VI_MTX(vp)));
need = 0;
if ((obj = vp->v_object) != NULL && (vp->v_vflag & VV_NOSYNC) == 0 &&
- (obj->flags & OBJ_MIGHTBEDIRTY) != 0)
+ vm_object_mightbedirty(obj))
need = 1;
return (need);
}
Modified: head/sys/ufs/ffs/ffs_rawread.c
==============================================================================
--- head/sys/ufs/ffs/ffs_rawread.c Tue Oct 29 20:58:46 2019 (r354157)
+++ head/sys/ufs/ffs/ffs_rawread.c Tue Oct 29 21:06:34 2019 (r354158)
@@ -109,7 +109,7 @@ ffs_rawread_sync(struct vnode *vp)
if (bo->bo_numoutput > 0 ||
bo->bo_dirty.bv_cnt > 0 ||
((obj = vp->v_object) != NULL &&
- (obj->flags & OBJ_MIGHTBEDIRTY) != 0)) {
+ vm_object_mightbedirty(obj))) {
VI_UNLOCK(vp);
BO_UNLOCK(bo);
@@ -140,7 +140,7 @@ ffs_rawread_sync(struct vnode *vp)
}
/* Attempt to msync mmap() regions to clean dirty mmap */
if ((obj = vp->v_object) != NULL &&
- (obj->flags & OBJ_MIGHTBEDIRTY) != 0) {
+ vm_object_mightbedirty(obj)) {
VI_UNLOCK(vp);
VM_OBJECT_WLOCK(obj);
vm_object_page_clean(obj, 0, 0, OBJPC_SYNC);
Modified: head/sys/vm/vm_fault.c
==============================================================================
--- head/sys/vm/vm_fault.c Tue Oct 29 20:58:46 2019 (r354157)
+++ head/sys/vm/vm_fault.c Tue Oct 29 21:06:34 2019 (r354158)
@@ -210,7 +210,7 @@ unlock_and_deallocate(struct faultstate *fs)
static void
vm_fault_dirty(vm_map_entry_t entry, vm_page_t m, vm_prot_t prot,
- vm_prot_t fault_type, int fault_flags, bool set_wd)
+ vm_prot_t fault_type, int fault_flags, bool excl)
{
bool need_dirty;
@@ -226,11 +226,11 @@ vm_fault_dirty(vm_map_entry_t entry, vm_page_t m, vm_p
(fault_flags & VM_FAULT_WIRE) == 0) ||
(fault_flags & VM_FAULT_DIRTY) != 0;
- if (set_wd)
- vm_object_set_writeable_dirty(m->object);
- else
+ vm_object_set_writeable_dirty(m->object);
+
+ if (!excl)
/*
- * If two callers of vm_fault_dirty() with set_wd ==
+ * If two callers of vm_fault_dirty() with excl ==
* FALSE, one for the map entry with MAP_ENTRY_NOSYNC
* flag set, other with flag clear, race, it is
* possible for the no-NOSYNC thread to see m->dirty
@@ -267,7 +267,7 @@ vm_fault_dirty(vm_map_entry_t entry, vm_page_t m, vm_p
*/
if (need_dirty)
vm_page_dirty(m);
- if (!set_wd)
+ if (!excl)
vm_page_unlock(m);
else if (need_dirty)
vm_pager_page_unswapped(m);
@@ -758,29 +758,17 @@ RetryFault_oom:
/*
* Try to avoid lock contention on the top-level object through
* special-case handling of some types of page faults, specifically,
- * those that are both (1) mapping an existing page from the top-
- * level object and (2) not having to mark that object as containing
- * dirty pages. Under these conditions, a read lock on the top-level
- * object suffices, allowing multiple page faults of a similar type to
- * run in parallel on the same top-level object.
+ * those that are mapping an existing page from the top-level object.
+ * Under this condition, a read lock on the object suffices, allowing
+ * multiple page faults of a similar type to run in parallel.
*/
if (fs.vp == NULL /* avoid locked vnode leak */ &&
- (fault_flags & (VM_FAULT_WIRE | VM_FAULT_DIRTY)) == 0 &&
- /* avoid calling vm_object_set_writeable_dirty() */
- ((prot & VM_PROT_WRITE) == 0 ||
- (fs.first_object->type != OBJT_VNODE &&
- (fs.first_object->flags & OBJ_TMPFS_NODE) == 0) ||
- (fs.first_object->flags & OBJ_MIGHTBEDIRTY) != 0)) {
+ (fault_flags & (VM_FAULT_WIRE | VM_FAULT_DIRTY)) == 0) {
VM_OBJECT_RLOCK(fs.first_object);
- if ((prot & VM_PROT_WRITE) == 0 ||
- (fs.first_object->type != OBJT_VNODE &&
- (fs.first_object->flags & OBJ_TMPFS_NODE) == 0) ||
- (fs.first_object->flags & OBJ_MIGHTBEDIRTY) != 0) {
- rv = vm_fault_soft_fast(&fs, vaddr, prot, fault_type,
- fault_flags, wired, m_hold);
- if (rv == KERN_SUCCESS)
- return (rv);
- }
+ rv = vm_fault_soft_fast(&fs, vaddr, prot, fault_type,
+ fault_flags, wired, m_hold);
+ if (rv == KERN_SUCCESS)
+ return (rv);
if (!VM_OBJECT_TRYUPGRADE(fs.first_object)) {
VM_OBJECT_RUNLOCK(fs.first_object);
VM_OBJECT_WLOCK(fs.first_object);
Modified: head/sys/vm/vm_object.c
==============================================================================
--- head/sys/vm/vm_object.c Tue Oct 29 20:58:46 2019 (r354157)
+++ head/sys/vm/vm_object.c Tue Oct 29 21:06:34 2019 (r354158)
@@ -112,10 +112,10 @@ SYSCTL_INT(_vm, OID_AUTO, old_msync, CTLFLAG_RW, &old_
"Use old (insecure) msync behavior");
static int vm_object_page_collect_flush(vm_object_t object, vm_page_t p,
- int pagerflags, int flags, boolean_t *clearobjflags,
+ int pagerflags, int flags, boolean_t *allclean,
boolean_t *eio);
static boolean_t vm_object_page_remove_write(vm_page_t p, int flags,
- boolean_t *clearobjflags);
+ boolean_t *allclean);
static void vm_object_qcollapse(vm_object_t object);
static void vm_object_vndeallocate(vm_object_t object);
@@ -282,6 +282,7 @@ _vm_object_allocate(objtype_t type, vm_pindex_t size,
object->size = size;
object->domain.dr_policy = NULL;
object->generation = 1;
+ object->cleangeneration = 1;
refcount_init(&object->ref_count, 1);
object->memattr = VM_MEMATTR_DEFAULT;
object->cred = NULL;
@@ -769,7 +770,7 @@ vm_object_terminate(vm_object_t object)
* page should be flushed, and FALSE otherwise.
*/
static boolean_t
-vm_object_page_remove_write(vm_page_t p, int flags, boolean_t *clearobjflags)
+vm_object_page_remove_write(vm_page_t p, int flags, boolean_t *allclean)
{
vm_page_assert_busied(p);
@@ -780,7 +781,7 @@ vm_object_page_remove_write(vm_page_t p, int flags, bo
* cleared in this case so we do not have to set them.
*/
if ((flags & OBJPC_NOSYNC) != 0 && (p->aflags & PGA_NOSYNC) != 0) {
- *clearobjflags = FALSE;
+ *allclean = FALSE;
return (FALSE);
} else {
pmap_remove_write(p);
@@ -813,16 +814,11 @@ vm_object_page_clean(vm_object_t object, vm_ooffset_t
vm_page_t np, p;
vm_pindex_t pi, tend, tstart;
int curgeneration, n, pagerflags;
- boolean_t clearobjflags, eio, res;
+ boolean_t eio, res, allclean;
VM_OBJECT_ASSERT_WLOCKED(object);
- /*
- * The OBJ_MIGHTBEDIRTY flag is only set for OBJT_VNODE
- * objects. The check below prevents the function from
- * operating on non-vnode objects.
- */
- if ((object->flags & OBJ_MIGHTBEDIRTY) == 0 ||
+ if (object->type != OBJT_VNODE || !vm_object_mightbedirty(object) ||
object->resident_page_count == 0)
return (TRUE);
@@ -832,7 +828,7 @@ vm_object_page_clean(vm_object_t object, vm_ooffset_t
tstart = OFF_TO_IDX(start);
tend = (end == 0) ? object->size : OFF_TO_IDX(end + PAGE_MASK);
- clearobjflags = tstart == 0 && tend >= object->size;
+ allclean = tstart == 0 && tend >= object->size;
res = TRUE;
rescan:
@@ -846,32 +842,26 @@ rescan:
if (vm_page_none_valid(p))
continue;
if (vm_page_busy_acquire(p, VM_ALLOC_WAITFAIL) == 0) {
- if (object->generation != curgeneration) {
- if ((flags & OBJPC_SYNC) != 0)
- goto rescan;
- else
- clearobjflags = FALSE;
- }
+ if (object->generation != curgeneration &&
+ (flags & OBJPC_SYNC) != 0)
+ goto rescan;
np = vm_page_find_least(object, pi);
continue;
}
- if (!vm_object_page_remove_write(p, flags, &clearobjflags)) {
+ if (!vm_object_page_remove_write(p, flags, &allclean)) {
vm_page_xunbusy(p);
continue;
}
n = vm_object_page_collect_flush(object, p, pagerflags,
- flags, &clearobjflags, &eio);
+ flags, &allclean, &eio);
if (eio) {
res = FALSE;
- clearobjflags = FALSE;
+ allclean = FALSE;
}
- if (object->generation != curgeneration) {
- if ((flags & OBJPC_SYNC) != 0)
- goto rescan;
- else
- clearobjflags = FALSE;
- }
+ if (object->generation != curgeneration &&
+ (flags & OBJPC_SYNC) != 0)
+ goto rescan;
/*
* If the VOP_PUTPAGES() did a truncated write, so
@@ -887,7 +877,7 @@ rescan:
*/
if (n == 0) {
n = 1;
- clearobjflags = FALSE;
+ allclean = FALSE;
}
np = vm_page_find_least(object, pi + n);
}
@@ -895,14 +885,14 @@ rescan:
VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC) ? MNT_WAIT : 0);
#endif
- if (clearobjflags)
- vm_object_clear_flag(object, OBJ_MIGHTBEDIRTY);
+ if (allclean)
+ object->cleangeneration = curgeneration;
return (res);
}
static int
vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags,
- int flags, boolean_t *clearobjflags, boolean_t *eio)
+ int flags, boolean_t *allclean, boolean_t *eio)
{
vm_page_t ma[vm_pageout_page_count], p_first, tp;
int count, i, mreq, runlen;
@@ -918,7 +908,7 @@ vm_object_page_collect_flush(vm_object_t object, vm_pa
tp = vm_page_next(tp);
if (tp == NULL || vm_page_tryxbusy(tp) == 0)
break;
- if (!vm_object_page_remove_write(tp, flags, clearobjflags)) {
+ if (!vm_object_page_remove_write(tp, flags, allclean)) {
vm_page_xunbusy(tp);
break;
}
@@ -928,7 +918,7 @@ vm_object_page_collect_flush(vm_object_t object, vm_pa
tp = vm_page_prev(p_first);
if (tp == NULL || vm_page_tryxbusy(tp) == 0)
break;
- if (!vm_object_page_remove_write(tp, flags, clearobjflags)) {
+ if (!vm_object_page_remove_write(tp, flags, allclean)) {
vm_page_xunbusy(tp);
break;
}
@@ -993,7 +983,7 @@ vm_object_sync(vm_object_t object, vm_ooffset_t offset
* I/O.
*/
if (object->type == OBJT_VNODE &&
- (object->flags & OBJ_MIGHTBEDIRTY) != 0 &&
+ vm_object_mightbedirty(object) != 0 &&
((vp = object->handle)->v_vflag & VV_NOSYNC) == 0) {
VM_OBJECT_WUNLOCK(object);
(void) vn_start_write(vp, &mp, V_WAIT);
@@ -2130,18 +2120,13 @@ void
vm_object_set_writeable_dirty(vm_object_t object)
{
- VM_OBJECT_ASSERT_WLOCKED(object);
- if (object->type != OBJT_VNODE) {
- if ((object->flags & OBJ_TMPFS_NODE) != 0) {
- KASSERT(object->type == OBJT_SWAP, ("non-swap tmpfs"));
- vm_object_set_flag(object, OBJ_TMPFS_DIRTY);
- }
+ VM_OBJECT_ASSERT_LOCKED(object);
+
+ /* Only set for vnodes & tmpfs */
+ if (object->type != OBJT_VNODE &&
+ (object->flags & OBJ_TMPFS_NODE) == 0)
return;
- }
- object->generation++;
- if ((object->flags & OBJ_MIGHTBEDIRTY) != 0)
- return;
- vm_object_set_flag(object, OBJ_MIGHTBEDIRTY);
+ atomic_add_int(&object->generation, 1);
}
/*
Modified: head/sys/vm/vm_object.h
==============================================================================
--- head/sys/vm/vm_object.h Tue Oct 29 20:58:46 2019 (r354157)
+++ head/sys/vm/vm_object.h Tue Oct 29 21:06:34 2019 (r354158)
@@ -105,7 +105,8 @@ struct vm_object {
struct vm_radix rtree; /* root of the resident page radix trie*/
vm_pindex_t size; /* Object size */
struct domainset_ref domain; /* NUMA policy. */
- int generation; /* generation ID */
+ volatile int generation; /* generation ID */
+ int cleangeneration; /* Generation at clean time */
volatile u_int ref_count; /* How many refs?? */
int shadow_count; /* how many objects that this is a shadow for */
vm_memattr_t memattr; /* default memory attribute for pages */
@@ -188,9 +189,7 @@ struct vm_object {
#define OBJ_UMTXDEAD 0x0020 /* umtx pshared was terminated */
#define OBJ_SIZEVNLOCK 0x0040 /* lock vnode to check obj size */
#define OBJ_PG_DTOR 0x0080 /* dont reset object, leave that for dtor */
-#define OBJ_MIGHTBEDIRTY 0x0100 /* object might be dirty, only for vnode */
#define OBJ_TMPFS_NODE 0x0200 /* object belongs to tmpfs VREG node */
-#define OBJ_TMPFS_DIRTY 0x0400 /* dirty tmpfs obj */
#define OBJ_COLORED 0x1000 /* pg_color is defined */
#define OBJ_ONEMAPPING 0x2000 /* One USE (a single, non-forked) mapping flag */
#define OBJ_TMPFS 0x8000 /* has tmpfs vnode allocated */
@@ -307,6 +306,14 @@ vm_object_reserv(vm_object_t object)
return (true);
}
return (false);
+}
+
+static __inline bool
+vm_object_mightbedirty(vm_object_t object)
+{
+
+ return (object->type == OBJT_VNODE &&
+ object->generation != object->cleangeneration);
}
void vm_object_clear_flag(vm_object_t object, u_short bits);
Modified: head/sys/vm/vm_page.c
==============================================================================
--- head/sys/vm/vm_page.c Tue Oct 29 20:58:46 2019 (r354157)
+++ head/sys/vm/vm_page.c Tue Oct 29 21:06:34 2019 (r354158)
@@ -1521,7 +1521,7 @@ vm_page_insert_radixdone(vm_page_t m, vm_object_t obje
/*
* Since we are inserting a new and possibly dirty page,
- * update the object's OBJ_MIGHTBEDIRTY flag.
+ * update the object's generation count.
*/
if (pmap_page_is_write_mapped(m))
vm_object_set_writeable_dirty(object);
@@ -1691,7 +1691,8 @@ vm_page_replace(vm_page_t mnew, vm_object_t object, vm
/*
* The object's resident_page_count does not change because we have
- * swapped one page for another, but OBJ_MIGHTBEDIRTY.
+ * swapped one page for another, but the generation count should
+ * change if the page is dirty.
*/
if (pmap_page_is_write_mapped(mnew))
vm_object_set_writeable_dirty(object);
More information about the svn-src-all
mailing list