svn commit: r355122 - in head/sys: sys vm
Jeff Roberson
jeff at FreeBSD.org
Wed Nov 27 00:39:24 UTC 2019
Author: jeff
Date: Wed Nov 27 00:39:23 2019
New Revision: 355122
URL: https://svnweb.freebsd.org/changeset/base/355122
Log:
Use atomics in more cases for object references. We now can completely
omit the object lock if we are above a certain threshold. Hold only a
single vnode reference when the vnode object has any ref > 0. This
allows us to only lock the object and vnode on 0-1 and 1-0 transitions.
Differential Revision: https://reviews.freebsd.org/D22452
Modified:
head/sys/sys/refcount.h
head/sys/vm/vm_object.c
head/sys/vm/vnode_pager.c
Modified: head/sys/sys/refcount.h
==============================================================================
--- head/sys/sys/refcount.h Tue Nov 26 22:17:02 2019 (r355121)
+++ head/sys/sys/refcount.h Wed Nov 27 00:39:23 2019 (r355122)
@@ -72,7 +72,7 @@ refcount_init(volatile u_int *count, u_int value)
*count = value;
}
-static __inline void
+static __inline u_int
refcount_acquire(volatile u_int *count)
{
u_int old;
@@ -80,9 +80,11 @@ refcount_acquire(volatile u_int *count)
old = atomic_fetchadd_int(count, 1);
if (__predict_false(REFCOUNT_SATURATED(old)))
_refcount_update_saturated(count);
+
+ return (old);
}
-static __inline void
+static __inline u_int
refcount_acquiren(volatile u_int *count, u_int n)
{
u_int old;
@@ -92,6 +94,8 @@ refcount_acquiren(volatile u_int *count, u_int n)
old = atomic_fetchadd_int(count, n);
if (__predict_false(REFCOUNT_SATURATED(old)))
_refcount_update_saturated(count);
+
+ return (old);
}
static __inline __result_use_check bool
@@ -144,13 +148,13 @@ refcount_wait(volatile u_int *count, const char *wmesg
* incremented. Else zero is returned.
*/
static __inline __result_use_check bool
-refcount_acquire_if_not_zero(volatile u_int *count)
+refcount_acquire_if_gt(volatile u_int *count, u_int n)
{
u_int old;
old = *count;
for (;;) {
- if (REFCOUNT_COUNT(old) == 0)
+ if (REFCOUNT_COUNT(old) <= n)
return (false);
if (__predict_false(REFCOUNT_SATURATED(old)))
return (true);
@@ -160,19 +164,10 @@ refcount_acquire_if_not_zero(volatile u_int *count)
}
static __inline __result_use_check bool
-refcount_release_if_not_last(volatile u_int *count)
+refcount_acquire_if_not_zero(volatile u_int *count)
{
- u_int old;
- old = *count;
- for (;;) {
- if (REFCOUNT_COUNT(old) == 1)
- return (false);
- if (__predict_false(REFCOUNT_SATURATED(old)))
- return (true);
- if (atomic_fcmpset_int(count, &old, old - 1))
- return (true);
- }
+ return refcount_acquire_if_gt(count, 0);
}
static __inline __result_use_check bool
@@ -193,4 +188,10 @@ refcount_release_if_gt(volatile u_int *count, u_int n)
}
}
+static __inline __result_use_check bool
+refcount_release_if_not_last(volatile u_int *count)
+{
+
+ return refcount_release_if_gt(count, 1);
+}
#endif /* ! __SYS_REFCOUNT_H__ */
Modified: head/sys/vm/vm_object.c
==============================================================================
--- head/sys/vm/vm_object.c Tue Nov 26 22:17:02 2019 (r355121)
+++ head/sys/vm/vm_object.c Wed Nov 27 00:39:23 2019 (r355122)
@@ -468,11 +468,28 @@ vm_object_allocate_anon(vm_pindex_t size)
void
vm_object_reference(vm_object_t object)
{
+ struct vnode *vp;
+ u_int old;
+
if (object == NULL)
return;
- VM_OBJECT_RLOCK(object);
- vm_object_reference_locked(object);
- VM_OBJECT_RUNLOCK(object);
+
+ /*
+ * Many places assume exclusive access to objects with a single
+ * ref. vm_object_collapse() in particular will directly mainpulate
+ * references for objects in this state. vnode objects only need
+ * the lock for the first ref to reference the vnode.
+ */
+ if (!refcount_acquire_if_gt(&object->ref_count,
+ object->type == OBJT_VNODE ? 0 : 1)) {
+ VM_OBJECT_RLOCK(object);
+ old = refcount_acquire(&object->ref_count);
+ if (object->type == OBJT_VNODE && old == 0) {
+ vp = object->handle;
+ vref(vp);
+ }
+ VM_OBJECT_RUNLOCK(object);
+ }
}
/*
@@ -486,10 +503,11 @@ void
vm_object_reference_locked(vm_object_t object)
{
struct vnode *vp;
+ u_int old;
VM_OBJECT_ASSERT_LOCKED(object);
- refcount_acquire(&object->ref_count);
- if (object->type == OBJT_VNODE) {
+ old = refcount_acquire(&object->ref_count);
+ if (object->type == OBJT_VNODE && old == 0) {
vp = object->handle;
vref(vp);
}
@@ -507,11 +525,10 @@ vm_object_vndeallocate(vm_object_t object)
("vm_object_vndeallocate: not a vnode object"));
KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp"));
- if (refcount_release(&object->ref_count) &&
- !umtx_shm_vnobj_persistent)
+ if (!umtx_shm_vnobj_persistent)
umtx_shm_object_terminated(object);
- VM_OBJECT_RUNLOCK(object);
+ VM_OBJECT_WUNLOCK(object);
/* vrele may need the vnode lock. */
vrele(vp);
}
@@ -531,15 +548,9 @@ void
vm_object_deallocate(vm_object_t object)
{
vm_object_t robject, temp;
- bool released;
+ bool last, released;
while (object != NULL) {
- VM_OBJECT_RLOCK(object);
- if (object->type == OBJT_VNODE) {
- vm_object_vndeallocate(object);
- return;
- }
-
/*
* If the reference count goes to 0 we start calling
* vm_object_terminate() on the object chain. A ref count
@@ -551,7 +562,6 @@ vm_object_deallocate(vm_object_t object)
released = refcount_release_if_gt(&object->ref_count, 1);
else
released = refcount_release_if_gt(&object->ref_count, 2);
- VM_OBJECT_RUNLOCK(object);
if (released)
return;
@@ -559,7 +569,14 @@ vm_object_deallocate(vm_object_t object)
KASSERT(object->ref_count != 0,
("vm_object_deallocate: object deallocated too many times: %d", object->type));
- refcount_release(&object->ref_count);
+ last = refcount_release(&object->ref_count);
+ if (object->type == OBJT_VNODE) {
+ if (last)
+ vm_object_vndeallocate(object);
+ else
+ VM_OBJECT_WUNLOCK(object);
+ return;
+ }
if (object->ref_count > 1) {
VM_OBJECT_WUNLOCK(object);
return;
@@ -629,7 +646,7 @@ retry:
VM_OBJECT_WUNLOCK(object);
if (robject->ref_count == 1) {
- robject->ref_count--;
+ refcount_release(&robject->ref_count);
object = robject;
goto doterm;
}
@@ -1838,7 +1855,7 @@ vm_object_collapse(vm_object_t object)
backing_object));
vm_object_pip_wakeup(backing_object);
backing_object->type = OBJT_DEAD;
- backing_object->ref_count = 0;
+ refcount_release(&backing_object->ref_count);
VM_OBJECT_WUNLOCK(backing_object);
vm_object_destroy(backing_object);
Modified: head/sys/vm/vnode_pager.c
==============================================================================
--- head/sys/vm/vnode_pager.c Tue Nov 26 22:17:02 2019 (r355121)
+++ head/sys/vm/vnode_pager.c Wed Nov 27 00:39:23 2019 (r355122)
@@ -150,6 +150,7 @@ vnode_create_vobject(struct vnode *vp, off_t isize, st
vm_object_t object;
vm_ooffset_t size = isize;
struct vattr va;
+ bool last;
if (!vn_isdisk(vp, NULL) && vn_canvmio(vp) == FALSE)
return (0);
@@ -171,12 +172,15 @@ vnode_create_vobject(struct vnode *vp, off_t isize, st
object = vnode_pager_alloc(vp, size, 0, 0, td->td_ucred);
/*
* Dereference the reference we just created. This assumes
- * that the object is associated with the vp.
+ * that the object is associated with the vp. We still have
+ * to serialize with vnode_pager_dealloc() for the last
+ * potential reference.
*/
VM_OBJECT_RLOCK(object);
- refcount_release(&object->ref_count);
+ last = refcount_release(&object->ref_count);
VM_OBJECT_RUNLOCK(object);
- vrele(vp);
+ if (last)
+ vrele(vp);
KASSERT(vp->v_object != NULL, ("vnode_create_vobject: NULL object"));
@@ -293,15 +297,17 @@ retry:
}
vp->v_object = object;
VI_UNLOCK(vp);
+ vrefact(vp);
} else {
- VM_OBJECT_WLOCK(object);
- refcount_acquire(&object->ref_count);
+ vm_object_reference(object);
#if VM_NRESERVLEVEL > 0
- vm_object_color(object, 0);
+ if ((object->flags & OBJ_COLORED) == 0) {
+ VM_OBJECT_WLOCK(object);
+ vm_object_color(object, 0);
+ VM_OBJECT_WUNLOCK(object);
+ }
#endif
- VM_OBJECT_WUNLOCK(object);
}
- vrefact(vp);
return (object);
}
@@ -345,7 +351,7 @@ vnode_pager_dealloc(vm_object_t object)
vp->v_writecount = 0;
VI_UNLOCK(vp);
VM_OBJECT_WUNLOCK(object);
- while (refs-- > 0)
+ if (refs > 0)
vunref(vp);
VM_OBJECT_WLOCK(object);
}
More information about the svn-src-head
mailing list