svn commit: r354157 - in head/sys: sys vm
Jeff Roberson
jeff at FreeBSD.org
Tue Oct 29 20:58:48 UTC 2019
Author: jeff
Date: Tue Oct 29 20:58:46 2019
New Revision: 354157
URL: https://svnweb.freebsd.org/changeset/base/354157
Log:
Use atomics and a shared object lock to protect the object reference count.
Certain consumers still need to guarantee a stable reference so we can not
switch entirely to atomics yet. Exclusive lock holders can still modify
and examine the refcount without using the ref api.
Reviewed by: kib
Tested by: pho
Sponsored by: Netflix, Intel
Differential Revision: https://reviews.freebsd.org/D21598
Modified:
head/sys/sys/refcount.h
head/sys/vm/vm_object.c
head/sys/vm/vm_object.h
head/sys/vm/vnode_pager.c
Modified: head/sys/sys/refcount.h
==============================================================================
--- head/sys/sys/refcount.h Tue Oct 29 20:46:25 2019 (r354156)
+++ head/sys/sys/refcount.h Tue Oct 29 20:58:46 2019 (r354157)
@@ -175,4 +175,22 @@ refcount_release_if_not_last(volatile u_int *count)
}
}
+static __inline __result_use_check bool
+refcount_release_if_gt(volatile u_int *count, u_int n)
+{
+ u_int old;
+
+ KASSERT(n > 0,
+ ("refcount_release_if_gt: Use refcount_release for final ref"));
+ old = *count;
+ for (;;) {
+ if (REFCOUNT_COUNT(old) <= n)
+ return (false);
+ if (__predict_false(REFCOUNT_SATURATED(old)))
+ return (true);
+ if (atomic_fcmpset_int(count, &old, old - 1))
+ return (true);
+ }
+}
+
#endif /* ! __SYS_REFCOUNT_H__ */
Modified: head/sys/vm/vm_object.c
==============================================================================
--- head/sys/vm/vm_object.c Tue Oct 29 20:46:25 2019 (r354156)
+++ head/sys/vm/vm_object.c Tue Oct 29 20:58:46 2019 (r354157)
@@ -224,8 +224,8 @@ vm_object_zinit(void *mem, int size, int flags)
/* These are true for any object that has been freed */
object->type = OBJT_DEAD;
- object->ref_count = 0;
vm_radix_init(&object->rtree);
+ refcount_init(&object->ref_count, 0);
refcount_init(&object->paging_in_progress, 0);
refcount_init(&object->busy, 0);
object->resident_page_count = 0;
@@ -282,7 +282,7 @@ _vm_object_allocate(objtype_t type, vm_pindex_t size,
object->size = size;
object->domain.dr_policy = NULL;
object->generation = 1;
- object->ref_count = 1;
+ refcount_init(&object->ref_count, 1);
object->memattr = VM_MEMATTR_DEFAULT;
object->cred = NULL;
object->charge = 0;
@@ -444,9 +444,9 @@ vm_object_reference(vm_object_t object)
{
if (object == NULL)
return;
- VM_OBJECT_WLOCK(object);
+ VM_OBJECT_RLOCK(object);
vm_object_reference_locked(object);
- VM_OBJECT_WUNLOCK(object);
+ VM_OBJECT_RUNLOCK(object);
}
/*
@@ -461,8 +461,8 @@ vm_object_reference_locked(vm_object_t object)
{
struct vnode *vp;
- VM_OBJECT_ASSERT_WLOCKED(object);
- object->ref_count++;
+ VM_OBJECT_ASSERT_LOCKED(object);
+ refcount_acquire(&object->ref_count);
if (object->type == OBJT_VNODE) {
vp = object->handle;
vref(vp);
@@ -477,24 +477,16 @@ vm_object_vndeallocate(vm_object_t object)
{
struct vnode *vp = (struct vnode *) object->handle;
- VM_OBJECT_ASSERT_WLOCKED(object);
KASSERT(object->type == OBJT_VNODE,
("vm_object_vndeallocate: not a vnode object"));
KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp"));
-#ifdef INVARIANTS
- if (object->ref_count == 0) {
- vn_printf(vp, "vm_object_vndeallocate ");
- panic("vm_object_vndeallocate: bad object reference count");
- }
-#endif
- if (!umtx_shm_vnobj_persistent && object->ref_count == 1)
+ if (refcount_release(&object->ref_count) &&
+ !umtx_shm_vnobj_persistent)
umtx_shm_object_terminated(object);
- object->ref_count--;
-
+ VM_OBJECT_RUNLOCK(object);
/* vrele may need the vnode lock. */
- VM_OBJECT_WUNLOCK(object);
vrele(vp);
}
@@ -513,24 +505,32 @@ void
vm_object_deallocate(vm_object_t object)
{
vm_object_t temp;
+ bool released;
while (object != NULL) {
- VM_OBJECT_WLOCK(object);
+ VM_OBJECT_RLOCK(object);
if (object->type == OBJT_VNODE) {
vm_object_vndeallocate(object);
return;
}
- KASSERT(object->ref_count != 0,
- ("vm_object_deallocate: object deallocated too many times: %d", object->type));
-
/*
* If the reference count goes to 0 we start calling
- * vm_object_terminate() on the object chain.
- * A ref count of 1 may be a special case depending on the
- * shadow count being 0 or 1.
+ * vm_object_terminate() on the object chain. A ref count
+ * of 1 may be a special case depending on the shadow count
+ * being 0 or 1. These cases require a write lock on the
+ * object.
*/
- object->ref_count--;
+ released = refcount_release_if_gt(&object->ref_count, 2);
+ VM_OBJECT_RUNLOCK(object);
+ if (released)
+ return;
+
+ VM_OBJECT_WLOCK(object);
+ KASSERT(object->ref_count != 0,
+ ("vm_object_deallocate: object deallocated too many times: %d", object->type));
+
+ refcount_release(&object->ref_count);
if (object->ref_count > 1) {
VM_OBJECT_WUNLOCK(object);
return;
@@ -558,7 +558,7 @@ vm_object_deallocate(vm_object_t object)
/*
* Avoid a potential deadlock.
*/
- object->ref_count++;
+ refcount_acquire(&object->ref_count);
VM_OBJECT_WUNLOCK(object);
/*
* More likely than not the thread
@@ -580,7 +580,7 @@ vm_object_deallocate(vm_object_t object)
(robject->type == OBJT_DEFAULT ||
robject->type == OBJT_SWAP)) {
- robject->ref_count++;
+ refcount_acquire(&robject->ref_count);
retry:
if (REFCOUNT_COUNT(robject->paging_in_progress) > 0) {
VM_OBJECT_WUNLOCK(object);
@@ -1223,15 +1223,15 @@ vm_object_shadow(
* Don't create the new object if the old object isn't shared.
*/
if (source != NULL) {
- VM_OBJECT_WLOCK(source);
+ VM_OBJECT_RLOCK(source);
if (source->ref_count == 1 &&
source->handle == NULL &&
(source->type == OBJT_DEFAULT ||
source->type == OBJT_SWAP)) {
- VM_OBJECT_WUNLOCK(source);
+ VM_OBJECT_RUNLOCK(source);
return;
}
- VM_OBJECT_WUNLOCK(source);
+ VM_OBJECT_RUNLOCK(source);
}
/*
@@ -1822,7 +1822,7 @@ vm_object_collapse(vm_object_t object)
* Drop the reference count on backing_object. Since
* its ref_count was at least 2, it will not vanish.
*/
- backing_object->ref_count--;
+ refcount_release(&backing_object->ref_count);
VM_OBJECT_WUNLOCK(backing_object);
counter_u64_add(object_bypasses, 1);
}
Modified: head/sys/vm/vm_object.h
==============================================================================
--- head/sys/vm/vm_object.h Tue Oct 29 20:46:25 2019 (r354156)
+++ head/sys/vm/vm_object.h Tue Oct 29 20:58:46 2019 (r354157)
@@ -106,7 +106,7 @@ struct vm_object {
vm_pindex_t size; /* Object size */
struct domainset_ref domain; /* NUMA policy. */
int generation; /* generation ID */
- int ref_count; /* How many refs?? */
+ volatile u_int ref_count; /* How many refs?? */
int shadow_count; /* how many objects that this is a shadow for */
vm_memattr_t memattr; /* default memory attribute for pages */
objtype_t type; /* type of pager */
Modified: head/sys/vm/vnode_pager.c
==============================================================================
--- head/sys/vm/vnode_pager.c Tue Oct 29 20:46:25 2019 (r354156)
+++ head/sys/vm/vnode_pager.c Tue Oct 29 20:58:46 2019 (r354157)
@@ -70,6 +70,7 @@ __FBSDID("$FreeBSD$");
#include <sys/ktr.h>
#include <sys/limits.h>
#include <sys/conf.h>
+#include <sys/refcount.h>
#include <sys/rwlock.h>
#include <sys/sf_buf.h>
#include <sys/domainset.h>
@@ -172,9 +173,9 @@ vnode_create_vobject(struct vnode *vp, off_t isize, st
* Dereference the reference we just created. This assumes
* that the object is associated with the vp.
*/
- VM_OBJECT_WLOCK(object);
- object->ref_count--;
- VM_OBJECT_WUNLOCK(object);
+ VM_OBJECT_RLOCK(object);
+ refcount_release(&object->ref_count);
+ VM_OBJECT_RUNLOCK(object);
vrele(vp);
KASSERT(vp->v_object != NULL, ("vnode_create_vobject: NULL object"));
@@ -285,7 +286,7 @@ retry:
KASSERT(object->ref_count == 1,
("leaked ref %p %d", object, object->ref_count));
object->type = OBJT_DEAD;
- object->ref_count = 0;
+ refcount_init(&object->ref_count, 0);
VM_OBJECT_WUNLOCK(object);
vm_object_destroy(object);
goto retry;
@@ -294,7 +295,7 @@ retry:
VI_UNLOCK(vp);
} else {
VM_OBJECT_WLOCK(object);
- object->ref_count++;
+ refcount_acquire(&object->ref_count);
#if VM_NRESERVLEVEL > 0
vm_object_color(object, 0);
#endif
More information about the svn-src-all
mailing list