svn commit: r354871 - head/sys/vm
Jeff Roberson
jeff at FreeBSD.org
Wed Nov 20 00:31:15 UTC 2019
Author: jeff
Date: Wed Nov 20 00:31:14 2019
New Revision: 354871
URL: https://svnweb.freebsd.org/changeset/base/354871
Log:
Only keep anonymous objects on shadow lists. This eliminates locking of
globally visible objects when they are part of a backing chain.
Reviewed by: kib, markj
Differential Revision: https://reviews.freebsd.org/D22423
Modified:
head/sys/vm/vm_object.c
head/sys/vm/vm_object.h
Modified: head/sys/vm/vm_object.c
==============================================================================
--- head/sys/vm/vm_object.c Tue Nov 19 23:30:09 2019 (r354870)
+++ head/sys/vm/vm_object.c Wed Nov 20 00:31:14 2019 (r354871)
@@ -118,6 +118,7 @@ static boolean_t vm_object_page_remove_write(vm_page_t
boolean_t *allclean);
static void vm_object_qcollapse(vm_object_t object);
static void vm_object_vndeallocate(vm_object_t object);
+static void vm_object_backing_remove(vm_object_t object);
/*
* Virtual memory objects maintain the actual data
@@ -651,11 +652,7 @@ doterm:
if (temp != NULL) {
KASSERT((object->flags & OBJ_TMPFS_NODE) == 0,
("shadowed tmpfs v_object 2 %p", object));
- VM_OBJECT_WLOCK(temp);
- LIST_REMOVE(object, shadow_list);
- temp->shadow_count--;
- VM_OBJECT_WUNLOCK(temp);
- object->backing_object = NULL;
+ vm_object_backing_remove(object);
}
/*
* Don't double-terminate, we could be in a termination
@@ -695,6 +692,70 @@ vm_object_destroy(vm_object_t object)
uma_zfree(obj_zone, object);
}
+static void
+vm_object_backing_remove_locked(vm_object_t object)
+{
+ vm_object_t backing_object;
+
+ backing_object = object->backing_object;
+ VM_OBJECT_ASSERT_WLOCKED(object);
+ VM_OBJECT_ASSERT_WLOCKED(backing_object);
+
+ if ((object->flags & OBJ_SHADOWLIST) != 0) {
+ LIST_REMOVE(object, shadow_list);
+ backing_object->shadow_count--;
+ object->flags &= ~OBJ_SHADOWLIST;
+ }
+ object->backing_object = NULL;
+}
+
+static void
+vm_object_backing_remove(vm_object_t object)
+{
+ vm_object_t backing_object;
+
+ VM_OBJECT_ASSERT_WLOCKED(object);
+
+ if ((object->flags & OBJ_SHADOWLIST) != 0) {
+ backing_object = object->backing_object;
+ VM_OBJECT_WLOCK(backing_object);
+ vm_object_backing_remove_locked(object);
+ VM_OBJECT_WUNLOCK(backing_object);
+ } else
+ object->backing_object = NULL;
+}
+
+static void
+vm_object_backing_insert_locked(vm_object_t object, vm_object_t backing_object)
+{
+
+ VM_OBJECT_ASSERT_WLOCKED(object);
+
+ if ((backing_object->flags & OBJ_ANON) != 0) {
+ VM_OBJECT_ASSERT_WLOCKED(backing_object);
+ LIST_INSERT_HEAD(&backing_object->shadow_head, object,
+ shadow_list);
+ backing_object->shadow_count++;
+ object->flags |= OBJ_SHADOWLIST;
+ }
+ object->backing_object = backing_object;
+}
+
+static void
+vm_object_backing_insert(vm_object_t object, vm_object_t backing_object)
+{
+
+ VM_OBJECT_ASSERT_WLOCKED(object);
+
+ if ((backing_object->flags & OBJ_ANON) != 0) {
+ VM_OBJECT_WLOCK(backing_object);
+ vm_object_backing_insert_locked(object, backing_object);
+ VM_OBJECT_WUNLOCK(backing_object);
+ } else
+ object->backing_object = backing_object;
+}
+
+
/*
* vm_object_terminate_pages removes any remaining pageable pages
* from the object and resets the object to an empty state.
@@ -1250,6 +1311,12 @@ vm_object_shadow(
result = vm_object_allocate_anon(atop(length));
/*
+ * Store the offset into the source object, and fix up the offset into
+ * the new object.
+ */
+ result->backing_object_offset = *offset;
+
+ /*
* The new object shadows the source object, adding a reference to it.
* Our caller changes his reference to point to the new object,
* removing a reference to the source object. Net result: no change
@@ -1259,26 +1326,18 @@ vm_object_shadow(
* in order to maintain page coloring consistency in the combined
* shadowed object.
*/
- result->backing_object = source;
- /*
- * Store the offset into the source object, and fix up the offset into
- * the new object.
- */
- result->backing_object_offset = *offset;
if (source != NULL) {
- VM_OBJECT_WLOCK(source);
+ VM_OBJECT_WLOCK(result);
+ vm_object_backing_insert(result, source);
result->domain = source->domain;
- LIST_INSERT_HEAD(&source->shadow_head, result, shadow_list);
- source->shadow_count++;
#if VM_NRESERVLEVEL > 0
result->flags |= source->flags & OBJ_COLORED;
result->pg_color = (source->pg_color + OFF_TO_IDX(*offset)) &
((1 << (VM_NFREEORDER - 1)) - 1);
#endif
- VM_OBJECT_WUNLOCK(source);
+ VM_OBJECT_WUNLOCK(result);
}
-
/*
* Return the new things
*/
@@ -1326,24 +1385,26 @@ vm_object_split(vm_map_entry_t entry)
new_object->domain = orig_object->domain;
source = orig_object->backing_object;
if (source != NULL) {
- VM_OBJECT_WLOCK(source);
- if ((source->flags & OBJ_DEAD) != 0) {
+ if ((source->flags & (OBJ_ANON | OBJ_DEAD)) != 0) {
+ VM_OBJECT_WLOCK(source);
+ if ((source->flags & OBJ_DEAD) != 0) {
+ VM_OBJECT_WUNLOCK(source);
+ VM_OBJECT_WUNLOCK(orig_object);
+ VM_OBJECT_WUNLOCK(new_object);
+ vm_object_deallocate(new_object);
+ VM_OBJECT_WLOCK(orig_object);
+ return;
+ }
+ vm_object_backing_insert_locked(new_object, source);
+ vm_object_reference_locked(source); /* for new_object */
+ vm_object_clear_flag(source, OBJ_ONEMAPPING);
VM_OBJECT_WUNLOCK(source);
- VM_OBJECT_WUNLOCK(orig_object);
- VM_OBJECT_WUNLOCK(new_object);
- vm_object_deallocate(new_object);
- VM_OBJECT_WLOCK(orig_object);
- return;
+ } else {
+ vm_object_backing_insert(new_object, source);
+ vm_object_reference(source);
}
- LIST_INSERT_HEAD(&source->shadow_head,
- new_object, shadow_list);
- source->shadow_count++;
- vm_object_reference_locked(source); /* for new_object */
- vm_object_clear_flag(source, OBJ_ONEMAPPING);
- VM_OBJECT_WUNLOCK(source);
new_object->backing_object_offset =
orig_object->backing_object_offset + entry->offset;
- new_object->backing_object = source;
}
if (orig_object->cred != NULL) {
new_object->cred = orig_object->cred;
@@ -1756,20 +1817,15 @@ vm_object_collapse(vm_object_t object)
* backing_object->backing_object moves from within
* backing_object to within object.
*/
- LIST_REMOVE(object, shadow_list);
- backing_object->shadow_count--;
- if (backing_object->backing_object) {
- VM_OBJECT_WLOCK(backing_object->backing_object);
- LIST_REMOVE(backing_object, shadow_list);
- LIST_INSERT_HEAD(
- &backing_object->backing_object->shadow_head,
- object, shadow_list);
- /*
- * The shadow_count has not changed.
- */
- VM_OBJECT_WUNLOCK(backing_object->backing_object);
+ vm_object_backing_remove_locked(object);
+ new_backing_object = backing_object->backing_object;
+ if (new_backing_object != NULL) {
+ VM_OBJECT_WLOCK(new_backing_object);
+ vm_object_backing_remove_locked(backing_object);
+ vm_object_backing_insert_locked(object,
+ new_backing_object);
+ VM_OBJECT_WUNLOCK(new_backing_object);
}
- object->backing_object = backing_object->backing_object;
object->backing_object_offset +=
backing_object->backing_object_offset;
@@ -1807,20 +1863,13 @@ vm_object_collapse(vm_object_t object)
* chain. Deallocating backing_object will not remove
* it, since its reference count is at least 2.
*/
- LIST_REMOVE(object, shadow_list);
- backing_object->shadow_count--;
+ vm_object_backing_remove_locked(object);
new_backing_object = backing_object->backing_object;
- if ((object->backing_object = new_backing_object) != NULL) {
- VM_OBJECT_WLOCK(new_backing_object);
- LIST_INSERT_HEAD(
- &new_backing_object->shadow_head,
- object,
- shadow_list
- );
- new_backing_object->shadow_count++;
- vm_object_reference_locked(new_backing_object);
- VM_OBJECT_WUNLOCK(new_backing_object);
+ if (new_backing_object != NULL) {
+ vm_object_backing_insert(object,
+ new_backing_object);
+ vm_object_reference(new_backing_object);
object->backing_object_offset +=
backing_object->backing_object_offset;
}
Modified: head/sys/vm/vm_object.h
==============================================================================
--- head/sys/vm/vm_object.h Tue Nov 19 23:30:09 2019 (r354870)
+++ head/sys/vm/vm_object.h Wed Nov 20 00:31:14 2019 (r354871)
@@ -192,6 +192,7 @@ struct vm_object {
#define OBJ_TMPFS_NODE 0x0200 /* object belongs to tmpfs VREG node */
#define OBJ_COLORED 0x1000 /* pg_color is defined */
#define OBJ_ONEMAPPING 0x2000 /* One USE (a single, non-forked) mapping flag */
+#define OBJ_SHADOWLIST 0x4000 /* Object is on the shadow list. */
#define OBJ_TMPFS 0x8000 /* has tmpfs vnode allocated */
/*
More information about the svn-src-all
mailing list