svn commit: r355147 - head/sys/vm
Doug Moore
dougm at FreeBSD.org
Wed Nov 27 21:00:44 UTC 2019
Author: dougm
Date: Wed Nov 27 21:00:44 2019
New Revision: 355147
URL: https://svnweb.freebsd.org/changeset/base/355147
Log:
Inline some splay helper functions to improve performance on a
micro-benchmark.
Reviewed by: markj
Tested by: pho
Differential Revision: https://reviews.freebsd.org/D22544
Modified:
head/sys/vm/vm_map.c
Modified: head/sys/vm/vm_map.c
==============================================================================
--- head/sys/vm/vm_map.c Wed Nov 27 20:33:53 2019 (r355146)
+++ head/sys/vm/vm_map.c Wed Nov 27 21:00:44 2019 (r355147)
@@ -1069,53 +1069,48 @@ vm_map_entry_pred(vm_map_entry_t entry)
* the subsequent call to vm_map_splay_merge, rely on the start and end address
* values in &map->header.
*/
-static vm_map_entry_t
+static __always_inline vm_map_entry_t
vm_map_splay_split(vm_map_t map, vm_offset_t addr, vm_size_t length,
- vm_map_entry_t *out_llist, vm_map_entry_t *out_rlist)
+ vm_map_entry_t *llist, vm_map_entry_t *rlist)
{
- vm_map_entry_t llist, rlist, root, y;
+ vm_map_entry_t root, y;
- llist = rlist = &map->header;
+ *llist = *rlist = &map->header;
root = map->root;
while (root != NULL && root->max_free >= length) {
- KASSERT(llist->end <= root->start && root->end <= rlist->start,
+ KASSERT((*llist)->end <= root->start &&
+ root->end <= (*rlist)->start,
("%s: root not within tree bounds", __func__));
if (addr < root->start) {
- SPLAY_LEFT_STEP(root, y, rlist,
+ SPLAY_LEFT_STEP(root, y, *rlist,
y->max_free >= length && addr < y->start);
} else if (addr >= root->end) {
- SPLAY_RIGHT_STEP(root, y, llist,
+ SPLAY_RIGHT_STEP(root, y, *llist,
y->max_free >= length && addr >= y->end);
} else
break;
}
- *out_llist = llist;
- *out_rlist = rlist;
return (root);
}
-static void
-vm_map_splay_findnext(vm_map_entry_t root, vm_map_entry_t *iolist)
+static __always_inline void
+vm_map_splay_findnext(vm_map_entry_t root, vm_map_entry_t *rlist)
{
- vm_map_entry_t rlist, y;
+ vm_map_entry_t hi, y;
- root = root->right;
- rlist = *iolist;
- while (root != NULL)
- SPLAY_LEFT_STEP(root, y, rlist, true);
- *iolist = rlist;
+ hi = root->right;
+ while (hi != NULL)
+ SPLAY_LEFT_STEP(hi, y, *rlist, true);
}
-static void
-vm_map_splay_findprev(vm_map_entry_t root, vm_map_entry_t *iolist)
+static __always_inline void
+vm_map_splay_findprev(vm_map_entry_t root, vm_map_entry_t *llist)
{
- vm_map_entry_t llist, y;
+ vm_map_entry_t lo, y;
- root = root->left;
- llist = *iolist;
- while (root != NULL)
- SPLAY_RIGHT_STEP(root, y, llist, true);
- *iolist = llist;
+ lo = root->left;
+ while (lo != NULL)
+ SPLAY_RIGHT_STEP(lo, y, *llist, true);
}
static inline void
@@ -1270,30 +1265,22 @@ vm_map_entry_unlink(vm_map_t map, vm_map_entry_t entry
KASSERT(root != NULL,
("vm_map_entry_unlink: unlink object not mapped"));
+ vm_map_splay_findprev(root, &llist);
vm_map_splay_findnext(root, &rlist);
- switch (op) {
- case UNLINK_MERGE_NEXT:
+ if (op == UNLINK_MERGE_NEXT) {
rlist->start = root->start;
rlist->offset = root->offset;
- y = root->left;
+ }
+ if (llist != &map->header) {
+ root = llist;
+ llist = root->right;
+ root->right = NULL;
+ } else if (rlist != &map->header) {
root = rlist;
rlist = root->left;
- root->left = y;
- break;
- case UNLINK_MERGE_NONE:
- vm_map_splay_findprev(root, &llist);
- if (llist != &map->header) {
- root = llist;
- llist = root->right;
- root->right = NULL;
- } else if (rlist != &map->header) {
- root = rlist;
- rlist = root->left;
- root->left = NULL;
- } else
- root = NULL;
- break;
- }
+ root->left = NULL;
+ } else
+ root = NULL;
y = entry->next;
y->prev = entry->prev;
y->prev->next = y;
@@ -1322,8 +1309,7 @@ vm_map_entry_resize(vm_map_t map, vm_map_entry_t entry
VM_MAP_ASSERT_LOCKED(map);
root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist);
- KASSERT(root != NULL,
- ("%s: resize object not mapped", __func__));
+ KASSERT(root != NULL, ("%s: resize object not mapped", __func__));
vm_map_splay_findnext(root, &rlist);
root->right = NULL;
entry->end += grow_amount;
More information about the svn-src-all
mailing list