svn commit: r326329 - head/sys/vm
Alan Cox
alc at FreeBSD.org
Tue Nov 28 17:46:05 UTC 2017
Author: alc
Date: Tue Nov 28 17:46:03 2017
New Revision: 326329
URL: https://svnweb.freebsd.org/changeset/base/326329
Log:
When the swap pager allocates space on disk, it requests contiguous
blocks in a single call to blist_alloc(). However, when it frees
that space, it previously called blist_free() on each block, one at a
time. With this change, the swap pager identifies ranges of
contiguous blocks to be freed, and calls blist_free() once per
range. In one extreme case, that is described in the review, the time
to perform an munmap(2) was reduced by 55%.
Submitted by: Doug Moore <dougm at rice.edu>
Reviewed by: kib
MFC after: 1 week
Differential Revision: https://reviews.freebsd.org/D12397
Modified:
head/sys/vm/swap_pager.c
Modified: head/sys/vm/swap_pager.c
==============================================================================
--- head/sys/vm/swap_pager.c Tue Nov 28 17:33:10 2017 (r326328)
+++ head/sys/vm/swap_pager.c Tue Nov 28 17:46:03 2017 (r326329)
@@ -390,6 +390,7 @@ SYSCTL_INT(_vm, OID_AUTO, dmmax, CTLFLAG_RD, &nsw_clus
static void swp_sizecheck(void);
static void swp_pager_async_iodone(struct buf *bp);
+static bool swp_pager_swblk_empty(struct swblk *sb, int start, int limit);
static int swapongeom(struct vnode *);
static int swaponvp(struct thread *, struct vnode *, u_long);
static int swapoff_one(struct swdevt *sp, struct ucred *cred);
@@ -397,7 +398,7 @@ static int swapoff_one(struct swdevt *sp, struct ucred
/*
* Swap bitmap functions
*/
-static void swp_pager_freeswapspace(daddr_t blk, int npages);
+static void swp_pager_freeswapspace(daddr_t blk, daddr_t npages);
static daddr_t swp_pager_getswapspace(int npages);
/*
@@ -768,10 +769,12 @@ swp_pager_strategy(struct buf *bp)
* This routine may not sleep.
*/
static void
-swp_pager_freeswapspace(daddr_t blk, int npages)
+swp_pager_freeswapspace(daddr_t blk, daddr_t npages)
{
struct swdevt *sp;
+ if (npages == 0)
+ return;
mtx_lock(&sw_dev_mtx);
TAILQ_FOREACH(sp, &swtailq, sw_list) {
if (blk >= sp->sw_first && blk < sp->sw_end) {
@@ -1761,6 +1764,22 @@ next_obj:
*/
/*
+ * SWP_PAGER_SWBLK_EMPTY() - is a range of blocks free?
+ */
+static bool
+swp_pager_swblk_empty(struct swblk *sb, int start, int limit)
+{
+ int i;
+
+ MPASS(0 <= start && start <= limit && limit <= SWAP_META_PAGES);
+ for (i = start; i < limit; i++) {
+ if (sb->d[i] != SWAPBLK_NONE)
+ return (false);
+ }
+ return (true);
+}
+
+/*
* SWP_PAGER_META_BUILD() - add swap block to swap meta data for object
*
* We first convert the object to a swap object if it is a default
@@ -1876,16 +1895,10 @@ allocated:
/*
* Free the swblk if we end up with the empty page run.
*/
- if (swapblk == SWAPBLK_NONE) {
- for (i = 0; i < SWAP_META_PAGES; i++) {
- if (sb->d[i] != SWAPBLK_NONE)
- break;
- }
- if (i == SWAP_META_PAGES) {
- SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,
- rdpi);
- uma_zfree(swblk_zone, sb);
- }
+ if (swapblk == SWAPBLK_NONE &&
+ swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {
+ SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rdpi);
+ uma_zfree(swblk_zone, sb);
}
}
@@ -1903,37 +1916,46 @@ static void
swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count)
{
struct swblk *sb;
+ daddr_t first_free, num_free;
vm_pindex_t last;
- int i;
- bool empty;
+ int i, limit, start;
VM_OBJECT_ASSERT_WLOCKED(object);
if (object->type != OBJT_SWAP || count == 0)
return;
- last = pindex + count - 1;
+ first_free = SWAPBLK_NONE;
+ num_free = 0;
+ last = pindex + count;
for (;;) {
sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks,
rounddown(pindex, SWAP_META_PAGES));
- if (sb == NULL || sb->p > last)
+ if (sb == NULL || sb->p >= last)
break;
- empty = true;
- for (i = 0; i < SWAP_META_PAGES; i++) {
+ start = pindex > sb->p ? pindex - sb->p : 0;
+ limit = last - sb->p < SWAP_META_PAGES ? last - sb->p :
+ SWAP_META_PAGES;
+ for (i = start; i < limit; i++) {
if (sb->d[i] == SWAPBLK_NONE)
continue;
- if (pindex <= sb->p + i && sb->p + i <= last) {
- swp_pager_freeswapspace(sb->d[i], 1);
- sb->d[i] = SWAPBLK_NONE;
- } else
- empty = false;
+ if (first_free + num_free == sb->d[i])
+ num_free++;
+ else {
+ swp_pager_freeswapspace(first_free, num_free);
+ first_free = sb->d[i];
+ num_free = 1;
+ }
+ sb->d[i] = SWAPBLK_NONE;
}
- pindex = sb->p + SWAP_META_PAGES;
- if (empty) {
+ if (swp_pager_swblk_empty(sb, 0, start) &&
+ swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) {
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,
sb->p);
uma_zfree(swblk_zone, sb);
}
+ pindex = sb->p + SWAP_META_PAGES;
}
+ swp_pager_freeswapspace(first_free, num_free);
}
/*
@@ -1946,6 +1968,7 @@ static void
swp_pager_meta_free_all(vm_object_t object)
{
struct swblk *sb;
+ daddr_t first_free, num_free;
vm_pindex_t pindex;
int i;
@@ -1953,16 +1976,26 @@ swp_pager_meta_free_all(vm_object_t object)
if (object->type != OBJT_SWAP)
return;
+ first_free = SWAPBLK_NONE;
+ num_free = 0;
for (pindex = 0; (sb = SWAP_PCTRIE_LOOKUP_GE(
&object->un_pager.swp.swp_blks, pindex)) != NULL;) {
pindex = sb->p + SWAP_META_PAGES;
for (i = 0; i < SWAP_META_PAGES; i++) {
- if (sb->d[i] != SWAPBLK_NONE)
- swp_pager_freeswapspace(sb->d[i], 1);
+ if (sb->d[i] == SWAPBLK_NONE)
+ continue;
+ if (first_free + num_free == sb->d[i])
+ num_free++;
+ else {
+ swp_pager_freeswapspace(first_free, num_free);
+ first_free = sb->d[i];
+ num_free = 1;
+ }
}
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p);
uma_zfree(swblk_zone, sb);
}
+ swp_pager_freeswapspace(first_free, num_free);
}
/*
@@ -1987,7 +2020,6 @@ swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pin
{
struct swblk *sb;
daddr_t r1;
- int i;
if ((flags & (SWM_FREE | SWM_POP)) != 0)
VM_OBJECT_ASSERT_WLOCKED(object);
@@ -2010,11 +2042,7 @@ swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pin
return (SWAPBLK_NONE);
if ((flags & (SWM_FREE | SWM_POP)) != 0) {
sb->d[pindex % SWAP_META_PAGES] = SWAPBLK_NONE;
- for (i = 0; i < SWAP_META_PAGES; i++) {
- if (sb->d[i] != SWAPBLK_NONE)
- break;
- }
- if (i == SWAP_META_PAGES) {
+ if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,
rounddown(pindex, SWAP_META_PAGES));
uma_zfree(swblk_zone, sb);
More information about the svn-src-all
mailing list