svn commit: r330410 - in head/sys: amd64/amd64 arm/arm arm64/arm64 i386/i386 riscv/riscv vm
Konstantin Belousov
kib at FreeBSD.org
Sun Mar 4 20:53:21 UTC 2018
Author: kib
Date: Sun Mar 4 20:53:20 2018
New Revision: 330410
URL: https://svnweb.freebsd.org/changeset/base/330410
Log:
Unify bulk free operations in several pmaps.
Submitted by: Yoshihiro Ota
Reviewed by: markj
MFC after: 2 weeks
Differential revision: https://reviews.freebsd.org/D13485
Modified:
head/sys/amd64/amd64/pmap.c
head/sys/arm/arm/pmap-v6.c
head/sys/arm64/arm64/pmap.c
head/sys/i386/i386/pmap.c
head/sys/riscv/riscv/pmap.c
head/sys/vm/vm_page.c
head/sys/vm/vm_page.h
Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c Sun Mar 4 20:06:02 2018 (r330409)
+++ head/sys/amd64/amd64/pmap.c Sun Mar 4 20:53:20 2018 (r330410)
@@ -2369,20 +2369,6 @@ pmap_qremove(vm_offset_t sva, int count)
/***************************************************
* Page table page management routines.....
***************************************************/
-static __inline void
-pmap_free_zero_pages(struct spglist *free)
-{
- vm_page_t m;
- int count;
-
- for (count = 0; (m = SLIST_FIRST(free)) != NULL; count++) {
- SLIST_REMOVE_HEAD(free, plinks.s.ss);
- /* Preserve the page's PG_ZERO setting. */
- vm_page_free_toq(m);
- }
- vm_wire_sub(count);
-}
-
/*
* Schedule the specified unused page table page to be freed. Specifically,
* add the page to the specified list of pages that will be released to the
@@ -3282,7 +3268,7 @@ next_chunk:
/* Recycle a freed page table page. */
m_pc->wire_count = 1;
}
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, true);
return (m_pc);
}
@@ -3781,7 +3767,7 @@ pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, v
pmap_remove_pde(pmap, pde, sva, &free, lockp);
if ((oldpde & PG_G) == 0)
pmap_invalidate_pde_page(pmap, sva, oldpde);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, true);
CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#lx"
" in pmap %p", va, pmap);
return (FALSE);
@@ -4183,7 +4169,7 @@ out:
pmap_invalidate_all(pmap);
PMAP_UNLOCK(pmap);
pmap_delayed_invl_finished();
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, true);
}
/*
@@ -4281,7 +4267,7 @@ retry:
vm_page_aflag_clear(m, PGA_WRITEABLE);
rw_wunlock(lock);
pmap_delayed_invl_wait(m);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, true);
}
/*
@@ -4949,7 +4935,7 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t
pmap_invalidate_all(pmap);
pmap_delayed_invl_finished();
}
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, true);
if (va >= VM_MAXUSER_ADDRESS) {
mt = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
if (pmap_insert_pt_page(pmap, mt)) {
@@ -4978,7 +4964,7 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t
* pages. Invalidate those entries.
*/
pmap_invalidate_page(pmap, va);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, true);
}
CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
" in pmap %p", va, pmap);
@@ -5159,7 +5145,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, v
* pages. Invalidate those entries.
*/
pmap_invalidate_page(pmap, va);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, true);
}
mpte = NULL;
}
@@ -5537,7 +5523,8 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_
*/
pmap_invalidate_page(dst_pmap,
addr);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free,
+ true);
}
goto out;
}
@@ -5955,7 +5942,7 @@ pmap_remove_pages(pmap_t pmap)
rw_wunlock(lock);
pmap_invalidate_all(pmap);
PMAP_UNLOCK(pmap);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, true);
}
static boolean_t
@@ -6440,7 +6427,7 @@ small_mappings:
not_cleared < PMAP_TS_REFERENCED_MAX);
out:
rw_wunlock(lock);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, true);
return (cleared + not_cleared);
}
Modified: head/sys/arm/arm/pmap-v6.c
==============================================================================
--- head/sys/arm/arm/pmap-v6.c Sun Mar 4 20:06:02 2018 (r330409)
+++ head/sys/arm/arm/pmap-v6.c Sun Mar 4 20:53:20 2018 (r330410)
@@ -2550,18 +2550,6 @@ retry:
return (m);
}
-static __inline void
-pmap_free_zero_pages(struct spglist *free)
-{
- vm_page_t m;
-
- while ((m = SLIST_FIRST(free)) != NULL) {
- SLIST_REMOVE_HEAD(free, plinks.s.ss);
- /* Preserve the page's PG_ZERO setting. */
- vm_page_free_toq(m);
- }
-}
-
/*
* Schedule the specified unused L2 page table page to be freed. Specifically,
* add the page to the specified list of pages that will be released to the
@@ -2948,7 +2936,7 @@ out:
m_pc->wire_count = 1;
vm_wire_add(1);
}
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, false);
return (m_pc);
}
@@ -3711,7 +3699,7 @@ pmap_demote_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_o
VM_ALLOC_NORMAL | VM_ALLOC_WIRED)) == NULL) {
SLIST_INIT(&free);
pmap_remove_pte1(pmap, pte1p, pte1_trunc(va), &free);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, false);
CTR3(KTR_PMAP, "%s: failure for va %#x in pmap %p",
__func__, va, pmap);
return (FALSE);
@@ -4235,7 +4223,7 @@ out:
sched_unpin();
rw_wunlock(&pvh_global_lock);
PMAP_UNLOCK(pmap);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, false);
}
/*
@@ -4309,7 +4297,7 @@ small_mappings:
vm_page_aflag_clear(m, PGA_WRITEABLE);
sched_unpin();
rw_wunlock(&pvh_global_lock);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, false);
}
/*
@@ -4496,7 +4484,7 @@ pmap_remove_pages(pmap_t pmap)
sched_unpin();
rw_wunlock(&pvh_global_lock);
PMAP_UNLOCK(pmap);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, false);
}
/*
@@ -4605,7 +4593,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, v
SLIST_INIT(&free);
if (pmap_unwire_pt2(pmap, va, mpt2pg, &free)) {
pmap_tlb_flush(pmap, va);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, false);
}
mpt2pg = NULL;
@@ -6079,7 +6067,8 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_
if (pmap_unwire_pt2(dst_pmap, addr,
dst_mpt2pg, &free)) {
pmap_tlb_flush(dst_pmap, addr);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free,
+ false);
}
goto out;
}
Modified: head/sys/arm64/arm64/pmap.c
==============================================================================
--- head/sys/arm64/arm64/pmap.c Sun Mar 4 20:06:02 2018 (r330409)
+++ head/sys/arm64/arm64/pmap.c Sun Mar 4 20:53:20 2018 (r330410)
@@ -1266,18 +1266,6 @@ pmap_qremove(vm_offset_t sva, int count)
/***************************************************
* Page table page management routines.....
***************************************************/
-static __inline void
-pmap_free_zero_pages(struct spglist *free)
-{
- vm_page_t m;
-
- while ((m = SLIST_FIRST(free)) != NULL) {
- SLIST_REMOVE_HEAD(free, plinks.s.ss);
- /* Preserve the page's PG_ZERO setting. */
- vm_page_free_toq(m);
- }
-}
-
/*
* Schedule the specified unused page table page to be freed. Specifically,
* add the page to the specified list of pages that will be released to the
@@ -1909,7 +1897,7 @@ reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **l
m_pc->wire_count = 1;
vm_wire_add(1);
}
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, false);
return (m_pc);
}
@@ -2417,7 +2405,7 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t
if (lock != NULL)
rw_wunlock(lock);
PMAP_UNLOCK(pmap);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, false);
}
/*
@@ -2522,7 +2510,7 @@ retry:
}
vm_page_aflag_clear(m, PGA_WRITEABLE);
rw_wunlock(lock);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, false);
}
/*
@@ -3230,7 +3218,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, v
SLIST_INIT(&free);
if (pmap_unwire_l3(pmap, va, mpte, &free)) {
pmap_invalidate_page(pmap, va);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, false);
}
mpte = NULL;
}
@@ -3750,7 +3738,7 @@ pmap_remove_pages(pmap_t pmap)
if (lock != NULL)
rw_wunlock(lock);
PMAP_UNLOCK(pmap);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, false);
}
/*
@@ -4211,7 +4199,7 @@ small_mappings:
not_cleared < PMAP_TS_REFERENCED_MAX);
out:
rw_wunlock(lock);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, false);
return (cleared + not_cleared);
}
Modified: head/sys/i386/i386/pmap.c
==============================================================================
--- head/sys/i386/i386/pmap.c Sun Mar 4 20:06:02 2018 (r330409)
+++ head/sys/i386/i386/pmap.c Sun Mar 4 20:53:20 2018 (r330410)
@@ -1711,20 +1711,6 @@ pmap_qremove(vm_offset_t sva, int count)
/***************************************************
* Page table page management routines.....
***************************************************/
-static __inline void
-pmap_free_zero_pages(struct spglist *free)
-{
- vm_page_t m;
- int count;
-
- for (count = 0; (m = SLIST_FIRST(free)) != NULL; count++) {
- SLIST_REMOVE_HEAD(free, plinks.s.ss);
- /* Preserve the page's PG_ZERO setting. */
- vm_page_free_toq(m);
- }
- vm_wire_sub(count);
-}
-
/*
* Schedule the specified unused page table page to be freed. Specifically,
* add the page to the specified list of pages that will be released to the
@@ -2324,7 +2310,7 @@ out:
/* Recycle a freed page table page. */
m_pc->wire_count = 1;
}
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, true);
return (m_pc);
}
@@ -2674,7 +2660,7 @@ pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offse
pmap_remove_pde(pmap, pde, sva, &free);
if ((oldpde & PG_G) == 0)
pmap_invalidate_pde_page(pmap, sva, oldpde);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, true);
CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#x"
" in pmap %p", va, pmap);
return (FALSE);
@@ -3044,7 +3030,7 @@ out:
pmap_invalidate_all(pmap);
rw_wunlock(&pvh_global_lock);
PMAP_UNLOCK(pmap);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, true);
}
/*
@@ -3118,7 +3104,7 @@ small_mappings:
vm_page_aflag_clear(m, PGA_WRITEABLE);
sched_unpin();
rw_wunlock(&pvh_global_lock);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, true);
}
/*
@@ -3886,7 +3872,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, v
SLIST_INIT(&free);
if (pmap_unwire_ptp(pmap, mpte, &free)) {
pmap_invalidate_page(pmap, va);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, true);
}
mpte = NULL;
@@ -4201,7 +4187,8 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_
&free)) {
pmap_invalidate_page(dst_pmap,
addr);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free,
+ true);
}
goto out;
}
@@ -4619,7 +4606,7 @@ pmap_remove_pages(pmap_t pmap)
pmap_invalidate_all(pmap);
rw_wunlock(&pvh_global_lock);
PMAP_UNLOCK(pmap);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, true);
}
/*
Modified: head/sys/riscv/riscv/pmap.c
==============================================================================
--- head/sys/riscv/riscv/pmap.c Sun Mar 4 20:06:02 2018 (r330409)
+++ head/sys/riscv/riscv/pmap.c Sun Mar 4 20:53:20 2018 (r330410)
@@ -1069,18 +1069,6 @@ pmap_qremove(vm_offset_t sva, int count)
/***************************************************
* Page table page management routines.....
***************************************************/
-static __inline void
-pmap_free_zero_pages(struct spglist *free)
-{
- vm_page_t m;
-
- while ((m = SLIST_FIRST(free)) != NULL) {
- SLIST_REMOVE_HEAD(free, plinks.s.ss);
- /* Preserve the page's PG_ZERO setting. */
- vm_page_free_toq(m);
- }
-}
-
/*
* Schedule the specified unused page table page to be freed. Specifically,
* add the page to the specified list of pages that will be released to the
@@ -1876,7 +1864,7 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t
rw_wunlock(lock);
rw_runlock(&pvh_global_lock);
PMAP_UNLOCK(pmap);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, false);
}
/*
@@ -1942,7 +1930,7 @@ pmap_remove_all(vm_page_t m)
}
vm_page_aflag_clear(m, PGA_WRITEABLE);
rw_wunlock(&pvh_global_lock);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, false);
}
/*
@@ -2377,7 +2365,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, v
SLIST_INIT(&free);
if (pmap_unwire_l3(pmap, va, mpte, &free)) {
pmap_invalidate_page(pmap, va);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, false);
}
mpte = NULL;
}
@@ -2783,7 +2771,7 @@ pmap_remove_pages(pmap_t pmap)
rw_wunlock(lock);
rw_runlock(&pvh_global_lock);
PMAP_UNLOCK(pmap);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, false);
}
/*
@@ -3085,7 +3073,7 @@ retry:
out:
rw_wunlock(lock);
rw_runlock(&pvh_global_lock);
- pmap_free_zero_pages(&free);
+ vm_page_free_pages_toq(&free, false);
return (cleared + not_cleared);
}
Modified: head/sys/vm/vm_page.c
==============================================================================
--- head/sys/vm/vm_page.c Sun Mar 4 20:06:02 2018 (r330409)
+++ head/sys/vm/vm_page.c Sun Mar 4 20:53:20 2018 (r330410)
@@ -2538,17 +2538,7 @@ unlock:
}
if (m_mtx != NULL)
mtx_unlock(m_mtx);
- if ((m = SLIST_FIRST(&free)) != NULL) {
- vmd = VM_DOMAIN(domain);
- vm_domain_free_lock(vmd);
- do {
- MPASS(vm_phys_domain(m) == domain);
- SLIST_REMOVE_HEAD(&free, plinks.s.ss);
- vm_page_free_phys(vmd, m);
- } while ((m = SLIST_FIRST(&free)) != NULL);
- vm_domain_free_wakeup(vmd);
- vm_domain_free_unlock(vmd);
- }
+ vm_page_free_pages_toq(&free, false);
return (error);
}
@@ -3248,7 +3238,42 @@ vm_page_free_toq(vm_page_t m)
}
/*
- * vm_page_wire:
+ * vm_page_free_pages_toq:
+ *
+ * Returns a list of pages to the free list, disassociating it
+ * from any VM object. In other words, this is equivalent to
+ * calling vm_page_free_toq() for each page of a list of VM objects.
+ *
+ * The objects must be locked. The pages must be locked if it is
+ * managed.
+ */
+void
+vm_page_free_pages_toq(struct spglist *free, bool update_wire_count)
+{
+ vm_page_t m;
+ struct pglist pgl;
+ int count;
+
+ if (SLIST_EMPTY(free))
+ return;
+
+ count = 0;
+ TAILQ_INIT(&pgl);
+ while ((m = SLIST_FIRST(free)) != NULL) {
+ count++;
+ SLIST_REMOVE_HEAD(free, plinks.s.ss);
+ if (vm_page_free_prep(m, false))
+ TAILQ_INSERT_TAIL(&pgl, m, listq);
+ }
+
+ vm_page_free_phys_pglist(&pgl);
+
+ if (update_wire_count && count > 0)
+ vm_wire_sub(count);
+}
+
+/*
+ * vm_page_wire:
*
* Mark this page as wired down. If the page is fictitious, then
* its wire count must remain one.
Modified: head/sys/vm/vm_page.h
==============================================================================
--- head/sys/vm/vm_page.h Sun Mar 4 20:06:02 2018 (r330409)
+++ head/sys/vm/vm_page.h Sun Mar 4 20:53:20 2018 (r330410)
@@ -535,6 +535,7 @@ void vm_page_test_dirty (vm_page_t);
vm_page_bits_t vm_page_bits(int base, int size);
void vm_page_zero_invalid(vm_page_t m, boolean_t setvalid);
void vm_page_free_toq(vm_page_t m);
+void vm_page_free_pages_toq(struct spglist *free, bool update_wire_count);
void vm_page_dirty_KBI(vm_page_t m);
void vm_page_lock_KBI(vm_page_t m, const char *file, int line);
More information about the svn-src-head
mailing list