svn commit: r207410 - in head: . sys/amd64/amd64 sys/amd64/include
sys/arm/arm sys/arm/include sys/dev/drm sys/i386/i386
sys/i386/include sys/i386/xen sys/ia64/ia64 sys/ia64/include
sys/kern sys/mi...
Kip Macy
kmacy at FreeBSD.org
Fri Apr 30 00:46:44 UTC 2010
Author: kmacy
Date: Fri Apr 30 00:46:43 2010
New Revision: 207410
URL: http://svn.freebsd.org/changeset/base/207410
Log:
On Alan's advice, rather than do a wholesale conversion on a single
architecture from page queue lock to a hashed array of page locks
(based on a patch by Jeff Roberson), I've implemented page lock
support in the MI code and have only moved vm_page's hold_count
out from under page queue mutex to page lock. This changes
pmap_extract_and_hold on all pmaps.
Supported by: Bitgravity Inc.
Discussed with: alc, jeffr, and kib
Modified:
head/UPDATING
head/sys/amd64/amd64/pmap.c
head/sys/amd64/include/pmap.h
head/sys/amd64/include/vmparam.h
head/sys/arm/arm/pmap.c
head/sys/arm/include/pmap.h
head/sys/dev/drm/via_dmablit.c
head/sys/i386/i386/pmap.c
head/sys/i386/include/pmap.h
head/sys/i386/xen/pmap.c
head/sys/ia64/ia64/pmap.c
head/sys/ia64/include/pmap.h
head/sys/kern/kern_exec.c
head/sys/kern/subr_witness.c
head/sys/kern/sys_pipe.c
head/sys/kern/sys_process.c
head/sys/kern/uipc_cow.c
head/sys/kern/vfs_bio.c
head/sys/mips/include/pmap.h
head/sys/mips/mips/pmap.c
head/sys/net/bpf_zerocopy.c
head/sys/powerpc/aim/mmu_oea.c
head/sys/powerpc/aim/mmu_oea64.c
head/sys/powerpc/booke/pmap.c
head/sys/powerpc/include/pmap.h
head/sys/sparc64/include/pmap.h
head/sys/sparc64/sparc64/pmap.c
head/sys/sun4v/include/pmap.h
head/sys/sun4v/sun4v/pmap.c
head/sys/sys/param.h
head/sys/vm/device_pager.c
head/sys/vm/sg_pager.c
head/sys/vm/swap_pager.c
head/sys/vm/uma_core.c
head/sys/vm/vm_contig.c
head/sys/vm/vm_fault.c
head/sys/vm/vm_glue.c
head/sys/vm/vm_kern.c
head/sys/vm/vm_mmap.c
head/sys/vm/vm_object.c
head/sys/vm/vm_page.c
head/sys/vm/vm_page.h
head/sys/vm/vm_pageout.c
head/sys/vm/vm_param.h
head/sys/vm/vnode_pager.c
Modified: head/UPDATING
==============================================================================
--- head/UPDATING Fri Apr 30 00:34:00 2010 (r207409)
+++ head/UPDATING Fri Apr 30 00:46:43 2010 (r207410)
@@ -22,6 +22,14 @@ NOTE TO PEOPLE WHO THINK THAT FreeBSD 9.
machines to maximize performance. (To disable malloc debugging, run
ln -s aj /etc/malloc.conf.)
+
+20100429:
+ 'vm_page's are now hashed by physical address to an array of mutexes.
+ Currently this is only used to serialize access to hold_count. Over
+ time the page queue mutex will be peeled away. This changes the size
+ of pmap on every architecture. And requires all callers of vm_page_hold
+ and vm_page_unhold to be updated.
+
20100402:
WITH_CTF can now be specified in src.conf (not recommended, there
are some problems with static executables), make.conf (would also
Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/amd64/amd64/pmap.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -793,7 +793,6 @@ static u_long pmap_pdpe_demotions;
SYSCTL_ULONG(_vm_pmap_pdpe, OID_AUTO, demotions, CTLFLAG_RD,
&pmap_pdpe_demotions, 0, "1GB page demotions");
-
/***************************************************
* Low level helper routines.....
***************************************************/
@@ -1200,15 +1199,20 @@ pmap_extract_and_hold(pmap_t pmap, vm_of
{
pd_entry_t pde, *pdep;
pt_entry_t pte;
+ vm_paddr_t pa;
vm_page_t m;
+ pa = 0;
m = NULL;
- vm_page_lock_queues();
PMAP_LOCK(pmap);
+retry:
pdep = pmap_pde(pmap, va);
if (pdep != NULL && (pde = *pdep)) {
if (pde & PG_PS) {
if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) {
+ if (vm_page_pa_tryrelock(pmap, (pde & PG_PS_FRAME) |
+ (va & PDRMASK), &pa))
+ goto retry;
m = PHYS_TO_VM_PAGE((pde & PG_PS_FRAME) |
(va & PDRMASK));
vm_page_hold(m);
@@ -1217,12 +1221,14 @@ pmap_extract_and_hold(pmap_t pmap, vm_of
pte = *pmap_pde_to_pte(pdep, va);
if ((pte & PG_V) &&
((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) {
+ if (vm_page_pa_tryrelock(pmap, pte & PG_FRAME, &pa))
+ goto retry;
m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
vm_page_hold(m);
}
}
}
- vm_page_unlock_queues();
+ PA_UNLOCK_COND(pa);
PMAP_UNLOCK(pmap);
return (m);
}
@@ -3143,9 +3149,8 @@ pmap_enter(pmap_t pmap, vm_offset_t va,
* In the case that a page table page is not
* resident, we are creating it here.
*/
- if (va < VM_MAXUSER_ADDRESS) {
+ if (va < VM_MAXUSER_ADDRESS)
mpte = pmap_allocpte(pmap, va, M_WAITOK);
- }
pde = pmap_pde(pmap, va);
if (pde != NULL && (*pde & PG_V) != 0) {
@@ -3393,7 +3398,7 @@ pmap_enter_object(pmap_t pmap, vm_offset
mpte);
m = TAILQ_NEXT(m, listq);
}
- PMAP_UNLOCK(pmap);
+ PMAP_UNLOCK(pmap);
}
/*
Modified: head/sys/amd64/include/pmap.h
==============================================================================
--- head/sys/amd64/include/pmap.h Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/amd64/include/pmap.h Fri Apr 30 00:46:43 2010 (r207410)
@@ -245,6 +245,8 @@ struct pmap {
pml4_entry_t *pm_pml4; /* KVA of level 4 page table */
TAILQ_HEAD(,pv_chunk) pm_pvchunk; /* list of mappings in pmap */
u_int pm_active; /* active on cpus */
+ uint32_t pm_gen_count; /* generation count (pmap lock dropped) */
+ u_int pm_retries;
/* spare u_int here due to padding */
struct pmap_statistics pm_stats; /* pmap statistics */
vm_page_t pm_root; /* spare page table pages */
Modified: head/sys/amd64/include/vmparam.h
==============================================================================
--- head/sys/amd64/include/vmparam.h Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/amd64/include/vmparam.h Fri Apr 30 00:46:43 2010 (r207410)
@@ -145,6 +145,10 @@
#define VM_LEVEL_0_ORDER 9
#endif
+#ifdef SMP
+#define PA_LOCK_COUNT 256
+#endif
+
/*
* Virtual addresses of things. Derived from the page directory and
* page table indexes from pmap.h for precision.
Modified: head/sys/arm/arm/pmap.c
==============================================================================
--- head/sys/arm/arm/pmap.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/arm/arm/pmap.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -3740,13 +3740,14 @@ pmap_extract_and_hold(pmap_t pmap, vm_of
struct l2_dtable *l2;
pd_entry_t l1pd;
pt_entry_t *ptep, pte;
- vm_paddr_t pa;
+ vm_paddr_t pa, paddr;
vm_page_t m = NULL;
u_int l1idx;
l1idx = L1_IDX(va);
+ paddr = 0;
- vm_page_lock_queues();
PMAP_LOCK(pmap);
+retry:
l1pd = pmap->pm_l1->l1_kva[l1idx];
if (l1pte_section_p(l1pd)) {
/*
@@ -3758,6 +3759,8 @@ pmap_extract_and_hold(pmap_t pmap, vm_of
pa = (l1pd & L1_SUP_FRAME) | (va & L1_SUP_OFFSET);
else
pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET);
+ if (vm_page_pa_tryrelock(pmap, pa & PG_FRAME, &paddr))
+ goto retry;
if (l1pd & L1_S_PROT_W || (prot & VM_PROT_WRITE) == 0) {
m = PHYS_TO_VM_PAGE(pa);
vm_page_hold(m);
@@ -3774,7 +3777,6 @@ pmap_extract_and_hold(pmap_t pmap, vm_of
if (l2 == NULL ||
(ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) {
PMAP_UNLOCK(pmap);
- vm_page_unlock_queues();
return (NULL);
}
@@ -3783,7 +3785,6 @@ pmap_extract_and_hold(pmap_t pmap, vm_of
if (pte == 0) {
PMAP_UNLOCK(pmap);
- vm_page_unlock_queues();
return (NULL);
}
if (pte & L2_S_PROT_W || (prot & VM_PROT_WRITE) == 0) {
@@ -3796,13 +3797,15 @@ pmap_extract_and_hold(pmap_t pmap, vm_of
pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET);
break;
}
+ if (vm_page_pa_tryrelock(pmap, pa & PG_FRAME, &paddr))
+ goto retry;
m = PHYS_TO_VM_PAGE(pa);
vm_page_hold(m);
}
}
PMAP_UNLOCK(pmap);
- vm_page_unlock_queues();
+ PA_UNLOCK_COND(paddr);
return (m);
}
Modified: head/sys/arm/include/pmap.h
==============================================================================
--- head/sys/arm/include/pmap.h Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/arm/include/pmap.h Fri Apr 30 00:46:43 2010 (r207410)
@@ -134,6 +134,8 @@ struct pmap {
struct l1_ttable *pm_l1;
struct l2_dtable *pm_l2[L2_SIZE];
pd_entry_t *pm_pdir; /* KVA of page directory */
+ uint32_t pm_gen_count; /* generation count (pmap lock dropped) */
+ u_int pm_retries;
int pm_active; /* active on cpus */
struct pmap_statistics pm_stats; /* pmap statictics */
TAILQ_HEAD(,pv_entry) pm_pvlist; /* list of mappings in pmap */
Modified: head/sys/dev/drm/via_dmablit.c
==============================================================================
--- head/sys/dev/drm/via_dmablit.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/dev/drm/via_dmablit.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -248,10 +248,12 @@ via_lock_all_dma_pages(drm_via_sg_info_t
(vm_offset_t)xfer->mem_addr + IDX_TO_OFF(i), VM_PROT_RW);
if (m == NULL)
break;
+ vm_page_lock(m);
vm_page_lock_queues();
vm_page_wire(m);
vm_page_unhold(m);
vm_page_unlock_queues();
+ vm_page_unlock(m);
vsg->pages[i] = m;
}
vsg->state = dr_via_pages_locked;
Modified: head/sys/i386/i386/pmap.c
==============================================================================
--- head/sys/i386/i386/pmap.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/i386/i386/pmap.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -1346,14 +1346,19 @@ pmap_extract_and_hold(pmap_t pmap, vm_of
pd_entry_t pde;
pt_entry_t pte;
vm_page_t m;
+ vm_paddr_t pa;
+ pa = 0;
m = NULL;
- vm_page_lock_queues();
PMAP_LOCK(pmap);
+retry:
pde = *pmap_pde(pmap, va);
if (pde != 0) {
if (pde & PG_PS) {
if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) {
+ if (vm_page_pa_tryrelock(pmap, (pde & PG_PS_FRAME) |
+ (va & PDRMASK), &pa))
+ goto retry;
m = PHYS_TO_VM_PAGE((pde & PG_PS_FRAME) |
(va & PDRMASK));
vm_page_hold(m);
@@ -1363,13 +1368,15 @@ pmap_extract_and_hold(pmap_t pmap, vm_of
pte = *pmap_pte_quick(pmap, va);
if (pte != 0 &&
((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) {
+ if (vm_page_pa_tryrelock(pmap, pte & PG_FRAME, &pa))
+ goto retry;
m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
vm_page_hold(m);
}
sched_unpin();
}
}
- vm_page_unlock_queues();
+ PA_UNLOCK_COND(pa);
PMAP_UNLOCK(pmap);
return (m);
}
Modified: head/sys/i386/include/pmap.h
==============================================================================
--- head/sys/i386/include/pmap.h Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/i386/include/pmap.h Fri Apr 30 00:46:43 2010 (r207410)
@@ -420,11 +420,14 @@ struct pmap {
u_int pm_active; /* active on cpus */
struct pmap_statistics pm_stats; /* pmap statistics */
LIST_ENTRY(pmap) pm_list; /* List of all pmaps */
+ uint32_t pm_gen_count; /* generation count (pmap lock dropped) */
+ u_int pm_retries;
#ifdef PAE
pdpt_entry_t *pm_pdpt; /* KVA of page director pointer
table */
#endif
vm_page_t pm_root; /* spare page table pages */
+
};
typedef struct pmap *pmap_t;
Modified: head/sys/i386/xen/pmap.c
==============================================================================
--- head/sys/i386/xen/pmap.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/i386/xen/pmap.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -1219,14 +1219,19 @@ pmap_extract_and_hold(pmap_t pmap, vm_of
pd_entry_t pde;
pt_entry_t pte;
vm_page_t m;
+ vm_paddr_t pa;
+ pa = 0;
m = NULL;
- vm_page_lock_queues();
PMAP_LOCK(pmap);
+retry:
pde = PT_GET(pmap_pde(pmap, va));
if (pde != 0) {
if (pde & PG_PS) {
if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) {
+ if (vm_page_pa_tryrelock(pmap, (pde & PG_PS_FRAME) |
+ (va & PDRMASK), &pa))
+ goto retry;
m = PHYS_TO_VM_PAGE((pde & PG_PS_FRAME) |
(va & PDRMASK));
vm_page_hold(m);
@@ -1238,13 +1243,15 @@ pmap_extract_and_hold(pmap_t pmap, vm_of
PT_SET_MA(PADDR1, 0);
if ((pte & PG_V) &&
((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) {
+ if (vm_page_pa_tryrelock(pmap, pte & PG_FRAME, &pa))
+ goto retry;
m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
vm_page_hold(m);
}
sched_unpin();
}
}
- vm_page_unlock_queues();
+ PA_UNLOCK_COND(pa);
PMAP_UNLOCK(pmap);
return (m);
}
Modified: head/sys/ia64/ia64/pmap.c
==============================================================================
--- head/sys/ia64/ia64/pmap.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/ia64/ia64/pmap.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -1028,18 +1028,22 @@ pmap_extract_and_hold(pmap_t pmap, vm_of
struct ia64_lpte *pte;
pmap_t oldpmap;
vm_page_t m;
+ vm_paddr_t pa;
+ pa = 0;
m = NULL;
- vm_page_lock_queues();
PMAP_LOCK(pmap);
oldpmap = pmap_switch(pmap);
+retry:
pte = pmap_find_vhpt(va);
if (pte != NULL && pmap_present(pte) &&
(pmap_prot(pte) & prot) == prot) {
m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
+ if (vm_page_pa_tryrelock(pmap, pmap_ppn(pte), &pa))
+ goto retry;
vm_page_hold(m);
}
- vm_page_unlock_queues();
+ PA_UNLOCK_COND(pa);
pmap_switch(oldpmap);
PMAP_UNLOCK(pmap);
return (m);
Modified: head/sys/ia64/include/pmap.h
==============================================================================
--- head/sys/ia64/include/pmap.h Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/ia64/include/pmap.h Fri Apr 30 00:46:43 2010 (r207410)
@@ -77,6 +77,8 @@ struct pmap {
TAILQ_HEAD(,pv_entry) pm_pvlist; /* list of mappings in pmap */
u_int32_t pm_rid[5]; /* base RID for pmap */
struct pmap_statistics pm_stats; /* pmap statistics */
+ uint32_t pm_gen_count; /* generation count (pmap lock dropped) */
+ u_int pm_retries;
};
typedef struct pmap *pmap_t;
Modified: head/sys/kern/kern_exec.c
==============================================================================
--- head/sys/kern/kern_exec.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/kern/kern_exec.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -957,9 +957,9 @@ exec_map_first_page(imgp)
return (EIO);
}
}
- vm_page_lock_queues();
+ vm_page_lock(ma[0]);
vm_page_hold(ma[0]);
- vm_page_unlock_queues();
+ vm_page_unlock(ma[0]);
vm_page_wakeup(ma[0]);
VM_OBJECT_UNLOCK(object);
@@ -979,9 +979,9 @@ exec_unmap_first_page(imgp)
m = sf_buf_page(imgp->firstpage);
sf_buf_free(imgp->firstpage);
imgp->firstpage = NULL;
- vm_page_lock_queues();
+ vm_page_lock(m);
vm_page_unhold(m);
- vm_page_unlock_queues();
+ vm_page_unlock(m);
}
}
Modified: head/sys/kern/subr_witness.c
==============================================================================
--- head/sys/kern/subr_witness.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/kern/subr_witness.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -597,6 +597,15 @@ static struct witness_order_list_entry o
{ "cdev", &lock_class_mtx_sleep },
{ NULL, NULL },
/*
+ * VM
+ *
+ */
+ { "vm object", &lock_class_mtx_sleep },
+ { "page lock", &lock_class_mtx_sleep },
+ { "vm page queue mutex", &lock_class_mtx_sleep },
+ { "pmap", &lock_class_mtx_sleep },
+ { NULL, NULL },
+ /*
* kqueue/VFS interaction
*/
{ "kqueue", &lock_class_mtx_sleep },
Modified: head/sys/kern/sys_pipe.c
==============================================================================
--- head/sys/kern/sys_pipe.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/kern/sys_pipe.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -773,10 +773,12 @@ pipe_build_write_buffer(wpipe, uio)
*/
race:
if (vm_fault_quick((caddr_t)addr, VM_PROT_READ) < 0) {
- vm_page_lock_queues();
- for (j = 0; j < i; j++)
+
+ for (j = 0; j < i; j++) {
+ vm_page_lock(wpipe->pipe_map.ms[j]);
vm_page_unhold(wpipe->pipe_map.ms[j]);
- vm_page_unlock_queues();
+ vm_page_unlock(wpipe->pipe_map.ms[j]);
+ }
return (EFAULT);
}
wpipe->pipe_map.ms[i] = pmap_extract_and_hold(pmap, addr,
@@ -816,11 +818,11 @@ pipe_destroy_write_buffer(wpipe)
int i;
PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
- vm_page_lock_queues();
for (i = 0; i < wpipe->pipe_map.npages; i++) {
+ vm_page_lock(wpipe->pipe_map.ms[i]);
vm_page_unhold(wpipe->pipe_map.ms[i]);
+ vm_page_unlock(wpipe->pipe_map.ms[i]);
}
- vm_page_unlock_queues();
wpipe->pipe_map.npages = 0;
}
Modified: head/sys/kern/sys_process.c
==============================================================================
--- head/sys/kern/sys_process.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/kern/sys_process.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -328,9 +328,9 @@ proc_rwmem(struct proc *p, struct uio *u
/*
* Hold the page in memory.
*/
- vm_page_lock_queues();
+ vm_page_lock(m);
vm_page_hold(m);
- vm_page_unlock_queues();
+ vm_page_unlock(m);
/*
* We're done with tmap now.
@@ -349,9 +349,9 @@ proc_rwmem(struct proc *p, struct uio *u
/*
* Release the page.
*/
- vm_page_lock_queues();
+ vm_page_lock(m);
vm_page_unhold(m);
- vm_page_unlock_queues();
+ vm_page_unlock(m);
} while (error == 0 && uio->uio_resid > 0);
Modified: head/sys/kern/uipc_cow.c
==============================================================================
--- head/sys/kern/uipc_cow.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/kern/uipc_cow.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -128,10 +128,12 @@ socow_setup(struct mbuf *m0, struct uio
/*
* set up COW
*/
+ vm_page_lock(pp);
vm_page_lock_queues();
if (vm_page_cowsetup(pp) != 0) {
vm_page_unhold(pp);
vm_page_unlock_queues();
+ vm_page_unlock(pp);
return (0);
}
@@ -141,7 +143,7 @@ socow_setup(struct mbuf *m0, struct uio
vm_page_wire(pp);
vm_page_unhold(pp);
vm_page_unlock_queues();
-
+ vm_page_unlock(pp);
/*
* Allocate an sf buf
*/
Modified: head/sys/kern/vfs_bio.c
==============================================================================
--- head/sys/kern/vfs_bio.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/kern/vfs_bio.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -3860,12 +3860,12 @@ vmapbuf(struct buf *bp)
retry:
if (vm_fault_quick(addr >= bp->b_data ? addr : bp->b_data,
prot) < 0) {
- vm_page_lock_queues();
for (i = 0; i < pidx; ++i) {
+ vm_page_lock(bp->b_pages[i]);
vm_page_unhold(bp->b_pages[i]);
+ vm_page_unlock(bp->b_pages[i]);
bp->b_pages[i] = NULL;
}
- vm_page_unlock_queues();
return(-1);
}
m = pmap_extract_and_hold(pmap, (vm_offset_t)addr, prot);
@@ -3896,11 +3896,12 @@ vunmapbuf(struct buf *bp)
npages = bp->b_npages;
pmap_qremove(trunc_page((vm_offset_t)bp->b_data), npages);
- vm_page_lock_queues();
- for (pidx = 0; pidx < npages; pidx++)
+ for (pidx = 0; pidx < npages; pidx++) {
+ vm_page_lock(bp->b_pages[pidx]);
vm_page_unhold(bp->b_pages[pidx]);
- vm_page_unlock_queues();
-
+ vm_page_unlock(bp->b_pages[pidx]);
+ }
+
bp->b_data = bp->b_saveaddr;
}
Modified: head/sys/mips/include/pmap.h
==============================================================================
--- head/sys/mips/include/pmap.h Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/mips/include/pmap.h Fri Apr 30 00:46:43 2010 (r207410)
@@ -88,6 +88,8 @@ struct pmap {
pd_entry_t *pm_segtab; /* KVA of segment table */
TAILQ_HEAD(, pv_entry) pm_pvlist; /* list of mappings in
* pmap */
+ uint32_t pm_gen_count; /* generation count (pmap lock dropped) */
+ u_int pm_retries;
int pm_active; /* active on cpus */
struct {
u_int32_t asid:ASID_BITS; /* TLB address space tag */
Modified: head/sys/mips/mips/pmap.c
==============================================================================
--- head/sys/mips/mips/pmap.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/mips/mips/pmap.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -147,7 +147,6 @@ unsigned pmap_max_asid; /* max ASID sup
#define PMAP_ASID_RESERVED 0
-
vm_offset_t kernel_vm_end;
static struct tlb tlbstash[MAXCPU][MIPS_MAX_TLB_ENTRIES];
@@ -710,18 +709,22 @@ pmap_extract_and_hold(pmap_t pmap, vm_of
{
pt_entry_t pte;
vm_page_t m;
+ vm_paddr_t pa;
m = NULL;
- vm_page_lock_queues();
+ pa = 0;
PMAP_LOCK(pmap);
-
+retry:
pte = *pmap_pte(pmap, va);
if (pte != 0 && pmap_pte_v(&pte) &&
((pte & PTE_RW) || (prot & VM_PROT_WRITE) == 0)) {
+ if (vm_page_pa_tryrelock(pmap, mips_tlbpfn_to_paddr(pte), &pa))
+ goto retry;
+
m = PHYS_TO_VM_PAGE(mips_tlbpfn_to_paddr(pte));
vm_page_hold(m);
}
- vm_page_unlock_queues();
+ PA_UNLOCK_COND(pa);
PMAP_UNLOCK(pmap);
return (m);
}
Modified: head/sys/net/bpf_zerocopy.c
==============================================================================
--- head/sys/net/bpf_zerocopy.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/net/bpf_zerocopy.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -168,10 +168,12 @@ zbuf_sfbuf_get(struct vm_map *map, vm_of
VM_PROT_WRITE);
if (pp == NULL)
return (NULL);
+ vm_page_lock(pp);
vm_page_lock_queues();
vm_page_wire(pp);
vm_page_unhold(pp);
vm_page_unlock_queues();
+ vm_page_unlock(pp);
sf = sf_buf_alloc(pp, SFB_NOWAIT);
if (sf == NULL) {
zbuf_page_free(pp);
Modified: head/sys/powerpc/aim/mmu_oea.c
==============================================================================
--- head/sys/powerpc/aim/mmu_oea.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/powerpc/aim/mmu_oea.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -1241,18 +1241,22 @@ moea_extract_and_hold(mmu_t mmu, pmap_t
{
struct pvo_entry *pvo;
vm_page_t m;
-
+ vm_paddr_t pa;
+
m = NULL;
- vm_page_lock_queues();
+ pa = 0;
PMAP_LOCK(pmap);
+retry:
pvo = moea_pvo_find_va(pmap, va & ~ADDR_POFF, NULL);
if (pvo != NULL && (pvo->pvo_pte.pte.pte_hi & PTE_VALID) &&
((pvo->pvo_pte.pte.pte_lo & PTE_PP) == PTE_RW ||
(prot & VM_PROT_WRITE) == 0)) {
+ if (vm_page_pa_tryrelock(pmap, pvo->pvo_pte.pte.pte_lo & PTE_RPGN, &pa))
+ goto retry;
m = PHYS_TO_VM_PAGE(pvo->pvo_pte.pte.pte_lo & PTE_RPGN);
vm_page_hold(m);
}
- vm_page_unlock_queues();
+ PA_UNLOCK_COND(pa);
PMAP_UNLOCK(pmap);
return (m);
}
Modified: head/sys/powerpc/aim/mmu_oea64.c
==============================================================================
--- head/sys/powerpc/aim/mmu_oea64.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/powerpc/aim/mmu_oea64.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -1374,18 +1374,23 @@ moea64_extract_and_hold(mmu_t mmu, pmap_
{
struct pvo_entry *pvo;
vm_page_t m;
+ vm_paddr_t pa;
m = NULL;
- vm_page_lock_queues();
+ pa = 0;
PMAP_LOCK(pmap);
+retry:
pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF, NULL);
if (pvo != NULL && (pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) &&
((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) == LPTE_RW ||
(prot & VM_PROT_WRITE) == 0)) {
+ if (vm_page_pa_tryrelock(pmap,
+ pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN, &pa))
+ goto retry;
m = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
vm_page_hold(m);
}
- vm_page_unlock_queues();
+ PA_UNLOCK_COND(pa);
PMAP_UNLOCK(pmap);
return (m);
}
Modified: head/sys/powerpc/booke/pmap.c
==============================================================================
--- head/sys/powerpc/booke/pmap.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/powerpc/booke/pmap.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -2034,11 +2034,12 @@ mmu_booke_extract_and_hold(mmu_t mmu, pm
pte_t *pte;
vm_page_t m;
uint32_t pte_wbit;
-
+ vm_paddr_t pa;
+
m = NULL;
- vm_page_lock_queues();
+ pa = 0;
PMAP_LOCK(pmap);
-
+retry:
pte = pte_find(mmu, pmap, va);
if ((pte != NULL) && PTE_ISVALID(pte)) {
if (pmap == kernel_pmap)
@@ -2047,12 +2048,14 @@ mmu_booke_extract_and_hold(mmu_t mmu, pm
pte_wbit = PTE_UW;
if ((pte->flags & pte_wbit) || ((prot & VM_PROT_WRITE) == 0)) {
+ if (vm_page_pa_tryrelock(pmap, PTE_PA(pte), &pa))
+ goto retry;
m = PHYS_TO_VM_PAGE(PTE_PA(pte));
vm_page_hold(m);
}
}
- vm_page_unlock_queues();
+ PA_UNLOCK_COND(pa);
PMAP_UNLOCK(pmap);
return (m);
}
Modified: head/sys/powerpc/include/pmap.h
==============================================================================
--- head/sys/powerpc/include/pmap.h Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/powerpc/include/pmap.h Fri Apr 30 00:46:43 2010 (r207410)
@@ -88,6 +88,8 @@ struct pmap {
struct mtx pm_mtx;
u_int pm_sr[16];
u_int pm_active;
+ uint32_t pm_gen_count; /* generation count (pmap lock dropped) */
+ u_int pm_retries;
u_int pm_context;
struct pmap *pmap_phys;
Modified: head/sys/sparc64/include/pmap.h
==============================================================================
--- head/sys/sparc64/include/pmap.h Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/sparc64/include/pmap.h Fri Apr 30 00:46:43 2010 (r207410)
@@ -62,6 +62,8 @@ struct pmap {
struct tte *pm_tsb;
vm_object_t pm_tsb_obj;
u_int pm_active;
+ uint32_t pm_gen_count; /* generation count (pmap lock dropped) */
+ u_int pm_retries;
u_int pm_context[MAXCPU];
struct pmap_statistics pm_stats;
};
Modified: head/sys/sparc64/sparc64/pmap.c
==============================================================================
--- head/sys/sparc64/sparc64/pmap.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/sparc64/sparc64/pmap.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -694,13 +694,17 @@ pmap_extract_and_hold(pmap_t pm, vm_offs
{
struct tte *tp;
vm_page_t m;
+ vm_paddr_t pa;
m = NULL;
- vm_page_lock_queues();
+ pa = 0;
+ PMAP_LOCK(pm);
+retry:
if (pm == kernel_pmap) {
if (va >= VM_MIN_DIRECT_ADDRESS) {
tp = NULL;
m = PHYS_TO_VM_PAGE(TLB_DIRECT_TO_PHYS(va));
+ (void)vm_page_pa_tryrelock(pm, TLB_DIRECT_TO_PHYS(va), &pa);
vm_page_hold(m);
} else {
tp = tsb_kvtotte(va);
@@ -708,17 +712,17 @@ pmap_extract_and_hold(pmap_t pm, vm_offs
tp = NULL;
}
} else {
- PMAP_LOCK(pm);
tp = tsb_tte_lookup(pm, va);
}
if (tp != NULL && ((tp->tte_data & TD_SW) ||
(prot & VM_PROT_WRITE) == 0)) {
+ if (vm_page_pa_tryrelock(pm, TTE_GET_PA(tp), &pa))
+ goto retry;
m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp));
vm_page_hold(m);
}
- vm_page_unlock_queues();
- if (pm != kernel_pmap)
- PMAP_UNLOCK(pm);
+ PA_UNLOCK_COND(pa);
+ PMAP_UNLOCK(pm);
return (m);
}
Modified: head/sys/sun4v/include/pmap.h
==============================================================================
--- head/sys/sun4v/include/pmap.h Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/sun4v/include/pmap.h Fri Apr 30 00:46:43 2010 (r207410)
@@ -75,6 +75,8 @@ struct pmap {
struct tte_hash *pm_hash;
TAILQ_HEAD(,pv_entry) pm_pvlist; /* list of mappings in pmap */
struct hv_tsb_info pm_tsb;
+ uint32_t pm_gen_count; /* generation count (pmap lock dropped) */
+ u_int pm_retries;
pmap_cpumask_t pm_active; /* mask of cpus currently using pmap */
pmap_cpumask_t pm_tlbactive; /* mask of cpus that have used this pmap */
struct pmap_statistics pm_stats;
Modified: head/sys/sun4v/sun4v/pmap.c
==============================================================================
--- head/sys/sun4v/sun4v/pmap.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/sun4v/sun4v/pmap.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -1275,17 +1275,21 @@ pmap_extract_and_hold(pmap_t pmap, vm_of
{
tte_t tte_data;
vm_page_t m;
+ vm_paddr_t pa;
m = NULL;
- vm_page_lock_queues();
+ pa = 0;
PMAP_LOCK(pmap);
+retry:
tte_data = tte_hash_lookup(pmap->pm_hash, va);
if (tte_data != 0 &&
((tte_data & VTD_SW_W) || (prot & VM_PROT_WRITE) == 0)) {
+ if (vm_page_pa_tryrelock(pmap, TTE_GET_PA(tte_data), &pa))
+ goto retry;
m = PHYS_TO_VM_PAGE(TTE_GET_PA(tte_data));
vm_page_hold(m);
}
- vm_page_unlock_queues();
+ PA_UNLOCK_COND(pa);
PMAP_UNLOCK(pmap);
return (m);
Modified: head/sys/sys/param.h
==============================================================================
--- head/sys/sys/param.h Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/sys/param.h Fri Apr 30 00:46:43 2010 (r207410)
@@ -58,7 +58,7 @@
* in the range 5 to 9.
*/
#undef __FreeBSD_version
-#define __FreeBSD_version 900010 /* Master, propagated to newvers */
+#define __FreeBSD_version 900011 /* Master, propagated to newvers */
#ifndef LOCORE
#include <sys/types.h>
Modified: head/sys/vm/device_pager.c
==============================================================================
--- head/sys/vm/device_pager.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/vm/device_pager.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -251,12 +251,16 @@ dev_pager_getpages(object, m, count, req
VM_OBJECT_LOCK(object);
dev_pager_updatefake(page, paddr, memattr);
if (count > 1) {
- vm_page_lock_queues();
+
for (i = 0; i < count; i++) {
- if (i != reqpage)
+ if (i != reqpage) {
+ vm_page_lock(m[i]);
+ vm_page_lock_queues();
vm_page_free(m[i]);
+ vm_page_unlock_queues();
+ vm_page_unlock(m[i]);
+ }
}
- vm_page_unlock_queues();
}
} else {
/*
@@ -266,10 +270,13 @@ dev_pager_getpages(object, m, count, req
page = dev_pager_getfake(paddr, memattr);
VM_OBJECT_LOCK(object);
TAILQ_INSERT_TAIL(&object->un_pager.devp.devp_pglist, page, pageq);
- vm_page_lock_queues();
- for (i = 0; i < count; i++)
+ for (i = 0; i < count; i++) {
+ vm_page_lock(m[i]);
+ vm_page_lock_queues();
vm_page_free(m[i]);
- vm_page_unlock_queues();
+ vm_page_unlock_queues();
+ vm_page_unlock(m[i]);
+ }
vm_page_insert(page, object, offset);
m[reqpage] = page;
}
Modified: head/sys/vm/sg_pager.c
==============================================================================
--- head/sys/vm/sg_pager.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/vm/sg_pager.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -198,10 +198,13 @@ sg_pager_getpages(vm_object_t object, vm
TAILQ_INSERT_TAIL(&object->un_pager.sgp.sgp_pglist, page, pageq);
/* Free the original pages and insert this fake page into the object. */
- vm_page_lock_queues();
- for (i = 0; i < count; i++)
+ for (i = 0; i < count; i++) {
+ vm_page_lock(m[i]);
+ vm_page_lock_queues();
vm_page_free(m[i]);
- vm_page_unlock_queues();
+ vm_page_unlock_queues();
+ vm_page_unlock(m[i]);
+ }
vm_page_insert(page, object, offset);
m[reqpage] = page;
page->valid = VM_PAGE_BITS_ALL;
Modified: head/sys/vm/swap_pager.c
==============================================================================
--- head/sys/vm/swap_pager.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/vm/swap_pager.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -1137,12 +1137,21 @@ swap_pager_getpages(vm_object_t object,
if (0 < i || j < count) {
int k;
- vm_page_lock_queues();
- for (k = 0; k < i; ++k)
+
+ for (k = 0; k < i; ++k) {
+ vm_page_lock(m[k]);
+ vm_page_lock_queues();
swp_pager_free_nrpage(m[k]);
- for (k = j; k < count; ++k)
+ vm_page_unlock_queues();
+ vm_page_unlock(m[k]);
+ }
+ for (k = j; k < count; ++k) {
+ vm_page_lock(m[k]);
+ vm_page_lock_queues();
swp_pager_free_nrpage(m[k]);
- vm_page_unlock_queues();
+ vm_page_unlock_queues();
+ vm_page_unlock(m[k]);
+ }
}
/*
@@ -1497,7 +1506,7 @@ swp_pager_async_iodone(struct buf *bp)
object = bp->b_pages[0]->object;
VM_OBJECT_LOCK(object);
}
- vm_page_lock_queues();
+
/*
* cleanup pages. If an error occurs writing to swap, we are in
* very serious trouble. If it happens to be a disk error, though,
@@ -1509,6 +1518,8 @@ swp_pager_async_iodone(struct buf *bp)
for (i = 0; i < bp->b_npages; ++i) {
vm_page_t m = bp->b_pages[i];
+ vm_page_lock(m);
+ vm_page_lock_queues();
m->oflags &= ~VPO_SWAPINPROG;
if (bp->b_ioflags & BIO_ERROR) {
@@ -1605,8 +1616,9 @@ swp_pager_async_iodone(struct buf *bp)
if (vm_page_count_severe())
vm_page_try_to_cache(m);
}
+ vm_page_unlock_queues();
+ vm_page_unlock(m);
}
- vm_page_unlock_queues();
/*
* adjust pip. NOTE: the original parent may still have its own
@@ -1702,10 +1714,12 @@ swp_pager_force_pagein(vm_object_t objec
m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL|VM_ALLOC_RETRY);
if (m->valid == VM_PAGE_BITS_ALL) {
vm_object_pip_subtract(object, 1);
+ vm_page_lock(m);
vm_page_lock_queues();
vm_page_activate(m);
vm_page_dirty(m);
vm_page_unlock_queues();
+ vm_page_unlock(m);
vm_page_wakeup(m);
vm_pager_page_unswapped(m);
return;
@@ -1714,10 +1728,12 @@ swp_pager_force_pagein(vm_object_t objec
if (swap_pager_getpages(object, &m, 1, 0) != VM_PAGER_OK)
panic("swap_pager_force_pagein: read from swap failed");/*XXX*/
vm_object_pip_subtract(object, 1);
+ vm_page_lock(m);
vm_page_lock_queues();
vm_page_dirty(m);
vm_page_dontneed(m);
vm_page_unlock_queues();
+ vm_page_unlock(m);
vm_page_wakeup(m);
vm_pager_page_unswapped(m);
}
Modified: head/sys/vm/uma_core.c
==============================================================================
--- head/sys/vm/uma_core.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/vm/uma_core.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -1022,10 +1022,12 @@ obj_alloc(uma_zone_t zone, int bytes, u_
while (pages != startpages) {
pages--;
p = TAILQ_LAST(&object->memq, pglist);
+ vm_page_lock(p);
vm_page_lock_queues();
vm_page_unwire(p, 0);
vm_page_free(p);
vm_page_unlock_queues();
+ vm_page_unlock(p);
}
retkva = 0;
goto done;
Modified: head/sys/vm/vm_contig.c
==============================================================================
--- head/sys/vm/vm_contig.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/vm/vm_contig.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -257,9 +257,11 @@ retry:
i -= PAGE_SIZE;
m = vm_page_lookup(object, OFF_TO_IDX(offset +
i));
+ vm_page_lock(m);
vm_page_lock_queues();
vm_page_free(m);
vm_page_unlock_queues();
+ vm_page_unlock(m);
}
VM_OBJECT_UNLOCK(object);
vm_map_delete(map, addr, addr + size);
Modified: head/sys/vm/vm_fault.c
==============================================================================
--- head/sys/vm/vm_fault.c Fri Apr 30 00:34:00 2010 (r207409)
+++ head/sys/vm/vm_fault.c Fri Apr 30 00:46:43 2010 (r207410)
@@ -137,9 +137,11 @@ release_page(struct faultstate *fs)
{
vm_page_wakeup(fs->m);
+ vm_page_lock(fs->m);
vm_page_lock_queues();
vm_page_deactivate(fs->m);
vm_page_unlock_queues();
+ vm_page_unlock(fs->m);
fs->m = NULL;
}
@@ -161,9 +163,11 @@ unlock_and_deallocate(struct faultstate
VM_OBJECT_UNLOCK(fs->object);
if (fs->object != fs->first_object) {
VM_OBJECT_LOCK(fs->first_object);
+ vm_page_lock(fs->first_m);
vm_page_lock_queues();
vm_page_free(fs->first_m);
vm_page_unlock_queues();
+ vm_page_unlock(fs->first_m);
vm_object_pip_wakeup(fs->first_object);
VM_OBJECT_UNLOCK(fs->first_object);
fs->first_m = NULL;
@@ -305,12 +309,14 @@ RetryFault:;
* removes the page from the backing object,
* which is not what we want.
*/
+ vm_page_lock(fs.m);
vm_page_lock_queues();
if ((fs.m->cow) &&
(fault_type & VM_PROT_WRITE) &&
(fs.object == fs.first_object)) {
vm_page_cowfault(fs.m);
vm_page_unlock_queues();
+ vm_page_unlock(fs.m);
unlock_and_deallocate(&fs);
goto RetryFault;
}
@@ -333,12 +339,15 @@ RetryFault:;
*/
if ((fs.m->oflags & VPO_BUSY) || fs.m->busy) {
vm_page_unlock_queues();
+ vm_page_unlock(fs.m);
VM_OBJECT_UNLOCK(fs.object);
if (fs.object != fs.first_object) {
VM_OBJECT_LOCK(fs.first_object);
+ vm_page_lock(fs.first_m);
vm_page_lock_queues();
vm_page_free(fs.first_m);
vm_page_unlock_queues();
+ vm_page_unlock(fs.first_m);
vm_object_pip_wakeup(fs.first_object);
VM_OBJECT_UNLOCK(fs.first_object);
fs.first_m = NULL;
@@ -358,6 +367,7 @@ RetryFault:;
}
vm_pageq_remove(fs.m);
vm_page_unlock_queues();
+ vm_page_unlock(fs.m);
/*
* Mark page busy for other processes, and the
@@ -481,17 +491,25 @@ readrest:
continue;
if (!are_queues_locked) {
are_queues_locked = TRUE;
+ vm_page_lock(mt);
+ vm_page_lock_queues();
+ } else {
+ vm_page_unlock_queues();
+ vm_page_lock(mt);
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-head
mailing list