svn commit: r311653 - in stable/11/sys/i386: i386 include
Jason A. Harmening
jah at FreeBSD.org
Sat Jan 7 18:54:59 UTC 2017
Author: jah
Date: Sat Jan 7 18:54:57 2017
New Revision: 311653
URL: https://svnweb.freebsd.org/changeset/base/311653
Log:
MFC r310481:
Move the objects used to create temporary mappings for i386 pmap zero
and copy operations to the MD PCPU region. Change sysmap
initialization to only allocate KVA pages for CPUs that are actually
present. As a minor optimization, this also prevents false sharing
between adjacent sysmap objects since the pcpu struct is already
cacheline-aligned.
While here, move pc_qmap_addr initialization for the BSP into
pmap_bootstrap(), which allows use of pmap_quick* functions during
early boot.
Reviewed by: kib
Differential Revision: https://reviews.freebsd.org/D8833
Modified:
stable/11/sys/i386/i386/pmap.c
stable/11/sys/i386/include/pcpu.h
Directory Properties:
stable/11/ (props changed)
Modified: stable/11/sys/i386/i386/pmap.c
==============================================================================
--- stable/11/sys/i386/i386/pmap.c Sat Jan 7 17:37:39 2017 (r311652)
+++ stable/11/sys/i386/i386/pmap.c Sat Jan 7 18:54:57 2017 (r311653)
@@ -257,14 +257,6 @@ vm_offset_t pv_vafree; /* freelist sto
/*
* All those kernel PT submaps that BSD is so fond of
*/
-struct sysmaps {
- struct mtx lock;
- pt_entry_t *CMAP1;
- pt_entry_t *CMAP2;
- caddr_t CADDR1;
- caddr_t CADDR2;
-};
-static struct sysmaps sysmaps_pcpu[MAXCPU];
pt_entry_t *CMAP3;
static pd_entry_t *KPTD;
caddr_t ptvmmap = 0;
@@ -380,7 +372,7 @@ pmap_bootstrap(vm_paddr_t firstaddr)
{
vm_offset_t va;
pt_entry_t *pte, *unused;
- struct sysmaps *sysmaps;
+ struct pcpu *pc;
int i;
/*
@@ -442,16 +434,19 @@ pmap_bootstrap(vm_paddr_t firstaddr)
va = virtual_avail;
pte = vtopte(va);
+
/*
+ * Initialize temporary map objects on the current CPU for use
+ * during early boot.
* CMAP1/CMAP2 are used for zeroing and copying pages.
* CMAP3 is used for the idle process page zeroing.
*/
- for (i = 0; i < MAXCPU; i++) {
- sysmaps = &sysmaps_pcpu[i];
- mtx_init(&sysmaps->lock, "SYSMAPS", NULL, MTX_DEF);
- SYSMAP(caddr_t, sysmaps->CMAP1, sysmaps->CADDR1, 1)
- SYSMAP(caddr_t, sysmaps->CMAP2, sysmaps->CADDR2, 1)
- }
+ pc = pcpu_find(curcpu);
+ mtx_init(&pc->pc_cmap_lock, "SYSMAPS", NULL, MTX_DEF);
+ SYSMAP(caddr_t, pc->pc_cmap_pte1, pc->pc_cmap_addr1, 1)
+ SYSMAP(caddr_t, pc->pc_cmap_pte2, pc->pc_cmap_addr2, 1)
+ SYSMAP(vm_offset_t, pte, pc->pc_qmap_addr, 1)
+
SYSMAP(caddr_t, CMAP3, CADDR3, 1)
/*
@@ -521,20 +516,33 @@ pmap_bootstrap(vm_paddr_t firstaddr)
}
static void
-pmap_init_qpages(void)
+pmap_init_reserved_pages(void)
{
struct pcpu *pc;
+ vm_offset_t pages;
int i;
CPU_FOREACH(i) {
pc = pcpu_find(i);
- pc->pc_qmap_addr = kva_alloc(PAGE_SIZE);
- if (pc->pc_qmap_addr == 0)
- panic("pmap_init_qpages: unable to allocate KVA");
+ /*
+ * Skip if the mapping has already been initialized,
+ * i.e. this is the BSP.
+ */
+ if (pc->pc_cmap_addr1 != 0)
+ continue;
+ mtx_init(&pc->pc_cmap_lock, "SYSMAPS", NULL, MTX_DEF);
+ pages = kva_alloc(PAGE_SIZE * 3);
+ if (pages == 0)
+ panic("%s: unable to allocate KVA", __func__);
+ pc->pc_cmap_pte1 = vtopte(pages);
+ pc->pc_cmap_pte2 = vtopte(pages + PAGE_SIZE);
+ pc->pc_cmap_addr1 = (caddr_t)pages;
+ pc->pc_cmap_addr2 = (caddr_t)(pages + PAGE_SIZE);
+ pc->pc_qmap_addr = pages + (PAGE_SIZE * 2);
}
}
-
-SYSINIT(qpages_init, SI_SUB_CPU, SI_ORDER_ANY, pmap_init_qpages, NULL);
+
+SYSINIT(rpages_init, SI_SUB_CPU, SI_ORDER_ANY, pmap_init_reserved_pages, NULL);
/*
* Setup the PAT MSR.
@@ -4209,20 +4217,22 @@ pagezero(void *page)
void
pmap_zero_page(vm_page_t m)
{
- struct sysmaps *sysmaps;
+ pt_entry_t *cmap_pte2;
+ struct pcpu *pc;
- sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
- mtx_lock(&sysmaps->lock);
- if (*sysmaps->CMAP2)
- panic("pmap_zero_page: CMAP2 busy");
sched_pin();
- *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M |
+ pc = pcpu_find(curcpu);
+ cmap_pte2 = pc->pc_cmap_pte2;
+ mtx_lock(&pc->pc_cmap_lock);
+ if (*cmap_pte2)
+ panic("pmap_zero_page: CMAP2 busy");
+ *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M |
pmap_cache_bits(m->md.pat_mode, 0);
- invlcaddr(sysmaps->CADDR2);
- pagezero(sysmaps->CADDR2);
- *sysmaps->CMAP2 = 0;
+ invlcaddr(pc->pc_cmap_addr2);
+ pagezero(pc->pc_cmap_addr2);
+ *cmap_pte2 = 0;
+ mtx_unlock(&pc->pc_cmap_lock);
sched_unpin();
- mtx_unlock(&sysmaps->lock);
}
/*
@@ -4234,23 +4244,25 @@ pmap_zero_page(vm_page_t m)
void
pmap_zero_page_area(vm_page_t m, int off, int size)
{
- struct sysmaps *sysmaps;
+ pt_entry_t *cmap_pte2;
+ struct pcpu *pc;
- sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
- mtx_lock(&sysmaps->lock);
- if (*sysmaps->CMAP2)
- panic("pmap_zero_page_area: CMAP2 busy");
sched_pin();
- *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M |
+ pc = pcpu_find(curcpu);
+ cmap_pte2 = pc->pc_cmap_pte2;
+ mtx_lock(&pc->pc_cmap_lock);
+ if (*cmap_pte2)
+ panic("pmap_zero_page_area: CMAP2 busy");
+ *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M |
pmap_cache_bits(m->md.pat_mode, 0);
- invlcaddr(sysmaps->CADDR2);
+ invlcaddr(pc->pc_cmap_addr2);
if (off == 0 && size == PAGE_SIZE)
- pagezero(sysmaps->CADDR2);
+ pagezero(pc->pc_cmap_addr2);
else
- bzero((char *)sysmaps->CADDR2 + off, size);
- *sysmaps->CMAP2 = 0;
+ bzero(pc->pc_cmap_addr2 + off, size);
+ *cmap_pte2 = 0;
+ mtx_unlock(&pc->pc_cmap_lock);
sched_unpin();
- mtx_unlock(&sysmaps->lock);
}
/*
@@ -4283,26 +4295,29 @@ pmap_zero_page_idle(vm_page_t m)
void
pmap_copy_page(vm_page_t src, vm_page_t dst)
{
- struct sysmaps *sysmaps;
+ pt_entry_t *cmap_pte1, *cmap_pte2;
+ struct pcpu *pc;
- sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
- mtx_lock(&sysmaps->lock);
- if (*sysmaps->CMAP1)
+ sched_pin();
+ pc = pcpu_find(curcpu);
+ cmap_pte1 = pc->pc_cmap_pte1;
+ cmap_pte2 = pc->pc_cmap_pte2;
+ mtx_lock(&pc->pc_cmap_lock);
+ if (*cmap_pte1)
panic("pmap_copy_page: CMAP1 busy");
- if (*sysmaps->CMAP2)
+ if (*cmap_pte2)
panic("pmap_copy_page: CMAP2 busy");
- sched_pin();
- *sysmaps->CMAP1 = PG_V | VM_PAGE_TO_PHYS(src) | PG_A |
+ *cmap_pte1 = PG_V | VM_PAGE_TO_PHYS(src) | PG_A |
pmap_cache_bits(src->md.pat_mode, 0);
- invlcaddr(sysmaps->CADDR1);
- *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(dst) | PG_A | PG_M |
+ invlcaddr(pc->pc_cmap_addr1);
+ *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(dst) | PG_A | PG_M |
pmap_cache_bits(dst->md.pat_mode, 0);
- invlcaddr(sysmaps->CADDR2);
- bcopy(sysmaps->CADDR1, sysmaps->CADDR2, PAGE_SIZE);
- *sysmaps->CMAP1 = 0;
- *sysmaps->CMAP2 = 0;
+ invlcaddr(pc->pc_cmap_addr2);
+ bcopy(pc->pc_cmap_addr1, pc->pc_cmap_addr2, PAGE_SIZE);
+ *cmap_pte1 = 0;
+ *cmap_pte2 = 0;
+ mtx_unlock(&pc->pc_cmap_lock);
sched_unpin();
- mtx_unlock(&sysmaps->lock);
}
int unmapped_buf_allowed = 1;
@@ -4311,19 +4326,22 @@ void
pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
vm_offset_t b_offset, int xfersize)
{
- struct sysmaps *sysmaps;
vm_page_t a_pg, b_pg;
char *a_cp, *b_cp;
vm_offset_t a_pg_offset, b_pg_offset;
+ pt_entry_t *cmap_pte1, *cmap_pte2;
+ struct pcpu *pc;
int cnt;
- sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
- mtx_lock(&sysmaps->lock);
- if (*sysmaps->CMAP1 != 0)
+ sched_pin();
+ pc = pcpu_find(curcpu);
+ cmap_pte1 = pc->pc_cmap_pte1;
+ cmap_pte2 = pc->pc_cmap_pte2;
+ mtx_lock(&pc->pc_cmap_lock);
+ if (*cmap_pte1 != 0)
panic("pmap_copy_pages: CMAP1 busy");
- if (*sysmaps->CMAP2 != 0)
+ if (*cmap_pte2 != 0)
panic("pmap_copy_pages: CMAP2 busy");
- sched_pin();
while (xfersize > 0) {
a_pg = ma[a_offset >> PAGE_SHIFT];
a_pg_offset = a_offset & PAGE_MASK;
@@ -4331,23 +4349,23 @@ pmap_copy_pages(vm_page_t ma[], vm_offse
b_pg = mb[b_offset >> PAGE_SHIFT];
b_pg_offset = b_offset & PAGE_MASK;
cnt = min(cnt, PAGE_SIZE - b_pg_offset);
- *sysmaps->CMAP1 = PG_V | VM_PAGE_TO_PHYS(a_pg) | PG_A |
+ *cmap_pte1 = PG_V | VM_PAGE_TO_PHYS(a_pg) | PG_A |
pmap_cache_bits(a_pg->md.pat_mode, 0);
- invlcaddr(sysmaps->CADDR1);
- *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(b_pg) | PG_A |
+ invlcaddr(pc->pc_cmap_addr1);
+ *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(b_pg) | PG_A |
PG_M | pmap_cache_bits(b_pg->md.pat_mode, 0);
- invlcaddr(sysmaps->CADDR2);
- a_cp = sysmaps->CADDR1 + a_pg_offset;
- b_cp = sysmaps->CADDR2 + b_pg_offset;
+ invlcaddr(pc->pc_cmap_addr2);
+ a_cp = pc->pc_cmap_addr1 + a_pg_offset;
+ b_cp = pc->pc_cmap_addr2 + b_pg_offset;
bcopy(a_cp, b_cp, cnt);
a_offset += cnt;
b_offset += cnt;
xfersize -= cnt;
}
- *sysmaps->CMAP1 = 0;
- *sysmaps->CMAP2 = 0;
+ *cmap_pte1 = 0;
+ *cmap_pte2 = 0;
+ mtx_unlock(&pc->pc_cmap_lock);
sched_unpin();
- mtx_unlock(&sysmaps->lock);
}
/*
@@ -5295,21 +5313,23 @@ pmap_page_set_memattr(vm_page_t m, vm_me
static void
pmap_flush_page(vm_page_t m)
{
- struct sysmaps *sysmaps;
+ pt_entry_t *cmap_pte2;
+ struct pcpu *pc;
vm_offset_t sva, eva;
bool useclflushopt;
useclflushopt = (cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0;
if (useclflushopt || (cpu_feature & CPUID_CLFSH) != 0) {
- sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
- mtx_lock(&sysmaps->lock);
- if (*sysmaps->CMAP2)
- panic("pmap_flush_page: CMAP2 busy");
sched_pin();
- *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) |
+ pc = pcpu_find(curcpu);
+ cmap_pte2 = pc->pc_cmap_pte2;
+ mtx_lock(&pc->pc_cmap_lock);
+ if (*cmap_pte2)
+ panic("pmap_flush_page: CMAP2 busy");
+ *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) |
PG_A | PG_M | pmap_cache_bits(m->md.pat_mode, 0);
- invlcaddr(sysmaps->CADDR2);
- sva = (vm_offset_t)sysmaps->CADDR2;
+ invlcaddr(pc->pc_cmap_addr2);
+ sva = (vm_offset_t)pc->pc_cmap_addr2;
eva = sva + PAGE_SIZE;
/*
@@ -5328,9 +5348,9 @@ pmap_flush_page(vm_page_t m)
}
if (useclflushopt || cpu_vendor_id != CPU_VENDOR_INTEL)
mfence();
- *sysmaps->CMAP2 = 0;
+ *cmap_pte2 = 0;
+ mtx_unlock(&pc->pc_cmap_lock);
sched_unpin();
- mtx_unlock(&sysmaps->lock);
} else
pmap_invalidate_cache();
}
Modified: stable/11/sys/i386/include/pcpu.h
==============================================================================
--- stable/11/sys/i386/include/pcpu.h Sat Jan 7 17:37:39 2017 (r311652)
+++ stable/11/sys/i386/include/pcpu.h Sat Jan 7 18:54:57 2017 (r311653)
@@ -36,6 +36,9 @@
#include <machine/segments.h>
#include <machine/tss.h>
+#include <sys/_lock.h>
+#include <sys/_mutex.h>
+
/*
* The SMP parts are setup in pmap.c and locore.s for the BSP, and
* mp_machdep.c sets up the data for the AP's to "see" when they awake.
@@ -58,9 +61,14 @@
int pc_private_tss; /* Flag indicating private tss*/\
u_int pc_cmci_mask; /* MCx banks for CMCI */ \
u_int pc_vcpu_id; /* Xen vCPU ID */ \
+ struct mtx pc_cmap_lock; \
+ void *pc_cmap_pte1; \
+ void *pc_cmap_pte2; \
+ caddr_t pc_cmap_addr1; \
+ caddr_t pc_cmap_addr2; \
vm_offset_t pc_qmap_addr; /* KVA for temporary mappings */\
uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \
- char __pad[225]
+ char __pad[189]
#ifdef _KERNEL
More information about the svn-src-all
mailing list