svn commit: r354049 - in stable/12/sys/amd64: amd64 include
Alexander Motin
mav at FreeBSD.org
Thu Oct 24 21:10:10 UTC 2019
Author: mav
Date: Thu Oct 24 21:10:09 2019
New Revision: 354049
URL: https://svnweb.freebsd.org/changeset/base/354049
Log:
MFC r351457 (by kib): amd64: rework PCPU allocation
Move pcpu KVA out of .bss into dynamically allocated VA at
pmap_bootstrap(). This avoids demoting superpage mapping .data/.bss.
Also it makes possible to use pmap_qenter() for installation of
domain-local pcpu page on NUMA configs.
Refactor pcpu and IST initialization by moving it to helper functions.
Modified:
stable/12/sys/amd64/amd64/machdep.c
stable/12/sys/amd64/amd64/mp_machdep.c
stable/12/sys/amd64/amd64/pmap.c
stable/12/sys/amd64/include/counter.h
stable/12/sys/amd64/include/md_var.h
Directory Properties:
stable/12/ (props changed)
Modified: stable/12/sys/amd64/amd64/machdep.c
==============================================================================
--- stable/12/sys/amd64/amd64/machdep.c Thu Oct 24 21:02:24 2019 (r354048)
+++ stable/12/sys/amd64/amd64/machdep.c Thu Oct 24 21:10:09 2019 (r354049)
@@ -230,7 +230,8 @@ struct kva_md_info kmi;
static struct trapframe proc0_tf;
struct region_descriptor r_gdt, r_idt;
-struct pcpu __pcpu[MAXCPU];
+struct pcpu *__pcpu;
+struct pcpu temp_bsp_pcpu;
struct mtx icu_lock;
@@ -1559,13 +1560,68 @@ amd64_conf_fast_syscall(void)
wrmsr(MSR_SF_MASK, PSL_NT | PSL_T | PSL_I | PSL_C | PSL_D | PSL_AC);
}
+void
+amd64_bsp_pcpu_init1(struct pcpu *pc)
+{
+
+ PCPU_SET(prvspace, pc);
+ PCPU_SET(curthread, &thread0);
+ PCPU_SET(tssp, &common_tss[0]);
+ PCPU_SET(commontssp, &common_tss[0]);
+ PCPU_SET(tss, (struct system_segment_descriptor *)&gdt[GPROC0_SEL]);
+ PCPU_SET(ldt, (struct system_segment_descriptor *)&gdt[GUSERLDT_SEL]);
+ PCPU_SET(fs32p, &gdt[GUFS32_SEL]);
+ PCPU_SET(gs32p, &gdt[GUGS32_SEL]);
+}
+
+void
+amd64_bsp_pcpu_init2(uint64_t rsp0)
+{
+
+ PCPU_SET(rsp0, rsp0);
+ PCPU_SET(pti_rsp0, ((vm_offset_t)PCPU_PTR(pti_stack) +
+ PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful);
+ PCPU_SET(curpcb, thread0.td_pcb);
+}
+
+void
+amd64_bsp_ist_init(struct pcpu *pc)
+{
+ struct nmi_pcpu *np;
+
+ /* doublefault stack space, runs on ist1 */
+ common_tss[0].tss_ist1 = (long)&dblfault_stack[sizeof(dblfault_stack)];
+
+ /*
+ * NMI stack, runs on ist2. The pcpu pointer is stored just
+ * above the start of the ist2 stack.
+ */
+ np = ((struct nmi_pcpu *)&nmi0_stack[sizeof(nmi0_stack)]) - 1;
+ np->np_pcpu = (register_t)pc;
+ common_tss[0].tss_ist2 = (long)np;
+
+ /*
+ * MC# stack, runs on ist3. The pcpu pointer is stored just
+ * above the start of the ist3 stack.
+ */
+ np = ((struct nmi_pcpu *)&mce0_stack[sizeof(mce0_stack)]) - 1;
+ np->np_pcpu = (register_t)pc;
+ common_tss[0].tss_ist3 = (long)np;
+
+ /*
+ * DB# stack, runs on ist4.
+ */
+ np = ((struct nmi_pcpu *)&dbg0_stack[sizeof(dbg0_stack)]) - 1;
+ np->np_pcpu = (register_t)pc;
+ common_tss[0].tss_ist4 = (long)np;
+}
+
u_int64_t
hammer_time(u_int64_t modulep, u_int64_t physfree)
{
caddr_t kmdp;
int gsel_tss, x;
struct pcpu *pc;
- struct nmi_pcpu *np;
struct xstate_hdr *xhdr;
u_int64_t rsp0;
char *env;
@@ -1639,7 +1695,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1;
r_gdt.rd_base = (long) gdt;
lgdt(&r_gdt);
- pc = &__pcpu[0];
+ pc = &temp_bsp_pcpu;
wrmsr(MSR_FSBASE, 0); /* User value */
wrmsr(MSR_GSBASE, (u_int64_t)pc);
@@ -1648,15 +1704,8 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
pcpu_init(pc, 0, sizeof(struct pcpu));
dpcpu_init((void *)(physfree + KERNBASE), 0);
physfree += DPCPU_SIZE;
- PCPU_SET(prvspace, pc);
- PCPU_SET(curthread, &thread0);
+ amd64_bsp_pcpu_init1(pc);
/* Non-late cninit() and printf() can be moved up to here. */
- PCPU_SET(tssp, &common_tss[0]);
- PCPU_SET(commontssp, &common_tss[0]);
- PCPU_SET(tss, (struct system_segment_descriptor *)&gdt[GPROC0_SEL]);
- PCPU_SET(ldt, (struct system_segment_descriptor *)&gdt[GUSERLDT_SEL]);
- PCPU_SET(fs32p, &gdt[GUFS32_SEL]);
- PCPU_SET(gs32p, &gdt[GUGS32_SEL]);
/*
* Initialize mutexes.
@@ -1745,31 +1794,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
finishidentcpu(); /* Final stage of CPU initialization */
initializecpu(); /* Initialize CPU registers */
- /* doublefault stack space, runs on ist1 */
- common_tss[0].tss_ist1 = (long)&dblfault_stack[sizeof(dblfault_stack)];
-
- /*
- * NMI stack, runs on ist2. The pcpu pointer is stored just
- * above the start of the ist2 stack.
- */
- np = ((struct nmi_pcpu *) &nmi0_stack[sizeof(nmi0_stack)]) - 1;
- np->np_pcpu = (register_t) pc;
- common_tss[0].tss_ist2 = (long) np;
-
- /*
- * MC# stack, runs on ist3. The pcpu pointer is stored just
- * above the start of the ist3 stack.
- */
- np = ((struct nmi_pcpu *) &mce0_stack[sizeof(mce0_stack)]) - 1;
- np->np_pcpu = (register_t) pc;
- common_tss[0].tss_ist3 = (long) np;
-
- /*
- * DB# stack, runs on ist4.
- */
- np = ((struct nmi_pcpu *) &dbg0_stack[sizeof(dbg0_stack)]) - 1;
- np->np_pcpu = (register_t) pc;
- common_tss[0].tss_ist4 = (long) np;
+ amd64_bsp_ist_init(pc);
/* Set the IO permission bitmap (empty due to tss seg limit) */
common_tss[0].tss_iobase = sizeof(struct amd64tss) + IOPERM_BITMAP_SIZE;
@@ -1858,10 +1883,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
/* Ensure the stack is aligned to 16 bytes */
rsp0 &= ~0xFul;
common_tss[0].tss_rsp0 = rsp0;
- PCPU_SET(rsp0, rsp0);
- PCPU_SET(pti_rsp0, ((vm_offset_t)PCPU_PTR(pti_stack) +
- PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful);
- PCPU_SET(curpcb, thread0.td_pcb);
+ amd64_bsp_pcpu_init2(rsp0);
/* transfer to user mode */
Modified: stable/12/sys/amd64/amd64/mp_machdep.c
==============================================================================
--- stable/12/sys/amd64/amd64/mp_machdep.c Thu Oct 24 21:02:24 2019 (r354048)
+++ stable/12/sys/amd64/amd64/mp_machdep.c Thu Oct 24 21:10:09 2019 (r354049)
@@ -94,7 +94,7 @@ __FBSDID("$FreeBSD$");
#define AP_BOOTPT_SZ (PAGE_SIZE * 3)
-extern struct pcpu __pcpu[];
+extern struct pcpu *__pcpu;
/* Temporary variables for init_secondary() */
char *doublefault_stack;
@@ -407,7 +407,7 @@ mp_realloc_pcpu(int cpuid, int domain)
return;
na = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
pagecopy((void *)oa, (void *)na);
- pmap_enter(kernel_pmap, oa, m, VM_PROT_READ | VM_PROT_WRITE, 0, 0);
+ pmap_qenter((vm_offset_t)&__pcpu[cpuid], &m, 1);
/* XXX old pcpu page leaked. */
}
#endif
Modified: stable/12/sys/amd64/amd64/pmap.c
==============================================================================
--- stable/12/sys/amd64/amd64/pmap.c Thu Oct 24 21:02:24 2019 (r354048)
+++ stable/12/sys/amd64/amd64/pmap.c Thu Oct 24 21:10:09 2019 (r354049)
@@ -446,6 +446,10 @@ static pml4_entry_t *pti_pml4;
static vm_pindex_t pti_pg_idx;
static bool pti_finalized;
+extern struct pcpu *__pcpu;
+extern struct pcpu temp_bsp_pcpu;
+extern pt_entry_t *pcpu_pte;
+
struct pmap_pkru_range {
struct rs_el pkru_rs_el;
u_int pkru_keyidx;
@@ -1595,8 +1599,8 @@ void
pmap_bootstrap(vm_paddr_t *firstaddr)
{
vm_offset_t va;
- pt_entry_t *pte;
- uint64_t cr4;
+ pt_entry_t *pte, *pcpu_pte;
+ uint64_t cr4, pcpu_phys;
u_long res;
int i;
@@ -1611,6 +1615,8 @@ pmap_bootstrap(vm_paddr_t *firstaddr)
*/
create_pagetables(firstaddr);
+ pcpu_phys = allocpages(firstaddr, MAXCPU);
+
/*
* Add a physical memory segment (vm_phys_seg) corresponding to the
* preallocated kernel page table pages so that vm_page structures
@@ -1678,7 +1684,20 @@ pmap_bootstrap(vm_paddr_t *firstaddr)
SYSMAP(caddr_t, CMAP1, crashdumpmap, MAXDUMPPGS)
CADDR1 = crashdumpmap;
+ SYSMAP(struct pcpu *, pcpu_pte, __pcpu, MAXCPU);
virtual_avail = va;
+
+ for (i = 0; i < MAXCPU; i++) {
+ pcpu_pte[i] = (pcpu_phys + ptoa(i)) | X86_PG_V | X86_PG_RW |
+ pg_g | pg_nx | X86_PG_M | X86_PG_A;
+ }
+ STAILQ_INIT(&cpuhead);
+ wrmsr(MSR_GSBASE, (uint64_t)&__pcpu[0]);
+ pcpu_init(&__pcpu[0], 0, sizeof(struct pcpu));
+ amd64_bsp_pcpu_init1(&__pcpu[0]);
+ amd64_bsp_ist_init(&__pcpu[0]);
+ __pcpu[0].pc_dynamic = temp_bsp_pcpu.pc_dynamic;
+ __pcpu[0].pc_acpi_id = temp_bsp_pcpu.pc_acpi_id;
/*
* Initialize the PAT MSR.
Modified: stable/12/sys/amd64/include/counter.h
==============================================================================
--- stable/12/sys/amd64/include/counter.h Thu Oct 24 21:02:24 2019 (r354048)
+++ stable/12/sys/amd64/include/counter.h Thu Oct 24 21:10:09 2019 (r354049)
@@ -33,9 +33,10 @@
#include <sys/pcpu.h>
-extern struct pcpu __pcpu[];
+extern struct pcpu *__pcpu;
+extern struct pcpu temp_bsp_pcpu;
-#define EARLY_COUNTER &__pcpu[0].pc_early_dummy_counter
+#define EARLY_COUNTER &temp_bsp_pcpu.pc_early_dummy_counter
#define counter_enter() do {} while (0)
#define counter_exit() do {} while (0)
Modified: stable/12/sys/amd64/include/md_var.h
==============================================================================
--- stable/12/sys/amd64/include/md_var.h Thu Oct 24 21:02:24 2019 (r354048)
+++ stable/12/sys/amd64/include/md_var.h Thu Oct 24 21:10:09 2019 (r354049)
@@ -58,6 +58,9 @@ struct sysentvec;
void amd64_conf_fast_syscall(void);
void amd64_db_resume_dbreg(void);
void amd64_lower_shared_page(struct sysentvec *);
+void amd64_bsp_pcpu_init1(struct pcpu *pc);
+void amd64_bsp_pcpu_init2(uint64_t rsp0);
+void amd64_bsp_ist_init(struct pcpu *pc);
void amd64_syscall(struct thread *td, int traced);
void amd64_syscall_ret_flush_l1d(int error);
void amd64_syscall_ret_flush_l1d_recalc(void);
More information about the svn-src-all
mailing list