svn commit: r354041 - in stable/12/sys: amd64/amd64 dev/acpica vm
Alexander Motin
mav at FreeBSD.org
Thu Oct 24 20:00:30 UTC 2019
Author: mav
Date: Thu Oct 24 20:00:28 2019
New Revision: 354041
URL: https://svnweb.freebsd.org/changeset/base/354041
Log:
MFC r351200 (by jeff), 351454,351456,351494-351495 (by kib):
Allocate all per-cpu datastructures in domain correct memory.
Modified:
stable/12/sys/amd64/amd64/mp_machdep.c
stable/12/sys/dev/acpica/acpi_pxm.c
stable/12/sys/dev/acpica/acpivar.h
stable/12/sys/vm/vm_phys.c
stable/12/sys/vm/vm_phys.h
Directory Properties:
stable/12/ (props changed)
Modified: stable/12/sys/amd64/amd64/mp_machdep.c
==============================================================================
--- stable/12/sys/amd64/amd64/mp_machdep.c Thu Oct 24 19:57:18 2019 (r354040)
+++ stable/12/sys/amd64/amd64/mp_machdep.c Thu Oct 24 20:00:28 2019 (r354041)
@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/cpuset.h>
+#include <sys/domainset.h>
#ifdef GPROF
#include <sys/gmon.h>
#endif
@@ -59,6 +60,8 @@ __FBSDID("$FreeBSD$");
#include <vm/pmap.h>
#include <vm/vm_kern.h>
#include <vm/vm_extern.h>
+#include <vm/vm_page.h>
+#include <vm/vm_phys.h>
#include <x86/apicreg.h>
#include <machine/clock.h>
@@ -75,6 +78,9 @@ __FBSDID("$FreeBSD$");
#include <machine/cpu.h>
#include <x86/init.h>
+#include <contrib/dev/acpica/include/acpi.h>
+#include <dev/acpica/acpivar.h>
+
#define WARMBOOT_TARGET 0
#define WARMBOOT_OFF (KERNBASE + 0x0467)
#define WARMBOOT_SEG (KERNBASE + 0x0469)
@@ -385,6 +391,27 @@ init_secondary(void)
* local functions and data
*/
+#ifdef NUMA
+static void
+mp_realloc_pcpu(int cpuid, int domain)
+{
+ vm_page_t m;
+ vm_offset_t oa, na;
+
+ oa = (vm_offset_t)&__pcpu[cpuid];
+ if (_vm_phys_domain(pmap_kextract(oa)) == domain)
+ return;
+ m = vm_page_alloc_domain(NULL, 0, domain,
+ VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ);
+ if (m == NULL)
+ return;
+ na = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
+ pagecopy((void *)oa, (void *)na);
+ pmap_enter(kernel_pmap, oa, m, VM_PROT_READ | VM_PROT_WRITE, 0, 0);
+ /* XXX old pcpu page leaked. */
+}
+#endif
+
/*
* start each AP in our list
*/
@@ -393,7 +420,7 @@ native_start_all_aps(void)
{
u_int64_t *pt4, *pt3, *pt2;
u_int32_t mpbioswarmvec;
- int apic_id, cpu, i;
+ int apic_id, cpu, domain, i;
u_char mpbiosreason;
mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
@@ -432,21 +459,39 @@ native_start_all_aps(void)
outb(CMOS_REG, BIOS_RESET);
outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */
+ /* Relocate pcpu areas to the correct domain. */
+#ifdef NUMA
+ if (vm_ndomains > 1)
+ for (cpu = 1; cpu < mp_ncpus; cpu++) {
+ apic_id = cpu_apic_ids[cpu];
+ domain = acpi_pxm_get_cpu_locality(apic_id);
+ mp_realloc_pcpu(cpu, domain);
+ }
+#endif
+
/* start each AP */
+ domain = 0;
for (cpu = 1; cpu < mp_ncpus; cpu++) {
apic_id = cpu_apic_ids[cpu];
-
+#ifdef NUMA
+ if (vm_ndomains > 1)
+ domain = acpi_pxm_get_cpu_locality(apic_id);
+#endif
/* allocate and set up an idle stack data page */
bootstacks[cpu] = (void *)kmem_malloc(kstack_pages * PAGE_SIZE,
M_WAITOK | M_ZERO);
doublefault_stack = (char *)kmem_malloc(PAGE_SIZE, M_WAITOK |
M_ZERO);
mce_stack = (char *)kmem_malloc(PAGE_SIZE, M_WAITOK | M_ZERO);
- nmi_stack = (char *)kmem_malloc(PAGE_SIZE, M_WAITOK | M_ZERO);
- dbg_stack = (char *)kmem_malloc(PAGE_SIZE, M_WAITOK | M_ZERO);
- dpcpu = (void *)kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO);
+ nmi_stack = (char *)kmem_malloc_domainset(
+ DOMAINSET_PREF(domain), PAGE_SIZE, M_WAITOK | M_ZERO);
+ dbg_stack = (char *)kmem_malloc_domainset(
+ DOMAINSET_PREF(domain), PAGE_SIZE, M_WAITOK | M_ZERO);
+ dpcpu = (void *)kmem_malloc_domainset(DOMAINSET_PREF(domain),
+ DPCPU_SIZE, M_WAITOK | M_ZERO);
- bootSTK = (char *)bootstacks[cpu] + kstack_pages * PAGE_SIZE - 8;
+ bootSTK = (char *)bootstacks[cpu] +
+ kstack_pages * PAGE_SIZE - 8;
bootAP = cpu;
/* attempt to start the Application Processor */
Modified: stable/12/sys/dev/acpica/acpi_pxm.c
==============================================================================
--- stable/12/sys/dev/acpica/acpi_pxm.c Thu Oct 24 19:57:18 2019 (r354040)
+++ stable/12/sys/dev/acpica/acpi_pxm.c Thu Oct 24 20:00:28 2019 (r354041)
@@ -653,6 +653,17 @@ acpi_pxm_set_cpu_locality(void)
}
}
+int
+acpi_pxm_get_cpu_locality(int apic_id)
+{
+ struct cpu_info *cpu;
+
+ cpu = cpu_find(apic_id);
+ if (cpu == NULL)
+ panic("SRAT: CPU with ID %u is not known", apic_id);
+ return (cpu->domain);
+}
+
/*
* Free data structures allocated during acpi_pxm_init.
*/
Modified: stable/12/sys/dev/acpica/acpivar.h
==============================================================================
--- stable/12/sys/dev/acpica/acpivar.h Thu Oct 24 19:57:18 2019 (r354040)
+++ stable/12/sys/dev/acpica/acpivar.h Thu Oct 24 20:00:28 2019 (r354041)
@@ -528,6 +528,7 @@ int acpi_pxm_init(int ncpus, vm_paddr_t maxphys);
void acpi_pxm_parse_tables(void);
void acpi_pxm_set_mem_locality(void);
void acpi_pxm_set_cpu_locality(void);
+int acpi_pxm_get_cpu_locality(int apic_id);
void acpi_pxm_free(void);
/*
Modified: stable/12/sys/vm/vm_phys.c
==============================================================================
--- stable/12/sys/vm/vm_phys.c Thu Oct 24 19:57:18 2019 (r354040)
+++ stable/12/sys/vm/vm_phys.c Thu Oct 24 20:00:28 2019 (r354041)
@@ -624,6 +624,26 @@ vm_phys_register_domains(int ndomains, struct mem_affi
#endif
}
+int
+_vm_phys_domain(vm_paddr_t pa)
+{
+#ifdef NUMA
+ int i;
+
+ if (vm_ndomains == 1 || mem_affinity == NULL)
+ return (0);
+
+ /*
+ * Check for any memory that overlaps.
+ */
+ for (i = 0; mem_affinity[i].end != 0; i++)
+ if (mem_affinity[i].start <= pa &&
+ mem_affinity[i].end >= pa)
+ return (mem_affinity[i].domain);
+#endif
+ return (0);
+}
+
/*
* Split a contiguous, power of two-sized set of physical pages.
*
Modified: stable/12/sys/vm/vm_phys.h
==============================================================================
--- stable/12/sys/vm/vm_phys.h Thu Oct 24 19:57:18 2019 (r354040)
+++ stable/12/sys/vm/vm_phys.h Thu Oct 24 20:00:28 2019 (r354041)
@@ -122,6 +122,7 @@ vm_phys_domain(vm_page_t m)
return (0);
#endif
}
+int _vm_phys_domain(vm_paddr_t pa);
#endif /* _KERNEL */
#endif /* !_VM_PHYS_H_ */
More information about the svn-src-all
mailing list