svn commit: r351200 - in head/sys: amd64/amd64 dev/acpica

[Jeff Roberson]

Author: jeff
Date: Sun Aug 18 23:44:23 2019
New Revision: 351200
URL: https://svnweb.freebsd.org/changeset/base/351200

Log:
  Allocate all per-cpu datastructures in domain correct memory.
  
  Reviewed by:	kib, gallatin (some objections)
  Tested by:	pho
  Sponsored by:	Netflix
  Differential Revision:	https://reviews.freebsd.org/D21242

Modified:
  head/sys/amd64/amd64/mp_machdep.c
  head/sys/dev/acpica/acpi_pxm.c
  head/sys/dev/acpica/acpivar.h

Modified: head/sys/amd64/amd64/mp_machdep.c
==============================================================================
--- head/sys/amd64/amd64/mp_machdep.c	Sun Aug 18 23:10:07 2019	(r351199)
+++ head/sys/amd64/amd64/mp_machdep.c	Sun Aug 18 23:44:23 2019	(r351200)
@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/bus.h>
 #include <sys/cpuset.h>
+#include <sys/domainset.h>
 #ifdef GPROF 
 #include <sys/gmon.h>
 #endif
@@ -59,6 +60,8 @@ __FBSDID("$FreeBSD$");
 #include <vm/pmap.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_extern.h>
+#include <vm/vm_page.h>
+#include <vm/vm_phys.h>
 
 #include <x86/apicreg.h>
 #include <machine/clock.h>
@@ -75,6 +78,9 @@ __FBSDID("$FreeBSD$");
 #include <machine/cpu.h>
 #include <x86/init.h>
 
+#include <contrib/dev/acpica/include/acpi.h>
+#include <dev/acpica/acpivar.h>
+
 #define WARMBOOT_TARGET		0
 #define WARMBOOT_OFF		(KERNBASE + 0x0467)
 #define WARMBOOT_SEG		(KERNBASE + 0x0469)
@@ -384,6 +390,25 @@ init_secondary(void)
  * local functions and data
  */
 
+#ifdef NUMA
+static void
+mp_realloc_pcpu(int cpuid, int domain)
+{
+	vm_page_t m;
+	vm_offset_t oa, na;
+
+	oa = (vm_offset_t)&__pcpu[cpuid];
+	if (_vm_phys_domain(pmap_kextract(oa)) == domain)
+		return;
+	m = vm_page_alloc_domain(NULL, 0, domain,
+	    VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_ZERO);
+	na = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
+	pagecopy((void *)oa, (void *)na);
+	pmap_enter(kernel_pmap, oa, m, VM_PROT_READ | VM_PROT_WRITE, 0, 0);
+	/* XXX old pcpu page leaked. */
+}
+#endif
+
 /*
  * start each AP in our list
  */
@@ -392,7 +417,7 @@ native_start_all_aps(void)
 {
 	u_int64_t *pt4, *pt3, *pt2;
 	u_int32_t mpbioswarmvec;
-	int apic_id, cpu, i;
+	int apic_id, cpu, domain, i;
 	u_char mpbiosreason;
 
 	mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
@@ -431,21 +456,41 @@ native_start_all_aps(void)
 	outb(CMOS_REG, BIOS_RESET);
 	outb(CMOS_DATA, BIOS_WARM);	/* 'warm-start' */
 
+	/* Relocate pcpu areas to the correct domain. */
+#ifdef NUMA
+	if (vm_ndomains > 1)
+		for (cpu = 1; cpu < mp_ncpus; cpu++) {
+			apic_id = cpu_apic_ids[cpu];
+			domain = acpi_pxm_get_cpu_locality(apic_id);
+			mp_realloc_pcpu(cpu, domain);
+		}
+#endif
+
 	/* start each AP */
+	domain = 0;
 	for (cpu = 1; cpu < mp_ncpus; cpu++) {
 		apic_id = cpu_apic_ids[cpu];
-
+#ifdef NUMA
+		if (vm_ndomains > 1)
+			domain = acpi_pxm_get_cpu_locality(apic_id);
+#endif
 		/* allocate and set up an idle stack data page */
-		bootstacks[cpu] = (void *)kmem_malloc(kstack_pages * PAGE_SIZE,
+		bootstacks[cpu] = (void *)kmem_malloc_domainset(
+		    DOMAINSET_FIXED(domain), kstack_pages * PAGE_SIZE,
 		    M_WAITOK | M_ZERO);
-		doublefault_stack = (char *)kmem_malloc(PAGE_SIZE, M_WAITOK |
-		    M_ZERO);
-		mce_stack = (char *)kmem_malloc(PAGE_SIZE, M_WAITOK | M_ZERO);
-		nmi_stack = (char *)kmem_malloc(PAGE_SIZE, M_WAITOK | M_ZERO);
-		dbg_stack = (char *)kmem_malloc(PAGE_SIZE, M_WAITOK | M_ZERO);
-		dpcpu = (void *)kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO);
+		doublefault_stack = (char *)kmem_malloc_domainset(
+		    DOMAINSET_FIXED(domain), PAGE_SIZE, M_WAITOK | M_ZERO);
+		mce_stack = (char *)kmem_malloc_domainset(
+		    DOMAINSET_FIXED(domain), PAGE_SIZE, M_WAITOK | M_ZERO);
+		nmi_stack = (char *)kmem_malloc_domainset(
+		    DOMAINSET_FIXED(domain), PAGE_SIZE, M_WAITOK | M_ZERO);
+		dbg_stack = (char *)kmem_malloc_domainset(
+		    DOMAINSET_FIXED(domain), PAGE_SIZE, M_WAITOK | M_ZERO);
+		dpcpu = (void *)kmem_malloc_domainset(DOMAINSET_FIXED(domain),
+		    DPCPU_SIZE, M_WAITOK | M_ZERO);
 
-		bootSTK = (char *)bootstacks[cpu] + kstack_pages * PAGE_SIZE - 8;
+		bootSTK = (char *)bootstacks[cpu] +
+		    kstack_pages * PAGE_SIZE - 8;
 		bootAP = cpu;
 
 		/* attempt to start the Application Processor */

Modified: head/sys/dev/acpica/acpi_pxm.c
==============================================================================
--- head/sys/dev/acpica/acpi_pxm.c	Sun Aug 18 23:10:07 2019	(r351199)
+++ head/sys/dev/acpica/acpi_pxm.c	Sun Aug 18 23:44:23 2019	(r351200)
@@ -653,6 +653,17 @@ acpi_pxm_set_cpu_locality(void)
 	}
 }
 
+int
+acpi_pxm_get_cpu_locality(int apic_id)
+{
+	struct cpu_info *cpu;
+
+	cpu = cpu_find(apic_id);
+	if (cpu == NULL)
+		panic("SRAT: CPU with ID %u is not known", apic_id);
+	return (cpu->domain);
+}
+
 /*
  * Free data structures allocated during acpi_pxm_init.
  */

Modified: head/sys/dev/acpica/acpivar.h
==============================================================================
--- head/sys/dev/acpica/acpivar.h	Sun Aug 18 23:10:07 2019	(r351199)
+++ head/sys/dev/acpica/acpivar.h	Sun Aug 18 23:44:23 2019	(r351200)
@@ -532,6 +532,7 @@ int		acpi_pxm_init(int ncpus, vm_paddr_t maxphys);
 void		acpi_pxm_parse_tables(void);
 void		acpi_pxm_set_mem_locality(void);
 void		acpi_pxm_set_cpu_locality(void);
+int		acpi_pxm_get_cpu_locality(int apic_id);
 void		acpi_pxm_free(void);
 
 /*