svn commit: r329849 - in user/jeff/numa/sys: amd64/include i386/include sys vm x86/x86 x86/xen
Jeff Roberson
jeff at FreeBSD.org
Fri Feb 23 01:20:16 UTC 2018
Author: jeff
Date: Fri Feb 23 01:20:14 2018
New Revision: 329849
URL: https://svnweb.freebsd.org/changeset/base/329849
Log:
By default, round-robin assign interrupts only within the domain that the
device is connected to.
Modified:
user/jeff/numa/sys/amd64/include/intr_machdep.h
user/jeff/numa/sys/i386/include/intr_machdep.h
user/jeff/numa/sys/sys/smp.h
user/jeff/numa/sys/vm/vm_phys.h
user/jeff/numa/sys/vm/vm_reserv.c
user/jeff/numa/sys/x86/x86/intr_machdep.c
user/jeff/numa/sys/x86/x86/io_apic.c
user/jeff/numa/sys/x86/x86/msi.c
user/jeff/numa/sys/x86/x86/nexus.c
user/jeff/numa/sys/x86/xen/xen_intr.c
Modified: user/jeff/numa/sys/amd64/include/intr_machdep.h
==============================================================================
--- user/jeff/numa/sys/amd64/include/intr_machdep.h Fri Feb 23 01:11:57 2018 (r329848)
+++ user/jeff/numa/sys/amd64/include/intr_machdep.h Fri Feb 23 01:20:14 2018 (r329849)
@@ -132,6 +132,7 @@ struct intsrc {
u_long *is_straycount;
u_int is_index;
u_int is_handlers;
+ u_int is_domain;
u_int is_cpu;
};
@@ -168,7 +169,7 @@ void intr_add_cpu(u_int cpu);
#endif
int intr_add_handler(const char *name, int vector, driver_filter_t filter,
driver_intr_t handler, void *arg, enum intr_type flags,
- void **cookiep);
+ void **cookiep, int domain);
#ifdef SMP
int intr_bind(u_int vector, u_char cpu);
#endif
@@ -176,7 +177,7 @@ int intr_config_intr(int vector, enum intr_trigger tri
enum intr_polarity pol);
int intr_describe(u_int vector, void *ih, const char *descr);
void intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame);
-u_int intr_next_cpu(void);
+u_int intr_next_cpu(int domain);
struct intsrc *intr_lookup_source(int vector);
int intr_register_pic(struct pic *pic);
int intr_register_source(struct intsrc *isrc);
Modified: user/jeff/numa/sys/i386/include/intr_machdep.h
==============================================================================
--- user/jeff/numa/sys/i386/include/intr_machdep.h Fri Feb 23 01:11:57 2018 (r329848)
+++ user/jeff/numa/sys/i386/include/intr_machdep.h Fri Feb 23 01:20:14 2018 (r329849)
@@ -132,6 +132,7 @@ struct intsrc {
u_long *is_straycount;
u_int is_index;
u_int is_handlers;
+ u_int is_domain;
u_int is_cpu;
};
@@ -158,7 +159,8 @@ void elcr_write_trigger(u_int irq, enum intr_trigger t
void intr_add_cpu(u_int cpu);
#endif
int intr_add_handler(const char *name, int vector, driver_filter_t filter,
- driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep);
+ driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep,
+ int domain);
#ifdef SMP
int intr_bind(u_int vector, u_char cpu);
#endif
@@ -166,7 +168,7 @@ int intr_config_intr(int vector, enum intr_trigger tri
enum intr_polarity pol);
int intr_describe(u_int vector, void *ih, const char *descr);
void intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame);
-u_int intr_next_cpu(void);
+u_int intr_next_cpu(int domain);
struct intsrc *intr_lookup_source(int vector);
int intr_register_pic(struct pic *pic);
int intr_register_source(struct intsrc *isrc);
Modified: user/jeff/numa/sys/sys/smp.h
==============================================================================
--- user/jeff/numa/sys/sys/smp.h Fri Feb 23 01:11:57 2018 (r329848)
+++ user/jeff/numa/sys/sys/smp.h Fri Feb 23 01:20:14 2018 (r329849)
@@ -169,6 +169,7 @@ extern u_int mp_maxid;
extern int mp_maxcpus;
extern int mp_ncpus;
extern volatile int smp_started;
+extern int vm_ndomains;
extern cpuset_t all_cpus;
extern cpuset_t cpuset_domain[MAXMEMDOM]; /* CPUs in each NUMA domain. */
Modified: user/jeff/numa/sys/vm/vm_phys.h
==============================================================================
--- user/jeff/numa/sys/vm/vm_phys.h Fri Feb 23 01:11:57 2018 (r329848)
+++ user/jeff/numa/sys/vm/vm_phys.h Fri Feb 23 01:20:14 2018 (r329849)
@@ -52,7 +52,6 @@ struct mem_affinity {
extern struct mem_affinity *mem_affinity;
extern int *mem_locality;
#endif
-extern int vm_ndomains;
struct vm_freelist {
struct pglist pl;
Modified: user/jeff/numa/sys/vm/vm_reserv.c
==============================================================================
--- user/jeff/numa/sys/vm/vm_reserv.c Fri Feb 23 01:11:57 2018 (r329848)
+++ user/jeff/numa/sys/vm/vm_reserv.c Fri Feb 23 01:20:14 2018 (r329849)
@@ -54,6 +54,7 @@ __FBSDID("$FreeBSD$");
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <sys/vmmeter.h>
+#include <sys/smp.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
Modified: user/jeff/numa/sys/x86/x86/intr_machdep.c
==============================================================================
--- user/jeff/numa/sys/x86/x86/intr_machdep.c Fri Feb 23 01:11:57 2018 (r329848)
+++ user/jeff/numa/sys/x86/x86/intr_machdep.c Fri Feb 23 01:20:14 2018 (r329849)
@@ -183,7 +183,8 @@ intr_lookup_source(int vector)
int
intr_add_handler(const char *name, int vector, driver_filter_t filter,
- driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep)
+ driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep,
+ int domain)
{
struct intsrc *isrc;
int error;
@@ -198,6 +199,7 @@ intr_add_handler(const char *name, int vector, driver_
intrcnt_updatename(isrc);
isrc->is_handlers++;
if (isrc->is_handlers == 1) {
+ isrc->is_domain = domain;
isrc->is_pic->pic_enable_intr(isrc);
isrc->is_pic->pic_enable_source(isrc);
}
@@ -505,14 +507,14 @@ DB_SHOW_COMMAND(irqs, db_show_irqs)
*/
cpuset_t intr_cpus = CPUSET_T_INITIALIZER(0x1);
-static int current_cpu;
+static int current_cpu[MAXMEMDOM];
/*
* Return the CPU that the next interrupt source should use. For now
* this just returns the next local APIC according to round-robin.
*/
u_int
-intr_next_cpu(void)
+intr_next_cpu(int domain)
{
u_int apic_id;
@@ -527,12 +529,13 @@ intr_next_cpu(void)
#endif
mtx_lock_spin(&icu_lock);
- apic_id = cpu_apic_ids[current_cpu];
+ apic_id = cpu_apic_ids[current_cpu[domain]];
do {
- current_cpu++;
- if (current_cpu > mp_maxid)
- current_cpu = 0;
- } while (!CPU_ISSET(current_cpu, &intr_cpus));
+ current_cpu[domain]++;
+ if (current_cpu[domain] > mp_maxid)
+ current_cpu[domain] = 0;
+ } while (!CPU_ISSET(current_cpu[domain], &intr_cpus) ||
+ !CPU_ISSET(current_cpu[domain], &cpuset_domain[domain]));
mtx_unlock_spin(&icu_lock);
return (apic_id);
}
@@ -597,12 +600,12 @@ intr_shuffle_irqs(void *arg __unused)
*/
cpu = isrc->is_event->ie_cpu;
if (cpu == NOCPU)
- cpu = current_cpu;
+ cpu = current_cpu[isrc->is_domain];
if (isrc->is_pic->pic_assign_cpu(isrc,
cpu_apic_ids[cpu]) == 0) {
isrc->is_cpu = cpu;
if (isrc->is_event->ie_cpu == NOCPU)
- intr_next_cpu();
+ intr_next_cpu(isrc->is_domain);
}
}
}
@@ -633,10 +636,11 @@ sysctl_hw_intrs(SYSCTL_HANDLER_ARGS)
isrc = interrupt_sources[i];
if (isrc == NULL)
continue;
- sbuf_printf(&sbuf, "%s:%d @%d: %ld\n",
+ sbuf_printf(&sbuf, "%s:%d @cpu%d(domain%d): %ld\n",
isrc->is_event->ie_fullname,
isrc->is_index,
isrc->is_cpu,
+ isrc->is_domain,
*isrc->is_count);
}
@@ -695,7 +699,8 @@ intr_balance(void *dummy __unused, int pending __unuse
* Restart the scan from the same location to avoid moving in the
* common case.
*/
- current_cpu = 0;
+ for (i = 0; i < vm_ndomains; i++)
+ current_cpu[i] = 0;
/*
* Assign round-robin from most loaded to least.
@@ -704,8 +709,8 @@ intr_balance(void *dummy __unused, int pending __unuse
isrc = interrupt_sorted[i];
if (isrc == NULL || isrc->is_event->ie_cpu != NOCPU)
continue;
- cpu = current_cpu;
- intr_next_cpu();
+ cpu = current_cpu[isrc->is_domain];
+ intr_next_cpu(isrc->is_domain);
if (isrc->is_cpu != cpu &&
isrc->is_pic->pic_assign_cpu(isrc,
cpu_apic_ids[cpu]) == 0)
@@ -733,7 +738,7 @@ SYSINIT(intr_balance_init, SI_SUB_SMP, SI_ORDER_ANY, i
* Always route interrupts to the current processor in the UP case.
*/
u_int
-intr_next_cpu(void)
+intr_next_cpu(int domain)
{
return (PCPU_GET(apic_id));
Modified: user/jeff/numa/sys/x86/x86/io_apic.c
==============================================================================
--- user/jeff/numa/sys/x86/x86/io_apic.c Fri Feb 23 01:11:57 2018 (r329848)
+++ user/jeff/numa/sys/x86/x86/io_apic.c Fri Feb 23 01:20:14 2018 (r329849)
@@ -499,7 +499,7 @@ ioapic_enable_intr(struct intsrc *isrc)
struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc;
if (intpin->io_vector == 0)
- if (ioapic_assign_cpu(isrc, intr_next_cpu()) != 0)
+ if (ioapic_assign_cpu(isrc, intr_next_cpu(isrc->is_domain)) != 0)
panic("Couldn't find an APIC vector for IRQ %d",
intpin->io_irq);
apic_enable_vector(intpin->io_cpu, intpin->io_vector);
Modified: user/jeff/numa/sys/x86/x86/msi.c
==============================================================================
--- user/jeff/numa/sys/x86/x86/msi.c Fri Feb 23 01:11:57 2018 (r329848)
+++ user/jeff/numa/sys/x86/x86/msi.c Fri Feb 23 01:20:14 2018 (r329849)
@@ -363,7 +363,7 @@ int
msi_alloc(device_t dev, int count, int maxcount, int *irqs)
{
struct msi_intsrc *msi, *fsrc;
- u_int cpu;
+ u_int cpu, domain;
int cnt, i, *mirqs, vector;
#ifdef ACPI_DMAR
u_int cookies[count];
@@ -373,6 +373,9 @@ msi_alloc(device_t dev, int count, int maxcount, int *
if (!msi_enabled)
return (ENXIO);
+ if (bus_get_domain(dev, &domain) != 0)
+ domain = 0;
+
if (count > 1)
mirqs = malloc(count * sizeof(*mirqs), M_MSI, M_WAITOK);
else
@@ -420,7 +423,7 @@ again:
KASSERT(cnt == count, ("count mismatch"));
/* Allocate 'count' IDT vectors. */
- cpu = intr_next_cpu();
+ cpu = intr_next_cpu(domain);
vector = apic_alloc_vectors(cpu, irqs, count, maxcount);
if (vector == 0) {
mtx_unlock(&msi_lock);
@@ -610,7 +613,7 @@ int
msix_alloc(device_t dev, int *irq)
{
struct msi_intsrc *msi;
- u_int cpu;
+ u_int cpu, domain;
int i, vector;
#ifdef ACPI_DMAR
u_int cookie;
@@ -620,6 +623,9 @@ msix_alloc(device_t dev, int *irq)
if (!msi_enabled)
return (ENXIO);
+ if (bus_get_domain(dev, &domain) != 0)
+ domain = 0;
+
again:
mtx_lock(&msi_lock);
@@ -651,7 +657,7 @@ again:
}
/* Allocate an IDT vector. */
- cpu = intr_next_cpu();
+ cpu = intr_next_cpu(domain);
vector = apic_alloc_vector(cpu, i);
if (vector == 0) {
mtx_unlock(&msi_lock);
Modified: user/jeff/numa/sys/x86/x86/nexus.c
==============================================================================
--- user/jeff/numa/sys/x86/x86/nexus.c Fri Feb 23 01:11:57 2018 (r329848)
+++ user/jeff/numa/sys/x86/x86/nexus.c Fri Feb 23 01:20:14 2018 (r329849)
@@ -573,7 +573,7 @@ nexus_setup_intr(device_t bus, device_t child, struct
int flags, driver_filter_t filter, void (*ihand)(void *),
void *arg, void **cookiep)
{
- int error;
+ int error, domain;
/* somebody tried to setup an irq that failed to allocate! */
if (irq == NULL)
@@ -589,9 +589,11 @@ nexus_setup_intr(device_t bus, device_t child, struct
error = rman_activate_resource(irq);
if (error)
return (error);
+ if (bus_get_domain(child, &domain) != 0)
+ domain = 0;
error = intr_add_handler(device_get_nameunit(child),
- rman_get_start(irq), filter, ihand, arg, flags, cookiep);
+ rman_get_start(irq), filter, ihand, arg, flags, cookiep, domain);
return (error);
}
Modified: user/jeff/numa/sys/x86/xen/xen_intr.c
==============================================================================
--- user/jeff/numa/sys/x86/xen/xen_intr.c Fri Feb 23 01:11:57 2018 (r329848)
+++ user/jeff/numa/sys/x86/xen/xen_intr.c Fri Feb 23 01:20:14 2018 (r329849)
@@ -430,7 +430,7 @@ xen_intr_bind_isrc(struct xenisrc **isrcp, evtchn_port
* unless specified otherwise, so shuffle them to balance
* the interrupt load.
*/
- xen_intr_assign_cpu(&isrc->xi_intsrc, intr_next_cpu());
+ xen_intr_assign_cpu(&isrc->xi_intsrc, intr_next_cpu(0));
}
#endif
@@ -1562,7 +1562,7 @@ xen_intr_add_handler(const char *name, driver_filter_t
return (EINVAL);
error = intr_add_handler(name, isrc->xi_vector,filter, handler, arg,
- flags|INTR_EXCL, &isrc->xi_cookie);
+ flags|INTR_EXCL, &isrc->xi_cookie, 0);
if (error != 0) {
printf(
"%s: xen_intr_add_handler: intr_add_handler failed: %d\n",
More information about the svn-src-user
mailing list