PERFORCE change 124874 for review
Kip Macy
kmacy at FreeBSD.org
Tue Aug 7 22:11:15 PDT 2007
http://perforce.freebsd.org/chv.cgi?CH=124874
Change 124874 by kmacy at kmacy_home:ethng on 2007/08/08 05:10:52
add intr_bind to i386 and amd64 to bind an interrupt to caller-specified
cpu rather than the default round robin
Submitted by: jhb
Affected files ...
.. //depot/projects/ethng/src/sys/amd64/amd64/intr_machdep.c#2 edit
.. //depot/projects/ethng/src/sys/amd64/amd64/sys_machdep.c#2 edit
.. //depot/projects/ethng/src/sys/amd64/include/intr_machdep.h#2 edit
.. //depot/projects/ethng/src/sys/amd64/include/sysarch.h#2 edit
.. //depot/projects/ethng/src/sys/conf/options#3 edit
.. //depot/projects/ethng/src/sys/i386/i386/intr_machdep.c#2 edit
.. //depot/projects/ethng/src/sys/i386/i386/sys_machdep.c#2 edit
.. //depot/projects/ethng/src/sys/i386/include/intr_machdep.h#2 edit
.. //depot/projects/ethng/src/sys/i386/include/sysarch.h#2 edit
.. //depot/projects/ethng/src/sys/kern/kern_intr.c#2 edit
.. //depot/projects/ethng/src/sys/sys/interrupt.h#2 edit
Differences ...
==== //depot/projects/ethng/src/sys/amd64/amd64/intr_machdep.c#2 (text+ko) ====
@@ -51,6 +51,7 @@
#include <sys/syslog.h>
#include <sys/systm.h>
#include <sys/sx.h>
+#include <sys/smp.h>
#include <machine/clock.h>
#include <machine/intr_machdep.h>
#include <machine/smp.h>
@@ -87,7 +88,7 @@
static void intr_assign_next_cpu(struct intsrc *isrc);
#endif
-
+static int intr_assign_cpu(void *arg, u_char cpu);
static void intr_init(void *__dummy);
static int intr_pic_registered(struct pic *pic);
static void intrcnt_setname(const char *name, int index);
@@ -145,10 +146,10 @@
#ifdef INTR_FILTER
error = intr_event_create(&isrc->is_event, isrc, 0,
(mask_fn)isrc->is_pic->pic_enable_source,
- intr_eoi_src, intr_disab_eoi_src, "irq%d:", vector);
+ intr_eoi_src, intr_disab_eoi_src, intr_assign_cpu, "irq%d:", vector);
#else
error = intr_event_create(&isrc->is_event, isrc, 0,
- (mask_fn)isrc->is_pic->pic_enable_source, "irq%d:", vector);
+ (mask_fn)isrc->is_pic->pic_enable_source, intr_assign_cpu, "irq%d:", vector);
#endif
if (error)
return (error);
@@ -430,6 +431,33 @@
sx_xunlock(&intr_table_lock);
}
+static int
+intr_assign_cpu(void *arg, u_char cpu)
+{
+#ifdef SMP
+ struct intsrc *isrc;
+
+ /*
+ * Don't do anything during early boot. We will pick up the
+ * assignment once the APs are started.
+ */
+ if (assign_cpu) {
+ isrc = arg;
+
+ if (bootverbose)
+ printf("assigning vector=%d to cpu=%d\n", isrc->is_pic->pic_vector(isrc), cpu);
+
+ sx_xlock(&intr_table_lock);
+ isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[cpu]);
+ sx_xunlock(&intr_table_lock);
+ }
+ return (0);
+#else
+ return (EOPNOTSUPP);
+#endif
+}
+
+
static void
intrcnt_setname(const char *name, int index)
{
@@ -477,12 +505,16 @@
static void
intr_init(void *dummy __unused)
{
-
+ int flags = 0;
+
intrcnt_setname("???", 0);
intrcnt_index = 1;
STAILQ_INIT(&pics);
- sx_init(&intr_table_lock, "intr sources");
- mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN);
+#ifdef BIND_ALL
+ flags = SX_RECURSE;
+#endif
+ sx_init_flags(&intr_table_lock, "intr sources", flags);
+ mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN);
}
SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL)
@@ -541,15 +573,14 @@
static void
intr_assign_next_cpu(struct intsrc *isrc)
{
- struct pic *pic;
- u_int apic_id;
-
/*
* Assign this source to a local APIC in a round-robin fashion.
*/
- pic = isrc->is_pic;
- apic_id = cpu_apic_ids[current_cpu];
- pic->pic_assign_cpu(isrc, apic_id);
+#ifdef BIND_ALL
+ intr_event_bind(isrc->is_event, current_cpu);
+#else
+ isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[current_cpu]);
+#endif
do {
current_cpu++;
if (current_cpu >= num_cpus)
@@ -557,6 +588,20 @@
} while (!(intr_cpus & (1 << current_cpu)));
}
+/* Attempt to bind the specified IRQ to the specified CPU. */
+int
+intr_bind(u_int vector, u_char cpu)
+{
+ struct intsrc *isrc;
+
+ isrc = intr_lookup_source(vector);
+ if (isrc == NULL)
+ return (EINVAL);
+ isrc->is_event->ie_source = isrc;
+
+ return (intr_event_bind(isrc->is_event, cpu));
+}
+
/*
* Add a CPU to our mask of valid CPUs that can be destinations of
* interrupts.
@@ -585,6 +630,10 @@
struct intsrc *isrc;
int i;
+ /*
+ * XXX should be mp_ncpus but this causes a crash
+ */
+
/* Don't bother on UP. */
if (num_cpus <= 1)
return;
@@ -594,8 +643,19 @@
assign_cpu = 1;
for (i = 0; i < NUM_IO_INTS; i++) {
isrc = interrupt_sources[i];
- if (isrc != NULL && isrc->is_handlers > 0)
- intr_assign_next_cpu(isrc);
+ if (isrc != NULL && isrc->is_handlers > 0) {
+ /*
+ * If this event is already bound to a CPU,
+ * then assign the source to that CPU instead
+ * of picking one via round-robin.
+ */
+ if (isrc->is_event->ie_cpu != NOCPU)
+ isrc->is_pic->pic_assign_cpu(isrc,
+ cpu_apic_ids[isrc->is_event->ie_cpu]);
+ else
+ intr_assign_next_cpu(isrc);
+
+ }
}
sx_xunlock(&intr_table_lock);
}
==== //depot/projects/ethng/src/sys/amd64/amd64/sys_machdep.c#2 (text+ko) ====
@@ -35,9 +35,11 @@
#include <sys/param.h>
#include <sys/systm.h>
-#include <sys/lock.h>
+#include <sys/bus.h>
+#include <sys/interrupt.h>
#include <sys/proc.h>
#include <sys/sysproto.h>
+#include <machine/intr_machdep.h>
#include <machine/specialreg.h>
#include <machine/sysarch.h>
#include <machine/pcb.h>
@@ -62,7 +64,10 @@
struct pcb *pcb = curthread->td_pcb;
uint32_t i386base;
uint64_t a64base;
-
+#ifdef SMP
+ struct amd64_intr_bind_args bargs;
+#endif
+
switch(uap->op) {
case I386_GET_FSBASE:
i386base = pcb->pcb_fsbase;
@@ -126,6 +131,15 @@
}
break;
+#ifdef SMP
+/* ABI and API compatible with I386_INTR_BIND. */
+ case AMD64_INTR_BIND:
+ error = copyin(uap->parms, &bargs,
+ sizeof(struct amd64_intr_bind_args));
+ if (error == 0)
+ error = intr_bind(bargs.vector, bargs.cpu);
+ break;
+#endif
default:
error = EINVAL;
break;
==== //depot/projects/ethng/src/sys/amd64/include/intr_machdep.h#2 (text+ko) ====
@@ -133,6 +133,7 @@
void elcr_write_trigger(u_int irq, enum intr_trigger trigger);
#ifdef SMP
void intr_add_cpu(u_int cpu);
+int intr_bind(u_int vector, u_char cpu);
#endif
int intr_add_handler(const char *name, int vector, driver_filter_t filter,
driver_intr_t handler, void *arg, enum intr_type flags,
==== //depot/projects/ethng/src/sys/amd64/include/sysarch.h#2 (text+ko) ====
@@ -39,6 +39,7 @@
#define I386_SET_FSBASE 8
#define I386_GET_GSBASE 9
#define I386_SET_GSBASE 10
+#define AMD64_INTR_BIND 11
/* Leave space for 0-127 for to avoid translating syscalls */
#define AMD64_GET_FSBASE 128
@@ -46,6 +47,11 @@
#define AMD64_GET_GSBASE 130
#define AMD64_SET_GSBASE 131
+struct amd64_intr_bind_args {
+ unsigned int vector;
+ unsigned int cpu;
+};
+
#ifndef _KERNEL
#include <sys/cdefs.h>
@@ -54,6 +60,7 @@
int amd64_get_gsbase(void **);
int amd64_set_fsbase(void *);
int amd64_set_gsbase(void *);
+int amd64_intr_bind(unsigned int, unsigned int);
int sysarch(int, void *);
__END_DECLS
#endif
==== //depot/projects/ethng/src/sys/conf/options#3 (text+ko) ====
@@ -547,6 +547,7 @@
SX_NOINLINE opt_global.h
VFS_BIO_DEBUG opt_global.h
IFNET_MULTIQUEUE opt_global.h
+BIND_ALL opt_global.h
# These are VM related options
VM_KMEM_SIZE opt_vm.h
==== //depot/projects/ethng/src/sys/i386/i386/intr_machdep.c#2 (text+ko) ====
@@ -79,6 +79,7 @@
static void intr_assign_next_cpu(struct intsrc *isrc);
#endif
+static int intr_assign_cpu(void *arg, u_char cpu);
static void intr_init(void *__dummy);
static int intr_pic_registered(struct pic *pic);
static void intrcnt_setname(const char *name, int index);
@@ -136,10 +137,10 @@
#ifdef INTR_FILTER
error = intr_event_create(&isrc->is_event, isrc, 0,
(mask_fn)isrc->is_pic->pic_enable_source,
- intr_eoi_src, intr_disab_eoi_src, "irq%d:", vector);
+ intr_eoi_src, intr_disab_eoi_src, intr_assign_cpu,, "irq%d:", vector);
#else
error = intr_event_create(&isrc->is_event, isrc, 0,
- (mask_fn)isrc->is_pic->pic_enable_source, "irq%d:", vector);
+ (mask_fn)isrc->is_pic->pic_enable_source, intr_assign_cpu, "irq%d:", vector);
#endif
if (error)
return (error);
@@ -428,6 +429,28 @@
sx_xunlock(&intr_table_lock);
}
+static int
+intr_assign_cpu(void *arg, u_char cpu)
+{
+#ifdef SMP
+ struct intsrc *isrc;
+
+ /*
+ * Don't do anything during early boot. We will pick up the
+ * assignment once the APs are started.
+ */
+ if (assign_cpu) {
+ isrc = arg;
+ sx_xlock(&intr_table_lock);
+ isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[cpu]);
+ sx_xunlock(&intr_table_lock);
+ }
+ return (0);
+#else
+ return (EOPNOTSUPP);
+#endif
+}
+
static void
intrcnt_setname(const char *name, int index)
{
@@ -475,11 +498,15 @@
static void
intr_init(void *dummy __unused)
{
-
+ int flags = 0;
+
intrcnt_setname("???", 0);
intrcnt_index = 1;
STAILQ_INIT(&pics);
- sx_init(&intr_table_lock, "intr sources");
+#ifdef BIND_ALL
+ flags = SX_RECURSE;
+#endif
+ sx_init_flags(&intr_table_lock, "intr sources", flags);
mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN);
}
SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL)
@@ -517,15 +544,15 @@
static void
intr_assign_next_cpu(struct intsrc *isrc)
{
- struct pic *pic;
- u_int apic_id;
/*
* Assign this source to a local APIC in a round-robin fashion.
*/
- pic = isrc->is_pic;
- apic_id = cpu_apic_ids[current_cpu];
- pic->pic_assign_cpu(isrc, apic_id);
+#ifdef BIND_ALL
+ intr_event_bind(isrc->is_event, current_cpu);
+#else
+ isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[current_cpu]);
+#endif
do {
current_cpu++;
if (current_cpu >= num_cpus)
@@ -533,6 +560,20 @@
} while (!(intr_cpus & (1 << current_cpu)));
}
++/* Attempt to bind the specified IRQ to the specified CPU. */
+int
+intr_bind(u_int vector, u_char cpu)
+{
+ struct intsrc *isrc;
+
+ isrc = intr_lookup_source(vector);
+ if (isrc == NULL)
+ return (EINVAL);
+ isrc->is_event->ie_source = isrc;
+
+ return (intr_event_bind(isrc->is_event, cpu));
+}
+
/*
* Add a CPU to our mask of valid CPUs that can be destinations of
* interrupts.
@@ -562,7 +603,7 @@
int i;
/* Don't bother on UP. */
- if (num_cpus <= 1)
+ if (mp_ncpus <= 1)
return;
/* Round-robin assign a CPU to each enabled source. */
@@ -570,8 +611,14 @@
assign_cpu = 1;
for (i = 0; i < NUM_IO_INTS; i++) {
isrc = interrupt_sources[i];
- if (isrc != NULL && isrc->is_handlers > 0)
- intr_assign_next_cpu(isrc);
+ if (isrc != NULL && isrc->is_handlers > 0) {
+ if (isrc->is_event->ie_cpu != NOCPU)
+ isrc->is_pic->pic_assign_cpu(isrc,
+ cpu_apic_ids[isrc->is_event->ie_cpu]);
+ else
+ intr_assign_next_cpu(isrc);
+
+ }
}
sx_xunlock(&intr_table_lock);
}
==== //depot/projects/ethng/src/sys/i386/i386/sys_machdep.c#2 (text+ko) ====
@@ -37,6 +37,8 @@
#include <sys/param.h>
#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/interrupt.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
@@ -44,6 +46,7 @@
#include <sys/proc.h>
#include <sys/smp.h>
#include <sys/sysproto.h>
+#include <machine/intr_machdep.h>
#include <vm/vm.h>
#include <vm/pmap.h>
@@ -90,6 +93,9 @@
union {
struct i386_ldt_args largs;
struct i386_ioperm_args iargs;
+#ifdef SMP
+ struct i386_intr_bind_args bargs;
+#endif
} kargs;
uint32_t base;
struct segment_descriptor sd, *sdp;
@@ -209,6 +215,14 @@
load_gs(GSEL(GUGS_SEL, SEL_UPL));
}
break;
+#ifdef SMP
+ case I386_INTR_BIND:
+ error = copyin(uap->parms, &kargs.bargs,
+ sizeof(struct i386_intr_bind_args));
+ if (error == 0)
+ error = intr_bind(kargs.bargs.vector, kargs.bargs.cpu);
+ break;
+#endif
default:
error = EINVAL;
break;
==== //depot/projects/ethng/src/sys/i386/include/intr_machdep.h#2 (text+ko) ====
@@ -130,6 +130,7 @@
void elcr_write_trigger(u_int irq, enum intr_trigger trigger);
#ifdef SMP
void intr_add_cpu(u_int cpu);
+int intr_bind(int vector, u_int cpu);
#endif
int intr_add_handler(const char *name, int vector, driver_filter_t filter,
driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep);
==== //depot/projects/ethng/src/sys/i386/include/sysarch.h#2 (text+ko) ====
@@ -47,6 +47,7 @@
#define I386_SET_FSBASE 8
#define I386_GET_GSBASE 9
#define I386_SET_GSBASE 10
+#define I386_INTR_BIND 11
/* These four only exist when running an i386 binary on amd64 */
#define _AMD64_GET_FSBASE 128
@@ -71,6 +72,11 @@
char *sub_args; /* args */
};
+struct i386_intr_bind_args {
+ unsigned int vector;
+ unsigned int cpu;
+};
+
#ifndef _KERNEL
#include <sys/cdefs.h>
==== //depot/projects/ethng/src/sys/kern/kern_intr.c#2 (text+ko) ====
@@ -46,6 +46,7 @@
#include <sys/random.h>
#include <sys/resourcevar.h>
#include <sys/sched.h>
+#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/unistd.h>
#include <sys/vmmeter.h>
@@ -240,7 +241,7 @@
#ifndef INTR_FILTER
int
intr_event_create(struct intr_event **event, void *source, int flags,
- void (*enable)(void *), const char *fmt, ...)
+ void (*enable)(void *), int (*assign_cpu)(void *, u_char), const char *fmt, ...)
{
struct intr_event *ie;
va_list ap;
@@ -251,6 +252,8 @@
ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO);
ie->ie_source = source;
ie->ie_enable = enable;
+ ie->ie_assign_cpu = assign_cpu;
+ ie->ie_cpu = NOCPU;
ie->ie_flags = flags;
TAILQ_INIT(&ie->ie_handlers);
mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF);
@@ -271,7 +274,7 @@
int
intr_event_create(struct intr_event **event, void *source, int flags,
void (*enable)(void *), void (*eoi)(void *), void (*disab)(void *),
- const char *fmt, ...)
+ int (*assign_cpu)(void *, u_char), const char *fmt, ...)
{
struct intr_event *ie;
va_list ap;
@@ -302,6 +305,32 @@
}
#endif
+/*
+ * Bind an interrupt event to the specified CPU.
+ */
+int
+intr_event_bind(struct intr_event *ie, u_char cpu)
+{
+ int error;
+ struct thread *td = curthread;
+
+ /* Need a CPU to bind to. */
+ if (cpu != NOCPU && CPU_ABSENT(cpu))
+ return (EINVAL);
+
+ if (ie->ie_assign_cpu == NULL)
+ return (EOPNOTSUPP);
+
+ error = ie->ie_assign_cpu(ie->ie_source, cpu);
+
+ thread_lock(td);
+ ie->ie_cpu = cpu;
+ thread_unlock(td);
+
+
+ return (0);
+}
+
int
intr_event_destroy(struct intr_event *ie)
{
@@ -895,10 +924,10 @@
} else {
#ifdef INTR_FILTER
error = intr_event_create(&ie, NULL, IE_SOFT,
- NULL, NULL, NULL, "swi%d:", pri);
+ NULL, NULL, NULL, NULL, "swi%d:", pri);
#else
error = intr_event_create(&ie, NULL, IE_SOFT,
- NULL, "swi%d:", pri);
+ NULL, NULL, "swi%d:", pri);
#endif
if (error)
return (error);
@@ -1081,6 +1110,7 @@
struct intr_event *ie;
struct thread *td;
struct proc *p;
+ u_char cpu;
td = curthread;
p = td->td_proc;
@@ -1089,7 +1119,8 @@
("%s: ithread and proc linkage out of sync", __func__));
ie = ithd->it_event;
ie->ie_count = 0;
-
+ cpu = NOCPU;
+
/*
* As long as we have interrupts outstanding, go through the
* list of handlers, giving each one a go at it.
@@ -1134,6 +1165,21 @@
ie->ie_count = 0;
mi_switch(SW_VOL, NULL);
}
+
+#ifdef SMP
+ /*
+ * Ensure we are bound to the correct CPU. We can't
+ * move ithreads until SMP is running however, so just
+ * leave interrupts on the boor CPU during boot.
+ */
+ if (ie->ie_cpu != cpu && smp_started) {
+ cpu = ie->ie_cpu;
+ if (cpu == NOCPU)
+ sched_unbind(td);
+ else
+ sched_bind(td, cpu);
+ }
+#endif
thread_unlock(td);
}
}
@@ -1463,6 +1509,8 @@
db_printf("ADDING_THREAD");
comma = 1;
}
+ if (ie->ie_cpu != NOCPU)
+ db_printf(" (CPU %d)", ie->ie_cpu);
if (it != NULL && it->it_need) {
if (comma)
db_printf(", ");
==== //depot/projects/ethng/src/sys/sys/interrupt.h#2 (text+ko) ====
@@ -73,6 +73,7 @@
void *ie_source; /* Cookie used by MD code. */
struct intr_thread *ie_thread; /* Thread we are connected to. */
void (*ie_enable)(void *);
+ int (*ie_assign_cpu)(void *, u_char);
#ifdef INTR_FILTER
void (*ie_eoi)(void *);
void (*ie_disab)(void *);
@@ -81,6 +82,7 @@
int ie_count; /* Loop counter. */
int ie_warncnt; /* Rate-check interrupt storm warns. */
struct timeval ie_warntm;
+ u_char ie_cpu;
};
/* Interrupt event flags kept in ie_flags. */
@@ -127,15 +129,16 @@
int intr_event_add_handler(struct intr_event *ie, const char *name,
driver_filter_t filter, driver_intr_t handler, void *arg,
u_char pri, enum intr_type flags, void **cookiep);
+int intr_event_bind(struct intr_event *ie, u_char cpu);
#ifndef INTR_FILTER
int intr_event_create(struct intr_event **event, void *source,
- int flags, void (*enable)(void *), const char *fmt, ...)
- __printflike(5, 6);
+ int flags, void (*enable)(void *), int (*assign_cpu)(void *, u_char), const char *fmt, ...)
+ __printflike(6, 7);
#else
int intr_event_create(struct intr_event **event, void *source,
int flags, void (*enable)(void *), void (*eoi)(void *),
- void (*disab)(void *), const char *fmt, ...)
- __printflike(7, 8);
+ void (*disab)(void *), int (*assign_cpu)(void *, u_char), const char *fmt, ...)
+ __printflike(8, 9);
#endif
int intr_event_destroy(struct intr_event *ie);
int intr_event_remove_handler(void *cookie);
More information about the p4-projects
mailing list