[patch and review please] 64 CPU Support
John Baldwin
jhb at freebsd.org
Tue Aug 3 20:50:31 UTC 2010
On Monday, July 26, 2010 10:43:59 am John Baldwin wrote:
> On Sunday, July 25, 2010 4:48:53 pm Sean Bruno wrote:
> > Reposting from -stable.
> >
> > Kind of a large patch, but in order to make an omlette, you need to
> > break a few servers.
> >
> > This is a diff against -CURRENT, not stable-8 as I didn't get a chance
> > to test it. It is directly based off of changes that peter@ made to the
> > Yahoo FreeBSD 7 tree.
> >
> > I have compile and boot tested this on my local machines, but I don't
> > have 64 CPU machines to test upon.
>
> I think IPI_AST in the first hunk should be using ipi_cpu(). I would perhaps
> tackle ipi_cpu() as a first step: introduce ipi_cpu() on both i386 and amd64
> (it should be ok to add a real version for i386 rather than the current macro,
> it should be a copy of the amd64 code). Other folks can help with other
> architectures. ipi_selected() should generally be a good clue as to how to
> implement ipi_cpu().
So I have a patch to add ipi_cpu() for HEAD. I've cross-compiled it for each
SMP architecture and have booted it on amd64. It uses ipi_cpu() instead of
ipi_selected() whenever possible.
--- //depot/vendor/freebsd/src/sys/amd64/amd64/mp_machdep.c 2010-06-22 16:25:14.000000000 0000
+++ //depot/projects/smpng/sys/amd64/amd64/mp_machdep.c 2010-08-01 18:48:37.000000000 0000
@@ -1239,15 +1239,51 @@
do {
old_pending = cpu_ipi_pending[cpu];
new_pending = old_pending | bitmap;
- } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],old_pending, new_pending));
-
+ } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
+ old_pending, new_pending));
if (old_pending)
continue;
}
+ lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
+ }
+}
- lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
+/*
+ * send an IPI to a specific CPU.
+ */
+void
+ipi_cpu(int cpu, u_int ipi)
+{
+ u_int bitmap = 0;
+ u_int old_pending;
+ u_int new_pending;
+
+ if (IPI_IS_BITMAPED(ipi)) {
+ bitmap = 1 << ipi;
+ ipi = IPI_BITMAP_VECTOR;
}
+ /*
+ * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
+ * of help in order to understand what is the source.
+ * Set the mask of receiving CPUs for this purpose.
+ */
+ if (ipi == IPI_STOP_HARD)
+ atomic_set_int(&ipi_nmi_pending, 1 << cpu);
+
+ CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi);
+ KASSERT(cpu_apic_ids[cpu] != -1, ("IPI to non-existent CPU %d", cpu));
+
+ if (bitmap) {
+ do {
+ old_pending = cpu_ipi_pending[cpu];
+ new_pending = old_pending | bitmap;
+ } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
+ old_pending, new_pending));
+ if (old_pending)
+ return;
+ }
+ lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
}
/*
--- //depot/vendor/freebsd/src/sys/amd64/include/smp.h 2010-06-17 12:00:13.000000000 0000
+++ //depot/projects/smpng/sys/amd64/include/smp.h 2010-08-01 18:48:37.000000000 0000
@@ -60,10 +60,11 @@
void cpustop_handler(void);
void cpususpend_handler(void);
void init_secondary(void);
+void ipi_all_but_self(u_int ipi);
+void ipi_bitmap_handler(struct trapframe frame);
+void ipi_cpu(int cpu, u_int ipi);
int ipi_nmi_handler(void);
void ipi_selected(cpumask_t cpus, u_int ipi);
-void ipi_all_but_self(u_int ipi);
-void ipi_bitmap_handler(struct trapframe frame);
u_int mp_bootaddress(u_int);
int mp_grab_cpu_hlt(void);
void smp_cache_flush(void);
--- //depot/vendor/freebsd/src/sys/i386/i386/mp_machdep.c 2010-06-22 16:25:14.000000000 0000
+++ //depot/projects/smpng/sys/i386/i386/mp_machdep.c 2010-08-01 18:48:37.000000000 0000
@@ -1327,15 +1327,51 @@
do {
old_pending = cpu_ipi_pending[cpu];
new_pending = old_pending | bitmap;
- } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],old_pending, new_pending));
-
+ } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
+ old_pending, new_pending));
if (old_pending)
continue;
}
+ lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
+ }
+}
- lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
+/*
+ * send an IPI to a specific CPU.
+ */
+void
+ipi_cpu(int cpu, u_int ipi)
+{
+ u_int bitmap = 0;
+ u_int old_pending;
+ u_int new_pending;
+
+ if (IPI_IS_BITMAPED(ipi)) {
+ bitmap = 1 << ipi;
+ ipi = IPI_BITMAP_VECTOR;
}
+ /*
+ * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
+ * of help in order to understand what is the source.
+ * Set the mask of receiving CPUs for this purpose.
+ */
+ if (ipi == IPI_STOP_HARD)
+ atomic_set_int(&ipi_nmi_pending, 1 << cpu);
+
+ CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi);
+ KASSERT(cpu_apic_ids[cpu] != -1, ("IPI to non-existent CPU %d", cpu));
+
+ if (bitmap) {
+ do {
+ old_pending = cpu_ipi_pending[cpu];
+ new_pending = old_pending | bitmap;
+ } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
+ old_pending, new_pending));
+ if (old_pending)
+ return;
+ }
+ lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
}
/*
--- //depot/vendor/freebsd/src/sys/i386/include/smp.h 2009-08-13 17:30:15.000000000 0000
+++ //depot/projects/smpng/sys/i386/include/smp.h 2010-08-01 18:48:37.000000000 0000
@@ -60,12 +60,13 @@
void cpu_add(u_int apic_id, char boot_cpu);
void cpustop_handler(void);
void init_secondary(void);
-int ipi_nmi_handler(void);
-void ipi_selected(cpumask_t cpus, u_int ipi);
void ipi_all_but_self(u_int ipi);
#ifndef XEN
void ipi_bitmap_handler(struct trapframe frame);
#endif
+void ipi_cpu(int cpu, u_int ipi);
+int ipi_nmi_handler(void);
+void ipi_selected(cpumask_t cpus, u_int ipi);
u_int mp_bootaddress(u_int);
int mp_grab_cpu_hlt(void);
void smp_cache_flush(void);
--- //depot/vendor/freebsd/src/sys/i386/xen/mp_machdep.c 2010-03-10 19:55:15.000000000 0000
+++ //depot/projects/smpng/sys/i386/xen/mp_machdep.c 2010-08-01 18:48:37.000000000 0000
@@ -1121,19 +1121,14 @@
cpu--;
cpus &= ~(1 << cpu);
- KASSERT(cpu_apic_ids[cpu] != -1,
- ("IPI to non-existent CPU %d", cpu));
-
if (bitmap) {
do {
old_pending = cpu_ipi_pending[cpu];
new_pending = old_pending | bitmap;
- } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],old_pending, new_pending));
-
+ } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
+ old_pending, new_pending));
if (!old_pending)
ipi_pcpu(cpu, RESCHEDULE_VECTOR);
- continue;
-
} else {
KASSERT(call_data != NULL, ("call_data not set"));
ipi_pcpu(cpu, CALL_FUNCTION_VECTOR);
@@ -1142,6 +1137,45 @@
}
/*
+ * send an IPI to a specific CPU.
+ */
+void
+ipi_cpu(int cpu, u_int ipi)
+{
+ u_int bitmap = 0;
+ u_int old_pending;
+ u_int new_pending;
+
+ if (IPI_IS_BITMAPED(ipi)) {
+ bitmap = 1 << ipi;
+ ipi = IPI_BITMAP_VECTOR;
+ }
+
+ /*
+ * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
+ * of help in order to understand what is the source.
+ * Set the mask of receiving CPUs for this purpose.
+ */
+ if (ipi == IPI_STOP_HARD)
+ atomic_set_int(&ipi_nmi_pending, 1 << cpu);
+
+ CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi);
+
+ if (bitmap) {
+ do {
+ old_pending = cpu_ipi_pending[cpu];
+ new_pending = old_pending | bitmap;
+ } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
+ old_pending, new_pending));
+ if (!old_pending)
+ ipi_pcpu(cpu, RESCHEDULE_VECTOR);
+ } else {
+ KASSERT(call_data != NULL, ("call_data not set"));
+ ipi_pcpu(cpu, CALL_FUNCTION_VECTOR);
+ }
+}
+
+/*
* send an IPI to all CPUs EXCEPT myself
*/
void
--- //depot/vendor/freebsd/src/sys/ia64/ia64/mp_machdep.c 2010-07-03 20:20:15.000000000 0000
+++ //depot/projects/smpng/sys/ia64/ia64/mp_machdep.c 2010-08-01 18:48:37.000000000 0000
@@ -408,6 +408,16 @@
}
/*
+ * send an IPI to a specific CPU.
+ */
+void
+ipi_cpu(int cpu, u_int ipi)
+{
+
+ ipi_send(cpuid_to_pcpu[cpu], ipi);
+}
+
+/*
* send an IPI to all CPUs EXCEPT myself.
*/
void
--- //depot/vendor/freebsd/src/sys/ia64/include/smp.h 2010-03-17 00:40:15.000000000 0000
+++ //depot/projects/smpng/sys/ia64/include/smp.h 2010-08-01 18:48:37.000000000 0000
@@ -25,6 +25,7 @@
extern int ia64_ipi_wakeup;
void ipi_all_but_self(int ipi);
+void ipi_cpu(int cpu, u_int ipi);
void ipi_selected(cpumask_t cpus, int ipi);
void ipi_send(struct pcpu *, int ipi);
--- //depot/vendor/freebsd/src/sys/kern/sched_4bsd.c 2010-06-11 18:50:16.000000000 0000
+++ //depot/projects/smpng/sys/kern/sched_4bsd.c 2010-08-01 18:48:37.000000000 0000
@@ -1154,7 +1154,7 @@
pcpu = pcpu_find(cpuid);
if (idle_cpus_mask & pcpu->pc_cpumask) {
forward_wakeups_delivered++;
- ipi_selected(pcpu->pc_cpumask, IPI_AST);
+ ipi_cpu(cpuid, IPI_AST);
return;
}
@@ -1167,13 +1167,13 @@
if (pri <= PRI_MAX_ITHD)
#endif /* ! FULL_PREEMPTION */
{
- ipi_selected(pcpu->pc_cpumask, IPI_PREEMPT);
+ ipi_cpu(cpuid, IPI_PREEMPT);
return;
}
#endif /* defined(IPI_PREEMPTION) && defined(PREEMPTION) */
pcpu->pc_curthread->td_flags |= TDF_NEEDRESCHED;
- ipi_selected(pcpu->pc_cpumask, IPI_AST);
+ ipi_cpu(cpuid, IPI_AST);
return;
}
#endif /* SMP */
@@ -1666,7 +1666,7 @@
td->td_flags |= TDF_NEEDRESCHED;
if (td != curthread)
- ipi_selected(1 << cpu, IPI_AST);
+ ipi_cpu(cpu, IPI_AST);
break;
default:
break;
--- //depot/vendor/freebsd/src/sys/kern/sched_ule.c 2010-07-15 13:50:13.000000000 0000
+++ //depot/projects/smpng/sys/kern/sched_ule.c 2010-08-01 18:48:37.000000000 0000
@@ -851,7 +851,7 @@
* IPI the target cpu to force it to reschedule with the new
* workload.
*/
- ipi_selected(1 << TDQ_ID(low), IPI_PREEMPT);
+ ipi_cpu(TDQ_ID(low), IPI_PREEMPT);
}
tdq_unlock_pair(high, low);
return (moved);
@@ -974,7 +974,7 @@
return;
}
tdq->tdq_ipipending = 1;
- ipi_selected(1 << cpu, IPI_PREEMPT);
+ ipi_cpu(cpu, IPI_PREEMPT);
}
/*
@@ -2411,7 +2411,7 @@
cpu = ts->ts_cpu;
ts->ts_cpu = sched_pickcpu(td, 0);
if (cpu != PCPU_GET(cpuid))
- ipi_selected(1 << cpu, IPI_PREEMPT);
+ ipi_cpu(cpu, IPI_PREEMPT);
#endif
}
--- //depot/vendor/freebsd/src/sys/kern/subr_smp.c 2010-06-11 18:50:16.000000000 0000
+++ //depot/projects/smpng/sys/kern/subr_smp.c 2010-08-01 18:48:37.000000000 0000
@@ -181,7 +181,7 @@
id = td->td_oncpu;
if (id == NOCPU)
return;
- ipi_selected(1 << id, IPI_AST);
+ ipi_cpu(id, IPI_AST);
}
/*
--- //depot/vendor/freebsd/src/sys/mips/include/smp.h 2010-05-16 19:50:44.000000000 0000
+++ //depot/projects/smpng/sys/mips/include/smp.h 2010-08-01 18:48:37.000000000 0000
@@ -30,6 +30,7 @@
#ifndef LOCORE
+void ipi_cpu(int cpu, u_int ipi);
void ipi_selected(cpumask_t cpus, int ipi);
void smp_init_secondary(u_int32_t cpuid);
void mpentry(void);
--- //depot/vendor/freebsd/src/sys/mips/mips/mp_machdep.c 2010-06-17 05:05:14.000000000 0000
+++ //depot/projects/smpng/sys/mips/mips/mp_machdep.c 2010-08-01 18:48:37.000000000 0000
@@ -85,6 +85,15 @@
}
}
+/* Send an IPI to a specific CPU. */
+void
+ipi_cpu(int cpu, u_int ipi)
+{
+
+ CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x\n", __func__, cpu, ipi);
+ ipi_send(cpuid_to_pcpu[cpu], ipi);
+}
+
/*
* Handle an IPI sent to this processor.
*/
--- //depot/vendor/freebsd/src/sys/powerpc/include/smp.h 2010-07-13 05:39:35.000000000 0000
+++ //depot/projects/smpng/sys/powerpc/include/smp.h 2010-08-01 18:48:37.000000000 0000
@@ -40,6 +40,7 @@
#ifndef LOCORE
void ipi_all_but_self(int ipi);
+void ipi_cpu(int cpu, u_int ipi);
void ipi_selected(cpumask_t cpus, int ipi);
struct cpuref {
--- //depot/vendor/freebsd/src/sys/powerpc/powerpc/mp_machdep.c 2010-07-13 05:39:35.000000000 0000
+++ //depot/projects/smpng/sys/powerpc/powerpc/mp_machdep.c 2010-08-01 18:48:37.000000000 0000
@@ -336,6 +336,14 @@
}
}
+/* Send an IPI to a specific CPU. */
+void
+ipi_cpu(int cpu, u_int ipi)
+{
+
+ ipi_send(cpuid_to_pcpu[cpu], ipi);
+}
+
/* Send an IPI to all CPUs EXCEPT myself. */
void
ipi_all_but_self(int ipi)
--- //depot/vendor/freebsd/src/sys/sparc64/include/smp.h 2010-07-04 12:45:48.000000000 0000
+++ //depot/projects/smpng/sys/sparc64/include/smp.h 2010-08-01 18:48:37.000000000 0000
@@ -133,6 +133,17 @@
cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_level, ipi);
}
+static __inline void
+ipi_cpu(int cpu, u_int ipi)
+{
+
+ /*
+ * XXX: Not ideal, but would require more work to add a cpu_ipi_cpu
+ * function pointer.
+ */
+ cpu_ipi_selected(1 << cpu, 0, (u_long)tl_ipi_level, ipi);
+}
+
#if defined(_MACHINE_PMAP_H_) && defined(_SYS_MUTEX_H_)
static __inline void *
--- //depot/vendor/freebsd/src/sys/sun4v/include/smp.h 2009-08-13 17:30:15.000000000 0000
+++ //depot/projects/smpng/sys/sun4v/include/smp.h 2010-08-01 18:48:37.000000000 0000
@@ -82,8 +82,9 @@
void cpu_ipi_stop(struct trapframe *tf);
void cpu_ipi_preempt(struct trapframe *tf);
+void ipi_all_but_self(u_int ipi);
+void ipi_cpu(int cpu, u_int ipi);
void ipi_selected(u_int cpus, u_int ipi);
-void ipi_all_but_self(u_int ipi);
vm_offset_t mp_tramp_alloc(void);
void mp_set_tsb_desc_ra(vm_paddr_t);
--- //depot/vendor/freebsd/src/sys/sun4v/sun4v/mp_machdep.c 2009-06-23 22:54:36.000000000 0000
+++ //depot/projects/smpng/sys/sun4v/sun4v/mp_machdep.c 2010-08-03 17:34:33.000000000 0000
@@ -518,7 +518,6 @@
}
}
-
void
ipi_selected(u_int icpus, u_int ipi)
{
@@ -533,7 +532,6 @@
* 4) handling 4-way threading vs 2-way threading should happen here
* and not in forward wakeup
*/
-
cpulist = PCPU_GET(cpulist);
cpus = (icpus & ~PCPU_GET(cpumask));
@@ -545,8 +543,32 @@
cpu_count++;
}
- cpu_ipi_selected(cpu_count, cpulist, (u_long)tl_ipi_level, ipi, 0, &ackmask);
-
+ cpu_ipi_selected(cpu_count, cpulist, (u_long)tl_ipi_level, ipi, 0,
+ &ackmask);
+}
+
+void
+ipi_cpu(int cpu, u_int ipi)
+{
+ int cpu_count;
+ uint16_t *cpulist;
+ uint64_t ackmask;
+
+ /*
+ *
+ * 3) forward_wakeup appears to abuse ASTs
+ * 4) handling 4-way threading vs 2-way threading should happen here
+ * and not in forward wakeup
+ */
+ cpulist = PCPU_GET(cpulist);
+ if (PCPU_GET(cpumask) & (1 << cpu))
+ cpu_count = 0;
+ else {
+ cpulist[0] = (uint16_t)cpu;
+ cpu_count = 1;
+ }
+ cpu_ipi_selected(cpu_count, cpulist, (u_long)tl_ipi_level, ipi, 0,
+ &ackmask);
}
void
--
John Baldwin
More information about the freebsd-current
mailing list