svn commit: r212690 - in stable/7/sys/sparc64: include sparc64
Marius Strobl
marius at FreeBSD.org
Wed Sep 15 19:28:06 UTC 2010
Author: marius
Date: Wed Sep 15 19:28:06 2010
New Revision: 212690
URL: http://svn.freebsd.org/changeset/base/212690
Log:
MFC: r211050 (partial)
- Introduce a cpu_ipi_single() function pointer in order to send IPIs
to single CPUs more efficiently with Cheetah(-class) and Jalapeno CPUs.
- Factor out the Jalapeno support from the Cheetah IPI send functions
in order to be able to more easily and efficiently implement support
for more than 32 target CPUs as well as a workaround for Cheetah+
erratum 25 for the latter.
Modified:
stable/7/sys/sparc64/include/smp.h
stable/7/sys/sparc64/sparc64/mp_machdep.c
Directory Properties:
stable/7/sys/ (props changed)
stable/7/sys/cddl/contrib/opensolaris/ (props changed)
stable/7/sys/contrib/dev/acpica/ (props changed)
stable/7/sys/contrib/pf/ (props changed)
Modified: stable/7/sys/sparc64/include/smp.h
==============================================================================
--- stable/7/sys/sparc64/include/smp.h Wed Sep 15 19:27:30 2010 (r212689)
+++ stable/7/sys/sparc64/include/smp.h Wed Sep 15 19:28:06 2010 (r212690)
@@ -95,6 +95,8 @@ void cpu_mp_shutdown(void);
typedef void cpu_ipi_selected_t(u_int, u_long, u_long, u_long);
extern cpu_ipi_selected_t *cpu_ipi_selected;
+typedef void cpu_ipi_single_t(u_int, u_long, u_long, u_long);
+extern cpu_ipi_single_t *cpu_ipi_single;
void mp_init(u_int cpu_impl);
@@ -229,7 +231,8 @@ ipi_tlb_range_demap(struct pmap *pm, vm_
ita->ita_pmap = pm;
ita->ita_start = start;
ita->ita_end = end;
- cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_range_demap, (u_long)ita);
+ cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_range_demap,
+ (u_long)ita);
return (&ita->ita_mask);
}
Modified: stable/7/sys/sparc64/sparc64/mp_machdep.c
==============================================================================
--- stable/7/sys/sparc64/sparc64/mp_machdep.c Wed Sep 15 19:27:30 2010 (r212689)
+++ stable/7/sys/sparc64/sparc64/mp_machdep.c Wed Sep 15 19:28:06 2010 (r212690)
@@ -29,7 +29,7 @@
*/
/*-
* Copyright (c) 2002 Jake Burkholder.
- * Copyright (c) 2007 Marius Strobl <marius at FreeBSD.org>
+ * Copyright (c) 2007 - 2010 Marius Strobl <marius at FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -113,6 +113,7 @@ struct pcb stoppcbs[MAXCPU];
struct mtx ipi_mtx;
cpu_ipi_selected_t *cpu_ipi_selected;
+cpu_ipi_single_t *cpu_ipi_single;
static vm_offset_t mp_tramp;
static u_int cpuid_to_mid[MAXCPU];
@@ -124,11 +125,14 @@ static void ap_start(phandle_t node, u_i
static void cpu_mp_unleash(void *v);
static void foreach_ap(phandle_t node, void (*func)(phandle_t node,
u_int mid, u_int cpu_impl));
-static void spitfire_ipi_send(u_int mid, u_long d0, u_long d1, u_long d2);
static void sun4u_startcpu(phandle_t cpu, void *func, u_long arg);
static cpu_ipi_selected_t cheetah_ipi_selected;
+static cpu_ipi_single_t cheetah_ipi_single;
+static cpu_ipi_selected_t jalapeno_ipi_selected;
+static cpu_ipi_single_t jalapeno_ipi_single;
static cpu_ipi_selected_t spitfire_ipi_selected;
+static cpu_ipi_single_t spitfire_ipi_single;
SYSINIT(cpu_mp_unleash, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL);
@@ -162,13 +166,18 @@ mp_init(u_int cpu_impl)
* cpu_mp_start() wasn't so initialize these here.
*/
if (cpu_impl == CPU_IMPL_ULTRASPARCIIIi ||
- cpu_impl == CPU_IMPL_ULTRASPARCIIIip)
+ cpu_impl == CPU_IMPL_ULTRASPARCIIIip) {
isjbus = 1;
- if (cpu_impl == CPU_IMPL_SPARC64V ||
- cpu_impl >= CPU_IMPL_ULTRASPARCIII)
+ cpu_ipi_selected = jalapeno_ipi_selected;
+ cpu_ipi_single = jalapeno_ipi_single;
+ } else if (cpu_impl == CPU_IMPL_SPARC64V ||
+ cpu_impl >= CPU_IMPL_ULTRASPARCIII) {
cpu_ipi_selected = cheetah_ipi_selected;
- else
+ cpu_ipi_single = cheetah_ipi_single;
+ } else {
cpu_ipi_selected = spitfire_ipi_selected;
+ cpu_ipi_single = spitfire_ipi_single;
+ }
}
static void
@@ -179,7 +188,7 @@ foreach_ap(phandle_t node, void (*func)(
phandle_t child;
u_int cpuid;
uint32_t cpu_impl;
-
+
/* There's no need to traverse the whole OFW tree twice. */
if (mp_maxid > 0 && mp_ncpus >= mp_maxid + 1)
return;
@@ -199,7 +208,7 @@ foreach_ap(phandle_t node, void (*func)(
panic("%s: couldn't determine CPU "
"implementation", __func__);
if (OF_getprop(node, cpu_cpuid_prop(cpu_impl), &cpuid,
- sizeof(cpuid)) <= 0)
+ sizeof(cpuid)) <= 0)
panic("%s: couldn't determine CPU module ID",
__func__);
if (cpuid == PCPU_GET(mid))
@@ -511,24 +520,25 @@ spitfire_ipi_selected(u_int cpus, u_long
{
u_int cpu;
- KASSERT((cpus & (1 << curcpu)) == 0,
- ("%s: CPU can't IPI itself", __func__));
while (cpus) {
cpu = ffs(cpus) - 1;
cpus &= ~(1 << cpu);
- spitfire_ipi_send(cpuid_to_mid[cpu], d0, d1, d2);
+ spitfire_ipi_single(cpu, d0, d1, d2);
}
}
static void
-spitfire_ipi_send(u_int mid, u_long d0, u_long d1, u_long d2)
+spitfire_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2)
{
register_t s;
u_long ids;
+ u_int mid;
int i;
+ KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__));
KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_BUSY) == 0,
("%s: outstanding dispatch", __func__));
+ mid = cpuid_to_mid[cpu];
for (i = 0; i < IPI_RETRIES; i++) {
s = intr_disable();
stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0);
@@ -569,6 +579,49 @@ spitfire_ipi_send(u_int mid, u_long d0,
}
static void
+cheetah_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2)
+{
+ register_t s;
+ u_long ids;
+ u_int mid;
+ int i;
+
+ KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__));
+ KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) &
+ IDR_CHEETAH_ALL_BUSY) == 0,
+ ("%s: outstanding dispatch", __func__));
+ mid = cpuid_to_mid[cpu];
+ for (i = 0; i < IPI_RETRIES; i++) {
+ s = intr_disable();
+ stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0);
+ stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1);
+ stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2);
+ membar(Sync);
+ stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT),
+ ASI_SDB_INTR_W, 0);
+ membar(Sync);
+ while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) &
+ IDR_BUSY) != 0)
+ ;
+ intr_restore(s);
+ if ((ids & (IDR_BUSY | IDR_NACK)) == 0)
+ return;
+ /*
+ * Leave interrupts enabled for a bit before retrying
+ * in order to avoid deadlocks if the other CPU is also
+ * trying to send an IPI.
+ */
+ DELAY(2);
+ }
+ if (kdb_active != 0 || panicstr != NULL)
+ printf("%s: couldn't send IPI to module 0x%u\n",
+ __func__, mid);
+ else
+ panic("%s: couldn't send IPI to module 0x%u",
+ __func__, mid);
+}
+
+static void
cheetah_ipi_selected(u_int cpus, u_long d0, u_long d1, u_long d2)
{
register_t s;
@@ -594,9 +647,8 @@ cheetah_ipi_selected(u_int cpus, u_long
bnp = 0;
for (cpu = 0; cpu < mp_ncpus; cpu++) {
if ((cpus & (1 << cpu)) != 0) {
- stxa(AA_INTR_SEND |
- (cpuid_to_mid[cpu] << IDC_ITID_SHIFT) |
- (isjbus ? 0 : bnp << IDC_BN_SHIFT),
+ stxa(AA_INTR_SEND | (cpuid_to_mid[cpu] <<
+ IDC_ITID_SHIFT) | bnp << IDC_BN_SHIFT,
ASI_SDB_INTR_W, 0);
membar(Sync);
bnp++;
@@ -606,14 +658,13 @@ cheetah_ipi_selected(u_int cpus, u_long
IDR_CHEETAH_ALL_BUSY) != 0)
;
intr_restore(s);
- if ((ids & (IDR_CHEETAH_ALL_BUSY | IDR_CHEETAH_ALL_NACK)) == 0)
+ if ((ids &
+ (IDR_CHEETAH_ALL_BUSY | IDR_CHEETAH_ALL_NACK)) == 0)
return;
bnp = 0;
for (cpu = 0; cpu < mp_ncpus; cpu++) {
if ((cpus & (1 << cpu)) != 0) {
- if ((ids & (IDR_NACK << (isjbus ?
- (2 * cpuid_to_mid[cpu]) :
- (2 * bnp)))) == 0)
+ if ((ids & (IDR_NACK << (2 * bnp))) == 0)
cpus &= ~(1 << cpu);
bnp++;
}
@@ -639,3 +690,104 @@ cheetah_ipi_selected(u_int cpus, u_long
panic("%s: couldn't send IPI (cpus=0x%u ids=0x%lu)",
__func__, cpus, ids);
}
+
+static void
+jalapeno_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2)
+{
+ register_t s;
+ u_long ids;
+ u_int busy, busynack, mid;
+ int i;
+
+ KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__));
+ KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) &
+ IDR_CHEETAH_ALL_BUSY) == 0,
+ ("%s: outstanding dispatch", __func__));
+ mid = cpuid_to_mid[cpu];
+ busy = IDR_BUSY << (2 * mid);
+ busynack = (IDR_BUSY | IDR_NACK) << (2 * mid);
+ for (i = 0; i < IPI_RETRIES; i++) {
+ s = intr_disable();
+ stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0);
+ stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1);
+ stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2);
+ membar(Sync);
+ stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT),
+ ASI_SDB_INTR_W, 0);
+ membar(Sync);
+ while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) &
+ busy) != 0)
+ ;
+ intr_restore(s);
+ if ((ids & busynack) == 0)
+ return;
+ /*
+ * Leave interrupts enabled for a bit before retrying
+ * in order to avoid deadlocks if the other CPU is also
+ * trying to send an IPI.
+ */
+ DELAY(2);
+ }
+ if (kdb_active != 0 || panicstr != NULL)
+ printf("%s: couldn't send IPI to module 0x%u\n",
+ __func__, mid);
+ else
+ panic("%s: couldn't send IPI to module 0x%u",
+ __func__, mid);
+}
+
+static void
+jalapeno_ipi_selected(u_int cpus, u_long d0, u_long d1, u_long d2)
+{
+ register_t s;
+ u_long ids;
+ u_int cpu;
+ int i;
+
+ KASSERT((cpus & (1 << curcpu)) == 0,
+ ("%s: CPU can't IPI itself", __func__));
+ KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) &
+ IDR_CHEETAH_ALL_BUSY) == 0,
+ ("%s: outstanding dispatch", __func__));
+ if (cpus == 0)
+ return;
+ ids = 0;
+ for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) {
+ s = intr_disable();
+ stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0);
+ stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1);
+ stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2);
+ membar(Sync);
+ for (cpu = 0; cpu < mp_ncpus; cpu++) {
+ if ((cpus & (1 << cpu)) != 0) {
+ stxa(AA_INTR_SEND | (cpuid_to_mid[cpu] <<
+ IDC_ITID_SHIFT), ASI_SDB_INTR_W, 0);
+ membar(Sync);
+ }
+ }
+ while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) &
+ IDR_CHEETAH_ALL_BUSY) != 0)
+ ;
+ intr_restore(s);
+ if ((ids &
+ (IDR_CHEETAH_ALL_BUSY | IDR_CHEETAH_ALL_NACK)) == 0)
+ return;
+ for (cpu = 0; cpu < mp_ncpus; cpu++)
+ if ((cpus & (1 << cpu)) != 0)
+ if ((ids & (IDR_NACK <<
+ (2 * cpuid_to_mid[cpu]))) == 0)
+ cpus &= ~(1 << cpu);
+ /*
+ * Leave interrupts enabled for a bit before retrying
+ * in order to avoid deadlocks if the other CPUs are
+ * also trying to send IPIs.
+ */
+ DELAY(2 * mp_ncpus);
+ }
+ if (kdb_active != 0 || panicstr != NULL)
+ printf("%s: couldn't send IPI (cpus=0x%u ids=0x%lu)\n",
+ __func__, cpus, ids);
+ else
+ panic("%s: couldn't send IPI (cpus=0x%u ids=0x%lu)",
+ __func__, cpus, ids);
+}
More information about the svn-src-stable-7
mailing list