number of CPUs and IPI panic
Nik Azim Azam
nskyline_r35 at yahoo.com
Wed Nov 10 10:23:14 PST 2004
I'm still getting this panic with the patch:
panic: APIC: Previous IPI is stuck
cpuid = 3
boot() called on cpu#0
Uptime: 4m16s
:(
--- Stephan Uphoff <ups at tree.com> wrote:
> On Wed, 2004-11-10 at 11:05, Nik Azim Azam wrote:
> > hmm, you've forgotten to include the patch with
> the
> > mail?
> >
> > --- Stephan Uphoff <ups at tree.com> wrote:
>
> > > OK - lets try this again.
> > > The attached patch should prevent blocking on
> > > redundant IPIs caused by
> > > the apic not being able to queue the request.
> > > If this does not work we will have to instrument
> the
> > > code.
> > >
> > > Stephan
>
> Sight - one more time ...
>
>
> > Index: sys/i386/i386/apic_vector.s
>
===================================================================
> RCS file: /cvsroot/src/sys/i386/i386/apic_vector.s,v
> retrieving revision 1.101
> diff -u -r1.101 apic_vector.s
> --- sys/i386/i386/apic_vector.s 26 May 2004 07:43:41
> -0000 1.101
> +++ sys/i386/i386/apic_vector.s 3 Nov 2004 15:43:07
> -0000
> @@ -286,7 +286,11 @@
> movl %eax, %es
> movl $KPSEL, %eax
> movl %eax, %fs
> -
> +
> + movl PCPU(CPUID), %edx
> + lock
> + btrl %edx,CNAME(ipi_ast_pending)
> +
> movl lapic, %edx
> movl $0, LA_EOI(%edx) /* End Of Interrupt to APIC
> */
>
> Index: sys/i386/i386/local_apic.c
>
===================================================================
> RCS file: /cvsroot/src/sys/i386/i386/local_apic.c,v
> retrieving revision 1.9
> diff -u -r1.9 local_apic.c
> --- sys/i386/i386/local_apic.c 14 Jul 2004 18:12:15
> -0000 1.9
> +++ sys/i386/i386/local_apic.c 4 Oct 2004 02:24:17
> -0000
> @@ -693,6 +693,62 @@
> intr_restore(eflags);
> }
>
> +
> +static void
> +lapic_ipi_wait_and_raw(int delay,register_t icrlo,
> u_int dest)
> +{
> + int x, incr,idle;
> + register_t value, eflags;
> +
> +
> +/* XXX: Need more sanity checking of icrlo? */
> + KASSERT(lapic != NULL, ("%s called too early",
> __func__));
> + KASSERT((dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT))
> == 0,
> + ("%s: invalid dest field", __func__));
> + KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0,
> + ("%s: reserved bits set in ICR LO register",
> __func__));
> +
> + if (delay == -1) {
> + incr = 0;
> + delay = 1;
> + } else
> + incr = 1;
> +
> + /* Set destination in ICR HI register if it is
> being used. */
> + eflags = intr_disable();
> +
> + idle = 0;
> +
> + for (x = 0; x < delay; x += incr) {
> + if ((lapic->icr_lo & APIC_DELSTAT_MASK) ==
> APIC_DELSTAT_IDLE)
> + {
> + idle = 1;
> + break;
> + }
> + intr_restore(eflags);
> + ia32_pause();
> + eflags = intr_disable();
> +
> + }
> +
> + if (!idle) panic("APIC: Previous IPI is stuck");
> +
> + if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD)
> {
> + value = lapic->icr_hi;
> + value &= ~APIC_ID_MASK;
> + value |= dest << APIC_ID_SHIFT;
> + lapic->icr_hi = value;
> + }
> +
> + /* Program the contents of the IPI and dispatch
> it. */
> + value = lapic->icr_lo;
> + value &= APIC_ICRLO_RESV_MASK;
> + value |= icrlo;
> + lapic->icr_lo = value;
> + intr_restore(eflags);
> +}
> +
> +
> #define BEFORE_SPIN 1000000
> #ifdef DETECT_DEADLOCK
> #define AFTER_SPIN 1000
> @@ -725,11 +781,8 @@
> destfield = dest;
> }
>
> - /* Wait for an earlier IPI to finish. */
> - if (!lapic_ipi_wait(BEFORE_SPIN))
> - panic("APIC: Previous IPI is stuck");
> -
> - lapic_ipi_raw(icrlo, destfield);
> +
> + lapic_ipi_wait_and_raw(BEFORE_SPIN,icrlo,
> destfield);
>
> #ifdef DETECT_DEADLOCK
> /* Wait for IPI to be delivered. */
> Index: sys/i386/i386/mp_machdep.c
>
===================================================================
> RCS file: /cvsroot/src/sys/i386/i386/mp_machdep.c,v
> retrieving revision 1.240
> diff -u -r1.240 mp_machdep.c
> --- sys/i386/i386/mp_machdep.c 1 Nov 2004 22:11:27
> -0000 1.240
> +++ sys/i386/i386/mp_machdep.c 10 Nov 2004 01:48:57
> -0000
> @@ -212,6 +212,14 @@
> static int hlt_logical_cpus;
> static struct sysctl_ctx_list logical_cpu_clist;
>
> +
> +static int ipi_statclock_pending;
> +static int ipi_hardclock_pending;
> +int ipi_ast_pending;
> +
> +
> +
> +
> static void
> mem_range_AP_init(void)
> {
> @@ -1017,12 +1025,24 @@
> smp_tlb_addr1 = addr1;
> smp_tlb_addr2 = addr2;
> atomic_store_rel_int(&smp_tlb_wait, 0);
> +
> +
> + /* Enable interrupts */
> + /* Thread switching still disabled */
> +
> + enable_intr();
> +
> if (mask == (u_int)-1)
> ipi_all_but_self(vector);
> else
> ipi_selected(mask, vector);
> +
> while (smp_tlb_wait < ncpu)
> ia32_pause();
> +
> + /* disable interrupts */
> + disable_intr();
> +
> }
>
> void
> @@ -1104,6 +1124,9 @@
> struct thread *td;
>
> CTR0(KTR_SMP, "forwarded_statclock");
> +
> +
>
atomic_clear_int(&ipi_statclock_pending,PCPU_GET(cpumask));
>
> +
> td = curthread;
> td->td_intr_nesting_level++;
> if (profprocs != 0)
> @@ -1123,9 +1146,11 @@
> if (!smp_started || cold || panicstr)
> return;
>
> - map = PCPU_GET(other_cpus) &
> ~(stopped_cpus|hlt_cpus_mask);
> - if (map != 0)
> + map = PCPU_GET(other_cpus) &
> ~(stopped_cpus|hlt_cpus_mask|ipi_statclock_pending);
> + if (map != 0) {
> + atomic_set_int(&ipi_statclock_pending,map);
> ipi_selected(map, IPI_STATCLOCK);
> + }
> }
>
> /*
> @@ -1141,6 +1166,9 @@
> struct thread *td;
>
> CTR0(KTR_SMP, "forwarded_hardclock");
> +
> +
>
atomic_clear_int(&ipi_hardclock_pending,PCPU_GET(cpumask));
=== message truncated ===
__________________________________
Do you Yahoo!?
Check out the new Yahoo! Front Page.
www.yahoo.com
More information about the freebsd-current
mailing list