SMP suspend/resume.
Vladimir Grebenschikov
vova at fbsd.ru
Wed May 21 17:23:39 UTC 2008
On Wed, 2008-05-21 at 18:21 +0400, Vladimir Grebenschikov wrote:
> On Tue, 2008-05-13 at 20:25 +0900, takawata at init-main.com wrote:
> > Hi, I managed to make suspend and resume work on SMP system.
> > The patch following is a bit crude patch, but it begin
> > to work on my ThinkPad X61 (core2duo system).
>
> Tried patch on T60, from text console,
> on acpiconf -s3
> it shows multiple times "forward_wakeup: Idle processor not found" on
> console and then friezed hard (even DDB does not works). But nothing was
> suspended actually ( before system goes to sleep but did not awakes)
>
> Any hints ?
In some conditions (single-user, no additional drivers) it shows:
forward_wakeup: Idle processor not found
PREP_WALK
LEAVE_SLEEP_STATE
DEVICE_RESUME
forward_wakeup: Idle processor not found
ENABLE_FIXED_EVENT
forward_wakeup: Idle processor not found
forward_wakeup: Idle processor not found
forward_wakeup: Idle processor not found
forward_wakeup: Idle processor not found
forward_wakeup: Idle processor not found
... etc ...
sometimes it shows just many
forward_wakeup: Idle processor not found
and in any case locks hard and did not enter S3
But RELENG_7 (without patch) normally enters to S3, but failed to wake.
> > TODO:
> > 1. Suspend/resume path it self is simular to AP boot path.
> > Some of code may be integrated.
> > 2. More context, like MTRR or npx context should be saved on
> > suspend.
> > 3. Make acpi suspend resume path more ABI aware: needless
> > register recoverly or special register context saving
> > (the value itself is usually constant) should be removed.
> > 4. Make same binary module work on both UP or SMP case.
> > (Or is it time to give up using acpi module on also on i386?)
> >
> >
> >
> >
> >
> > Index: i386/acpica/acpi_wakeup.c
> > ===================================================================
> > RCS file: /home/ncvs/src/sys/i386/acpica/acpi_wakeup.c,v
> > retrieving revision 1.47
> > diff -u -r1.47 acpi_wakeup.c
> > --- i386/acpica/acpi_wakeup.c 16 Mar 2008 10:58:03 -0000 1.47
> > +++ i386/acpica/acpi_wakeup.c 13 May 2008 09:12:18 -0000
> > @@ -27,6 +27,7 @@
> >
> > #include <sys/cdefs.h>
> > __FBSDID("$FreeBSD: src/sys/i386/acpica/acpi_wakeup.c,v 1.47 2008/03/16 10:58:03 rwatson Exp $");
> > +#define SMP
> >
> > #include <sys/param.h>
> > #include <sys/systm.h>
> > @@ -49,6 +50,11 @@
> >
> > #include <contrib/dev/acpica/acpi.h>
> > #include <dev/acpica/acpivar.h>
> > +#include <sys/smp.h>
> > +#include <machine/apicreg.h>
> > +#include <machine/apicvar.h>
> > +#include <machine/smp.h>
> > +#include <sys/sched.h>
> >
> > #include "acpi_wakecode.h"
> >
> > @@ -71,7 +77,9 @@
> >
> > static uint16_t r_cs, r_ds, r_es, r_fs, r_gs, r_ss, r_tr;
> > static uint32_t r_esp;
> > -
> > +extern void *bootstacks[];
> > +static char *bootSTK;
> > +void restore_sub(void);
> > static void acpi_printcpu(void);
> > static void acpi_realmodeinst(void *arg, bus_dma_segment_t *segs,
> > int nsegs, int error);
> > @@ -80,6 +88,7 @@
> > /* XXX shut gcc up */
> > extern int acpi_savecpu(void);
> > extern int acpi_restorecpu(void);
> > +extern void acpi_kicksub(void);
> >
> > #ifdef __GNUCLIKE_ASM
> > __asm__(" \n\
> > @@ -104,6 +113,15 @@
> > movl %eax,(%esp) \n\
> > xorl %eax,%eax \n\
> > ret \n\
> > + \n\
> > + .text \n\
> > + .p2align 2, 0x90 \n\
> > + .type acpi_kicksub, @function \n\
> > +acpi_kicksub: \n\
> > + .align 4 \n\
> > + movl bootSTK,%esp \n\
> > + jmp restore_sub \n\
> > + ret \n\
> > \n\
> > .text \n\
> > .p2align 2, 0x90 \n\
> > @@ -149,6 +167,24 @@
> > ret \n\
> > ");
> > #endif /* __GNUCLIKE_ASM */
> > +int acpi_cpu_resumed[MAXCPU];
> > +int acpi_curcpu;
> > +extern int switch_debug;
> > +
> > +void restore_sub()
> > +{
> > + ACPI_DISABLE_IRQS();
> > + printf("RESTORE_SUB\n");
> > + lapic_disable();
> > + printf("LAPIC_SETUP\n");
> > + lapic_setup(0);
> > + lapic_dump("RESTORE_SUB");
> > + printf("RESTORE_SUB2\n");
> > + ACPI_ENABLE_IRQS();
> > +
> > + acpi_cpu_resumed[acpi_curcpu]= 1;
> > + acpi_restorecpu();
> > +}
> >
> > static void
> > acpi_printcpu(void)
> > @@ -187,6 +223,119 @@
> > outb(0x61, inb(0x61) & ~0x3);
> > }
> >
> > +
> > +int resume_other_cpu(struct acpi_softc *sc, int cpu);
> > +int resume_other_cpu(struct acpi_softc *sc, int cpu)
> > +{
> > + int ms;
> > + int apic_id = cpu_apic_ids[cpu];
> > + int gsel_tss;
> > +
> > + gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
> > + acpi_curcpu = cpu;
> > + bootSTK= (char *)bootstacks[cpu] + KSTACK_PAGES * PAGE_SIZE - 4;
> > + printf("%p\n", bootSTK);
> > + p_gdt = (struct region_descriptor *)
> > + (sc->acpi_wakeaddr + physical_gdt);
> > + saved_gdt.rd_limit = NGDT * sizeof(gdt[0]) -1;
> > + saved_gdt.rd_base = (int )&gdt[cpu*NGDT];
> > + p_gdt->rd_limit = saved_gdt.rd_limit;
> > + p_gdt->rd_base = vtophys(saved_gdt.rd_base);
> > + r_esp = stoppcbs[cpu].pcb_esp;
> > + r_ebp = stoppcbs[cpu].pcb_ebp;
> > + r_esi = stoppcbs[cpu].pcb_esi;
> > + r_edi = stoppcbs[cpu].pcb_edi;
> > + r_efl = stoppcbs[cpu].pcb_psl;
> > + ret_addr = stoppcbs[cpu].pcb_eip;
> > + WAKECODE_FIXUP(physical_esp, uint32_t, vtophys(bootSTK) );
> > + WAKECODE_FIXUP(previous_cr0, uint32_t, r_cr0);
> > + WAKECODE_FIXUP(previous_cr2, uint32_t, r_cr2);
> > + WAKECODE_FIXUP(previous_cr3, uint32_t, r_cr3);
> > + WAKECODE_FIXUP(previous_cr4, uint32_t, r_cr4);
> > +
> > + WAKECODE_FIXUP(resume_beep, uint32_t, 0);
> > + WAKECODE_FIXUP(reset_video, uint32_t, 0);
> > +
> > + WAKECODE_FIXUP(previous_tr, uint16_t, gsel_tss);
> > + WAKECODE_BCOPY(previous_gdt, struct region_descriptor, saved_gdt);
> > + WAKECODE_FIXUP(previous_ldt, uint16_t, saved_ldt);
> > + WAKECODE_BCOPY(previous_idt, struct region_descriptor, saved_idt);
> > +
> > + WAKECODE_FIXUP(where_to_recover, void *, acpi_kicksub);
> > +
> > + WAKECODE_FIXUP(previous_ds, uint16_t, r_ds);
> > + WAKECODE_FIXUP(previous_es, uint16_t, r_es);
> > + WAKECODE_FIXUP(previous_fs, uint16_t, r_fs);
> > + WAKECODE_FIXUP(previous_gs, uint16_t, 0);
> > + WAKECODE_FIXUP(previous_ss, uint16_t, r_ss);
> > +
> > + /* do an INIT IPI: assert RESET */
> > + lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
> > + APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, apic_id);
> > +
> > + /* wait for pending status end */
> > + lapic_ipi_wait(-1);
> > +
> > + /* do an INIT IPI: deassert RESET */
> > + lapic_ipi_raw(APIC_DEST_ALLESELF | APIC_TRIGMOD_LEVEL |
> > + APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, 0);
> > +
> > + /* wait for pending status end */
> > + DELAY(10000); /* wait ~10mS */
> > + lapic_ipi_wait(-1);
> > + /*
> > + * next we do a STARTUP IPI: the previous INIT IPI might still be
> > + * latched, (P5 bug) this 1st STARTUP would then terminate
> > + * immediately, and the previously started INIT IPI would continue. OR
> > + * the previous INIT IPI has already run. and this STARTUP IPI will
> > + * run. OR the previous INIT IPI was ignored. and this STARTUP IPI
> > + * will run.
> > + */
> > +
> > + /* do a STARTUP IPI */
> > + lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
> > + APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP |
> > + ((sc->acpi_wakephys >>12)&0xff), apic_id);
> > + lapic_ipi_wait(-1);
> > + DELAY(200); /* wait ~200uS */
> > +
> > + /*
> > + * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF
> > + * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR
> > + * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
> > + * recognized after hardware RESET or INIT IPI.
> > + */
> > +
> > + lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
> > + APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP |
> > + ((sc->acpi_wakephys >>12)&0xff), apic_id);
> > + lapic_ipi_wait(-1);
> > + DELAY(200); /* wait ~200uS */
> > +
> > + /* Wait up to 5 seconds for it to start. */
> > + for (ms = 0; ms < 5000; ms++) {
> > + if(acpi_cpu_resumed[cpu]){
> > + acpi_cpu_resumed[cpu]= 0;
> > + return 0;
> > + }
> > + DELAY(1000);
> > + }
> > + return -1; /* return FAILURE */
> > +
> > +}
> > +int resume_other_cpus(struct acpi_softc *sc);
> > +int resume_other_cpus(struct acpi_softc *sc)
> > +{
> > + int i;
> > + printf("RESUME_OTHER_CPUS");
> > + *((volatile u_short *) 0x467) = 0;
> > + *((volatile u_short *) 0x468) = (sc->acpi_wakephys&0xffff0)>>4;
> > +
> > + for(i = 1; i < mp_ncpus; i++){
> > + resume_other_cpu(sc, i);
> > + }
> > + return 0;
> > +}
> > int
> > acpi_sleep_machdep(struct acpi_softc *sc, int state)
> > {
> > @@ -270,14 +419,15 @@
> > for (;;) ;
> > } else {
> > /* Execute Wakeup */
> > - intr_resume();
> > -
> > if (bootverbose) {
> > acpi_savecpu();
> > acpi_printcpu();
> > }
> > + resume_other_cpus(sc);
> > + restart_cpus(stopped_cpus);
> > + intr_resume();
> > + lapic_dump("MAIN");
> > }
> > -
> > out:
> > load_cr3(cr3);
> > write_eflags(ef);
> > @@ -285,7 +435,7 @@
> > /* If we beeped, turn it off after a delay. */
> > if (acpi_resume_beep)
> > timeout(acpi_stop_beep, NULL, 3 * hz);
> > -
> > + printf("FUGAFUGA\n");
> > return (ret);
> > }
> >
> > Index: i386/i386/io_apic.c
> > ===================================================================
> > RCS file: /home/ncvs/src/sys/i386/i386/io_apic.c,v
> > retrieving revision 1.35
> > diff -u -r1.35 io_apic.c
> > --- i386/i386/io_apic.c 5 Jun 2007 18:57:48 -0000 1.35
> > +++ i386/i386/io_apic.c 13 May 2008 08:22:55 -0000
> > @@ -444,8 +444,9 @@
> > struct ioapic *io = (struct ioapic *)pic;
> > int i;
> >
> > - for (i = 0; i < io->io_numintr; i++)
> > + for (i = 0; i < io->io_numintr; i++){
> > ioapic_program_intpin(&io->io_pins[i]);
> > + }
> > }
> >
> > /*
> > Index: i386/i386/mp_machdep.c
> > ===================================================================
> > RCS file: /home/ncvs/src/sys/i386/i386/mp_machdep.c,v
> > retrieving revision 1.286
> > diff -u -r1.286 mp_machdep.c
> > --- i386/i386/mp_machdep.c 10 Apr 2008 18:38:31 -0000 1.286
> > +++ i386/i386/mp_machdep.c 13 May 2008 07:08:29 -0000
> > @@ -1299,18 +1299,19 @@
> > int cpu = PCPU_GET(cpuid);
> > int cpumask = PCPU_GET(cpumask);
> >
> > - savectx(&stoppcbs[cpu]);
> > -
> > - /* Indicate that we are stopped */
> > - atomic_set_int(&stopped_cpus, cpumask);
> > + if(savectx(&stoppcbs[cpu])){
> > + /* Indicate that we are stopped */
> > + atomic_set_int(&stopped_cpus, cpumask);
> > + wbinvd();
> > + }
> >
> > /* Wait for restart */
> > - while (!(started_cpus & cpumask))
> > - ia32_pause();
> > -
> > + while (!(started_cpus & cpumask)){
> > + ia32_pause();
> > + }
> > atomic_clear_int(&started_cpus, cpumask);
> > atomic_clear_int(&stopped_cpus, cpumask);
> > -
> > +
> > if (cpu == 0 && cpustop_restartfunc != NULL) {
> > cpustop_restartfunc();
> > cpustop_restartfunc = NULL;
> > Index: i386/i386/swtch.s
> > ===================================================================
> > RCS file: /home/ncvs/src/sys/i386/i386/swtch.s,v
> > retrieving revision 1.156
> > diff -u -r1.156 swtch.s
> > --- i386/i386/swtch.s 22 Aug 2007 05:06:14 -0000 1.156
> > +++ i386/i386/swtch.s 9 May 2008 15:16:03 -0000
> > @@ -413,6 +413,6 @@
> > 1:
> > popfl
> > #endif /* DEV_NPX */
> > -
> > + movl $1, %eax
> > ret
> > END(savectx)
> > Index: i386/include/pcb.h
> > ===================================================================
> > RCS file: /home/ncvs/src/sys/i386/include/pcb.h,v
> > retrieving revision 1.56
> > diff -u -r1.56 pcb.h
> > --- i386/include/pcb.h 29 Dec 2005 13:23:48 -0000 1.56
> > +++ i386/include/pcb.h 24 Apr 2008 06:46:59 -0000
> > @@ -81,7 +81,7 @@
> > struct trapframe;
> >
> > void makectx(struct trapframe *, struct pcb *);
> > -void savectx(struct pcb *);
> > +int savectx(struct pcb *);
> > #endif
> >
> > #endif /* _I386_PCB_H_ */
> > Index: dev/acpica/acpi.c
> > ===================================================================
> > RCS file: /home/ncvs/src/sys/dev/acpica/acpi.c,v
> > retrieving revision 1.247
> > diff -u -r1.247 acpi.c
> > --- dev/acpica/acpi.c 13 Mar 2008 20:39:03 -0000 1.247
> > +++ dev/acpica/acpi.c 30 Apr 2008 13:14:48 -0000
> > @@ -29,7 +29,7 @@
> >
> > #include <sys/cdefs.h>
> > __FBSDID("$FreeBSD: src/sys/dev/acpica/acpi.c,v 1.247 2008/03/13 20:39:03 jhb Exp $");
> > -
> > +#define SMP
> > #include "opt_acpi.h"
> > #include <sys/param.h>
> > #include <sys/kernel.h>
> > @@ -47,6 +47,7 @@
> > #include <sys/power.h>
> > #include <sys/sbuf.h>
> > #include <sys/smp.h>
> > +#include <sys/sched.h>
> >
> > #include <machine/resource.h>
> > #include <machine/bus.h>
> > @@ -2339,6 +2340,8 @@
> > * drivers need this.
> > */
> > mtx_lock(&Giant);
> > + sched_bind(curthread, 0);
> > + stop_cpus(PCPU_GET(other_cpus));
> > slp_state = ACPI_SS_NONE;
> > switch (state) {
> > case ACPI_STATE_S1:
> > @@ -2430,13 +2433,16 @@
> > acpi_wake_prep_walk(state);
> > sc->acpi_sstate = ACPI_STATE_S0;
> > }
> > + printf("PREP WALK\n");
> > if (slp_state >= ACPI_SS_SLP_PREP)
> > AcpiLeaveSleepState(state);
> > + printf("LEAVE_SLEEP_STATE\n");
> > if (slp_state >= ACPI_SS_DEV_SUSPEND)
> > DEVICE_RESUME(root_bus);
> > + printf("DEVICE_RESUME\n");
> > if (slp_state >= ACPI_SS_SLEPT)
> > acpi_enable_fixed_events(sc);
> > -
> > + printf("ENABLE_FIXED_EVENT\n");
> > /* Allow another sleep request after a while. */
> > if (state != ACPI_STATE_S5)
> > timeout(acpi_sleep_enable, sc, hz * ACPI_MINIMUM_AWAKETIME);
> > @@ -2445,6 +2451,7 @@
> > acpi_UserNotify("Resume", ACPI_ROOT_OBJECT, state);
> >
> > mtx_unlock(&Giant);
> > + sched_unbind(curthread);
> > return_ACPI_STATUS (status);
> > }
> >
> > Index: dev/acpica/acpi_ec.c
> > ===================================================================
> > RCS file: /home/ncvs/src/sys/dev/acpica/acpi_ec.c,v
> > retrieving revision 1.80
> > diff -u -r1.80 acpi_ec.c
> > --- dev/acpica/acpi_ec.c 8 Nov 2007 21:20:34 -0000 1.80
> > +++ dev/acpica/acpi_ec.c 7 May 2008 17:07:11 -0000
> > @@ -747,7 +747,7 @@
> > * If booting, check if we need to run the query handler. If so, we
> > * we call it directly here since our thread taskq is not active yet.
> > */
> > - if (cold || rebooting) {
> > + if (cold || rebooting||sc->ec_suspending) {
> > if ((EC_GET_CSR(sc) & EC_EVENT_SCI)) {
> > CTR0(KTR_ACPI, "ec running gpe handler directly");
> > EcGpeQueryHandler(sc);
> > _______________________________________________
> > freebsd-acpi at freebsd.org mailing list
> > http://lists.freebsd.org/mailman/listinfo/freebsd-acpi
> > To unsubscribe, send any mail to "freebsd-acpi-unsubscribe at freebsd.org"
--
Vladimir B. Grebenschikov
vova at fbsd.ru
More information about the freebsd-acpi
mailing list