svn commit: r278473 - in head/sys: amd64/amd64 amd64/include amd64/vmm contrib/dev/acpica/include i386/i386 i386/include x86/acpica x86/include x86/x86 x86/xen

[Konstantin Belousov]

On Thu, Feb 26, 2015 at 05:26:26PM -0800, Adrian Chadd wrote:
> On 26 February 2015 at 17:17, Peter Grehan <grehan at freebsd.org> wrote:
> >> Anyway - I have that model laptop, it hangs after calling
> >> ipi_startup() to the first AP with x2apic enabled. What can I do to
> >> continue debugging?
> >
> >
> >  Please provide the info that Kostik requested here, try out the patch, and
> > wait for the timezone in Ukraine to be a little more reasonable for a reply.
> >
> >  http://docs.FreeBSD.org/cgi/mid.cgi?20150216174658.GI34251
> 
> Hi,
> 
> That patch is already in -HEAD; I'm already running it and no, it
> doesn't help. The details are the same as what koop provided.
> 
> >  Use the workaround of setting hw.x2apic_enable=0 in the loader to fall back
> > to stock APIC mode.
> 
> Yup - I'm using it to at least boot the laptop to get debugging output.
> 
Sigh.  

I need verbose dmesg output of the successfull boot, acpidump -t output,
and report what happen when you are trying to boot with the following
patch, without x2apic knob disabling x2apic.

The patch below increases the time wait for the AP init.

diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index b767691..9d33ca5 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -569,6 +569,7 @@ cpu_mp_start(void)
 
 	/* Start each Application Processor */
 	init_ops.start_all_aps();
+printf("start_all_aps finished\n");
 
 	set_interrupt_apic_ids();
 }
@@ -973,6 +974,7 @@ native_start_all_aps(void)
 		bootSTK = (char *)bootstacks[cpu] + KSTACK_PAGES * PAGE_SIZE - 8;
 		bootAP = cpu;
 
+printf("cpu %d start\n", cpu);
 		/* attempt to start the Application Processor */
 		if (!start_ap(apic_id)) {
 			/* restore the warmstart vector */
@@ -981,6 +983,7 @@ native_start_all_aps(void)
 		}
 
 		CPU_SET(cpu, &all_cpus);	/* record AP in CPU map */
+printf("cpu %d done\n", cpu);
 	}
 
 	/* restore the warmstart vector */
@@ -1083,6 +1086,7 @@ ipi_startup(int apic_id, int vector)
 	lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_LEVEL |
 	    APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, apic_id);
 	lapic_ipi_wait(20);
+	DELAY(200);
 
 	/* Explicitly deassert the INIT IPI. */
 	lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_LEVEL |
@@ -1105,7 +1109,7 @@ ipi_startup(int apic_id, int vector)
 	if (!lapic_ipi_wait(20))
 		panic("Failed to deliver first STARTUP IPI to APIC %d",
 		    apic_id);
-	DELAY(200);		/* wait ~200uS */
+	DELAY(400);		/* wait ~200uS */
 
 	/*
 	 * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF
@@ -1120,7 +1124,7 @@ ipi_startup(int apic_id, int vector)
 		panic("Failed to deliver second STARTUP IPI to APIC %d",
 		    apic_id);
 
-	DELAY(200);		/* wait ~200uS */
+	DELAY(400);		/* wait ~200uS */
 }
 
 /*
diff --git a/sys/x86/x86/local_apic.c b/sys/x86/x86/local_apic.c
index 1b66674..dffb639 100644
--- a/sys/x86/x86/local_apic.c
+++ b/sys/x86/x86/local_apic.c
@@ -165,11 +165,25 @@ static struct eventtimer lapic_et;
 SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG_RD, 0, "APIC options");
 SYSCTL_INT(_hw_apic, OID_AUTO, x2apic_mode, CTLFLAG_RD, &x2apic_mode, 0, "");
 
+static int x2apic_inited;
+static void
+lapic_report(int read, int reg, int val)
+{
+
+	if (!IS_BSP())
+		return;
+	if (read)
+		printf("lapic read %x; %d %d\n", reg, x2apic_mode, x2apic_inited);
+	else
+		printf("lapic write %x %x; %d %d\n", reg, val,x2apic_mode, x2apic_inited);
+}
+
 static uint32_t
 lapic_read32(enum LAPIC_REGISTERS reg)
 {
 	uint32_t res;
 
+lapic_report(1, reg, 0);
 	if (x2apic_mode) {
 		res = rdmsr32(MSR_APIC_000 + reg);
 	} else {
@@ -182,6 +196,7 @@ static void
 lapic_write32(enum LAPIC_REGISTERS reg, uint32_t val)
 {
 
+lapic_report(0, reg, val);
 	if (x2apic_mode) {
 		mfence();
 		wrmsr(MSR_APIC_000 + reg, val);
@@ -194,6 +209,7 @@ static void
 lapic_write32_nofence(enum LAPIC_REGISTERS reg, uint32_t val)
 {
 
+lapic_report(0, reg, val);
 	if (x2apic_mode) {
 		wrmsr(MSR_APIC_000 + reg, val);
 	} else {
@@ -207,6 +223,7 @@ lapic_read_icr(void)
 	uint64_t v;
 	uint32_t vhi, vlo;
 
+lapic_report(1, LAPIC_ICR_LO, 0);
 	if (x2apic_mode) {
 		v = rdmsr(MSR_APIC_000 + LAPIC_ICR_LO);
 	} else {
@@ -221,6 +238,7 @@ static uint64_t
 lapic_read_icr_lo(void)
 {
 
+lapic_report(1, LAPIC_ICR_LO, 0);
 	return (lapic_read32(LAPIC_ICR_LO));
 }
 
@@ -229,6 +247,7 @@ lapic_write_icr(uint32_t vhi, uint32_t vlo)
 {
 	uint64_t v;
 
+lapic_report(0, LAPIC_ICR_LO, vlo);
 	if (x2apic_mode) {
 		v = ((uint64_t)vhi << 32) | vlo;
 		mfence();
@@ -247,6 +266,7 @@ native_lapic_enable_x2apic(void)
 	apic_base = rdmsr(MSR_APICBASE);
 	apic_base |= APICBASE_X2APIC | APICBASE_ENABLED;
 	wrmsr(MSR_APICBASE, apic_base);
+x2apic_inited = 1;
 }
 
 static void	lapic_enable(void);