SCSI, SMP, and Supermicro, problems!
John Baldwin
jhb at FreeBSD.org
Wed Oct 8 13:17:34 PDT 2003
On 08-Oct-2003 David Raistrick wrote:
>
>> The machine booted, and ran, with no problems. Hyperthreading was enabled
>> in the bios, but disabled in the kernel (no options HTT).
> ...
>> I rebooted, disabled HT in the bios, and attempted to boot again.
>>
>> This time around we were back to the old failure:
>>
>> Waiting 15 seconds for SCSI devices to settle
>> >>>>>>>>>>>>>>>>>> Dump Card State Begins <<<<<<<<<<<<<<<<<
>
>
> An update: I added options HTT to the kernel and disabled HTT in the
> bios. System boots and runs fine, still with 4 CPUs.
>
>
> I'd really like to be able to use this system /without/ HTT, but WITH
> 4.8-RELEASE. (it's been suggested that in 5.1 I might have better luck,
> being able to disable the additional CPUs. Alas, my environment for this
> server outright /requires/ 4.x. 4.8-R until we later move it to 4.9-R.
>
> Anyone?
You can try using the attached patch. It removes the HTT option and
HTT is on by default, but it halts the CPUs the same way that we do
in current now.
--
John Baldwin <jhb at FreeBSD.org> <>< http://www.FreeBSD.org/~jhb/
"Power Users Use the Power to Serve!" - http://www.FreeBSD.org/
-------------- next part --------------
Index: sys/conf/options.i386
===========================================================================
--- sys/conf/options.i386 2003/10/06 09:30:31 #16
+++ sys/conf/options.i386 2003/10/06 09:30:31
@@ -32,7 +33,6 @@
# i386 SMP options
APIC_IO opt_global.h
-HTT
# Change KVM size. Changes things all over the kernel.
KVA_PAGES opt_global.h
Index: sys/conf/options.pc98
===========================================================================
--- sys/conf/options.pc98 2003/10/06 09:30:31 #14
+++ sys/conf/options.pc98 2003/10/06 09:30:31
@@ -32,7 +32,6 @@
# i386 SMP options
APIC_IO opt_global.h
-HTT
# Change KVM size. Changes things all over the kernel.
KVA_PAGES opt_global.h
Index: sys/i386/conf/LINT
===========================================================================
--- sys/i386/conf/LINT 2003/10/06 09:30:31 #95
+++ sys/i386/conf/LINT 2003/10/06 09:30:31
@@ -122,7 +122,6 @@
# Mandatory:
options SMP # Symmetric MultiProcessor Kernel
options APIC_IO # Symmetric (APIC) I/O
-options HTT # HyperThreading Technology
#
# Rogue SMP hardware:
Index: sys/i386/i386/machdep.c
===========================================================================
--- sys/i386/i386/machdep.c 2003/10/06 09:30:31 #20
+++ sys/i386/i386/machdep.c 2003/10/06 09:30:31
@@ -975,17 +987,19 @@
* On -stable, cpu_idle() is called with interrupts disabled and must
* return with them enabled.
*/
-#ifdef SMP
-static int cpu_idle_hlt = 0;
-#else
static int cpu_idle_hlt = 1;
-#endif
SYSCTL_INT(_machdep, OID_AUTO, cpu_idle_hlt, CTLFLAG_RW,
&cpu_idle_hlt, 0, "Idle loop HLT enable");
void
cpu_idle(void)
{
+
+#ifdef SMP
+ if (mp_grab_cpu_hlt())
+ return;
+#endif
+
if (cpu_idle_hlt) {
/*
* We must guarentee that hlt is exactly the instruction
Index: sys/i386/i386/mp_machdep.c
===========================================================================
--- sys/i386/i386/mp_machdep.c 2003/10/06 09:30:32 #11
+++ sys/i386/i386/mp_machdep.c 2003/10/06 09:30:32
@@ -26,7 +26,6 @@
*/
#include "opt_cpu.h"
-#include "opt_htt.h"
#include "opt_user_ldt.h"
#ifdef SMP
@@ -236,10 +235,9 @@
#define MP_ANNOUNCE_POST 0x19
-#ifdef HTT
static int need_hyperthreading_fixup;
static u_int logical_cpus;
-#endif
+u_int logical_cpus_mask; /* bit mask of logical cpu's */
/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
int current_postcode;
@@ -330,9 +328,7 @@
static int search_for_sig(u_int32_t target, int count);
static void mp_enable(u_int boot_addr);
-#ifdef HTT
static void mptable_hyperthread_fixup(u_int id_mask);
-#endif
static void mptable_pass1(void);
static int mptable_pass2(void);
static void default_mp_table(int type);
@@ -344,6 +340,10 @@
static int start_ap(int logicalCpu, u_int boot_addr);
static int apic_int_is_bus_type(int intr, int bus_type);
+static int hlt_cpus_mask;
+static int hlt_logical_cpus = 1;
+static struct sysctl_ctx_list logical_cpu_clist;
+
/*
* Calculate usable address in base memory for AP trampoline code.
*/
@@ -765,9 +765,7 @@
void* position;
int count;
int type;
-#ifdef HTT
u_int id_mask;
-#endif
POSTCODE(MPTABLE_PASS1_POST);
@@ -781,9 +779,7 @@
mp_nbusses = 0;
mp_napics = 0;
nintrs = 0;
-#ifdef HTT
id_mask = 0;
-#endif
/* check for use of 'default' configuration */
if (MPFPS_MPFB1 != 0) {
@@ -816,10 +812,8 @@
if (((proc_entry_ptr)position)->cpu_flags
& PROCENTRY_FLAG_EN) {
++mp_naps;
-#ifdef HTT
id_mask |= 1 <<
((proc_entry_ptr)position)->apic_id;
-#endif
}
break;
case 1: /* bus_entry */
@@ -854,10 +848,8 @@
mp_naps = MAXCPU;
}
-#ifdef HTT
/* See if we need to fixup HT logical CPUs. */
mptable_hyperthread_fixup(id_mask);
-#endif
/*
* Count the BSP.
@@ -883,9 +875,7 @@
static int
mptable_pass2(void)
{
-#ifdef HTT
struct PROCENTRY proc;
-#endif
int x;
mpcth_t cth;
int totalSize;
@@ -898,12 +888,10 @@
POSTCODE(MPTABLE_PASS2_POST);
-#ifdef HTT
/* Initialize fake proc entry for use with HT fixup. */
bzero(&proc, sizeof(proc));
proc.type = 0;
proc.cpu_flags = PROCENTRY_FLAG_EN;
-#endif
pgeflag = 0; /* XXX - Not used under SMP yet. */
@@ -983,7 +971,6 @@
if (processor_entry(position, cpu))
++cpu;
-#ifdef HTT
if (need_hyperthreading_fixup) {
/*
* Create fake mptable processor entries
@@ -994,10 +981,10 @@
for (i = 1; i < logical_cpus; i++) {
proc.apic_id++;
(void)processor_entry(&proc, cpu);
+ logical_cpus_mask |= (1 << cpu);
cpu++;
}
}
-#endif
break;
case 1:
if (bus_entry(position, bus))
@@ -1030,7 +1017,6 @@
return 0;
}
-#ifdef HTT
/*
* Check if we should perform a hyperthreading "fix-up" to
* enumerate any logical CPU's that aren't already listed
@@ -1079,7 +1065,6 @@
need_hyperthreading_fixup = 1;
mp_naps *= logical_cpus;
}
-#endif
void
assign_apic_irq(int apic, int intpin, int irq)
@@ -2707,7 +2692,7 @@
/* Step 1: Probe state (user, cpu, interrupt, spinlock, idle ) */
- map = other_cpus & ~stopped_cpus ;
+ map = other_cpus & ~(stopped_cpus|hlt_cpus_mask);
checkstate_probed_cpus = 0;
if (map != 0)
selected_apic_ipi(map,
@@ -2782,7 +2767,7 @@
/* Step 1: Probe state (user, cpu, interrupt, spinlock, idle) */
- map = other_cpus & ~stopped_cpus ;
+ map = other_cpus & ~(stopped_cpus|hlt_cpus_mask);
checkstate_probed_cpus = 0;
if (map != 0)
selected_apic_ipi(map,
@@ -2911,7 +2896,7 @@
if (!forward_roundrobin_enabled)
return;
resched_cpus |= other_cpus;
- map = other_cpus & ~stopped_cpus ;
+ map = other_cpus & ~(stopped_cpus|hlt_cpus_mask);
#if 1
selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED);
#else
@@ -3020,3 +3005,90 @@
/* release lock */
s_unlock(&smp_rv_lock);
}
+
+static int
+sysctl_htl_cpus(SYSCTL_HANDLER_ARGS)
+{
+ u_int mask;
+ int error;
+
+ mask = hlt_cpus_mask;
+ error = sysctl_handle_int(oidp, &mask, 0, req);
+ if (error || !req->newptr)
+ return (error);
+
+ if (logical_cpus_mask != 0 &&
+ (mask & logical_cpus_mask) == logical_cpus_mask)
+ hlt_logical_cpus = 1;
+ else
+ hlt_logical_cpus = 0;
+
+ if ((mask & all_cpus) == all_cpus)
+ mask &= ~(1<<0);
+ hlt_cpus_mask = mask;
+ return (error);
+}
+SYSCTL_PROC(_machdep, OID_AUTO, hlt_cpus, CTLTYPE_INT|CTLFLAG_RW,
+ 0, 0, sysctl_htl_cpus, "IU", "");
+
+static int
+sysctl_hlt_logical_cpus(SYSCTL_HANDLER_ARGS)
+{
+ int disable, error;
+
+ disable = hlt_logical_cpus;
+ error = sysctl_handle_int(oidp, &disable, 0, req);
+ if (error || !req->newptr)
+ return (error);
+
+ if (disable)
+ hlt_cpus_mask |= logical_cpus_mask;
+ else
+ hlt_cpus_mask &= ~logical_cpus_mask;
+
+ if ((hlt_cpus_mask & all_cpus) == all_cpus)
+ hlt_cpus_mask &= ~(1<<0);
+
+ hlt_logical_cpus = disable;
+ return (error);
+}
+
+static void
+cpu_hlt_setup(void *dummy __unused)
+{
+
+ if (logical_cpus_mask != 0) {
+ TUNABLE_INT_FETCH("machdep.hlt_logical_cpus",
+ &hlt_logical_cpus);
+ sysctl_ctx_init(&logical_cpu_clist);
+ SYSCTL_ADD_PROC(&logical_cpu_clist,
+ SYSCTL_STATIC_CHILDREN(_machdep), OID_AUTO,
+ "hlt_logical_cpus", CTLTYPE_INT|CTLFLAG_RW, 0, 0,
+ sysctl_hlt_logical_cpus, "IU", "");
+ SYSCTL_ADD_UINT(&logical_cpu_clist,
+ SYSCTL_STATIC_CHILDREN(_machdep), OID_AUTO,
+ "logical_cpus_mask", CTLTYPE_INT|CTLFLAG_RD,
+ &logical_cpus_mask, 0, "");
+
+ if (hlt_logical_cpus)
+ hlt_cpus_mask |= logical_cpus_mask;
+ }
+}
+SYSINIT(cpu_hlt, SI_SUB_SMP, SI_ORDER_ANY, cpu_hlt_setup, NULL);
+
+int
+mp_grab_cpu_hlt(void)
+{
+ u_int mask = 1 << cpuid;
+ u_int temp;
+ int retval;
+
+ retval = mask & hlt_cpus_mask;
+ while (mask & hlt_cpus_mask) {
+ temp = lapic.tpr;
+ lapic.tpr = LOPRIO_LEVEL;
+ __asm __volatile("sti; hlt" : : : "memory");
+ lapic.tpr = temp;
+ }
+ return (retval);
+}
Index: sys/i386/include/smp.h
===========================================================================
--- sys/i386/include/smp.h 2003/10/06 09:30:32 #2
+++ sys/i386/include/smp.h 2003/10/06 09:30:32
@@ -121,6 +121,7 @@
int apic_int_type __P((int, int));
int apic_trigger __P((int, int));
int apic_polarity __P((int, int));
+int mp_grab_cpu_hlt __P((void));
void assign_apic_irq __P((int apic, int intpin, int irq));
void revoke_apic_irq __P((int irq));
void bsp_apic_configure __P((void));
More information about the freebsd-scsi
mailing list