PERFORCE change 37941 for review
Peter Wemm
peter at FreeBSD.org
Thu Sep 11 16:16:57 PDT 2003
http://perforce.freebsd.org/chv.cgi?CH=37941
Change 37941 by peter at peter_work on 2003/09/11 16:16:06
another smp pass (not finished still)
Affected files ...
.. //depot/projects/hammer/sys/amd64/amd64/amd64_mem.c#2 edit
.. //depot/projects/hammer/sys/amd64/amd64/mp_machdep.c#2 edit
.. //depot/projects/hammer/sys/amd64/amd64/mptable.c#2 edit
.. //depot/projects/hammer/sys/amd64/amd64/pmap.c#32 edit
.. //depot/projects/hammer/sys/amd64/include/smp.h#7 edit
.. //depot/projects/hammer/sys/amd64/include/smptests.h#2 edit
Differences ...
==== //depot/projects/hammer/sys/amd64/amd64/amd64_mem.c#2 (text+ko) ====
==== //depot/projects/hammer/sys/amd64/amd64/mp_machdep.c#2 (text+ko) ====
@@ -35,15 +35,9 @@
#error How did you get here?
#endif
-#if defined(I386_CPU) && !defined(COMPILING_LINT)
-#error SMP not supported with I386_CPU
-#endif
#ifndef DEV_APIC
#error The apic device is required for SMP, add "device apic" to your config file.
#endif
-#if defined(CPU_DISABLE_CMPXCHG) && !defined(COMPILING_LINT)
-#error SMP not supported with CPU_DISABLE_CMPXCHG
-#endif
#endif /* not lint */
#include <sys/param.h>
@@ -63,41 +57,19 @@
#include <sys/proc.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
-#if 0
-#include <sys/user.h>
-#endif
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/pmap.h>
#include <vm/vm_kern.h>
#include <vm/vm_extern.h>
-#if 0
-#include <vm/vm_map.h>
-#endif
#include <machine/apicreg.h>
-#if 0
-#include <machine/atomic.h>
-#endif
#include <machine/clock.h>
-#if 0
-#include <machine/cpu.h>
-#include <machine/cpufunc.h>
-#endif
#include <machine/md_var.h>
#include <machine/pcb.h>
-#if 0
-#include <machine/psl.h>
-#include <machine/segments.h>
-#endif
#include <machine/smp.h>
-#include <machine/smptests.h> /** COUNT_XINVLTLB_HITS, USE_COMLOCK */
-#if 0
-#include <machine/tss.h>
-#endif
#include <machine/specialreg.h>
-#include <machine/privatespace.h>
#define WARMBOOT_TARGET 0
#define WARMBOOT_OFF (KERNBASE + 0x0467)
@@ -109,43 +81,6 @@
#define BIOS_WARM (0x0a)
/*
- * this code MUST be enabled here and in mpboot.s.
- * it follows the very early stages of AP boot by placing values in CMOS ram.
- * it NORMALLY will never be needed and thus the primitive method for enabling.
- *
-#define CHECK_POINTS
- */
-
-#if defined(CHECK_POINTS) && !defined(PC98)
-#define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA))
-#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D)))
-
-#define CHECK_INIT(D); \
- CHECK_WRITE(0x34, (D)); \
- CHECK_WRITE(0x35, (D)); \
- CHECK_WRITE(0x36, (D)); \
- CHECK_WRITE(0x37, (D)); \
- CHECK_WRITE(0x38, (D)); \
- CHECK_WRITE(0x39, (D));
-
-#define CHECK_PRINT(S); \
- printf("%s: %d, %d, %d, %d, %d, %d\n", \
- (S), \
- CHECK_READ(0x34), \
- CHECK_READ(0x35), \
- CHECK_READ(0x36), \
- CHECK_READ(0x37), \
- CHECK_READ(0x38), \
- CHECK_READ(0x39));
-
-#else /* CHECK_POINTS */
-
-#define CHECK_INIT(D)
-#define CHECK_PRINT(S)
-
-#endif /* CHECK_POINTS */
-
-/*
* Values to send to the POST hardware.
*/
#define MP_BOOTADDRESS_POST 0x10
@@ -165,11 +100,6 @@
/* lock region used by kernel profiling */
int mcount_lock;
-#ifdef USE_COMLOCK
-/* locks com (tty) data/hardware accesses: a FASTINTR() */
-struct mtx com_mtx;
-#endif
-
/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
int current_postcode;
@@ -247,7 +177,6 @@
u_int
mp_bootaddress(u_int basemem)
{
- POSTCODE(MP_BOOTADDRESS_POST);
boot_address = basemem & ~0xfff; /* round down to 4k boundary */
if ((basemem - boot_address) < bootMP_size)
@@ -321,8 +250,6 @@
cpu_mp_start(void)
{
- POSTCODE(MP_START_POST);
-
/* Install an inter-CPU IPI for TLB invalidation */
setidt(IPI_INVLTLB, IDTVEC(invltlb),
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
@@ -355,9 +282,6 @@
setidt(IPI_STOP, IDTVEC(cpustop),
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
-#ifdef USE_COMLOCK
- mtx_init(&com_mtx, "com", NULL, MTX_SPIN);
-#endif
mtx_init(&smp_tlb_mtx, "tlb", NULL, MTX_SPIN);
/* Setup BSP apic ID */
@@ -389,8 +313,6 @@
{
int i, x;
- POSTCODE(MP_ANNOUNCE_POST);
-
/* List CPUs */
printf(" cpu0 (BSP): APIC ID: %2d\n", boot_cpu_id);
for (i = 1, x = 0; x < MAXCPU; x++) {
@@ -502,8 +424,6 @@
uintptr_t kptbase;
int i, pg, apic_id, cpu;
- POSTCODE(START_ALL_APS_POST);
-
mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
/* install the AP 1st level boot code */
@@ -563,16 +483,13 @@
bootAP = cpu;
/* attempt to start the Application Processor */
- CHECK_INIT(99); /* setup checkpoints */
if (!start_ap(apic_id, boot_addr)) {
printf("AP #%d (PHY# %d) failed!\n", cpu, apic_id);
- CHECK_PRINT("trace"); /* show checkpoints */
/* better panic as the AP may be running loose */
printf("panic y/n? [y] ");
if (cngetc() != 'n')
panic("bye-bye");
}
- CHECK_PRINT("trace"); /* show checkpoints */
all_cpus |= (1 << cpu); /* record AP in CPU map */
}
@@ -632,8 +549,6 @@
u_int16_t *dst16;
u_int32_t *dst32;
- POSTCODE(INSTALL_AP_TRAMP_POST);
-
for (x = 0; x < size; ++x)
*dst++ = *src++;
@@ -680,8 +595,6 @@
int vector, ms;
int cpus;
- POSTCODE(START_AP_POST);
-
/* calculate the vector */
vector = (boot_addr >> 12) & 0xff;
==== //depot/projects/hammer/sys/amd64/amd64/mptable.c#2 (text+ko) ====
@@ -225,10 +225,6 @@
u_long segment;
u_int32_t target;
-#if 0
- POSTCODE(MP_PROBE_POST);
-#endif
-
/* see if EBDA exists */
if ((segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) != 0) {
/* search first 1K of EBDA */
==== //depot/projects/hammer/sys/amd64/amd64/pmap.c#32 (text+ko) ====
@@ -116,6 +116,9 @@
#include <sys/user.h>
#include <sys/vmmeter.h>
#include <sys/sysctl.h>
+#ifdef SMP
+#include <sys/smp.h>
+#endif
#include <vm/vm.h>
#include <vm/vm_param.h>
@@ -165,6 +168,9 @@
LIST_HEAD(pmaplist, pmap);
static struct pmaplist allpmaps;
static struct mtx allpmaps_lock;
+#ifdef SMP
+static struct mtx lazypmap_lock;
+#endif
vm_paddr_t avail_start; /* PA of first available physical page */
vm_paddr_t avail_end; /* PA of last available physical page */
@@ -479,6 +485,9 @@
kernel_pmap->pm_active = -1; /* don't allow deactivation */
TAILQ_INIT(&kernel_pmap->pm_pvlist);
LIST_INIT(&allpmaps);
+#ifdef SMP
+ mtx_init(&lazypmap_lock, "lazypmap", NULL, MTX_SPIN);
+#endif
mtx_init(&allpmaps_lock, "allpmaps", NULL, MTX_SPIN);
mtx_lock_spin(&allpmaps_lock);
LIST_INSERT_HEAD(&allpmaps, kernel_pmap, pm_list);
@@ -678,10 +687,121 @@
return 0;
}
-XXXX SMP VERSIONS
+#ifdef SMP
+/*
+ * For SMP, these functions have to use the IPI mechanism for coherence.
+ */
+void
+pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
+{
+ u_int cpumask;
+ u_int other_cpus;
+
+ if (smp_started) {
+ if (!(read_eflags() & PSL_I))
+ panic("%s: interrupts disabled", __func__);
+ mtx_lock_spin(&smp_tlb_mtx);
+ } else
+ critical_enter();
+ /*
+ * We need to disable interrupt preemption but MUST NOT have
+ * interrupts disabled here.
+ * XXX we may need to hold schedlock to get a coherent pm_active
+ * XXX critical sections disable interrupts again
+ */
+ if (pmap->pm_active == -1 || pmap->pm_active == all_cpus) {
+ invlpg(va);
+ smp_invlpg(va);
+ } else {
+ cpumask = PCPU_GET(cpumask);
+ other_cpus = PCPU_GET(other_cpus);
+ if (pmap->pm_active & cpumask)
+ invlpg(va);
+ if (pmap->pm_active & other_cpus)
+ smp_masked_invlpg(pmap->pm_active & other_cpus, va);
+ }
+ if (smp_started)
+ mtx_unlock_spin(&smp_tlb_mtx);
+ else
+ critical_exit();
+}
+
+void
+pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
+{
+ u_int cpumask;
+ u_int other_cpus;
+ vm_offset_t addr;
+
+ if (smp_started) {
+ if (!(read_eflags() & PSL_I))
+ panic("%s: interrupts disabled", __func__);
+ mtx_lock_spin(&smp_tlb_mtx);
+ } else
+ critical_enter();
+ /*
+ * We need to disable interrupt preemption but MUST NOT have
+ * interrupts disabled here.
+ * XXX we may need to hold schedlock to get a coherent pm_active
+ * XXX critical sections disable interrupts again
+ */
+ if (pmap->pm_active == -1 || pmap->pm_active == all_cpus) {
+ for (addr = sva; addr < eva; addr += PAGE_SIZE)
+ invlpg(addr);
+ smp_invlpg_range(sva, eva);
+ } else {
+ cpumask = PCPU_GET(cpumask);
+ other_cpus = PCPU_GET(other_cpus);
+ if (pmap->pm_active & cpumask)
+ for (addr = sva; addr < eva; addr += PAGE_SIZE)
+ invlpg(addr);
+ if (pmap->pm_active & other_cpus)
+ smp_masked_invlpg_range(pmap->pm_active & other_cpus,
+ sva, eva);
+ }
+ if (smp_started)
+ mtx_unlock_spin(&smp_tlb_mtx);
+ else
+ critical_exit();
+}
+
+void
+pmap_invalidate_all(pmap_t pmap)
+{
+ u_int cpumask;
+ u_int other_cpus;
+ if (smp_started) {
+ if (!(read_eflags() & PSL_I))
+ panic("%s: interrupts disabled", __func__);
+ mtx_lock_spin(&smp_tlb_mtx);
+ } else
+ critical_enter();
+ /*
+ * We need to disable interrupt preemption but MUST NOT have
+ * interrupts disabled here.
+ * XXX we may need to hold schedlock to get a coherent pm_active
+ * XXX critical sections disable interrupts again
+ */
+ if (pmap->pm_active == -1 || pmap->pm_active == all_cpus) {
+ invltlb();
+ smp_invltlb();
+ } else {
+ cpumask = PCPU_GET(cpumask);
+ other_cpus = PCPU_GET(other_cpus);
+ if (pmap->pm_active & cpumask)
+ invltlb();
+ if (pmap->pm_active & other_cpus)
+ smp_masked_invltlb(pmap->pm_active & other_cpus);
+ }
+ if (smp_started)
+ mtx_unlock_spin(&smp_tlb_mtx);
+ else
+ critical_exit();
+}
+#else /* !SMP */
/*
- * Normal invalidation functions.
+ * Normal, non-SMP, invalidation functions.
* We inline these within pmap.c for speed.
*/
PMAP_INLINE void
@@ -709,6 +829,7 @@
if (pmap == kernel_pmap || pmap->pm_active)
invltlb();
}
+#endif /* !SMP */
/*
* Are we current address space or kernel?
@@ -1241,6 +1362,95 @@
* Pmap allocation/deallocation routines.
***************************************************/
+#ifdef SMP
+/*
+ * Deal with a SMP shootdown of other users of the pmap that we are
+ * trying to dispose of. This can be a bit hairy.
+ */
+static u_int *lazymask;
+static u_int lazyptd;
+static volatile u_int lazywait;
+
+void pmap_lazyfix_action(void);
+
+void
+pmap_lazyfix_action(void)
+{
+ u_int mymask = PCPU_GET(cpumask);
+
+ if (rcr3() == lazyptd)
+ load_cr3(PCPU_GET(curpcb)->pcb_cr3);
+ atomic_clear_int(lazymask, mymask);
+ atomic_store_rel_int(&lazywait, 1);
+}
+
+static void
+pmap_lazyfix_self(u_int mymask)
+{
+
+ if (rcr3() == lazyptd)
+ load_cr3(PCPU_GET(curpcb)->pcb_cr3);
+ atomic_clear_int(lazymask, mymask);
+}
+
+
+static void
+pmap_lazyfix(pmap_t pmap)
+{
+ u_int mymask = PCPU_GET(cpumask);
+ u_int mask;
+ register u_int spins;
+
+ while ((mask = pmap->pm_active) != 0) {
+ spins = 50000000;
+ mask = mask & -mask; /* Find least significant set bit */
+ mtx_lock_spin(&lazypmap_lock);
+#ifdef PAE
+ lazyptd = vtophys(pmap->pm_pdpt);
+#else
+ lazyptd = vtophys(pmap->pm_pdir);
+#endif
+ if (mask == mymask) {
+ lazymask = &pmap->pm_active;
+ pmap_lazyfix_self(mymask);
+ } else {
+ atomic_store_rel_int((u_int *)&lazymask,
+ (u_int)&pmap->pm_active);
+ atomic_store_rel_int(&lazywait, 0);
+ ipi_selected(mask, IPI_LAZYPMAP);
+ while (lazywait == 0) {
+ ia32_pause();
+ if (--spins == 0)
+ break;
+ }
+ }
+ mtx_unlock_spin(&lazypmap_lock);
+ if (spins == 0)
+ printf("pmap_lazyfix: spun for 50000000\n");
+ }
+}
+
+#else /* SMP */
+
+/*
+ * Cleaning up on uniprocessor is easy. For various reasons, we're
+ * unlikely to have to even execute this code, including the fact
+ * that the cleanup is deferred until the parent does a wait(2), which
+ * means that another userland process has run.
+ */
+static void
+pmap_lazyfix(pmap_t pmap)
+{
+ u_int cr3;
+
+ cr3 = vtophys(pmap->pm_pdir);
+ if (cr3 == rcr3()) {
+ load_cr3(PCPU_GET(curpcb)->pcb_cr3);
+ pmap->pm_active &= ~(PCPU_GET(cpumask));
+ }
+}
+#endif /* SMP */
+
/*
* Release any resources held by the given physical map.
* Called when a pmap initialized by pmap_pinit is being released.
@@ -1261,6 +1471,7 @@
("pmap_release: pmap resident count %ld != 0",
pmap->pm_stats.resident_count));
+ pmap_lazyfix(pmap);
mtx_lock_spin(&allpmaps_lock);
LIST_REMOVE(pmap, pm_list);
mtx_unlock_spin(&allpmaps_lock);
@@ -2899,12 +3110,19 @@
pmap_activate(struct thread *td)
{
struct proc *p = td->td_proc;
- pmap_t pmap;
+ pmap_t pmap, oldpmap;
u_int64_t cr3;
critical_enter();
pmap = vmspace_pmap(td->td_proc->p_vmspace);
+ oldpmap = PCPU_GET(curpmap);
+#ifdef SMP
+ atomic_clear_long(&oldpmap->pm_active, PCPU_GET(cpumask));
+ atomic_set_long(&pmap->pm_active, PCPU_GET(cpumask));
+#else
+ oldpmap->pm_active &= ~PCPU_GET(cpumask);
pmap->pm_active |= PCPU_GET(cpumask);
+#endif
cr3 = vtophys(pmap->pm_pml4);
/* XXXKSE this is wrong.
* pmap_activate is for the current thread on the current cpu
==== //depot/projects/hammer/sys/amd64/include/smp.h#7 (text+ko) ====
@@ -19,20 +19,6 @@
#ifndef LOCORE
-/*
- * For sending values to POST displays.
- * XXX FIXME: where does this really belong, isa.h/isa.c perhaps?
- */
-extern int current_postcode; /** XXX currently in mp_machdep.c */
-#define POSTCODE(X) current_postcode = (X), \
- outb(0x80, current_postcode)
-#define POSTCODE_LO(X) current_postcode &= 0xf0, \
- current_postcode |= ((X) & 0x0f), \
- outb(0x80, current_postcode)
-#define POSTCODE_HI(X) current_postcode &= 0x0f, \
- current_postcode |= (((X) << 4) & 0xf0), \
- outb(0x80, current_postcode)
-
#include <sys/bus.h>
#include <machine/frame.h>
#include <machine/intr_machdep.h>
==== //depot/projects/hammer/sys/amd64/include/smptests.h#2 (text+ko) ====
@@ -24,99 +24,3 @@
*
* $FreeBSD: src/sys/i386/include/smptests.h,v 1.44 2003/01/23 01:04:27 peter Exp $
*/
-
-#ifndef _MACHINE_SMPTESTS_H_
-#define _MACHINE_SMPTESTS_H_
-
-
-/*
- * Various 'tests in progress' and configuration parameters.
- */
-
-/*
- * These defines enable critical region locking of areas that were
- * protected via cli/sti in the UP kernel.
- *
- * COMLOCK protects the sio/cy drivers.
- * known to be incomplete:
- * joystick lkm
- * ?
- */
-#define USE_COMLOCK
-
-
-/*
- * Send CPUSTOP IPI for stop/restart of other CPUs on DDB break.
-#define VERBOSE_CPUSTOP_ON_DDBBREAK
- */
-#define CPUSTOP_ON_DDBBREAK
-
-/*
- * Misc. counters.
- *
-#define COUNT_XINVLTLB_HITS
- */
-
-/*
- * Address of POST hardware port.
- * Defining this enables POSTCODE macros.
- *
-#define POST_ADDR 0x80
- */
-
-
-/*
- * POST hardware macros.
- */
-#ifdef POST_ADDR
-#define ASMPOSTCODE_INC \
- pushl %eax ; \
- movl _current_postcode, %eax ; \
- incl %eax ; \
- andl $0xff, %eax ; \
- movl %eax, _current_postcode ; \
- outb %al, $POST_ADDR ; \
- popl %eax
-
-/*
- * Overwrite the current_postcode value.
- */
-#define ASMPOSTCODE(X) \
- pushl %eax ; \
- movl $X, %eax ; \
- movl %eax, _current_postcode ; \
- outb %al, $POST_ADDR ; \
- popl %eax
-
-/*
- * Overwrite the current_postcode low nibble.
- */
-#define ASMPOSTCODE_LO(X) \
- pushl %eax ; \
- movl _current_postcode, %eax ; \
- andl $0xf0, %eax ; \
- orl $X, %eax ; \
- movl %eax, _current_postcode ; \
- outb %al, $POST_ADDR ; \
- popl %eax
-
-/*
- * Overwrite the current_postcode high nibble.
- */
-#define ASMPOSTCODE_HI(X) \
- pushl %eax ; \
- movl _current_postcode, %eax ; \
- andl $0x0f, %eax ; \
- orl $(X<<4), %eax ; \
- movl %eax, _current_postcode ; \
- outb %al, $POST_ADDR ; \
- popl %eax
-#else
-#define ASMPOSTCODE_INC
-#define ASMPOSTCODE(X)
-#define ASMPOSTCODE_LO(X)
-#define ASMPOSTCODE_HI(X)
-#endif /* POST_ADDR */
-
-
-#endif /* _MACHINE_SMPTESTS_H_ */
More information about the p4-projects
mailing list