svn commit: r184246 - in projects/release_6_3_xen/sys: conf
i386/i386 i386/include i386/include/xen i386/xen xen/evtchn
Kip Macy
kmacy at FreeBSD.org
Sat Oct 25 04:40:25 UTC 2008
Author: kmacy
Date: Sat Oct 25 04:40:22 2008
New Revision: 184246
URL: http://svn.freebsd.org/changeset/base/184246
Log:
MFC SMP support
Added:
projects/release_6_3_xen/sys/i386/xen/mptable.c
Modified:
projects/release_6_3_xen/sys/conf/files.i386
projects/release_6_3_xen/sys/i386/i386/apic_vector.s
projects/release_6_3_xen/sys/i386/i386/local_apic.c
projects/release_6_3_xen/sys/i386/include/apicvar.h
projects/release_6_3_xen/sys/i386/include/pcpu.h
projects/release_6_3_xen/sys/i386/include/smp.h
projects/release_6_3_xen/sys/i386/include/xen/evtchn.h
projects/release_6_3_xen/sys/i386/include/xen/xen-os.h
projects/release_6_3_xen/sys/i386/include/xen/xen_intr.h
projects/release_6_3_xen/sys/i386/include/xen/xenfunc.h
projects/release_6_3_xen/sys/i386/xen/clock.c
projects/release_6_3_xen/sys/i386/xen/exception.s
projects/release_6_3_xen/sys/i386/xen/mp_machdep.c
projects/release_6_3_xen/sys/i386/xen/xen_machdep.c
projects/release_6_3_xen/sys/xen/evtchn/evtchn.c
Modified: projects/release_6_3_xen/sys/conf/files.i386
==============================================================================
--- projects/release_6_3_xen/sys/conf/files.i386 Sat Oct 25 03:41:36 2008 (r184245)
+++ projects/release_6_3_xen/sys/conf/files.i386 Sat Oct 25 04:40:22 2008 (r184246)
@@ -323,7 +323,8 @@ i386/i386/mp_machdep.c optional native
i386/xen/mp_machdep.c optional xen smp
i386/i386/mp_watchdog.c optional mp_watchdog smp
i386/i386/mpboot.s optional native smp
-i386/i386/mptable.c optional apic
+i386/xen/mptable.c optional apic xen
+i386/i386/mptable.c optional apic native
i386/i386/mptable_pci.c optional apic pci
i386/i386/msi.c optional apic pci
i386/i386/nexus.c standard
Modified: projects/release_6_3_xen/sys/i386/i386/apic_vector.s
==============================================================================
--- projects/release_6_3_xen/sys/i386/i386/apic_vector.s Sat Oct 25 03:41:36 2008 (r184245)
+++ projects/release_6_3_xen/sys/i386/i386/apic_vector.s Sat Oct 25 04:40:22 2008 (r184246)
@@ -299,6 +299,7 @@ IDTVEC(invlcache)
/*
* Handler for IPIs sent via the per-cpu IPI bitmap.
*/
+#ifndef XEN
.text
SUPERALIGN_TEXT
IDTVEC(ipi_intr_bitmap_handler)
@@ -320,7 +321,7 @@ IDTVEC(ipi_intr_bitmap_handler)
addl $4, %esp /* XXX convert clockframe to trapframe */
MEXITCOUNT
jmp doreti
-
+#endif
/*
* Executed by a CPU when it receives an Xcpustop IPI from another CPU,
*
Modified: projects/release_6_3_xen/sys/i386/i386/local_apic.c
==============================================================================
--- projects/release_6_3_xen/sys/i386/i386/local_apic.c Sat Oct 25 03:41:36 2008 (r184245)
+++ projects/release_6_3_xen/sys/i386/i386/local_apic.c Sat Oct 25 04:40:22 2008 (r184246)
@@ -1068,7 +1068,9 @@ apic_setup_io(void *dummy __unused)
if (retval != 0)
printf("%s: Failed to setup I/O APICs: returned %d\n",
best_enum->apic_name, retval);
-
+#ifdef XEN
+ return;
+#endif
/*
* Finish setting up the local APIC on the BSP once we know how to
* properly program the LINT pins.
Modified: projects/release_6_3_xen/sys/i386/include/apicvar.h
==============================================================================
--- projects/release_6_3_xen/sys/i386/include/apicvar.h Sat Oct 25 03:41:36 2008 (r184245)
+++ projects/release_6_3_xen/sys/i386/include/apicvar.h Sat Oct 25 04:40:22 2008 (r184246)
@@ -113,6 +113,18 @@
#define APIC_THERMAL_INT (APIC_LOCAL_INTS + 1)
#define APIC_IPI_INTS (APIC_LOCAL_INTS + 2)
+#ifdef XEN
+#define IPI_RENDEZVOUS (2) /* Inter-CPU rendezvous. */
+#define IPI_INVLTLB (3) /* TLB Shootdown IPIs */
+#define IPI_INVLPG (4)
+#define IPI_INVLRNG (5)
+#define IPI_INVLCACHE (6)
+#define IPI_LAZYPMAP (7) /* Lazy pmap release. */
+/* Vector to handle bitmap based IPIs */
+#define IPI_BITMAP_VECTOR (8)
+#define IPI_STOP (9)
+
+#else
#define IPI_RENDEZVOUS (APIC_IPI_INTS) /* Inter-CPU rendezvous. */
#define IPI_INVLTLB (APIC_IPI_INTS + 1) /* TLB Shootdown IPIs */
#define IPI_INVLPG (APIC_IPI_INTS + 2)
@@ -121,6 +133,8 @@
#define IPI_LAZYPMAP (APIC_IPI_INTS + 5) /* Lazy pmap release. */
/* Vector to handle bitmap based IPIs */
#define IPI_BITMAP_VECTOR (APIC_IPI_INTS + 6)
+#define IPI_STOP (APIC_IPI_INTS + 7) /* Stop CPU until restarted. */
+#endif
/* IPIs handled by IPI_BITMAPED_VECTOR (XXX ups is there a better place?) */
#define IPI_AST 0 /* Generate software trap. */
@@ -128,7 +142,7 @@
#define IPI_BITMAP_LAST IPI_PREEMPT
#define IPI_IS_BITMAPED(x) ((x) <= IPI_BITMAP_LAST)
-#define IPI_STOP (APIC_IPI_INTS + 7) /* Stop CPU until restarted. */
+
/*
* The spurious interrupt can share the priority class with the IPIs since
Modified: projects/release_6_3_xen/sys/i386/include/pcpu.h
==============================================================================
--- projects/release_6_3_xen/sys/i386/include/pcpu.h Sat Oct 25 03:41:36 2008 (r184245)
+++ projects/release_6_3_xen/sys/i386/include/pcpu.h Sat Oct 25 04:40:22 2008 (r184246)
@@ -47,6 +47,24 @@
*/
#ifdef XEN
+#ifndef NR_VIRQS
+#define NR_VIRQS 24
+#endif
+#ifndef NR_IPIS
+#define NR_IPIS 2
+#endif
+
+/* These are peridically updated in shared_info, and then copied here. */
+struct shadow_time_info {
+ uint64_t tsc_timestamp; /* TSC at last update of time vals. */
+ uint64_t system_timestamp; /* Time, in nanosecs, since boot. */
+ uint32_t tsc_to_nsec_mul;
+ uint32_t tsc_to_usec_mul;
+ int tsc_shift;
+ uint32_t version;
+};
+
+
#define PCPU_MD_FIELDS \
struct pcpu *pc_prvspace; /* Self-reference */ \
struct pmap *pc_curpmap; \
@@ -63,7 +81,14 @@
u_int pc_pdir; \
u_int pc_lazypmap; \
u_int pc_rendezvous; \
- u_int pc_cpuast
+ u_int pc_cpuast; \
+ uint64_t pc_processed_system_time; \
+ struct shadow_time_info pc_shadow_time; \
+ int pc_resched_irq; \
+ int pc_callfunc_irq; \
+ int pc_virq_to_irq[NR_VIRQS]; \
+ int pc_ipi_to_irq[NR_IPIS]
+
#else
Modified: projects/release_6_3_xen/sys/i386/include/smp.h
==============================================================================
--- projects/release_6_3_xen/sys/i386/include/smp.h Sat Oct 25 03:41:36 2008 (r184245)
+++ projects/release_6_3_xen/sys/i386/include/smp.h Sat Oct 25 04:40:22 2008 (r184246)
@@ -68,7 +68,9 @@ void ipi_selected(u_int cpus, u_int ipi)
void ipi_all(u_int ipi);
void ipi_all_but_self(u_int ipi);
void ipi_self(u_int ipi);
+#ifndef XEN
void ipi_bitmap_handler(struct clockframe frame);
+#endif
u_int mp_bootaddress(u_int);
int mp_grab_cpu_hlt(void);
void mp_topology(void);
@@ -85,7 +87,14 @@ void smp_masked_invltlb(u_int mask);
int ipi_nmi_handler(void);
void ipi_nmi_selected(u_int32_t cpus);
#endif
+#ifdef XEN
+void ipi_to_irq_init(void);
+
+#define RESCHEDULE_VECTOR 0
+#define CALL_FUNCTION_VECTOR 1
+#define NR_IPIS 2
+#endif
#endif /* !LOCORE */
#endif /* SMP */
Modified: projects/release_6_3_xen/sys/i386/include/xen/evtchn.h
==============================================================================
--- projects/release_6_3_xen/sys/i386/include/xen/evtchn.h Sat Oct 25 03:41:36 2008 (r184245)
+++ projects/release_6_3_xen/sys/i386/include/xen/evtchn.h Sat Oct 25 04:40:22 2008 (r184246)
@@ -35,13 +35,24 @@ void mask_evtchn(int port);
void unmask_evtchn(int port);
+#ifdef SMP
+void rebind_evtchn_to_cpu(int port, unsigned int cpu);
+#else
+#define rebind_evtchn_to_cpu(port, cpu) ((void)0)
+#endif
+static inline
+int test_and_set_evtchn_mask(int port)
+{
+ shared_info_t *s = HYPERVISOR_shared_info;
+ return synch_test_and_set_bit(port, s->evtchn_mask);
+}
static inline void
clear_evtchn(int port)
{
- shared_info_t *s = HYPERVISOR_shared_info;
- synch_clear_bit(port, &s->evtchn_pending[0]);
+ shared_info_t *s = HYPERVISOR_shared_info;
+ synch_clear_bit(port, &s->evtchn_pending[0]);
}
static inline void
Modified: projects/release_6_3_xen/sys/i386/include/xen/xen-os.h
==============================================================================
--- projects/release_6_3_xen/sys/i386/include/xen/xen-os.h Sat Oct 25 03:41:36 2008 (r184245)
+++ projects/release_6_3_xen/sys/i386/include/xen/xen-os.h Sat Oct 25 04:40:22 2008 (r184246)
@@ -73,10 +73,7 @@ static inline void rep_nop(void)
#define __builtin_expect(x, expected_value) (x)
#endif
-#define DEFINE_PER_CPU(type, name) \
- __typeof__(type) per_cpu__##name
-
-#define per_cpu(var, cpu) (*((void)cpu, &per_cpu__##var))
+#define per_cpu(var, cpu) (pcpu_find((cpu))->pc_ ## var)
/* crude memory allocator for memory allocation early in
* boot
@@ -94,12 +91,6 @@ void printk(const char *fmt, ...);
/* some function prototypes */
void trap_init(void);
-extern int preemptable;
-#define preempt_disable() (preemptable = 0)
-#define preempt_enable() (preemptable = 1)
-#define preempt_enable_no_resched() (preemptable = 1)
-
-
/*
* STI/CLI equivalents. These basically set and clear the virtual
* event_enable flag in teh shared_info structure. Note that when
@@ -114,10 +105,8 @@ extern int preemptable;
#define __cli() \
do { \
vcpu_info_t *_vcpu; \
- preempt_disable(); \
_vcpu = &HYPERVISOR_shared_info->vcpu_info[smp_processor_id()]; \
_vcpu->evtchn_upcall_mask = 1; \
- preempt_enable_no_resched(); \
barrier(); \
} while (0)
@@ -125,36 +114,23 @@ do {
do { \
vcpu_info_t *_vcpu; \
barrier(); \
- preempt_disable(); \
_vcpu = &HYPERVISOR_shared_info->vcpu_info[smp_processor_id()]; \
_vcpu->evtchn_upcall_mask = 0; \
barrier(); /* unmask then check (avoid races) */ \
if ( unlikely(_vcpu->evtchn_upcall_pending) ) \
force_evtchn_callback(); \
- preempt_enable(); \
-} while (0)
-
-
-#define __save_flags(x) \
-do { \
- vcpu_info_t *vcpu; \
- vcpu = HYPERVISOR_shared_info->vcpu_info[smp_processor_id()]; \
- (x) = _vcpu->evtchn_upcall_mask; \
} while (0)
#define __restore_flags(x) \
do { \
vcpu_info_t *_vcpu; \
barrier(); \
- preempt_disable(); \
_vcpu = &HYPERVISOR_shared_info->vcpu_info[smp_processor_id()]; \
if ((_vcpu->evtchn_upcall_mask = (x)) == 0) { \
barrier(); /* unmask then check (avoid races) */ \
if ( unlikely(_vcpu->evtchn_upcall_pending) ) \
force_evtchn_callback(); \
- preempt_enable(); \
- } else \
- preempt_enable_no_resched(); \
+ } \
} while (0)
/*
Modified: projects/release_6_3_xen/sys/i386/include/xen/xen_intr.h
==============================================================================
--- projects/release_6_3_xen/sys/i386/include/xen/xen_intr.h Sat Oct 25 03:41:36 2008 (r184245)
+++ projects/release_6_3_xen/sys/i386/include/xen/xen_intr.h Sat Oct 25 04:40:22 2008 (r184246)
@@ -29,7 +29,6 @@
/* Dynamic binding of event channels and VIRQ sources to Linux IRQ space. */
extern void unbind_from_irq(int irq);
-extern void bind_evtchn_to_cpu(unsigned int chn, unsigned int cpu);
extern int bind_caller_port_to_irqhandler(unsigned int caller_port,
const char *devname, driver_intr_t handler, void *arg,
unsigned long irqflags, void **cookiep);
@@ -38,8 +37,12 @@ extern int bind_listening_port_to_irqhan
void **cookiep);
extern int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu, const char *devname,
driver_intr_t handler, unsigned long irqflags);
-extern int bind_ipi_to_irqhandler(unsigned int ipi, unsigned int cpu, const char *devname,
- driver_intr_t handler, unsigned long irqflags);
+extern int bind_ipi_to_irqhandler(unsigned int ipi,
+ unsigned int cpu,
+ const char *devname,
+ driver_intr_t handler,
+ unsigned long irqflags);
+
extern int bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain,
unsigned int remote_port,
const char *devname,
@@ -61,7 +64,7 @@ extern void enable_irq(unsigned int);
extern void irq_suspend(void);
extern void irq_resume(void);
-extern void idle_block(void);
-
+extern void idle_block(void);
+extern int ap_cpu_initclocks(int cpu);
#endif /* _XEN_INTR_H_ */
Modified: projects/release_6_3_xen/sys/i386/include/xen/xenfunc.h
==============================================================================
--- projects/release_6_3_xen/sys/i386/include/xen/xenfunc.h Sat Oct 25 03:41:36 2008 (r184245)
+++ projects/release_6_3_xen/sys/i386/include/xen/xenfunc.h Sat Oct 25 04:40:22 2008 (r184246)
@@ -65,8 +65,6 @@ void _xen_machphys_update(vm_paddr_t, vm
void xen_update_descriptor(union descriptor *, union descriptor *);
-void ap_cpu_initclocks(void);
-
extern struct mtx balloon_lock;
#if 0
#define balloon_lock(__flags) mtx_lock_irqsave(&balloon_lock, __flags)
Modified: projects/release_6_3_xen/sys/i386/xen/clock.c
==============================================================================
--- projects/release_6_3_xen/sys/i386/xen/clock.c Sat Oct 25 03:41:36 2008 (r184245)
+++ projects/release_6_3_xen/sys/i386/xen/clock.c Sat Oct 25 04:40:22 2008 (r184246)
@@ -161,19 +161,6 @@ SYSCTL_INT(_machdep, OID_AUTO, xen_disab
})
-/* These are peridically updated in shared_info, and then copied here. */
-struct shadow_time_info {
- uint64_t tsc_timestamp; /* TSC at last update of time vals. */
- uint64_t system_timestamp; /* Time, in nanosecs, since boot. */
- uint32_t tsc_to_nsec_mul;
- uint32_t tsc_to_usec_mul;
- int tsc_shift;
- uint32_t version;
-};
-static DEFINE_PER_CPU(uint64_t, processed_system_time);
-static DEFINE_PER_CPU(struct shadow_time_info, shadow_time);
-
-
#define NS_PER_TICK (1000000000ULL/hz)
#define rdtscll(val) \
@@ -868,25 +855,26 @@ cpu_initclocks(void)
/* should fast clock be enabled ? */
}
-/*
- *
- * XXX
- */
-#if 0 && defined(SMP)
-void
-ap_cpu_initclocks(void)
+
+int
+ap_cpu_initclocks(int cpu)
{
- int irq;
- int cpu = smp_processor_id();
-
- per_cpu(processed_system_time, cpu) = processed_system_time;
+ int time_irq;
+
+ xen_set_periodic_tick.period_ns = NS_PER_TICK;
- irq = bind_virq_to_irq(VIRQ_TIMER);
- PCPU_SET(time_irq, irq);
- PANIC_IF(intr_add_handler("clk", irq, (driver_intr_t *)clkintr, NULL,
- NULL, INTR_TYPE_CLK | INTR_FAST, NULL));
+ HYPERVISOR_vcpu_op(VCPUOP_set_periodic_timer, cpu,
+ &xen_set_periodic_tick);
+
+ if ((time_irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, "clk",
+ (driver_intr_t *)clkintr,
+ INTR_TYPE_CLK | INTR_FAST)) < 0) {
+ panic("failed to register clock interrupt\n");
+ }
+
+ return (0);
}
-#endif
+
void
cpu_startprofclock(void)
Modified: projects/release_6_3_xen/sys/i386/xen/exception.s
==============================================================================
--- projects/release_6_3_xen/sys/i386/xen/exception.s Sat Oct 25 03:41:36 2008 (r184245)
+++ projects/release_6_3_xen/sys/i386/xen/exception.s Sat Oct 25 04:40:22 2008 (r184246)
@@ -37,18 +37,34 @@
#include <machine/psl.h>
#include <machine/trap.h>
-
#include "assym.s"
#define SEL_RPL_MASK 0x0002
#define __HYPERVISOR_iret 23
/* Offsets into shared_info_t. */
+
#define evtchn_upcall_pending /* 0 */
#define evtchn_upcall_mask 1
-#define XEN_BLOCK_EVENTS(reg) movb $1,evtchn_upcall_mask(reg)
-#define XEN_UNBLOCK_EVENTS(reg) movb $0,evtchn_upcall_mask(reg)
-#define XEN_TEST_PENDING(reg) testb $0x1,evtchn_upcall_pending(reg)
+
+#define sizeof_vcpu_shift 6
+
+
+#ifdef SMP
+#define GET_VCPU_INFO(reg) movl PCPU(CPUID),reg ; \
+ shl $sizeof_vcpu_shift,reg ; \
+ addl HYPERVISOR_shared_info,reg
+#else
+#define GET_VCPU_INFO(reg) movl HYPERVISOR_shared_info,reg
+#endif
+
+#define __DISABLE_INTERRUPTS(reg) movb $1,evtchn_upcall_mask(reg)
+#define __ENABLE_INTERRUPTS(reg) movb $0,evtchn_upcall_mask(reg)
+#define DISABLE_INTERRUPTS(reg) GET_VCPU_INFO(reg) ; \
+ __DISABLE_INTERRUPTS(reg)
+#define ENABLE_INTERRUPTS(reg) GET_VCPU_INFO(reg) ; \
+ __ENABLE_INTERRUPTS(reg)
+#define __TEST_PENDING(reg) testb $0xFF,evtchn_upcall_pending(reg)
#define POPA \
popl %edi; \
@@ -161,8 +177,7 @@ call_evtchn_upcall:
hypervisor_callback_pending:
- movl HYPERVISOR_shared_info,%esi
- XEN_BLOCK_EVENTS(%esi) /* cli */
+ DISABLE_INTERRUPTS(%esi) /* cli */
jmp 10b
/*
@@ -327,12 +342,11 @@ doreti_ast:
* interrupts provides sufficient locking even in the SMP case,
* since we will be informed of any new ASTs by an IPI.
*/
- movl HYPERVISOR_shared_info,%esi
- XEN_BLOCK_EVENTS(%esi) /* cli */
+ DISABLE_INTERRUPTS(%esi) /* cli */
movl PCPU(CURTHREAD),%eax
testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%eax)
je doreti_exit
- XEN_UNBLOCK_EVENTS(%esi) /* sti */
+ ENABLE_INTERRUPTS(%esi) /* sti */
pushl %esp /* pass a pointer to the trapframe */
call ast
add $4,%esp
@@ -346,12 +360,11 @@ doreti_ast:
* registers. The fault is handled in trap.c.
*/
doreti_exit:
- movl HYPERVISOR_shared_info,%esi
- XEN_UNBLOCK_EVENTS(%esi) # reenable event callbacks (sti)
+ ENABLE_INTERRUPTS(%esi) # reenable event callbacks (sti)
.globl scrit
scrit:
- XEN_TEST_PENDING(%esi)
+ __TEST_PENDING(%esi)
jnz hypervisor_callback_pending /* More to go */
MEXITCOUNT
Modified: projects/release_6_3_xen/sys/i386/xen/mp_machdep.c
==============================================================================
--- projects/release_6_3_xen/sys/i386/xen/mp_machdep.c Sat Oct 25 03:41:36 2008 (r184245)
+++ projects/release_6_3_xen/sys/i386/xen/mp_machdep.c Sat Oct 25 04:40:22 2008 (r184246)
@@ -83,18 +83,10 @@ __FBSDID("$FreeBSD$");
#include <machine/specialreg.h>
#include <machine/xen/hypervisor.h>
+#include <machine/xen/xen_intr.h>
#include <machine/xen/evtchn.h>
#include <xen/interface/vcpu.h>
-#define WARMBOOT_TARGET 0
-#define WARMBOOT_OFF (KERNBASE + 0x0467)
-#define WARMBOOT_SEG (KERNBASE + 0x0469)
-
-#define CMOS_REG (0x70)
-#define CMOS_DATA (0x71)
-#define BIOS_RESET (0x0f)
-#define BIOS_WARM (0x0a)
-
/*
* this code MUST be enabled here and in mpboot.s.
* it follows the very early stages of AP boot by placing values in CMOS ram.
@@ -126,6 +118,9 @@ char *bootSTK;
static int bootAP;
static union descriptor *bootAPgdt;
+static char resched_name[NR_CPUS][15];
+static char callfunc_name[NR_CPUS][15];
+
/* Free these after use */
void *bootstacks[MAXCPU];
@@ -139,6 +134,9 @@ vm_offset_t smp_tlb_addr1;
vm_offset_t smp_tlb_addr2;
volatile int smp_tlb_wait;
+typedef void call_data_func_t(uintptr_t , uintptr_t);
+
+
#ifdef COUNT_IPIS
/* Interrupt counts. */
#ifdef IPI_PREEMPTION
@@ -179,8 +177,6 @@ static int cpu_apic_ids[MAXCPU];
/* Holds pending bitmap based IPIs per CPU */
static volatile u_int cpu_ipi_pending[MAXCPU];
-static u_int boot_address;
-
static void assign_cpu_ids(void);
static void set_interrupt_apic_ids(void);
static int start_all_aps(void);
@@ -191,6 +187,7 @@ static u_int hyperthreading_cpus;
static cpumask_t hyperthreading_cpus_mask;
extern void Xhypervisor_callback(void);
extern void failsafe_callback(void);
+extern void pmap_lazyfix_action(void);
void
mp_topology(void)
@@ -320,40 +317,6 @@ cpu_mp_start(void)
cpu_ipi_pending[i] = 0;
}
-#if 0
- /*
- * IPI list that has to be converted to Xen
- *
- */
- /* Install an inter-CPU IPI for TLB invalidation */
- setidt(IPI_INVLTLB, IDTVEC(invltlb),
- SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
- setidt(IPI_INVLPG, IDTVEC(invlpg),
- SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
- setidt(IPI_INVLRNG, IDTVEC(invlrng),
- SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
-
- /* Install an inter-CPU IPI for cache invalidation. */
- setidt(IPI_INVLCACHE, IDTVEC(invlcache),
- SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
-
- /* Install an inter-CPU IPI for lazy pmap release */
- setidt(IPI_LAZYPMAP, IDTVEC(lazypmap),
- SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
-
- /* Install an inter-CPU IPI for all-CPU rendezvous */
- setidt(IPI_RENDEZVOUS, IDTVEC(rendezvous),
- SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
-
- /* Install generic inter-CPU IPI handler */
- setidt(IPI_BITMAP_VECTOR, IDTVEC(ipi_intr_bitmap_handler),
- SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
-
- /* Install an inter-CPU IPI for CPU stop/restart */
- setidt(IPI_STOP, IDTVEC(cpustop),
- SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
-#endif
-
/* Set boot_cpu_id if needed. */
if (boot_cpu_id == -1) {
boot_cpu_id = PCPU_GET(apic_id);
@@ -376,6 +339,151 @@ cpu_mp_start(void)
set_interrupt_apic_ids();
}
+
+static void
+iv_rendezvous(uintptr_t a, uintptr_t b)
+{
+ smp_rendezvous_action();
+}
+
+static void
+iv_invltlb(uintptr_t a, uintptr_t b)
+{
+ xen_tlb_flush();
+}
+
+static void
+iv_invlpg(uintptr_t a, uintptr_t b)
+{
+ xen_invlpg(a);
+}
+
+static void
+iv_invlrng(uintptr_t a, uintptr_t b)
+{
+ vm_offset_t start = (vm_offset_t)a;
+ vm_offset_t end = (vm_offset_t)b;
+
+ while (start < end) {
+ xen_invlpg(start);
+ start += PAGE_SIZE;
+ }
+}
+
+
+static void
+iv_invlcache(uintptr_t a, uintptr_t b)
+{
+
+ wbinvd();
+}
+
+static void
+iv_lazypmap(uintptr_t a, uintptr_t b)
+{
+ pmap_lazyfix_action();
+}
+
+
+static void
+iv_noop(uintptr_t a, uintptr_t b)
+{
+}
+
+static call_data_func_t *ipi_vectors[IPI_BITMAP_VECTOR] =
+{
+ iv_noop,
+ iv_noop,
+ iv_rendezvous,
+ iv_invltlb,
+ iv_invlpg,
+ iv_invlrng,
+ iv_invlcache,
+ iv_lazypmap,
+};
+
+/*
+ * Reschedule call back. Nothing to do,
+ * all the work is done automatically when
+ * we return from the interrupt.
+ */
+static void
+smp_reschedule_interrupt(void *unused)
+{
+ int cpu = PCPU_GET(cpuid);
+ u_int ipi_bitmap;
+
+ ipi_bitmap = atomic_readandclear_int(&cpu_ipi_pending[cpu]);
+
+#ifdef IPI_PREEMPTION
+ if (ipi_bitmap & (1 << IPI_PREEMPT)) {
+#ifdef COUNT_IPIS
+ *ipi_preempt_counts[cpu]++;
+#endif
+ mtx_lock_spin(&sched_lock);
+ /* Don't preempt the idle thread */
+ if (curthread != PCPU_GET(idlethread)) {
+ struct thread *running_thread = curthread;
+ if (running_thread->td_critnest > 1)
+ running_thread->td_owepreempt = 1;
+ else
+ mi_switch(SW_INVOL | SW_PREEMPT, NULL);
+ }
+ mtx_unlock_spin(&sched_lock);
+ }
+#endif
+
+ if (ipi_bitmap & (1 << IPI_AST)) {
+#ifdef COUNT_IPIS
+ *ipi_ast_counts[cpu]++;
+#endif
+ /* Nothing to do for AST */
+ }
+}
+
+struct _call_data {
+ uint16_t func_id;
+ uint16_t wait;
+ uintptr_t arg1;
+ uintptr_t arg2;
+ atomic_t started;
+ atomic_t finished;
+};
+
+static struct _call_data *call_data;
+
+static void
+smp_call_function_interrupt(void *arg)
+{
+ call_data_func_t *func;
+ uintptr_t arg1 = call_data->arg1;
+ uintptr_t arg2 = call_data->arg2;
+ int wait = call_data->wait;
+ atomic_t *started = &call_data->started;
+ atomic_t *finished = &call_data->finished;
+
+ if (call_data->func_id > IPI_BITMAP_VECTOR)
+ panic("invalid function id %u", call_data->func_id);
+
+ func = ipi_vectors[call_data->func_id];
+ /*
+ * Notify initiating CPU that I've grabbed the data and am
+ * about to execute the function
+ */
+ mb();
+ atomic_inc(started);
+ /*
+ * At this point the info structure may be out of scope unless wait==1
+ */
+ (*func)(arg1, arg2);
+
+ if (wait) {
+ mb();
+ atomic_inc(finished);
+ }
+ atomic_add_int(&smp_tlb_wait, 1);
+}
+
/*
* Print various information about the SMP system hardware and setup.
*/
@@ -399,6 +507,61 @@ cpu_mp_announce(void)
}
}
+static int
+xen_smp_intr_init(unsigned int cpu)
+{
+ int rc;
+
+ per_cpu(resched_irq, cpu) = per_cpu(callfunc_irq, cpu) = -1;
+
+ sprintf(resched_name[cpu], "resched%u", cpu);
+ rc = bind_ipi_to_irqhandler(RESCHEDULE_VECTOR,
+ cpu,
+ resched_name[cpu],
+ smp_reschedule_interrupt,
+ INTR_FAST|INTR_TYPE_TTY|INTR_MPSAFE);
+
+ printf("cpu=%d irq=%d vector=%d\n",
+ cpu, rc, RESCHEDULE_VECTOR);
+
+ per_cpu(resched_irq, cpu) = rc;
+
+ sprintf(callfunc_name[cpu], "callfunc%u", cpu);
+ rc = bind_ipi_to_irqhandler(CALL_FUNCTION_VECTOR,
+ cpu,
+ callfunc_name[cpu],
+ smp_call_function_interrupt,
+ INTR_FAST|INTR_TYPE_TTY|INTR_MPSAFE);
+ if (rc < 0)
+ goto fail;
+ per_cpu(callfunc_irq, cpu) = rc;
+
+ printf("cpu=%d irq=%d vector=%d\n",
+ cpu, rc, CALL_FUNCTION_VECTOR);
+
+
+ if ((cpu != 0) && ((rc = ap_cpu_initclocks(cpu)) != 0))
+ goto fail;
+
+ return 0;
+
+ fail:
+ if (per_cpu(resched_irq, cpu) >= 0)
+ unbind_from_irqhandler(per_cpu(resched_irq, cpu), NULL);
+ if (per_cpu(callfunc_irq, cpu) >= 0)
+ unbind_from_irqhandler(per_cpu(callfunc_irq, cpu), NULL);
+ return rc;
+}
+
+static void
+xen_smp_intr_init_cpus(void *unused)
+{
+ int i;
+
+ for (i = 0; i < mp_ncpus; i++)
+ xen_smp_intr_init(i);
+}
+
#define MTOPSIZE (1<<(14 + PAGE_SHIFT))
/*
* AP CPU's call this to initialize themselves.
@@ -613,10 +776,6 @@ start_all_aps(void)
bootstacks[cpu] = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE);
- /* setup a vector to our boot code */
- *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
- *((volatile u_short *) WARMBOOT_SEG) = (boot_address >> 4);
-
bootSTK = (char *)bootstacks[cpu] + KSTACK_PAGES * PAGE_SIZE - 4;
bootAP = cpu;
bootAPgdt = gdt + (512*cpu);
@@ -703,7 +862,6 @@ cpu_initialize_context(unsigned int cpu)
* Page 0,[0-3] PTD
* Page 1, [4] boot stack
* Page [5] PDPT
-
*
*/
for (i = 0; i < NPGPTD + 2; i++) {
@@ -881,19 +1039,24 @@ static void
smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2)
{
u_int ncpu;
+ struct _call_data data;
+ call_data = &data;
+
ncpu = mp_ncpus - 1; /* does not shootdown self */
if (ncpu < 1)
return; /* no other cpus */
if (!(read_eflags() & PSL_I))
panic("%s: interrupts disabled", __func__);
mtx_lock_spin(&smp_ipi_mtx);
- smp_tlb_addr1 = addr1;
- smp_tlb_addr2 = addr2;
+ call_data->func_id = vector;
+ call_data->arg1 = addr1;
+ call_data->arg2 = addr2;
atomic_store_rel_int(&smp_tlb_wait, 0);
ipi_all_but_self(vector);
while (smp_tlb_wait < ncpu)
ia32_pause();
+ call_data = NULL;
mtx_unlock_spin(&smp_ipi_mtx);
}
@@ -901,6 +1064,7 @@ static void
smp_targeted_tlb_shootdown(u_int mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2)
{
int ncpu, othercpus;
+ struct _call_data data;
othercpus = mp_ncpus - 1;
if (mask == (u_int)-1) {
@@ -925,8 +1089,10 @@ smp_targeted_tlb_shootdown(u_int mask, u
if (!(read_eflags() & PSL_I))
panic("%s: interrupts disabled", __func__);
mtx_lock_spin(&smp_ipi_mtx);
- smp_tlb_addr1 = addr1;
- smp_tlb_addr2 = addr2;
+ call_data = &data;
+ call_data->func_id = vector;
+ call_data->arg1 = addr1;
+ call_data->arg2 = addr2;
atomic_store_rel_int(&smp_tlb_wait, 0);
if (mask == (u_int)-1)
ipi_all_but_self(vector);
@@ -934,6 +1100,7 @@ smp_targeted_tlb_shootdown(u_int mask, u
ipi_selected(mask, vector);
while (smp_tlb_wait < ncpu)
ia32_pause();
+ call_data = NULL;
mtx_unlock_spin(&smp_ipi_mtx);
}
@@ -1019,56 +1186,21 @@ smp_masked_invlpg_range(u_int mask, vm_o
}
}
-
-void
-ipi_bitmap_handler(struct clockframe frame)
-{
- int cpu = PCPU_GET(cpuid);
- u_int ipi_bitmap;
-
- ipi_bitmap = atomic_readandclear_int(&cpu_ipi_pending[cpu]);
-
-#ifdef IPI_PREEMPTION
- if (ipi_bitmap & (1 << IPI_PREEMPT)) {
-#ifdef COUNT_IPIS
- *ipi_preempt_counts[cpu]++;
-#endif
- mtx_lock_spin(&sched_lock);
- /* Don't preempt the idle thread */
- if (curthread != PCPU_GET(idlethread)) {
- struct thread *running_thread = curthread;
- if (running_thread->td_critnest > 1)
- running_thread->td_owepreempt = 1;
- else
- mi_switch(SW_INVOL | SW_PREEMPT, NULL);
- }
- mtx_unlock_spin(&sched_lock);
- }
-#endif
-
- if (ipi_bitmap & (1 << IPI_AST)) {
-#ifdef COUNT_IPIS
- *ipi_ast_counts[cpu]++;
-#endif
- /* Nothing to do for AST */
- }
-}
-
/*
* send an IPI to a set of cpus.
*/
void
-ipi_selected(u_int32_t cpus, u_int ipi)
+ipi_selected(uint32_t cpus, u_int ipi)
{
int cpu;
u_int bitmap = 0;
u_int old_pending;
u_int new_pending;
-
+
if (IPI_IS_BITMAPED(ipi)) {
bitmap = 1 << ipi;
ipi = IPI_BITMAP_VECTOR;
- }
+ }
CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi);
while ((cpu = ffs(cpus)) != 0) {
@@ -1084,11 +1216,15 @@ ipi_selected(u_int32_t cpus, u_int ipi)
new_pending = old_pending | bitmap;
} while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],old_pending, new_pending));
- if (old_pending)
- continue;
+ if (!old_pending)
+ ipi_pcpu(cpu, RESCHEDULE_VECTOR);
+ continue;
+
}
- ipi_pcpu(cpu, ipi);
+ KASSERT(call_data != NULL, ("call_data not set"));
+
+ ipi_pcpu(cpu, CALL_FUNCTION_VECTOR);
}
}
@@ -1112,7 +1248,7 @@ ipi_all_but_self(u_int ipi)
{
CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
- ipi_selected(all_cpus & ~(1<<curcpu), ipi);
+ ipi_selected(PCPU_GET(other_cpus), ipi);
}
/*
@@ -1123,7 +1259,7 @@ ipi_self(u_int ipi)
{
CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
- lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF);
+ ipi_selected(PCPU_GET(cpumask), ipi);
}
/*
@@ -1143,6 +1279,7 @@ release_aps(void *dummy __unused)
mtx_unlock_spin(&sched_lock);
}
SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);
+SYSINIT(start_ipis, SI_SUB_INTR, SI_ORDER_ANY, xen_smp_intr_init_cpus, NULL);
#ifdef COUNT_IPIS
/*
Added: projects/release_6_3_xen/sys/i386/xen/mptable.c
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ projects/release_6_3_xen/sys/i386/xen/mptable.c Sat Oct 25 04:40:22 2008 (r184246)
@@ -0,0 +1,130 @@
+/*-
+ * Copyright (c) 2003 John Baldwin <jhb at FreeBSD.org>
+ * Copyright (c) 1996, by Steve Passe
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. The name of the developer may NOT be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+#include <vm/pmap.h>
+
+#include <machine/apicreg.h>
+#include <machine/frame.h>
+#include <machine/intr_machdep.h>
+#include <machine/apicvar.h>
+#include <machine/md_var.h>
+#include <machine/mptable.h>
+#include <machine/specialreg.h>
+
+#include <machine/xen/hypervisor.h>
+#include <machine/xen/xen-os.h>
+#include <machine/smp.h>
+#include <xen/interface/vcpu.h>
+
+
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-projects
mailing list