svn commit: r263001 - in head/sys: amd64/amd64 amd64/include i386/i386 i386/include i386/xen x86/xen

Roger Pau Monné royger at FreeBSD.org
Tue Mar 11 10:03:32 UTC 2014


Author: royger
Date: Tue Mar 11 10:03:29 2014
New Revision: 263001
URL: http://svnweb.freebsd.org/changeset/base/263001

Log:
  amd64/i386: switch IPI handlers to C code.
  
  Move asm IPIs handlers to C code, so both Xen and native IPI handlers
  share the same code.
  
  Reviewed by: jhb
  Approved by: gibbs
  Sponsored by: Citrix Systems R&D
  
  amd64/amd64/apic_vector.S:
  i386/i386/apic_vector.s:
   - Remove asm coded IPI handlers and instead call the newly introduced
     C variants.
  
  amd64/amd64/mp_machdep.c:
  i386/i386/mp_machdep.c:
   - Add C coded clones to the asm IPI handlers (moved from
     x86/xen/hvm.c).
  
  i386/include/smp.h:
  amd64/include/smp.h:
   - Add prototypes for the C IPI handlers.
  
  x86/xen/hvm.c:
   - Move the C IPI handlers to mp_machdep and call those in the Xen IPI
     handlers.
  
  i386/xen/mp_machdep.c:
   - Add dummy IPI handlers to the i386 Xen PV port (this port doesn't
     support SMP).

Modified:
  head/sys/amd64/amd64/apic_vector.S
  head/sys/amd64/amd64/mp_machdep.c
  head/sys/amd64/include/smp.h
  head/sys/i386/i386/apic_vector.s
  head/sys/i386/i386/mp_machdep.c
  head/sys/i386/include/smp.h
  head/sys/i386/xen/mp_machdep.c
  head/sys/x86/xen/hvm.c

Modified: head/sys/amd64/amd64/apic_vector.S
==============================================================================
--- head/sys/amd64/amd64/apic_vector.S	Tue Mar 11 07:33:33 2014	(r263000)
+++ head/sys/amd64/amd64/apic_vector.S	Tue Mar 11 10:03:29 2014	(r263001)
@@ -159,101 +159,26 @@ IDTVEC(xen_intr_upcall)
 #define	NAKE_INTR_CS	24
 
 	SUPERALIGN_TEXT
-global_invltlb:
-	movq	%cr4,%rax
-	andq	$~0x80,%rax	/* PGE */
-	movq	%rax,%cr4
-	orq	$0x80,%rax
-	movq	%rax,%cr4
-invltlb_ret_clear_pm_save:
-	movq	smp_tlb_pmap,%rdx
-	testq	%rdx,%rdx
-	jz	invltlb_ret_rdx
-	testb	$SEL_RPL_MASK,NAKE_INTR_CS(%rsp)
-	jz	1f
-	swapgs
-1:
-	movl	PCPU(CPUID),%eax
-	jz	2f
-	swapgs
-2:
-	LK btcl	%eax,PM_SAVE(%rdx)
-	SUPERALIGN_TEXT
-invltlb_ret_rdx:
-	popq	%rdx
-invltlb_ret_rax:
+invltlb_ret:
 	movq	lapic, %rax
 	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
-	LK incl	smp_tlb_wait
-	popq	%rax
+	POP_FRAME
 	jmp	doreti_iret
 
 	SUPERALIGN_TEXT
 IDTVEC(invltlb_pcid)
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
 	PUSH_FRAME
-	movl	PCPU(CPUID), %eax
-#ifdef COUNT_XINVLTLB_HITS
-	incl	xhits_gbl(,%rax,4)
-#endif
-#ifdef COUNT_IPIS
-	movq	ipi_invltlb_counts(,%rax,8),%rax
-	incq	(%rax)
-#endif
-	POP_FRAME
-#endif
-
-	pushq	%rax
-	pushq	%rdx
 
-	movq	%cr3,%rax
+	call	invltlb_pcid_handler
+	jmp	invltlb_ret
 
-	movq	$smp_tlb_invpcid,%rdx
-	cmpl	$0,(%rdx)
-	je	global_invltlb
-	cmpl	$-1,(%rdx)
-	je	global_invltlb
-
-	/*
-	 * Only invalidate TLB for entries with current PCID.
-	 */
-	cmpl	$0,invpcid_works
-	je	1f
-	/* Use invpcid if available. */
-	movl	$1,%eax /* INVPCID_CTX */
-	/* invpcid (%rdx),%rax */
-	.byte 0x66,0x0f,0x38,0x82,0x02
-	jmp	invltlb_ret_clear_pm_save
-1:
-	/* Otherwise reload %cr3 twice. */
-	movq	pcid_cr3,%rdx
-	cmpq	%rax,%rdx
-	je	2f
-	movq	%rdx,%cr3	/* Invalidate, bit 63 is zero. */
-	btsq	$63,%rax
-2:
-	movq	%rax,%cr3
-	jmp	invltlb_ret_clear_pm_save
 
 	SUPERALIGN_TEXT
 IDTVEC(invltlb)
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
 	PUSH_FRAME
-	movl	PCPU(CPUID), %eax
-#ifdef COUNT_XINVLTLB_HITS
-	incl	xhits_gbl(,%rax,4)
-#endif
-#ifdef COUNT_IPIS
-	movq	ipi_invltlb_counts(,%rax,8),%rax
-	incq	(%rax)
-#endif
-	POP_FRAME
-#endif
 
-	pushq	%rax
-	movq	%cr3, %rax		/* invalidate the TLB */
-	movq	%rax, %cr3
-	jmp	invltlb_ret_rax
+	call	invltlb_handler
+	jmp	invltlb_ret
 
 /*
  * Single page TLB shootdown
@@ -261,86 +186,17 @@ IDTVEC(invltlb)
 	.text
 	SUPERALIGN_TEXT
 IDTVEC(invlpg_pcid)
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
 	PUSH_FRAME
-	movl	PCPU(CPUID), %eax
-#ifdef COUNT_XINVLTLB_HITS
-	incl	xhits_pg(,%rax,4)
-#endif
-#ifdef COUNT_IPIS
-	movq	ipi_invlpg_counts(,%rax,8),%rax
-	incq	(%rax)
-#endif
-	POP_FRAME
-#endif
 
-	pushq	%rax
-	pushq	%rdx
-	movq	$smp_tlb_invpcid,%rdx
-	cmpl	$0,invpcid_works
-	jne	2f
-
-	/* kernel pmap - use invlpg to invalidate global mapping */
-	cmpl	$0,(%rdx)
-	je	3f
-	cmpl	$-1,(%rdx)
-	je	global_invltlb
-
-	/*
-	 * PCID supported, but INVPCID is not.
-	 * Temporarily switch to the target address space and do INVLPG.
-	 */
-	pushq	%rcx
-	movq	%cr3,%rcx
-	movq	pcid_cr3,%rax
-	cmp	%rcx,%rax
-	je	1f
-	btsq	$63,%rax
-	movq	%rax,%cr3
-1:	movq	8(%rdx),%rax
-	invlpg	(%rax)
-	btsq	$63,%rcx
-	movq	%rcx,%cr3
-	popq	%rcx
-	jmp	invltlb_ret_rdx
-
-	/*
-	 * Invalidate the TLB entry using INVPCID_ADDR.
-	 */
-2:
-	xorl	%eax,%eax
-/*	invpcid	(%rdx),%rax */
-	.byte	0x66,0x0f,0x38,0x82,0x02
-	jmp	invltlb_ret_rdx
-
-	/*
-	 * PCID is not supported or kernel pmap.
-	 * Invalidate single page using INVLPG.
-	 */
-3:
-	movq	8(%rdx),%rax
-	invlpg	(%rax)
-	jmp	invltlb_ret_rdx
+	call	invlpg_pcid_handler
+	jmp	invltlb_ret
 
 	SUPERALIGN_TEXT
 IDTVEC(invlpg)
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
 	PUSH_FRAME
-	movl	PCPU(CPUID), %eax
-#ifdef COUNT_XINVLTLB_HITS
-	incl	xhits_pg(,%rax,4)
-#endif
-#ifdef COUNT_IPIS
-	movq	ipi_invlpg_counts(,%rax,8),%rax
-	incq	(%rax)
-#endif
-	POP_FRAME
-#endif
 
-	pushq	%rax
-	movq	smp_tlb_invpcid+8,%rax
-	invlpg	(%rax)			/* invalidate single page */
-	jmp	invltlb_ret_rax
+	call	invlpg_handler
+	jmp	invltlb_ret
 
 /*
  * Page range TLB shootdown.
@@ -348,81 +204,10 @@ IDTVEC(invlpg)
 	.text
 	SUPERALIGN_TEXT
 IDTVEC(invlrng)
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
 	PUSH_FRAME
-	movl	PCPU(CPUID), %eax
-#ifdef COUNT_XINVLTLB_HITS
-	incl	xhits_rng(,%rax,4)
-#endif
-#ifdef COUNT_IPIS
-	movq	ipi_invlrng_counts(,%rax,8),%rax
-	incq	(%rax)
-#endif
-	POP_FRAME
-#endif
 
-	pushq	%rax
-	pushq	%rdx
-	movq	$smp_tlb_invpcid,%rdx
-	cmpl	$0,pmap_pcid_enabled
-	je	invlrng_single_page
-
-	/* kernel pmap - use invlpg to invalidate global mapping */
-	cmpl	$0,(%rdx)
-	je	invlrng_single_page
-	cmpl	$-1,(%rdx)
-	je	global_invltlb
-	cmpl	$0,invpcid_works
-	jne	invlrng_invpcid
-
-	pushq	%rcx
-	movq	%cr3,%rcx
-	movq	pcid_cr3,%rax
-	cmpq	%rcx,%rax
-	je	1f
-	btsq	$63,%rax
-	movq	%rax,%cr3
-1:
-	movq	8(%rdx),%rdx
-	movq	smp_tlb_addr2,%rax
-2:
-	invlpg	(%rdx)
-	addq	$PAGE_SIZE,%rdx
-	cmpq	%rax,%rdx
-	jb	2b
-	btsq	$63,%rcx
-	movq	%rcx,%cr3
-	popq	%rcx
-	jmp	invltlb_ret_rdx
-
-invlrng_invpcid:
-	pushq	%rcx
-	subq	$16,%rsp
-	movq	(%rdx),%rcx
-	movq	%rcx,(%rsp)
-	movq	8(%rdx),%rax
-	movq	%rax,8(%rsp)
-	movq	smp_tlb_addr2,%rcx
-	subq	%rax,%rcx
-	shrq	$PAGE_SHIFT,%rcx
-1:
-//	invpcid	(%rdx),%rax
-	.byte	0x66,0x0f,0x38,0x82,0x02
-	addq	$PAGE_SIZE,8(%rsp)
-	dec	%rcx
-	jne	1b
-	addq	$16,%rsp
-	popq	%rcx
-	jmp	invltlb_ret_rdx
-
-invlrng_single_page:
-	movq	8(%rdx),%rdx
-	movq	smp_tlb_addr2,%rax
-1:	invlpg	(%rdx)			/* invalidate single page */
-	addq	$PAGE_SIZE,%rdx
-	cmpq	%rax,%rdx
-	jb	1b
-	jmp	invltlb_ret_rdx
+	call	invlrng_handler
+	jmp	invltlb_ret
 
 /*
  * Invalidate cache.
@@ -430,17 +215,10 @@ invlrng_single_page:
 	.text
 	SUPERALIGN_TEXT
 IDTVEC(invlcache)
-#ifdef COUNT_IPIS
 	PUSH_FRAME
-	movl	PCPU(CPUID), %eax
-	movq	ipi_invlcache_counts(,%rax,8),%rax
-	incq	(%rax)
-	POP_FRAME
-#endif
 
-	pushq	%rax
-	wbinvd
-	jmp	invltlb_ret_rax
+	call	invlcache_handler
+	jmp	invltlb_ret
 
 /*
  * Handler for IPIs sent via the per-cpu IPI bitmap.

Modified: head/sys/amd64/amd64/mp_machdep.c
==============================================================================
--- head/sys/amd64/amd64/mp_machdep.c	Tue Mar 11 07:33:33 2014	(r263000)
+++ head/sys/amd64/amd64/mp_machdep.c	Tue Mar 11 10:03:29 2014	(r263001)
@@ -109,6 +109,7 @@ struct invpcid_descr smp_tlb_invpcid;
 volatile int smp_tlb_wait;
 uint64_t pcid_cr3;
 pmap_t smp_tlb_pmap;
+extern int invpcid_works;
 
 #ifdef COUNT_IPIS
 /* Interrupt counts. */
@@ -1496,6 +1497,175 @@ cpususpend_handler(void)
 }
 
 /*
+ * Handlers for TLB related IPIs
+ */
+void
+invltlb_handler(void)
+{
+#ifdef COUNT_XINVLTLB_HITS
+	xhits_gbl[PCPU_GET(cpuid)]++;
+#endif /* COUNT_XINVLTLB_HITS */
+#ifdef COUNT_IPIS
+	(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+	invltlb();
+	atomic_add_int(&smp_tlb_wait, 1);
+}
+
+void
+invltlb_pcid_handler(void)
+{
+	uint64_t cr3;
+#ifdef COUNT_XINVLTLB_HITS
+	xhits_gbl[PCPU_GET(cpuid)]++;
+#endif /* COUNT_XINVLTLB_HITS */
+#ifdef COUNT_IPIS
+	(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+	cr3 = rcr3();
+	if (smp_tlb_invpcid.pcid != (uint64_t)-1 &&
+	    smp_tlb_invpcid.pcid != 0) {
+
+		if (invpcid_works) {
+			invpcid(&smp_tlb_invpcid, INVPCID_CTX);
+		} else {
+			/* Otherwise reload %cr3 twice. */
+			if (cr3 != pcid_cr3) {
+				load_cr3(pcid_cr3);
+				cr3 |= CR3_PCID_SAVE;
+			}
+			load_cr3(cr3);
+		}
+	} else {
+		invltlb_globpcid();
+	}
+	if (smp_tlb_pmap != NULL)
+		CPU_CLR_ATOMIC(PCPU_GET(cpuid), &smp_tlb_pmap->pm_save);
+
+	atomic_add_int(&smp_tlb_wait, 1);
+}
+
+void
+invlpg_handler(void)
+{
+#ifdef COUNT_XINVLTLB_HITS
+	xhits_pg[PCPU_GET(cpuid)]++;
+#endif /* COUNT_XINVLTLB_HITS */
+#ifdef COUNT_IPIS
+	(*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+	invlpg(smp_tlb_invpcid.addr);
+	atomic_add_int(&smp_tlb_wait, 1);
+}
+
+void
+invlpg_pcid_handler(void)
+{
+#ifdef COUNT_XINVLTLB_HITS
+	xhits_pg[PCPU_GET(cpuid)]++;
+#endif /* COUNT_XINVLTLB_HITS */
+#ifdef COUNT_IPIS
+	(*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+	if (invpcid_works) {
+		invpcid(&smp_tlb_invpcid, INVPCID_ADDR);
+	} else if (smp_tlb_invpcid.pcid == 0) {
+		invlpg(smp_tlb_invpcid.addr);
+	} else if (smp_tlb_invpcid.pcid == (uint64_t)-1) {
+		invltlb_globpcid();
+	} else {
+		uint64_t cr3;
+
+		/*
+		 * PCID supported, but INVPCID is not.
+		 * Temporarily switch to the target address
+		 * space and do INVLPG.
+		 */
+		cr3 = rcr3();
+		if (cr3 != pcid_cr3)
+			load_cr3(pcid_cr3 | CR3_PCID_SAVE);
+		invlpg(smp_tlb_invpcid.addr);
+		load_cr3(cr3 | CR3_PCID_SAVE);
+	}
+
+	atomic_add_int(&smp_tlb_wait, 1);
+}
+
+static inline void
+invlpg_range(vm_offset_t start, vm_offset_t end)
+{
+
+	do {
+		invlpg(start);
+		start += PAGE_SIZE;
+	} while (start < end);
+}
+
+void
+invlrng_handler(void)
+{
+	vm_offset_t addr;
+#ifdef COUNT_XINVLTLB_HITS
+	xhits_rng[PCPU_GET(cpuid)]++;
+#endif /* COUNT_XINVLTLB_HITS */
+#ifdef COUNT_IPIS
+	(*ipi_invlrng_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+	addr = smp_tlb_invpcid.addr;
+	if (pmap_pcid_enabled) {
+		if (invpcid_works) {
+			struct invpcid_descr d;
+
+			d = smp_tlb_invpcid;
+			do {
+				invpcid(&d, INVPCID_ADDR);
+				d.addr += PAGE_SIZE;
+			} while (d.addr < smp_tlb_addr2);
+		} else if (smp_tlb_invpcid.pcid == 0) {
+			/*
+			 * kernel pmap - use invlpg to invalidate
+			 * global mapping.
+			 */
+			invlpg_range(addr, smp_tlb_addr2);
+		} else if (smp_tlb_invpcid.pcid == (uint64_t)-1) {
+			invltlb_globpcid();
+			if (smp_tlb_pmap != NULL) {
+				CPU_CLR_ATOMIC(PCPU_GET(cpuid),
+				    &smp_tlb_pmap->pm_save);
+			}
+		} else {
+			uint64_t cr3;
+
+			cr3 = rcr3();
+			if (cr3 != pcid_cr3)
+				load_cr3(pcid_cr3 | CR3_PCID_SAVE);
+			invlpg_range(addr, smp_tlb_addr2);
+			load_cr3(cr3 | CR3_PCID_SAVE);
+		}
+	} else {
+		invlpg_range(addr, smp_tlb_addr2);
+	}
+
+	atomic_add_int(&smp_tlb_wait, 1);
+}
+
+void
+invlcache_handler(void)
+{
+#ifdef COUNT_IPIS
+	(*ipi_invlcache_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+	wbinvd();
+	atomic_add_int(&smp_tlb_wait, 1);
+}
+
+/*
  * This is called once the rest of the system is up and running and we're
  * ready to let the AP's out of the pen.
  */

Modified: head/sys/amd64/include/smp.h
==============================================================================
--- head/sys/amd64/include/smp.h	Tue Mar 11 07:33:33 2014	(r263000)
+++ head/sys/amd64/include/smp.h	Tue Mar 11 10:03:29 2014	(r263001)
@@ -62,6 +62,12 @@ struct pmap;
 void	cpu_add(u_int apic_id, char boot_cpu);
 void	cpustop_handler(void);
 void	cpususpend_handler(void);
+void	invltlb_handler(void);
+void	invltlb_pcid_handler(void);
+void	invlpg_handler(void);
+void	invlpg_pcid_handler(void);
+void	invlrng_handler(void);
+void	invlcache_handler(void);
 void	init_secondary(void);
 void	ipi_startup(int apic_id, int vector);
 void	ipi_all_but_self(u_int ipi);

Modified: head/sys/i386/i386/apic_vector.s
==============================================================================
--- head/sys/i386/i386/apic_vector.s	Tue Mar 11 07:33:33 2014	(r263000)
+++ head/sys/i386/i386/apic_vector.s	Tue Mar 11 10:03:29 2014	(r263001)
@@ -163,39 +163,21 @@ IDTVEC(xen_intr_upcall)
  */
 	.text
 	SUPERALIGN_TEXT
-IDTVEC(invltlb)
-	pushl	%eax
-	pushl	%ds
-	movl	$KDSEL, %eax		/* Kernel data selector */
-	movl	%eax, %ds
-
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
-	pushl	%fs
-	movl	$KPSEL, %eax		/* Private space selector */
-	movl	%eax, %fs
-	movl	PCPU(CPUID), %eax
-	popl	%fs
-#ifdef COUNT_XINVLTLB_HITS
-	incl	xhits_gbl(,%eax,4)
-#endif
-#ifdef COUNT_IPIS
-	movl	ipi_invltlb_counts(,%eax,4),%eax
-	incl	(%eax)
-#endif
-#endif
-
-	movl	%cr3, %eax		/* invalidate the TLB */
-	movl	%eax, %cr3
-
+invltlb_ret:
 	movl	lapic, %eax
 	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
+	POP_FRAME
+	iret
+
+	SUPERALIGN_TEXT
+IDTVEC(invltlb)
+	PUSH_FRAME
+	SET_KERNEL_SREGS
+	cld
 
-	lock
-	incl	smp_tlb_wait
+	call	invltlb_handler
 
-	popl	%ds
-	popl	%eax
-	iret
+	jmp	invltlb_ret
 
 /*
  * Single page TLB shootdown
@@ -203,38 +185,13 @@ IDTVEC(invltlb)
 	.text
 	SUPERALIGN_TEXT
 IDTVEC(invlpg)
-	pushl	%eax
-	pushl	%ds
-	movl	$KDSEL, %eax		/* Kernel data selector */
-	movl	%eax, %ds
-
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
-	pushl	%fs
-	movl	$KPSEL, %eax		/* Private space selector */
-	movl	%eax, %fs
-	movl	PCPU(CPUID), %eax
-	popl	%fs
-#ifdef COUNT_XINVLTLB_HITS
-	incl	xhits_pg(,%eax,4)
-#endif
-#ifdef COUNT_IPIS
-	movl	ipi_invlpg_counts(,%eax,4),%eax
-	incl	(%eax)
-#endif
-#endif
-
-	movl	smp_tlb_addr1, %eax
-	invlpg	(%eax)			/* invalidate single page */
-
-	movl	lapic, %eax
-	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
+	PUSH_FRAME
+	SET_KERNEL_SREGS
+	cld
 
-	lock
-	incl	smp_tlb_wait
+	call	invlpg_handler
 
-	popl	%ds
-	popl	%eax
-	iret
+	jmp	invltlb_ret
 
 /*
  * Page range TLB shootdown.
@@ -242,44 +199,13 @@ IDTVEC(invlpg)
 	.text
 	SUPERALIGN_TEXT
 IDTVEC(invlrng)
-	pushl	%eax
-	pushl	%edx
-	pushl	%ds
-	movl	$KDSEL, %eax		/* Kernel data selector */
-	movl	%eax, %ds
-
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
-	pushl	%fs
-	movl	$KPSEL, %eax		/* Private space selector */
-	movl	%eax, %fs
-	movl	PCPU(CPUID), %eax
-	popl	%fs
-#ifdef COUNT_XINVLTLB_HITS
-	incl	xhits_rng(,%eax,4)
-#endif
-#ifdef COUNT_IPIS
-	movl	ipi_invlrng_counts(,%eax,4),%eax
-	incl	(%eax)
-#endif
-#endif
+	PUSH_FRAME
+	SET_KERNEL_SREGS
+	cld
 
-	movl	smp_tlb_addr1, %edx
-	movl	smp_tlb_addr2, %eax
-1:	invlpg	(%edx)			/* invalidate single page */
-	addl	$PAGE_SIZE, %edx
-	cmpl	%eax, %edx
-	jb	1b
+	call	invlrng_handler
 
-	movl	lapic, %eax
-	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
-
-	lock
-	incl	smp_tlb_wait
-
-	popl	%ds
-	popl	%edx
-	popl	%eax
-	iret
+	jmp	invltlb_ret
 
 /*
  * Invalidate cache.
@@ -287,32 +213,13 @@ IDTVEC(invlrng)
 	.text
 	SUPERALIGN_TEXT
 IDTVEC(invlcache)
-	pushl	%eax
-	pushl	%ds
-	movl	$KDSEL, %eax		/* Kernel data selector */
-	movl	%eax, %ds
-
-#ifdef COUNT_IPIS
-	pushl	%fs
-	movl	$KPSEL, %eax		/* Private space selector */
-	movl	%eax, %fs
-	movl	PCPU(CPUID), %eax
-	popl	%fs
-	movl	ipi_invlcache_counts(,%eax,4),%eax
-	incl	(%eax)
-#endif
-
-	wbinvd
-
-	movl	lapic, %eax
-	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
+	PUSH_FRAME
+	SET_KERNEL_SREGS
+	cld
 
-	lock
-	incl	smp_tlb_wait
+	call	invlcache_handler
 
-	popl	%ds
-	popl	%eax
-	iret
+	jmp	invltlb_ret
 
 /*
  * Handler for IPIs sent via the per-cpu IPI bitmap.

Modified: head/sys/i386/i386/mp_machdep.c
==============================================================================
--- head/sys/i386/i386/mp_machdep.c	Tue Mar 11 07:33:33 2014	(r263000)
+++ head/sys/i386/i386/mp_machdep.c	Tue Mar 11 10:03:29 2014	(r263001)
@@ -1551,6 +1551,72 @@ cpususpend_handler(void)
 	CPU_CLR_ATOMIC(cpu, &suspended_cpus);
 	CPU_CLR_ATOMIC(cpu, &started_cpus);
 }
+
+/*
+ * Handlers for TLB related IPIs
+ */
+void
+invltlb_handler(void)
+{
+	uint64_t cr3;
+#ifdef COUNT_XINVLTLB_HITS
+	xhits_gbl[PCPU_GET(cpuid)]++;
+#endif /* COUNT_XINVLTLB_HITS */
+#ifdef COUNT_IPIS
+	(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+	cr3 = rcr3();
+	load_cr3(cr3);
+	atomic_add_int(&smp_tlb_wait, 1);
+}
+
+void
+invlpg_handler(void)
+{
+#ifdef COUNT_XINVLTLB_HITS
+	xhits_pg[PCPU_GET(cpuid)]++;
+#endif /* COUNT_XINVLTLB_HITS */
+#ifdef COUNT_IPIS
+	(*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+	invlpg(smp_tlb_addr1);
+
+	atomic_add_int(&smp_tlb_wait, 1);
+}
+
+void
+invlrng_handler(void)
+{
+	vm_offset_t addr;
+#ifdef COUNT_XINVLTLB_HITS
+	xhits_rng[PCPU_GET(cpuid)]++;
+#endif /* COUNT_XINVLTLB_HITS */
+#ifdef COUNT_IPIS
+	(*ipi_invlrng_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+	addr = smp_tlb_addr1;
+	do {
+		invlpg(addr);
+		addr += PAGE_SIZE;
+	} while (addr < smp_tlb_addr2);
+
+	atomic_add_int(&smp_tlb_wait, 1);
+}
+
+void
+invlcache_handler(void)
+{
+#ifdef COUNT_IPIS
+	(*ipi_invlcache_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+	wbinvd();
+	atomic_add_int(&smp_tlb_wait, 1);
+}
+
 /*
  * This is called once the rest of the system is up and running and we're
  * ready to let the AP's out of the pen.

Modified: head/sys/i386/include/smp.h
==============================================================================
--- head/sys/i386/include/smp.h	Tue Mar 11 07:33:33 2014	(r263000)
+++ head/sys/i386/include/smp.h	Tue Mar 11 10:03:29 2014	(r263001)
@@ -63,6 +63,10 @@ void	cpustop_handler(void);
 #ifndef XEN
 void	cpususpend_handler(void);
 #endif
+void	invltlb_handler(void);
+void	invlpg_handler(void);
+void	invlrng_handler(void);
+void	invlcache_handler(void);
 void	init_secondary(void);
 void	ipi_startup(int apic_id, int vector);
 void	ipi_all_but_self(u_int ipi);

Modified: head/sys/i386/xen/mp_machdep.c
==============================================================================
--- head/sys/i386/xen/mp_machdep.c	Tue Mar 11 07:33:33 2014	(r263000)
+++ head/sys/i386/xen/mp_machdep.c	Tue Mar 11 10:03:29 2014	(r263001)
@@ -1276,6 +1276,31 @@ cpustop_handler(void)
 }
 
 /*
+ * Handlers for TLB related IPIs
+ *
+ * On i386 Xen PV this are no-ops since this port doesn't support SMP.
+ */
+void
+invltlb_handler(void)
+{
+}
+
+void
+invlpg_handler(void)
+{
+}
+
+void
+invlrng_handler(void)
+{
+}
+
+void
+invlcache_handler(void)
+{
+}
+
+/*
  * This is called once the rest of the system is up and running and we're
  * ready to let the AP's out of the pen.
  */

Modified: head/sys/x86/xen/hvm.c
==============================================================================
--- head/sys/x86/xen/hvm.c	Tue Mar 11 07:33:33 2014	(r263000)
+++ head/sys/x86/xen/hvm.c	Tue Mar 11 10:03:29 2014	(r263001)
@@ -77,22 +77,12 @@ static void xen_hvm_cpu_resume(void);
 static void xen_hvm_cpu_init(void);
 
 /*---------------------------- Extern Declarations ---------------------------*/
-/* Variables used by mp_machdep to perform the MMU related IPIs */
-extern volatile int smp_tlb_wait;
-extern vm_offset_t smp_tlb_addr2;
-#ifdef __i386__
-extern vm_offset_t smp_tlb_addr1;
-#else
-extern struct invpcid_descr smp_tlb_invpcid;
-extern uint64_t pcid_cr3;
-extern int invpcid_works;
-extern int pmap_pcid_enabled;
-extern pmap_t smp_tlb_pmap;
-#endif
-
 #ifdef __i386__
 extern void pmap_lazyfix_action(void);
 #endif
+#ifdef __amd64__
+extern int pmap_pcid_enabled;
+#endif
 
 /* Variables used by mp_machdep to perform the bitmap IPI */
 extern volatile u_int cpu_ipi_pending[MAXCPU];
@@ -179,10 +169,7 @@ static int
 xen_smp_rendezvous_action(void *arg)
 {
 #ifdef COUNT_IPIS
-	int cpu;
-
-	cpu = PCPU_GET(cpuid);
-	(*ipi_rendezvous_counts[cpu])++;
+	(*ipi_rendezvous_counts[PCPU_GET(cpuid)])++;
 #endif /* COUNT_IPIS */
 
 	smp_rendezvous_action();
@@ -192,20 +179,8 @@ xen_smp_rendezvous_action(void *arg)
 static int
 xen_invltlb(void *arg)
 {
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
-	int cpu;
-
-	cpu = PCPU_GET(cpuid);
-#ifdef COUNT_XINVLTLB_HITS
-	xhits_gbl[cpu]++;
-#endif /* COUNT_XINVLTLB_HITS */
-#ifdef COUNT_IPIS
-	(*ipi_invltlb_counts[cpu])++;
-#endif /* COUNT_IPIS */
-#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */
 
-	invltlb();
-	atomic_add_int(&smp_tlb_wait, 1);
+	invltlb_handler();
 	return (FILTER_HANDLED);
 }
 
@@ -213,40 +188,8 @@ xen_invltlb(void *arg)
 static int
 xen_invltlb_pcid(void *arg)
 {
-	uint64_t cr3;
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
-	int cpu;
-
-	cpu = PCPU_GET(cpuid);
-#ifdef COUNT_XINVLTLB_HITS
-	xhits_gbl[cpu]++;
-#endif /* COUNT_XINVLTLB_HITS */
-#ifdef COUNT_IPIS
-	(*ipi_invltlb_counts[cpu])++;
-#endif /* COUNT_IPIS */
-#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */
 
-	cr3 = rcr3();
-	if (smp_tlb_invpcid.pcid != (uint64_t)-1 &&
-	    smp_tlb_invpcid.pcid != 0) {
-
-		if (invpcid_works) {
-			invpcid(&smp_tlb_invpcid, INVPCID_CTX);
-		} else {
-			/* Otherwise reload %cr3 twice. */
-			if (cr3 != pcid_cr3) {
-				load_cr3(pcid_cr3);
-				cr3 |= CR3_PCID_SAVE;
-			}
-			load_cr3(cr3);
-		}
-	} else {
-		invltlb_globpcid();
-	}
-	if (smp_tlb_pmap != NULL)
-		CPU_CLR_ATOMIC(PCPU_GET(cpuid), &smp_tlb_pmap->pm_save);
-
-	atomic_add_int(&smp_tlb_wait, 1);
+	invltlb_pcid_handler();
 	return (FILTER_HANDLED);
 }
 #endif
@@ -254,24 +197,8 @@ xen_invltlb_pcid(void *arg)
 static int
 xen_invlpg(void *arg)
 {
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
-	int cpu;
-
-	cpu = PCPU_GET(cpuid);
-#ifdef COUNT_XINVLTLB_HITS
-	xhits_pg[cpu]++;
-#endif /* COUNT_XINVLTLB_HITS */
-#ifdef COUNT_IPIS
-	(*ipi_invlpg_counts[cpu])++;
-#endif /* COUNT_IPIS */
-#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */
 
-#ifdef __i386__
-	invlpg(smp_tlb_addr1);
-#else
-	invlpg(smp_tlb_invpcid.addr);
-#endif
-	atomic_add_int(&smp_tlb_wait, 1);
+	invlpg_handler();
 	return (FILTER_HANDLED);
 }
 
@@ -279,125 +206,25 @@ xen_invlpg(void *arg)
 static int
 xen_invlpg_pcid(void *arg)
 {
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
-	int cpu;
-
-	cpu = PCPU_GET(cpuid);
-#ifdef COUNT_XINVLTLB_HITS
-	xhits_pg[cpu]++;
-#endif /* COUNT_XINVLTLB_HITS */
-#ifdef COUNT_IPIS
-	(*ipi_invlpg_counts[cpu])++;
-#endif /* COUNT_IPIS */
-#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */
-
-	if (invpcid_works) {
-		invpcid(&smp_tlb_invpcid, INVPCID_ADDR);
-	} else if (smp_tlb_invpcid.pcid == 0) {
-		invlpg(smp_tlb_invpcid.addr);
-	} else if (smp_tlb_invpcid.pcid == (uint64_t)-1) {
-		invltlb_globpcid();
-	} else {
-		uint64_t cr3;
-
-		/*
-		 * PCID supported, but INVPCID is not.
-		 * Temporarily switch to the target address
-		 * space and do INVLPG.
-		 */
-		cr3 = rcr3();
-		if (cr3 != pcid_cr3)
-			load_cr3(pcid_cr3 | CR3_PCID_SAVE);
-		invlpg(smp_tlb_invpcid.addr);
-		load_cr3(cr3 | CR3_PCID_SAVE);
-	}
 
-	atomic_add_int(&smp_tlb_wait, 1);
+	invlpg_pcid_handler();
 	return (FILTER_HANDLED);
 }
 #endif
 
-static inline void
-invlpg_range(vm_offset_t start, vm_offset_t end)
-{
-	do {
-		invlpg(start);
-		start += PAGE_SIZE;
-	} while (start < end);
-}
-
 static int
 xen_invlrng(void *arg)
 {
-	vm_offset_t addr;
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***


More information about the svn-src-head mailing list