svn commit: r357805 - head/sys/amd64/include

[Mateusz Guzik]

Author: mjg
Date: Wed Feb 12 11:12:13 2020
New Revision: 357805
URL: https://svnweb.freebsd.org/changeset/base/357805

Log:
  amd64: store per-cpu allocations subtracted by __pcpu
  
  This eliminates a runtime subtraction from counter_u64_add.
  
  before:
  mov    0x4f00ed(%rip),%rax        # 0xffffffff80c01788 <numfullpathfail4>
  sub    0x808ff6(%rip),%rax        # 0xffffffff80f1a698 <__pcpu>
  addq   $0x1,%gs:(%rax)
  
  after:
  mov    0x4f02fd(%rip),%rax        # 0xffffffff80c01788 <numfullpathfail4>
  addq   $0x1,%gs:(%rax)
  
  Reviewed by:	jeff
  Differential Revision:	https://reviews.freebsd.org/D23570

Modified:
  head/sys/amd64/include/counter.h
  head/sys/amd64/include/pcpu.h

Modified: head/sys/amd64/include/counter.h
==============================================================================
--- head/sys/amd64/include/counter.h	Wed Feb 12 11:11:22 2020	(r357804)
+++ head/sys/amd64/include/counter.h	Wed Feb 12 11:12:13 2020	(r357805)
@@ -33,7 +33,7 @@
 
 #include <sys/pcpu.h>
 
-#define	EARLY_COUNTER	&temp_bsp_pcpu.pc_early_dummy_counter
+#define	EARLY_COUNTER	(void *)__offsetof(struct pcpu, pc_early_dummy_counter)
 
 #define	counter_enter()	do {} while (0)
 #define	counter_exit()	do {} while (0)
@@ -43,6 +43,7 @@ static inline uint64_t
 counter_u64_read_one(counter_u64_t c, int cpu)
 {
 
+	MPASS(c != EARLY_COUNTER);
 	return (*zpcpu_get_cpu(c, cpu));
 }
 
@@ -65,6 +66,7 @@ counter_u64_zero_one_cpu(void *arg)
 	counter_u64_t c;
 
 	c = arg;
+	MPASS(c != EARLY_COUNTER);
 	*(zpcpu_get(c)) = 0;
 }
 
@@ -86,7 +88,7 @@ counter_u64_add(counter_u64_t c, int64_t inc)
 	KASSERT(IS_BSP() || c != EARLY_COUNTER, ("EARLY_COUNTER used on AP"));
 	__asm __volatile("addq\t%1,%%gs:(%0)"
 	    :
-	    : "r" ((char *)c - (char *)&__pcpu[0]), "ri" (inc)
+	    : "r" (c), "ri" (inc)
 	    : "memory", "cc");
 }
 

Modified: head/sys/amd64/include/pcpu.h
==============================================================================
--- head/sys/amd64/include/pcpu.h	Wed Feb 12 11:11:22 2020	(r357804)
+++ head/sys/amd64/include/pcpu.h	Wed Feb 12 11:12:13 2020	(r357805)
@@ -240,6 +240,10 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x c
 
 #define	IS_BSP()	(PCPU_GET(cpuid) == 0)
 
+#define zpcpu_offset_cpu(cpu)	((uintptr_t)&__pcpu[0] + UMA_PCPU_ALLOC_SIZE * cpu)
+#define zpcpu_base_to_offset(base) (void *)((uintptr_t)(base) - (uintptr_t)&__pcpu[0])
+#define zpcpu_offset_to_base(base) (void *)((uintptr_t)(base) + (uintptr_t)&__pcpu[0])
+
 #else /* !__GNUCLIKE_ASM || !__GNUCLIKE___TYPEOF */
 
 #error "this file needs to be ported to your compiler"