svn commit: r313930 - in head/sys: amd64/include kern mips/include mips/mips net powerpc/include sparc64/include
Jason A. Harmening
jah at FreeBSD.org
Sun Feb 19 02:03:12 UTC 2017
Author: jah
Date: Sun Feb 19 02:03:09 2017
New Revision: 313930
URL: https://svnweb.freebsd.org/changeset/base/313930
Log:
Bring back r313037, with fixes for mips:
Implement get_pcpu() for amd64/sparc64/mips/powerpc, and use it to
replace pcpu_find(curcpu) in MI code.
Reviewed by: andreast, kan, lidl
Tested by: lidl(mips, sparc64), andreast(powerpc)
Differential Revision: https://reviews.freebsd.org/D9587
Modified:
head/sys/amd64/include/pcpu.h
head/sys/kern/kern_rmlock.c
head/sys/mips/include/pcpu.h
head/sys/mips/mips/machdep.c
head/sys/net/netisr.c
head/sys/powerpc/include/cpufunc.h
head/sys/powerpc/include/pcpu.h
head/sys/sparc64/include/pcpu.h
Modified: head/sys/amd64/include/pcpu.h
==============================================================================
--- head/sys/amd64/include/pcpu.h Sat Feb 18 22:13:28 2017 (r313929)
+++ head/sys/amd64/include/pcpu.h Sun Feb 19 02:03:09 2017 (r313930)
@@ -78,6 +78,7 @@
extern struct pcpu *pcpup;
+#define get_pcpu() (pcpup)
#define PCPU_GET(member) (pcpup->pc_ ## member)
#define PCPU_ADD(member, val) (pcpup->pc_ ## member += (val))
#define PCPU_INC(member) PCPU_ADD(member, 1)
@@ -203,6 +204,15 @@ extern struct pcpu *pcpup;
} \
}
+#define get_pcpu() __extension__ ({ \
+ struct pcpu *__pc; \
+ \
+ __asm __volatile("movq %%gs:%1,%0" \
+ : "=r" (__pc) \
+ : "m" (*(struct pcpu *)(__pcpu_offset(pc_prvspace)))); \
+ __pc; \
+})
+
#define PCPU_GET(member) __PCPU_GET(pc_ ## member)
#define PCPU_ADD(member, val) __PCPU_ADD(pc_ ## member, val)
#define PCPU_INC(member) __PCPU_INC(pc_ ## member)
Modified: head/sys/kern/kern_rmlock.c
==============================================================================
--- head/sys/kern/kern_rmlock.c Sat Feb 18 22:13:28 2017 (r313929)
+++ head/sys/kern/kern_rmlock.c Sun Feb 19 02:03:09 2017 (r313930)
@@ -156,7 +156,7 @@ unlock_rm(struct lock_object *lock)
*/
critical_enter();
td = curthread;
- pc = pcpu_find(curcpu);
+ pc = get_pcpu();
for (queue = pc->pc_rm_queue.rmq_next;
queue != &pc->pc_rm_queue; queue = queue->rmq_next) {
tracker = (struct rm_priotracker *)queue;
@@ -258,7 +258,7 @@ rm_cleanIPI(void *arg)
struct rmlock *rm = arg;
struct rm_priotracker *tracker;
struct rm_queue *queue;
- pc = pcpu_find(curcpu);
+ pc = get_pcpu();
for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue;
queue = queue->rmq_next) {
@@ -355,7 +355,7 @@ _rm_rlock_hard(struct rmlock *rm, struct
struct pcpu *pc;
critical_enter();
- pc = pcpu_find(curcpu);
+ pc = get_pcpu();
/* Check if we just need to do a proper critical_exit. */
if (!CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)) {
@@ -416,7 +416,7 @@ _rm_rlock_hard(struct rmlock *rm, struct
}
critical_enter();
- pc = pcpu_find(curcpu);
+ pc = get_pcpu();
CPU_CLR(pc->pc_cpuid, &rm->rm_writecpus);
rm_tracker_add(pc, tracker);
sched_pin();
@@ -641,7 +641,7 @@ _rm_rlock_debug(struct rmlock *rm, struc
#ifdef INVARIANTS
if (!(rm->lock_object.lo_flags & LO_RECURSABLE) && !trylock) {
critical_enter();
- KASSERT(rm_trackers_present(pcpu_find(curcpu), rm,
+ KASSERT(rm_trackers_present(get_pcpu(), rm,
curthread) == 0,
("rm_rlock: recursed on non-recursive rmlock %s @ %s:%d\n",
rm->lock_object.lo_name, file, line));
@@ -771,7 +771,7 @@ _rm_assert(const struct rmlock *rm, int
}
critical_enter();
- count = rm_trackers_present(pcpu_find(curcpu), rm, curthread);
+ count = rm_trackers_present(get_pcpu(), rm, curthread);
critical_exit();
if (count == 0)
@@ -797,7 +797,7 @@ _rm_assert(const struct rmlock *rm, int
rm->lock_object.lo_name, file, line);
critical_enter();
- count = rm_trackers_present(pcpu_find(curcpu), rm, curthread);
+ count = rm_trackers_present(get_pcpu(), rm, curthread);
critical_exit();
if (count != 0)
Modified: head/sys/mips/include/pcpu.h
==============================================================================
--- head/sys/mips/include/pcpu.h Sat Feb 18 22:13:28 2017 (r313929)
+++ head/sys/mips/include/pcpu.h Sun Feb 19 02:03:09 2017 (r313930)
@@ -39,16 +39,17 @@
struct pmap *pc_curpmap; /* pmap of curthread */ \
u_int32_t pc_next_asid; /* next ASID to alloc */ \
u_int32_t pc_asid_generation; /* current ASID generation */ \
- u_int pc_pending_ipis; /* IPIs pending to this CPU */
+ u_int pc_pending_ipis; /* IPIs pending to this CPU */ \
+ struct pcpu *pc_self; /* globally-uniqe self pointer */
#ifdef __mips_n64
#define PCPU_MD_MIPS64_FIELDS \
PCPU_MD_COMMON_FIELDS \
- char __pad[61]
+ char __pad[53]
#else
#define PCPU_MD_MIPS32_FIELDS \
PCPU_MD_COMMON_FIELDS \
- char __pad[193]
+ char __pad[189]
#endif
#ifdef __mips_n64
@@ -65,6 +66,13 @@ extern char pcpu_space[MAXCPU][PAGE_SIZE
extern struct pcpu *pcpup;
#define PCPUP pcpup
+/*
+ * Since we use a wired TLB entry to map the same VA to a different
+ * physical page for each CPU, get_pcpu() must use the pc_self
+ * field to obtain a globally-unique pointer.
+ */
+#define get_pcpu() (PCPUP->pc_self)
+
#define PCPU_ADD(member, value) (PCPUP->pc_ ## member += (value))
#define PCPU_GET(member) (PCPUP->pc_ ## member)
#define PCPU_INC(member) PCPU_ADD(member, 1)
Modified: head/sys/mips/mips/machdep.c
==============================================================================
--- head/sys/mips/mips/machdep.c Sat Feb 18 22:13:28 2017 (r313929)
+++ head/sys/mips/mips/machdep.c Sun Feb 19 02:03:09 2017 (r313930)
@@ -475,6 +475,7 @@ cpu_pcpu_init(struct pcpu *pcpu, int cpu
pcpu->pc_next_asid = 1;
pcpu->pc_asid_generation = 1;
+ pcpu->pc_self = pcpu;
#ifdef SMP
if ((vm_offset_t)pcpup >= VM_MIN_KERNEL_ADDRESS &&
(vm_offset_t)pcpup <= VM_MAX_KERNEL_ADDRESS) {
Modified: head/sys/net/netisr.c
==============================================================================
--- head/sys/net/netisr.c Sat Feb 18 22:13:28 2017 (r313929)
+++ head/sys/net/netisr.c Sun Feb 19 02:03:09 2017 (r313930)
@@ -1268,9 +1268,7 @@ netisr_start_swi(u_int cpuid, struct pcp
static void
netisr_init(void *arg)
{
-#ifdef EARLY_AP_STARTUP
struct pcpu *pc;
-#endif
NETISR_LOCK_INIT();
if (netisr_maxthreads == 0 || netisr_maxthreads < -1 )
@@ -1308,7 +1306,8 @@ netisr_init(void *arg)
netisr_start_swi(pc->pc_cpuid, pc);
}
#else
- netisr_start_swi(curcpu, pcpu_find(curcpu));
+ pc = get_pcpu();
+ netisr_start_swi(pc->pc_cpuid, pc);
#endif
}
SYSINIT(netisr_init, SI_SUB_SOFTINTR, SI_ORDER_FIRST, netisr_init, NULL);
Modified: head/sys/powerpc/include/cpufunc.h
==============================================================================
--- head/sys/powerpc/include/cpufunc.h Sat Feb 18 22:13:28 2017 (r313929)
+++ head/sys/powerpc/include/cpufunc.h Sun Feb 19 02:03:09 2017 (r313930)
@@ -201,7 +201,7 @@ intr_restore(register_t msr)
}
static __inline struct pcpu *
-powerpc_get_pcpup(void)
+get_pcpu(void)
{
struct pcpu *ret;
Modified: head/sys/powerpc/include/pcpu.h
==============================================================================
--- head/sys/powerpc/include/pcpu.h Sat Feb 18 22:13:28 2017 (r313929)
+++ head/sys/powerpc/include/pcpu.h Sun Feb 19 02:03:09 2017 (r313930)
@@ -142,7 +142,7 @@ struct pvo_entry;
#ifdef _KERNEL
-#define pcpup ((struct pcpu *) powerpc_get_pcpup())
+#define pcpup (get_pcpu())
static __inline __pure2 struct thread *
__curthread(void)
Modified: head/sys/sparc64/include/pcpu.h
==============================================================================
--- head/sys/sparc64/include/pcpu.h Sat Feb 18 22:13:28 2017 (r313929)
+++ head/sys/sparc64/include/pcpu.h Sun Feb 19 02:03:09 2017 (r313930)
@@ -74,6 +74,7 @@ struct pcpu;
register struct pcb *curpcb __asm__(__XSTRING(PCB_REG));
register struct pcpu *pcpup __asm__(__XSTRING(PCPU_REG));
+#define get_pcpu() (pcpup)
#define PCPU_GET(member) (pcpup->pc_ ## member)
static __inline __pure2 struct thread *
More information about the svn-src-all
mailing list