svn commit: r220404 - in head/sys: i386/i386 i386/include pc98/pc98
Jung-uk Kim
jkim at FreeBSD.org
Wed Apr 6 23:59:59 UTC 2011
Author: jkim
Date: Wed Apr 6 23:59:59 2011
New Revision: 220404
URL: http://svn.freebsd.org/changeset/base/220404
Log:
Implement atomic_load_acq_64(9) and atomic_store_rel_64(9) for i386. These
functions are implemented with CMPXCHG8B instruction where it is available,
i. e., all Pentium-class and later processors. Note this instruction is
also used for atomic_store_rel_64() because a simple XCHG-like instruction
for 64-bit memory access does not exist, unfortunately. If the processor
lacks the instruction, i. e., 80486-class CPUs, two 32-bit load/store are
performed with interrupt temporarily disabled, assuming it does not support
SMP. Although this assumption may be little naive, it is true in reality.
This implementation is inspired by Linux.
Modified:
head/sys/i386/i386/machdep.c
head/sys/i386/include/atomic.h
head/sys/pc98/pc98/machdep.c
Modified: head/sys/i386/i386/machdep.c
==============================================================================
--- head/sys/i386/i386/machdep.c Wed Apr 6 20:54:26 2011 (r220403)
+++ head/sys/i386/i386/machdep.c Wed Apr 6 23:59:59 2011 (r220404)
@@ -1497,6 +1497,22 @@ idle_sysctl(SYSCTL_HANDLER_ARGS)
SYSCTL_PROC(_machdep, OID_AUTO, idle, CTLTYPE_STRING | CTLFLAG_RW, 0, 0,
idle_sysctl, "A", "currently selected idle function");
+uint64_t (*atomic_load_acq_64)(volatile uint64_t *) =
+ atomic_load_acq_64_i386;
+void (*atomic_store_rel_64)(volatile uint64_t *, uint64_t) =
+ atomic_store_rel_64_i386;
+
+static void
+cpu_probe_cmpxchg8b(void)
+{
+
+ if ((cpu_feature & CPUID_CX8) != 0 ||
+ cpu_vendor_id == CPU_VENDOR_RISE) {
+ atomic_load_acq_64 = atomic_load_acq_64_i586;
+ atomic_store_rel_64 = atomic_store_rel_64_i586;
+ }
+}
+
/*
* Reset registers to default values on exec.
*/
@@ -2730,6 +2746,7 @@ init386(first)
thread0.td_pcb->pcb_gsd = PCPU_GET(fsgs_gdt)[1];
cpu_probe_amdc1e();
+ cpu_probe_cmpxchg8b();
}
#else
@@ -3006,6 +3023,7 @@ init386(first)
thread0.td_frame = &proc0_tf;
cpu_probe_amdc1e();
+ cpu_probe_cmpxchg8b();
}
#endif
Modified: head/sys/i386/include/atomic.h
==============================================================================
--- head/sys/i386/include/atomic.h Wed Apr 6 20:54:26 2011 (r220403)
+++ head/sys/i386/include/atomic.h Wed Apr 6 23:59:59 2011 (r220404)
@@ -120,6 +120,87 @@ atomic_##NAME##_barr_##TYPE(volatile u_#
} \
struct __hack
+#if defined(_KERNEL) && !defined(WANT_FUNCTIONS)
+
+/* I486 does not support SMP or CMPXCHG8B. */
+static __inline uint64_t
+atomic_load_acq_64_i386(volatile uint64_t *p)
+{
+ volatile uint32_t *high, *low;
+ uint64_t res;
+
+ low = (volatile uint32_t *)p;
+ high = (volatile uint32_t *)p + 1;
+ __asm __volatile(
+ " pushfl ; "
+ " cli ; "
+ " movl %1,%%eax ; "
+ " movl %2,%%edx ; "
+ " popfl"
+ : "=&A" (res) /* 0 */
+ : "m" (*low), /* 1 */
+ "m" (*high) /* 2 */
+ : "memory");
+
+ return (res);
+}
+
+static __inline void
+atomic_store_rel_64_i386(volatile uint64_t *p, uint64_t v)
+{
+ volatile uint32_t *high, *low;
+
+ low = (volatile uint32_t *)p;
+ high = (volatile uint32_t *)p + 1;
+ __asm __volatile(
+ " pushfl ; "
+ " cli ; "
+ " movl %%eax,%0 ; "
+ " movl %%edx,%1 ; "
+ " popfl"
+ : "=m" (*low), /* 0 */
+ "=m" (*high) /* 1 */
+ : "A" (v) /* 2 */
+ : "memory");
+}
+
+static __inline uint64_t
+atomic_load_acq_64_i586(volatile uint64_t *p)
+{
+ uint64_t res;
+
+ __asm __volatile(
+ " movl %%ebx,%%eax ; "
+ " movl %%ecx,%%edx ; "
+ " " MPLOCKED " "
+ " cmpxchg8b %2"
+ : "=&A" (res), /* 0 */
+ "=m" (*p) /* 1 */
+ : "m" (*p) /* 2 */
+ : "memory", "cc");
+
+ return (res);
+}
+
+static __inline void
+atomic_store_rel_64_i586(volatile uint64_t *p, uint64_t v)
+{
+
+ __asm __volatile(
+ " movl %%eax,%%ebx ; "
+ " movl %%edx,%%ecx ; "
+ "1: "
+ " " MPLOCKED " "
+ " cmpxchg8b %2 ; "
+ " jne 1b"
+ : "=m" (*p), /* 0 */
+ "+A" (v) /* 1 */
+ : "m" (*p) /* 2 */
+ : "ebx", "ecx", "memory", "cc");
+}
+
+#endif /* _KERNEL && !WANT_FUNCTIONS */
+
/*
* Atomic compare and set, used by the mutex functions
*
@@ -292,6 +373,11 @@ ATOMIC_STORE_LOAD(long, "cmpxchgl %0,%1"
#ifndef WANT_FUNCTIONS
+#ifdef _KERNEL
+extern uint64_t (*atomic_load_acq_64)(volatile uint64_t *);
+extern void (*atomic_store_rel_64)(volatile uint64_t *, uint64_t);
+#endif
+
static __inline int
atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src)
{
Modified: head/sys/pc98/pc98/machdep.c
==============================================================================
--- head/sys/pc98/pc98/machdep.c Wed Apr 6 20:54:26 2011 (r220403)
+++ head/sys/pc98/pc98/machdep.c Wed Apr 6 23:59:59 2011 (r220404)
@@ -1330,6 +1330,21 @@ idle_sysctl(SYSCTL_HANDLER_ARGS)
SYSCTL_PROC(_machdep, OID_AUTO, idle, CTLTYPE_STRING | CTLFLAG_RW, 0, 0,
idle_sysctl, "A", "currently selected idle function");
+uint64_t (*atomic_load_acq_64)(volatile uint64_t *) =
+ atomic_load_acq_64_i386;
+void (*atomic_store_rel_64)(volatile uint64_t *, uint64_t) =
+ atomic_store_rel_64_i386;
+
+static void
+cpu_probe_cmpxchg8b(void)
+{
+
+ if ((cpu_feature & CPUID_CX8) != 0) {
+ atomic_load_acq_64 = atomic_load_acq_64_i586;
+ atomic_store_rel_64 = atomic_store_rel_64_i586;
+ }
+}
+
/*
* Reset registers to default values on exec.
*/
@@ -2344,6 +2359,8 @@ init386(first)
thread0.td_pcb->pcb_cr3 = (int)IdlePTD;
thread0.td_pcb->pcb_ext = 0;
thread0.td_frame = &proc0_tf;
+
+ cpu_probe_cmpxchg8b();
}
void
More information about the svn-src-all
mailing list