svn commit: r357460 - in head/sys: arm64/arm64 arm64/include conf
Mark Johnston
markj at FreeBSD.org
Mon Feb 3 18:23:51 UTC 2020
Author: markj
Date: Mon Feb 3 18:23:50 2020
New Revision: 357460
URL: https://svnweb.freebsd.org/changeset/base/357460
Log:
Dynamically select LSE-based atomic(9)s on arm64.
Once all CPUs are online, determine if they all support LSE atomics and
set lse_supported to indicate this. For now the atomic(9)
implementations are still always inlined, though it would be preferable
to create out-of-line functions to avoid text bloat. This was not done
here since big.little systems exist in which some CPUs implement LSE
while others do not, and ifunc resolution must occur well before this
scenario can be detected. It does seem unlikely that FreeBSD will
ever run on such platforms, however, so converting atomic(9) to use
ifuncs is probably a good next step.
Add a LSE_ATOMICS arm64 kernel configuration option to unconditionally
select LSE-based atomic(9) implementations when the target system is
known.
Reviewed by: andrew, kib
MFC after: 1 month
Sponsored by: The FreeBSD Foundation, Amazon (hardware)
Differential Revision: https://reviews.freebsd.org/D23325
Modified:
head/sys/arm64/arm64/identcpu.c
head/sys/arm64/include/atomic.h
head/sys/conf/options.arm64
Modified: head/sys/arm64/arm64/identcpu.c
==============================================================================
--- head/sys/arm64/arm64/identcpu.c Mon Feb 3 18:23:35 2020 (r357459)
+++ head/sys/arm64/arm64/identcpu.c Mon Feb 3 18:23:50 2020 (r357460)
@@ -114,8 +114,8 @@ struct cpu_desc {
uint64_t id_aa64pfr1;
};
-struct cpu_desc cpu_desc[MAXCPU];
-struct cpu_desc user_cpu_desc;
+static struct cpu_desc cpu_desc[MAXCPU];
+static struct cpu_desc user_cpu_desc;
static u_int cpu_print_regs;
#define PRINT_ID_AA64_AFR0 0x00000001
#define PRINT_ID_AA64_AFR1 0x00000002
@@ -961,6 +961,7 @@ update_user_regs(u_int cpu)
/* HWCAP */
extern u_long elf_hwcap;
+bool __read_frequently lse_supported = false;
static void
identify_cpu_sysinit(void *dummy __unused)
@@ -975,7 +976,6 @@ identify_cpu_sysinit(void *dummy __unused)
ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 | ID_AA64PFR0_EL0_64;
user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8;
-
CPU_FOREACH(cpu) {
print_cpu_features(cpu);
hwcap = parse_cpu_features_hwcap(cpu);
@@ -985,6 +985,16 @@ identify_cpu_sysinit(void *dummy __unused)
elf_hwcap &= hwcap;
update_user_regs(cpu);
}
+
+ if ((elf_hwcap & HWCAP_ATOMICS) != 0) {
+ lse_supported = true;
+ if (bootverbose)
+ printf("Enabling LSE atomics in the kernel\n");
+ }
+#ifdef LSE_ATOMICS
+ if (!lse_supported)
+ panic("CPU does not support LSE atomic instructions");
+#endif
install_undef_handler(true, user_mrs_handler);
}
Modified: head/sys/arm64/include/atomic.h
==============================================================================
--- head/sys/arm64/include/atomic.h Mon Feb 3 18:23:35 2020 (r357459)
+++ head/sys/arm64/include/atomic.h Mon Feb 3 18:23:50 2020 (r357460)
@@ -59,6 +59,18 @@
#include <sys/atomic_common.h>
+#ifdef _KERNEL
+extern bool lse_supported;
+
+#ifdef LSE_ATOMICS
+#define _ATOMIC_LSE_SUPPORTED 1
+#else
+#define _ATOMIC_LSE_SUPPORTED lse_supported
+#endif
+#else
+#define _ATOMIC_LSE_SUPPORTED 0
+#endif
+
#define _ATOMIC_OP_PROTO(t, op, bar, flav) \
static __inline void \
atomic_##op##_##bar##t##flav(volatile uint##t##_t *p, uint##t##_t val)
@@ -98,7 +110,10 @@ _ATOMIC_OP_PROTO(t, op, bar, _lse) \
\
_ATOMIC_OP_PROTO(t, op, bar, ) \
{ \
- atomic_##op##_##bar##t##_llsc(p, val); \
+ if (_ATOMIC_LSE_SUPPORTED) \
+ atomic_##op##_##bar##t##_lse(p, val); \
+ else \
+ atomic_##op##_##bar##t##_llsc(p, val); \
}
#define __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \
@@ -175,7 +190,12 @@ _ATOMIC_CMPSET_PROTO(t, bar, _lse) \
\
_ATOMIC_CMPSET_PROTO(t, bar, ) \
{ \
- return (atomic_cmpset_##bar##t##_llsc(p, cmpval, newval)); \
+ if (_ATOMIC_LSE_SUPPORTED) \
+ return (atomic_cmpset_##bar##t##_lse(p, cmpval, \
+ newval)); \
+ else \
+ return (atomic_cmpset_##bar##t##_llsc(p, cmpval, \
+ newval)); \
} \
\
_ATOMIC_FCMPSET_PROTO(t, bar, _llsc) \
@@ -223,7 +243,12 @@ _ATOMIC_FCMPSET_PROTO(t, bar, _lse) \
\
_ATOMIC_FCMPSET_PROTO(t, bar, ) \
{ \
- return (atomic_fcmpset_##bar##t##_llsc(p, cmpval, newval)); \
+ if (_ATOMIC_LSE_SUPPORTED) \
+ return (atomic_fcmpset_##bar##t##_lse(p, cmpval, \
+ newval)); \
+ else \
+ return (atomic_fcmpset_##bar##t##_llsc(p, cmpval, \
+ newval)); \
}
#define _ATOMIC_CMPSET(bar, a, l) \
@@ -277,7 +302,10 @@ _ATOMIC_FETCHADD_PROTO(t, _lse) \
\
_ATOMIC_FETCHADD_PROTO(t, ) \
{ \
- return (atomic_fetchadd_##t##_llsc(p, val)); \
+ if (_ATOMIC_LSE_SUPPORTED) \
+ return (atomic_fetchadd_##t##_lse(p, val)); \
+ else \
+ return (atomic_fetchadd_##t##_llsc(p, val)); \
}
_ATOMIC_FETCHADD_IMPL(32, w)
@@ -327,7 +355,10 @@ _ATOMIC_SWAP_PROTO(t, _lse) \
\
_ATOMIC_SWAP_PROTO(t, ) \
{ \
- return (atomic_swap_##t##_llsc(p, val)); \
+ if (_ATOMIC_LSE_SUPPORTED) \
+ return (atomic_swap_##t##_lse(p, val)); \
+ else \
+ return (atomic_swap_##t##_llsc(p, val)); \
} \
\
_ATOMIC_READANDCLEAR_PROTO(t, _llsc) \
@@ -354,7 +385,10 @@ _ATOMIC_READANDCLEAR_PROTO(t, _lse) \
\
_ATOMIC_READANDCLEAR_PROTO(t, ) \
{ \
- return (atomic_readandclear_##t##_llsc(p)); \
+ if (_ATOMIC_LSE_SUPPORTED) \
+ return (atomic_readandclear_##t##_lse(p)); \
+ else \
+ return (atomic_readandclear_##t##_llsc(p)); \
}
_ATOMIC_SWAP_IMPL(32, w, wzr)
@@ -403,7 +437,10 @@ _ATOMIC_TEST_OP_PROTO(t, op, _lse) \
\
_ATOMIC_TEST_OP_PROTO(t, op, ) \
{ \
- return (atomic_testand##op##_##t##_llsc(p, val)); \
+ if (_ATOMIC_LSE_SUPPORTED) \
+ return (atomic_testand##op##_##t##_lse(p, val)); \
+ else \
+ return (atomic_testand##op##_##t##_llsc(p, val)); \
}
#define _ATOMIC_TEST_OP(op, llsc_asm_op, lse_asm_op) \
Modified: head/sys/conf/options.arm64
==============================================================================
--- head/sys/conf/options.arm64 Mon Feb 3 18:23:35 2020 (r357459)
+++ head/sys/conf/options.arm64 Mon Feb 3 18:23:50 2020 (r357460)
@@ -7,6 +7,7 @@ SOCDEV_VA opt_global.h
THUNDERX_PASS_1_1_ERRATA opt_global.h
VFP opt_global.h
LINUX_BOOT_ABI opt_global.h
+LSE_ATOMICS opt_global.h
# Binary compatibility
COMPAT_FREEBSD32 opt_global.h
More information about the svn-src-head
mailing list