svn commit: r354942 - in head/sys: amd64/amd64 amd64/conf amd64/include arm64/arm64 arm64/conf arm64/include conf kern libkern modules sys x86/include x86/x86
Andrew Turner
andrew at FreeBSD.org
Thu Nov 21 11:22:12 UTC 2019
Author: andrew
Date: Thu Nov 21 11:22:08 2019
New Revision: 354942
URL: https://svnweb.freebsd.org/changeset/base/354942
Log:
Port the NetBSD KCSAN runtime to FreeBSD.
Update the NetBSD Kernel Concurrency Sanitizer (KCSAN) runtime to work in
the FreeBSD kernel. It is a useful tool for finding data races between
threads executing on different CPUs.
This can be enabled by enabling KCSAN in the kernel config, or by using the
GENERIC-KCSAN amd64 kernel. It works on amd64 and arm64, however the later
needs a compiler change to allow -fsanitize=thread that KCSAN uses.
Sponsored by: DARPA, AFRL
Differential Revision: https://reviews.freebsd.org/D22315
Added:
head/sys/amd64/conf/GENERIC-KCSAN (contents, props changed)
head/sys/amd64/include/csan.h (contents, props changed)
head/sys/arm64/include/csan.h (contents, props changed)
head/sys/sys/_cscan_atomic.h (contents, props changed)
head/sys/sys/_cscan_bus.h (contents, props changed)
Modified:
head/sys/amd64/amd64/copyout.c
head/sys/amd64/amd64/machdep.c
head/sys/amd64/conf/GENERIC
head/sys/amd64/include/atomic.h
head/sys/arm64/arm64/bus_machdep.c
head/sys/arm64/arm64/copystr.c
head/sys/arm64/arm64/machdep.c
head/sys/arm64/arm64/mp_machdep.c
head/sys/arm64/conf/GENERIC
head/sys/arm64/include/atomic.h
head/sys/arm64/include/bus.h
head/sys/conf/files
head/sys/conf/files.arm64
head/sys/conf/kern.post.mk
head/sys/conf/kern.pre.mk
head/sys/conf/options
head/sys/kern/subr_csan.c (contents, props changed)
head/sys/kern/vfs_aio.c
head/sys/libkern/strcmp.c
head/sys/libkern/strcpy.c
head/sys/libkern/strlen.c
head/sys/modules/Makefile
head/sys/sys/csan.h (contents, props changed)
head/sys/sys/libkern.h
head/sys/sys/systm.h
head/sys/x86/include/bus.h
head/sys/x86/x86/bus_machdep.c
head/sys/x86/x86/mp_x86.c
Modified: head/sys/amd64/amd64/copyout.c
==============================================================================
--- head/sys/amd64/amd64/copyout.c Thu Nov 21 08:20:05 2019 (r354941)
+++ head/sys/amd64/amd64/copyout.c Thu Nov 21 11:22:08 2019 (r354942)
@@ -146,6 +146,10 @@ DEFINE_IFUNC(, int, casueword, (volatile u_long *, u_l
casueword_smap : casueword_nosmap);
}
+#undef copyinstr
+#undef copyin
+#undef copyout
+
int copyinstr_nosmap(const void *udaddr, void *kaddr, size_t len,
size_t *lencopied);
int copyinstr_smap(const void *udaddr, void *kaddr, size_t len,
Modified: head/sys/amd64/amd64/machdep.c
==============================================================================
--- head/sys/amd64/amd64/machdep.c Thu Nov 21 08:20:05 2019 (r354941)
+++ head/sys/amd64/amd64/machdep.c Thu Nov 21 11:22:08 2019 (r354942)
@@ -64,6 +64,7 @@ __FBSDID("$FreeBSD$");
#include <sys/callout.h>
#include <sys/cons.h>
#include <sys/cpu.h>
+#include <sys/csan.h>
#include <sys/efi.h>
#include <sys/eventhandler.h>
#include <sys/exec.h>
@@ -1899,6 +1900,8 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
cpu_probe_amdc1e();
+ kcsan_cpu_init(0);
+
#ifdef FDT
x86_init_fdt();
#endif
@@ -2722,6 +2725,40 @@ outb_(u_short port, u_char data)
void *memset_std(void *buf, int c, size_t len);
void *memset_erms(void *buf, int c, size_t len);
+void *memmove_std(void * _Nonnull dst, const void * _Nonnull src,
+ size_t len);
+void *memmove_erms(void * _Nonnull dst, const void * _Nonnull src,
+ size_t len);
+void *memcpy_std(void * _Nonnull dst, const void * _Nonnull src,
+ size_t len);
+void *memcpy_erms(void * _Nonnull dst, const void * _Nonnull src,
+ size_t len);
+
+#ifdef KCSAN
+/*
+ * These fail to build as ifuncs when used with KCSAN.
+ */
+void *
+memset(void *buf, int c, size_t len)
+{
+
+ return memset_std(buf, c, len);
+}
+
+void *
+memmove(void * _Nonnull dst, const void * _Nonnull src, size_t len)
+{
+
+ return memmove_std(dst, src, len);
+}
+
+void *
+memcpy(void * _Nonnull dst, const void * _Nonnull src, size_t len)
+{
+
+ return memcpy_std(dst, src, len);
+}
+#else
DEFINE_IFUNC(, void *, memset, (void *, int, size_t))
{
@@ -2729,10 +2766,6 @@ DEFINE_IFUNC(, void *, memset, (void *, int, size_t))
memset_erms : memset_std);
}
-void *memmove_std(void * _Nonnull dst, const void * _Nonnull src,
- size_t len);
-void *memmove_erms(void * _Nonnull dst, const void * _Nonnull src,
- size_t len);
DEFINE_IFUNC(, void *, memmove, (void * _Nonnull, const void * _Nonnull,
size_t))
{
@@ -2741,16 +2774,13 @@ DEFINE_IFUNC(, void *, memmove, (void * _Nonnull, cons
memmove_erms : memmove_std);
}
-void *memcpy_std(void * _Nonnull dst, const void * _Nonnull src,
- size_t len);
-void *memcpy_erms(void * _Nonnull dst, const void * _Nonnull src,
- size_t len);
DEFINE_IFUNC(, void *, memcpy, (void * _Nonnull, const void * _Nonnull,size_t))
{
return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ?
memcpy_erms : memcpy_std);
}
+#endif
void pagezero_std(void *addr);
void pagezero_erms(void *addr);
Modified: head/sys/amd64/conf/GENERIC
==============================================================================
--- head/sys/amd64/conf/GENERIC Thu Nov 21 08:20:05 2019 (r354941)
+++ head/sys/amd64/conf/GENERIC Thu Nov 21 11:22:08 2019 (r354942)
@@ -106,6 +106,7 @@ options VERBOSE_SYSINIT=0 # Support debug.verbose_sys
#options KCOV # Kernel Coverage Sanitizer
# Warning: KUBSAN can result in a kernel too large for loader to load
#options KUBSAN # Kernel Undefined Behavior Sanitizer
+#options KCSAN # Kernel Concurrency Sanitizer
# Kernel dump features.
options EKCD # Support for encrypted kernel dumps
Added: head/sys/amd64/conf/GENERIC-KCSAN
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/sys/amd64/conf/GENERIC-KCSAN Thu Nov 21 11:22:08 2019 (r354942)
@@ -0,0 +1,33 @@
+#
+# GENERIC-KCSAN -- Kernel Concurrency Sanitizer kernel configuration file
+# for FreeBSD/amd64
+#
+# This configuration file removes several debugging options, including
+# WITNESS and INVARIANTS checking, which are known to have significant
+# performance impact on running systems. When benchmarking new features
+# this kernel should be used instead of the standard GENERIC.
+# This kernel configuration should never appear outside of the HEAD
+# of the FreeBSD tree.
+#
+# For more information on this file, please read the config(5) manual page,
+# and/or the handbook section on Kernel Configuration Files:
+#
+# https://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html
+#
+# The handbook is also available locally in /usr/share/doc/handbook
+# if you've installed the doc distribution, otherwise always see the
+# FreeBSD World Wide Web server (https://www.FreeBSD.org/) for the
+# latest information.
+#
+# An exhaustive list of options and more detailed explanations of the
+# device lines is also present in the ../../conf/NOTES and NOTES files.
+# If you are in doubt as to the purpose or necessity of a line, check first
+# in NOTES.
+#
+# $FreeBSD$
+
+include GENERIC
+
+ident GENERIC-KCSAN
+
+options KCSAN
Modified: head/sys/amd64/include/atomic.h
==============================================================================
--- head/sys/amd64/include/atomic.h Thu Nov 21 08:20:05 2019 (r354941)
+++ head/sys/amd64/include/atomic.h Thu Nov 21 11:22:08 2019 (r354942)
@@ -57,6 +57,20 @@
#define wmb() __asm __volatile("sfence;" : : : "memory")
#define rmb() __asm __volatile("lfence;" : : : "memory")
+#ifdef _KERNEL
+/*
+ * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf).
+ *
+ * The open-coded number is used instead of the symbolic expression to
+ * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers.
+ * An assertion in amd64/vm_machdep.c ensures that the value is correct.
+ */
+#define OFFSETOF_MONITORBUF 0x100
+#endif
+
+#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
+#include <sys/_cscan_atomic.h>
+#else
#include <sys/atomic_common.h>
/*
@@ -345,15 +359,6 @@ atomic_testandclear_long(volatile u_long *p, u_int v)
#if defined(_KERNEL)
-/*
- * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf).
- *
- * The open-coded number is used instead of the symbolic expression to
- * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers.
- * An assertion in amd64/vm_machdep.c ensures that the value is correct.
- */
-#define OFFSETOF_MONITORBUF 0x100
-
#if defined(SMP) || defined(KLD_MODULE)
static __inline void
__storeload_barrier(void)
@@ -678,5 +683,7 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
#define atomic_readandclear_ptr atomic_readandclear_long
#endif /* !WANT_FUNCTIONS */
+
+#endif /* KCSAN && !KCSAN_RUNTIME */
#endif /* !_MACHINE_ATOMIC_H_ */
Added: head/sys/amd64/include/csan.h
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/sys/amd64/include/csan.h Thu Nov 21 11:22:08 2019 (r354942)
@@ -0,0 +1,67 @@
+/* $NetBSD: csan.h,v 1.2 2019/11/06 06:57:22 maxv Exp $ */
+
+/*
+ * Copyright (c) 2019 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Maxime Villard.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <machine/cpufunc.h>
+#include <machine/stack.h>
+#include <machine/vmparam.h>
+
+static inline bool
+kcsan_md_is_avail(void)
+{
+ return true;
+}
+
+static inline void
+kcsan_md_disable_intrs(uint64_t *state)
+{
+
+ *state = intr_disable();
+}
+
+static inline void
+kcsan_md_enable_intrs(uint64_t *state)
+{
+
+ intr_restore(*state);
+}
+
+static inline void
+kcsan_md_delay(uint64_t us)
+{
+ DELAY(us);
+}
+
+static void
+kcsan_md_unwind(void)
+{
+}
Modified: head/sys/arm64/arm64/bus_machdep.c
==============================================================================
--- head/sys/arm64/arm64/bus_machdep.c Thu Nov 21 08:20:05 2019 (r354941)
+++ head/sys/arm64/arm64/bus_machdep.c Thu Nov 21 11:22:08 2019 (r354942)
@@ -25,6 +25,8 @@
*
*/
+#define KCSAN_RUNTIME
+
#include "opt_platform.h"
#include <sys/param.h>
Modified: head/sys/arm64/arm64/copystr.c
==============================================================================
--- head/sys/arm64/arm64/copystr.c Thu Nov 21 08:20:05 2019 (r354941)
+++ head/sys/arm64/arm64/copystr.c Thu Nov 21 11:22:08 2019 (r354942)
@@ -32,7 +32,7 @@ __FBSDID("$FreeBSD$");
#include <sys/systm.h>
int
-copystr(const void * __restrict kfaddr, void * __restrict kdaddr, size_t len,
+(copystr)(const void * __restrict kfaddr, void * __restrict kdaddr, size_t len,
size_t * __restrict lencopied)
{
const char *src;
Modified: head/sys/arm64/arm64/machdep.c
==============================================================================
--- head/sys/arm64/arm64/machdep.c Thu Nov 21 08:20:05 2019 (r354941)
+++ head/sys/arm64/arm64/machdep.c Thu Nov 21 11:22:08 2019 (r354942)
@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
#include <sys/bus.h>
#include <sys/cons.h>
#include <sys/cpu.h>
+#include <sys/csan.h>
#include <sys/devmap.h>
#include <sys/efi.h>
#include <sys/exec.h>
@@ -1208,6 +1209,8 @@ initarm(struct arm64_bootparams *abp)
dbg_init();
kdb_init();
pan_enable();
+
+ kcsan_cpu_init(0);
env = kern_getenv("kernelname");
if (env != NULL)
Modified: head/sys/arm64/arm64/mp_machdep.c
==============================================================================
--- head/sys/arm64/arm64/mp_machdep.c Thu Nov 21 08:20:05 2019 (r354941)
+++ head/sys/arm64/arm64/mp_machdep.c Thu Nov 21 11:22:08 2019 (r354942)
@@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/cpu.h>
+#include <sys/csan.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/malloc.h>
@@ -252,6 +253,8 @@ init_secondary(uint64_t cpu)
}
mtx_unlock_spin(&ap_boot_mtx);
+
+ kcsan_cpu_init(cpu);
/* Enter the scheduler */
sched_throw(NULL);
Modified: head/sys/arm64/conf/GENERIC
==============================================================================
--- head/sys/arm64/conf/GENERIC Thu Nov 21 08:20:05 2019 (r354941)
+++ head/sys/arm64/conf/GENERIC Thu Nov 21 11:22:08 2019 (r354942)
@@ -100,6 +100,7 @@ options VERBOSE_SYSINIT=0 # Support debug.verbose_sys
#options KCOV # Kernel Coverage Sanitizer
# Warning: KUBSAN can result in a kernel too large for loader to load
#options KUBSAN # Kernel Undefined Behavior Sanitizer
+#options KCSAN # Kernel Concurrency Sanitizer
# Kernel dump features.
options EKCD # Support for encrypted kernel dumps
Modified: head/sys/arm64/include/atomic.h
==============================================================================
--- head/sys/arm64/include/atomic.h Thu Nov 21 08:20:05 2019 (r354941)
+++ head/sys/arm64/include/atomic.h Thu Nov 21 11:22:08 2019 (r354942)
@@ -29,8 +29,6 @@
#ifndef _MACHINE_ATOMIC_H_
#define _MACHINE_ATOMIC_H_
-#include <sys/atomic_common.h>
-
#define isb() __asm __volatile("isb" : : : "memory")
/*
@@ -55,6 +53,12 @@
#define wmb() dmb(st) /* Full system memory barrier store */
#define rmb() dmb(ld) /* Full system memory barrier load */
+#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
+#include <sys/_cscan_atomic.h>
+#else
+
+#include <sys/atomic_common.h>
+
#define ATOMIC_OP(op, asm_op, bar, a, l) \
static __inline void \
atomic_##op##_##bar##8(volatile uint8_t *p, uint8_t val) \
@@ -600,6 +604,8 @@ atomic_thread_fence_seq_cst(void)
dmb(sy);
}
+
+#endif /* KCSAN && !KCSAN_RUNTIME */
#endif /* _MACHINE_ATOMIC_H_ */
Modified: head/sys/arm64/include/bus.h
==============================================================================
--- head/sys/arm64/include/bus.h Thu Nov 21 08:20:05 2019 (r354941)
+++ head/sys/arm64/include/bus.h Thu Nov 21 11:22:08 2019 (r354942)
@@ -89,6 +89,9 @@
#define BUS_SPACE_BARRIER_READ 0x01
#define BUS_SPACE_BARRIER_WRITE 0x02
+#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
+#include <sys/_cscan_bus.h>
+#else
struct bus_space {
/* cookie */
@@ -463,6 +466,8 @@ struct bus_space {
__bs_copy(4, t, h1, o1, h2, o2, c)
#define bus_space_copy_region_8(t, h1, o1, h2, o2, c) \
__bs_copy(8, t, h1, o1, h2, o2, c)
+
+#endif
#include <machine/bus_dma.h>
Added: head/sys/arm64/include/csan.h
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/sys/arm64/include/csan.h Thu Nov 21 11:22:08 2019 (r354942)
@@ -0,0 +1,104 @@
+/* $NetBSD: csan.h,v 1.2 2019/11/06 06:57:22 maxv Exp $ */
+
+/*
+ * Copyright (c) 2019 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Maxime Villard.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <machine/cpufunc.h>
+#include <machine/stack.h>
+#include <machine/vmparam.h>
+
+static inline bool
+kcsan_md_is_avail(void)
+{
+ return true;
+}
+
+static inline void
+kcsan_md_disable_intrs(uint64_t *state)
+{
+
+ *state = intr_disable();
+}
+
+static inline void
+kcsan_md_enable_intrs(uint64_t *state)
+{
+
+ intr_restore(*state);
+}
+
+static inline void
+kcsan_md_delay(uint64_t us)
+{
+ DELAY(us);
+}
+
+static void
+kcsan_md_unwind(void)
+{
+#ifdef DDB
+ c_db_sym_t sym;
+ db_expr_t offset;
+ const char *symname;
+#endif
+ struct unwind_state frame;
+ uint64_t sp;
+ int nsym;
+
+ __asm __volatile("mov %0, sp" : "=&r" (sp));
+
+ frame.sp = sp;
+ frame.fp = (uint64_t)__builtin_frame_address(0);
+ frame.pc = (uint64_t)kcsan_md_unwind;
+ nsym = 0;
+
+ while (1) {
+ unwind_frame(&frame);
+ if (!INKERNEL((vm_offset_t)frame.fp) ||
+ !INKERNEL((vm_offset_t)frame.pc))
+ break;
+
+#ifdef DDB
+ sym = db_search_symbol((vm_offset_t)frame.pc, DB_STGY_PROC,
+ &offset);
+ db_symbol_values(sym, &symname, NULL);
+ printf("#%d %p in %s+%#lx\n", nsym, (void *)frame.pc,
+ symname, offset);
+#else
+ printf("#%d %p\n", nsym, (void *)frame.pc);
+#endif
+ nsym++;
+
+ if (nsym >= 15) {
+ break;
+ }
+ }
+}
Modified: head/sys/conf/files
==============================================================================
--- head/sys/conf/files Thu Nov 21 08:20:05 2019 (r354941)
+++ head/sys/conf/files Thu Nov 21 11:22:08 2019 (r354942)
@@ -3805,6 +3805,8 @@ kern/subr_compressor.c standard \
kern/subr_coverage.c optional coverage \
compile-with "${NORMAL_C:N-fsanitize*}"
kern/subr_counter.c standard
+kern/subr_csan.c optional kcsan \
+ compile-with "${NORMAL_C:N-fsanitize*}"
kern/subr_devstat.c standard
kern/subr_disk.c standard
kern/subr_early.c standard
Modified: head/sys/conf/files.arm64
==============================================================================
--- head/sys/conf/files.arm64 Thu Nov 21 08:20:05 2019 (r354941)
+++ head/sys/conf/files.arm64 Thu Nov 21 11:22:08 2019 (r354942)
@@ -286,8 +286,10 @@ kern/pic_if.m optional intrng
kern/subr_devmap.c standard
kern/subr_intr.c optional intrng
libkern/bcmp.c standard
-libkern/memcmp.c standard
-libkern/memset.c standard
+libkern/memcmp.c standard \
+ compile-with "${NORMAL_C:N-fsanitize*}"
+libkern/memset.c standard \
+ compile-with "${NORMAL_C:N-fsanitize*}"
libkern/arm64/crc32c_armv8.S standard
cddl/dev/dtrace/aarch64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}"
cddl/dev/dtrace/aarch64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}"
Modified: head/sys/conf/kern.post.mk
==============================================================================
--- head/sys/conf/kern.post.mk Thu Nov 21 08:20:05 2019 (r354941)
+++ head/sys/conf/kern.post.mk Thu Nov 21 11:22:08 2019 (r354942)
@@ -38,6 +38,10 @@ MKMODULESENV+= WITH_CTF="${WITH_CTF}"
MKMODULESENV+= WITH_EXTRA_TCP_STACKS="${WITH_EXTRA_TCP_STACKS}"
.endif
+.if defined(KCSAN_ENABLED)
+MKMODULESENV+= KCSAN_ENABLED="yes"
+.endif
+
.if defined(SAN_CFLAGS)
MKMODULESENV+= SAN_CFLAGS="${SAN_CFLAGS}"
.endif
Modified: head/sys/conf/kern.pre.mk
==============================================================================
--- head/sys/conf/kern.pre.mk Thu Nov 21 08:20:05 2019 (r354941)
+++ head/sys/conf/kern.pre.mk Thu Nov 21 11:22:08 2019 (r354942)
@@ -117,6 +117,11 @@ PROF= -pg
.endif
DEFINED_PROF= ${PROF}
+KCSAN_ENABLED!= grep KCSAN opt_global.h || true ; echo
+.if !empty(KCSAN_ENABLED)
+SAN_CFLAGS+= -fsanitize=thread
+.endif
+
KUBSAN_ENABLED!= grep KUBSAN opt_global.h || true ; echo
.if !empty(KUBSAN_ENABLED)
SAN_CFLAGS+= -fsanitize=undefined
Modified: head/sys/conf/options
==============================================================================
--- head/sys/conf/options Thu Nov 21 08:20:05 2019 (r354941)
+++ head/sys/conf/options Thu Nov 21 11:22:08 2019 (r354942)
@@ -230,6 +230,7 @@ ZSTDIO opt_zstdio.h
# Sanitizers
COVERAGE opt_global.h
KCOV
+KCSAN opt_global.h
KUBSAN opt_global.h
# POSIX kernel options
Modified: head/sys/kern/subr_csan.c
==============================================================================
--- head/sys/kern/subr_csan.c Thu Nov 21 08:20:05 2019 (r354941)
+++ head/sys/kern/subr_csan.c Thu Nov 21 11:22:08 2019 (r354942)
@@ -3,6 +3,7 @@
/*
* Copyright (c) 2019 The NetBSD Foundation, Inc.
* All rights reserved.
+ * Copyright (c) 2019 Andrew Turner
*
* This code is derived from software contributed to The NetBSD Foundation
* by Maxime Villard.
@@ -29,19 +30,26 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
+#define KCSAN_RUNTIME
+
+#include "opt_ddb.h"
+
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $");
+__FBSDID("$FreeBSD$");
#include <sys/param.h>
-#include <sys/device.h>
#include <sys/kernel.h>
-#include <sys/param.h>
+#include <sys/bus.h>
#include <sys/conf.h>
-#include <sys/systm.h>
-#include <sys/types.h>
-#include <sys/csan.h>
#include <sys/cpu.h>
+#include <sys/csan.h>
+#include <sys/proc.h>
+#include <sys/smp.h>
+#include <sys/systm.h>
+#include <ddb/ddb.h>
+#include <ddb/db_sym.h>
+
#ifdef KCSAN_PANIC
#define REPORT panic
#else
@@ -62,7 +70,7 @@ typedef struct {
csan_cell_t cell;
} csan_cpu_t;
-static csan_cpu_t kcsan_cpus[MAXCPUS];
+static csan_cpu_t kcsan_cpus[MAXCPU];
static bool kcsan_enabled __read_mostly;
#define __RET_ADDR (uintptr_t)__builtin_return_address(0)
@@ -77,34 +85,43 @@ static bool kcsan_enabled __read_mostly;
/* -------------------------------------------------------------------------- */
-void
-kcsan_init(void)
+static void
+kcsan_enable(void *dummy __unused)
{
+
+ printf("Enabling KCSCAN, expect reduced performance.\n");
kcsan_enabled = true;
}
+SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL);
void
-kcsan_cpu_init(struct cpu_info *ci)
+kcsan_cpu_init(u_int cpu)
{
- kcsan_cpus[cpu_index(ci)].inited = true;
+ kcsan_cpus[cpu].inited = true;
}
/* -------------------------------------------------------------------------- */
static inline void
-kcsan_report(csan_cell_t *new, cpuid_t newcpu, csan_cell_t *old, cpuid_t oldcpu)
+kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu)
{
const char *newsym, *oldsym;
+#ifdef DDB
+ c_db_sym_t sym;
+ db_expr_t offset;
- if (ksyms_getname(NULL, &newsym, (vaddr_t)new->pc, KSYMS_PROC) != 0) {
- newsym = "Unknown";
- }
- if (ksyms_getname(NULL, &oldsym, (vaddr_t)old->pc, KSYMS_PROC) != 0) {
- oldsym = "Unknown";
- }
+ sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset);
+ db_symbol_values(sym, &newsym, NULL);
+
+ sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset);
+ db_symbol_values(sym, &oldsym, NULL);
+#else
+ newsym = "";
+ oldsym = "";
+#endif
REPORT("CSan: Racy Access "
- "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>] "
- "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>]\n",
+ "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] "
+ "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n",
newcpu,
(new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"),
(void *)new->addr, new->size, (void *)new->pc, newsym,
@@ -134,8 +151,6 @@ kcsan_access(uintptr_t addr, size_t size, bool write,
if (__predict_false(!kcsan_enabled))
return;
- if (__predict_false(kcsan_md_unsupported((vaddr_t)addr)))
- return;
new.addr = addr;
new.size = size;
@@ -143,7 +158,7 @@ kcsan_access(uintptr_t addr, size_t size, bool write,
new.atomic = atomic;
new.pc = pc;
- for (i = 0; i < ncpu; i++) {
+ CPU_FOREACH(i) {
__builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old));
if (old.addr + old.size <= new.addr)
@@ -155,7 +170,7 @@ kcsan_access(uintptr_t addr, size_t size, bool write,
if (__predict_true(kcsan_access_is_atomic(&new, &old)))
continue;
- kcsan_report(&new, cpu_number(), &old, i);
+ kcsan_report(&new, PCPU_GET(cpuid), &old, i);
break;
}
@@ -164,7 +179,7 @@ kcsan_access(uintptr_t addr, size_t size, bool write,
kcsan_md_disable_intrs(&intr);
- cpu = &kcsan_cpus[cpu_number()];
+ cpu = &kcsan_cpus[PCPU_GET(cpuid)];
if (__predict_false(!cpu->inited))
goto out;
cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES;
@@ -184,6 +199,11 @@ out:
void __tsan_read##size(uintptr_t addr) \
{ \
kcsan_access(addr, size, false, false, __RET_ADDR); \
+ } \
+ void __tsan_unaligned_read##size(uintptr_t); \
+ void __tsan_unaligned_read##size(uintptr_t addr) \
+ { \
+ kcsan_access(addr, size, false, false, __RET_ADDR); \
}
CSAN_READ(1)
@@ -197,6 +217,11 @@ CSAN_READ(16)
void __tsan_write##size(uintptr_t addr) \
{ \
kcsan_access(addr, size, true, false, __RET_ADDR); \
+ } \
+ void __tsan_unaligned_write##size(uintptr_t); \
+ void __tsan_unaligned_write##size(uintptr_t addr) \
+ { \
+ kcsan_access(addr, size, true, false, __RET_ADDR); \
}
CSAN_WRITE(1)
@@ -321,35 +346,14 @@ kcsan_strlen(const char *str)
return (s - str);
}
-#undef kcopy
#undef copystr
-#undef copyinstr
-#undef copyoutstr
#undef copyin
+#undef copyin_nofault
+#undef copyinstr
#undef copyout
+#undef copyout_nofault
-int kcsan_kcopy(const void *, void *, size_t);
-int kcsan_copystr(const void *, void *, size_t, size_t *);
-int kcsan_copyinstr(const void *, void *, size_t, size_t *);
-int kcsan_copyoutstr(const void *, void *, size_t, size_t *);
-int kcsan_copyin(const void *, void *, size_t);
-int kcsan_copyout(const void *, void *, size_t);
-int kcopy(const void *, void *, size_t);
-int copystr(const void *, void *, size_t, size_t *);
-int copyinstr(const void *, void *, size_t, size_t *);
-int copyoutstr(const void *, void *, size_t, size_t *);
-int copyin(const void *, void *, size_t);
-int copyout(const void *, void *, size_t);
-
int
-kcsan_kcopy(const void *src, void *dst, size_t len)
-{
- kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
- kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
- return kcopy(src, dst, len);
-}
-
-int
kcsan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done)
{
kcsan_access((uintptr_t)kdaddr, len, true, false, __RET_ADDR);
@@ -364,13 +368,6 @@ kcsan_copyin(const void *uaddr, void *kaddr, size_t le
}
int
-kcsan_copyout(const void *kaddr, void *uaddr, size_t len)
-{
- kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR);
- return copyout(kaddr, uaddr, len);
-}
-
-int
kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
{
kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
@@ -378,377 +375,477 @@ kcsan_copyinstr(const void *uaddr, void *kaddr, size_t
}
int
-kcsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done)
+kcsan_copyout(const void *kaddr, void *uaddr, size_t len)
{
kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR);
- return copyoutstr(kaddr, uaddr, len, done);
+ return copyout(kaddr, uaddr, len);
}
/* -------------------------------------------------------------------------- */
-#undef atomic_add_32
-#undef atomic_add_int
-#undef atomic_add_long
-#undef atomic_add_ptr
-#undef atomic_add_64
-#undef atomic_add_32_nv
-#undef atomic_add_int_nv
-#undef atomic_add_long_nv
-#undef atomic_add_ptr_nv
-#undef atomic_add_64_nv
-#undef atomic_and_32
-#undef atomic_and_uint
-#undef atomic_and_ulong
-#undef atomic_and_64
-#undef atomic_and_32_nv
-#undef atomic_and_uint_nv
-#undef atomic_and_ulong_nv
-#undef atomic_and_64_nv
-#undef atomic_or_32
-#undef atomic_or_uint
-#undef atomic_or_ulong
-#undef atomic_or_64
-#undef atomic_or_32_nv
-#undef atomic_or_uint_nv
-#undef atomic_or_ulong_nv
-#undef atomic_or_64_nv
-#undef atomic_cas_32
-#undef atomic_cas_uint
-#undef atomic_cas_ulong
-#undef atomic_cas_ptr
-#undef atomic_cas_64
-#undef atomic_cas_32_ni
-#undef atomic_cas_uint_ni
-#undef atomic_cas_ulong_ni
-#undef atomic_cas_ptr_ni
-#undef atomic_cas_64_ni
-#undef atomic_swap_32
-#undef atomic_swap_uint
-#undef atomic_swap_ulong
-#undef atomic_swap_ptr
-#undef atomic_swap_64
-#undef atomic_dec_32
-#undef atomic_dec_uint
-#undef atomic_dec_ulong
-#undef atomic_dec_ptr
-#undef atomic_dec_64
-#undef atomic_dec_32_nv
-#undef atomic_dec_uint_nv
-#undef atomic_dec_ulong_nv
-#undef atomic_dec_ptr_nv
-#undef atomic_dec_64_nv
-#undef atomic_inc_32
-#undef atomic_inc_uint
-#undef atomic_inc_ulong
-#undef atomic_inc_ptr
-#undef atomic_inc_64
-#undef atomic_inc_32_nv
-#undef atomic_inc_uint_nv
-#undef atomic_inc_ulong_nv
-#undef atomic_inc_ptr_nv
-#undef atomic_inc_64_nv
+#include <machine/atomic.h>
+#include <sys/_cscan_atomic.h>
-#define CSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \
- void atomic_add_##name(volatile targ1 *, targ2); \
- void kcsan_atomic_add_##name(volatile targ1 *, targ2); \
- void kcsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- atomic_add_##name(ptr, val); \
- } \
- tret atomic_add_##name##_nv(volatile targ1 *, targ2); \
- tret kcsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \
- tret kcsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- return atomic_add_##name##_nv(ptr, val); \
+#define _CSAN_ATOMIC_FUNC_ADD(name, type) \
+ void kcsan_atomic_add_##name(volatile type *ptr, type val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ atomic_add_##name(ptr, val); \
}
-#define CSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \
- void atomic_and_##name(volatile targ1 *, targ2); \
- void kcsan_atomic_and_##name(volatile targ1 *, targ2); \
- void kcsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- atomic_and_##name(ptr, val); \
- } \
- tret atomic_and_##name##_nv(volatile targ1 *, targ2); \
- tret kcsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \
- tret kcsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- return atomic_and_##name##_nv(ptr, val); \
+#define CSAN_ATOMIC_FUNC_ADD(name, type) \
+ _CSAN_ATOMIC_FUNC_ADD(name, type) \
+ _CSAN_ATOMIC_FUNC_ADD(acq_##name, type) \
+ _CSAN_ATOMIC_FUNC_ADD(rel_##name, type)
+
+#define _CSAN_ATOMIC_FUNC_CLEAR(name, type) \
+ void kcsan_atomic_clear_##name(volatile type *ptr, type val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ atomic_clear_##name(ptr, val); \
}
-#define CSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \
- void atomic_or_##name(volatile targ1 *, targ2); \
- void kcsan_atomic_or_##name(volatile targ1 *, targ2); \
- void kcsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- atomic_or_##name(ptr, val); \
- } \
- tret atomic_or_##name##_nv(volatile targ1 *, targ2); \
- tret kcsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \
- tret kcsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- return atomic_or_##name##_nv(ptr, val); \
+#define CSAN_ATOMIC_FUNC_CLEAR(name, type) \
+ _CSAN_ATOMIC_FUNC_CLEAR(name, type) \
+ _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \
+ _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
+
+#define _CSAN_ATOMIC_FUNC_CMPSET(name, type) \
+ int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1, \
+ type val2) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ return (atomic_cmpset_##name(ptr, val1, val2)); \
}
-#define CSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \
- tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \
- tret kcsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
- tret kcsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- return atomic_cas_##name(ptr, exp, new); \
- } \
- tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
- tret kcsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
- tret kcsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- return atomic_cas_##name##_ni(ptr, exp, new); \
+#define CSAN_ATOMIC_FUNC_CMPSET(name, type) \
+ _CSAN_ATOMIC_FUNC_CMPSET(name, type) \
+ _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \
+ _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
+
+#define _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
+ int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \
+ type val2) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ return (atomic_fcmpset_##name(ptr, val1, val2)); \
}
-#define CSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \
- tret atomic_swap_##name(volatile targ1 *, targ2); \
- tret kcsan_atomic_swap_##name(volatile targ1 *, targ2); \
- tret kcsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- return atomic_swap_##name(ptr, val); \
+#define CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
+ _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
+ _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \
+ _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
+
+#define CSAN_ATOMIC_FUNC_FETCHADD(name, type) \
+ type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ return (atomic_fetchadd_##name(ptr, val)); \
}
-#define CSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \
- void atomic_dec_##name(volatile targ1 *); \
- void kcsan_atomic_dec_##name(volatile targ1 *); \
- void kcsan_atomic_dec_##name(volatile targ1 *ptr) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-all
mailing list