svn commit: r354942 - in head/sys: amd64/amd64 amd64/conf amd64/include arm64/arm64 arm64/conf arm64/include conf kern libkern modules sys x86/include x86/x86

Andrew Turner andrew at FreeBSD.org
Thu Nov 21 11:22:12 UTC 2019


Author: andrew
Date: Thu Nov 21 11:22:08 2019
New Revision: 354942
URL: https://svnweb.freebsd.org/changeset/base/354942

Log:
  Port the NetBSD KCSAN runtime to FreeBSD.
  
  Update the NetBSD Kernel Concurrency Sanitizer (KCSAN) runtime to work in
  the FreeBSD kernel. It is a useful tool for finding data races between
  threads executing on different CPUs.
  
  This can be enabled by enabling KCSAN in the kernel config, or by using the
  GENERIC-KCSAN amd64 kernel. It works on amd64 and arm64, however the later
  needs a compiler change to allow -fsanitize=thread that KCSAN uses.
  
  Sponsored by:	DARPA, AFRL
  Differential Revision:	https://reviews.freebsd.org/D22315

Added:
  head/sys/amd64/conf/GENERIC-KCSAN   (contents, props changed)
  head/sys/amd64/include/csan.h   (contents, props changed)
  head/sys/arm64/include/csan.h   (contents, props changed)
  head/sys/sys/_cscan_atomic.h   (contents, props changed)
  head/sys/sys/_cscan_bus.h   (contents, props changed)
Modified:
  head/sys/amd64/amd64/copyout.c
  head/sys/amd64/amd64/machdep.c
  head/sys/amd64/conf/GENERIC
  head/sys/amd64/include/atomic.h
  head/sys/arm64/arm64/bus_machdep.c
  head/sys/arm64/arm64/copystr.c
  head/sys/arm64/arm64/machdep.c
  head/sys/arm64/arm64/mp_machdep.c
  head/sys/arm64/conf/GENERIC
  head/sys/arm64/include/atomic.h
  head/sys/arm64/include/bus.h
  head/sys/conf/files
  head/sys/conf/files.arm64
  head/sys/conf/kern.post.mk
  head/sys/conf/kern.pre.mk
  head/sys/conf/options
  head/sys/kern/subr_csan.c   (contents, props changed)
  head/sys/kern/vfs_aio.c
  head/sys/libkern/strcmp.c
  head/sys/libkern/strcpy.c
  head/sys/libkern/strlen.c
  head/sys/modules/Makefile
  head/sys/sys/csan.h   (contents, props changed)
  head/sys/sys/libkern.h
  head/sys/sys/systm.h
  head/sys/x86/include/bus.h
  head/sys/x86/x86/bus_machdep.c
  head/sys/x86/x86/mp_x86.c

Modified: head/sys/amd64/amd64/copyout.c
==============================================================================
--- head/sys/amd64/amd64/copyout.c	Thu Nov 21 08:20:05 2019	(r354941)
+++ head/sys/amd64/amd64/copyout.c	Thu Nov 21 11:22:08 2019	(r354942)
@@ -146,6 +146,10 @@ DEFINE_IFUNC(, int, casueword, (volatile u_long *, u_l
 	    casueword_smap : casueword_nosmap);
 }
 
+#undef copyinstr
+#undef copyin
+#undef copyout
+
 int	copyinstr_nosmap(const void *udaddr, void *kaddr, size_t len,
 	    size_t *lencopied);
 int	copyinstr_smap(const void *udaddr, void *kaddr, size_t len,

Modified: head/sys/amd64/amd64/machdep.c
==============================================================================
--- head/sys/amd64/amd64/machdep.c	Thu Nov 21 08:20:05 2019	(r354941)
+++ head/sys/amd64/amd64/machdep.c	Thu Nov 21 11:22:08 2019	(r354942)
@@ -64,6 +64,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/callout.h>
 #include <sys/cons.h>
 #include <sys/cpu.h>
+#include <sys/csan.h>
 #include <sys/efi.h>
 #include <sys/eventhandler.h>
 #include <sys/exec.h>
@@ -1899,6 +1900,8 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 
 	cpu_probe_amdc1e();
 
+	kcsan_cpu_init(0);
+
 #ifdef FDT
 	x86_init_fdt();
 #endif
@@ -2722,6 +2725,40 @@ outb_(u_short port, u_char data)
 
 void	*memset_std(void *buf, int c, size_t len);
 void	*memset_erms(void *buf, int c, size_t len);
+void    *memmove_std(void * _Nonnull dst, const void * _Nonnull src,
+	    size_t len);
+void    *memmove_erms(void * _Nonnull dst, const void * _Nonnull src,
+	    size_t len);
+void    *memcpy_std(void * _Nonnull dst, const void * _Nonnull src,
+	    size_t len);
+void    *memcpy_erms(void * _Nonnull dst, const void * _Nonnull src,
+	    size_t len);
+
+#ifdef KCSAN
+/*
+ * These fail to build as ifuncs when used with KCSAN.
+ */
+void *
+memset(void *buf, int c, size_t len)
+{
+
+	return memset_std(buf, c, len);
+}
+
+void *
+memmove(void * _Nonnull dst, const void * _Nonnull src, size_t len)
+{
+
+	return memmove_std(dst, src, len);
+}
+
+void *
+memcpy(void * _Nonnull dst, const void * _Nonnull src, size_t len)
+{
+
+	return memcpy_std(dst, src, len);
+}
+#else
 DEFINE_IFUNC(, void *, memset, (void *, int, size_t))
 {
 
@@ -2729,10 +2766,6 @@ DEFINE_IFUNC(, void *, memset, (void *, int, size_t))
 	    memset_erms : memset_std);
 }
 
-void    *memmove_std(void * _Nonnull dst, const void * _Nonnull src,
-	    size_t len);
-void    *memmove_erms(void * _Nonnull dst, const void * _Nonnull src,
-	    size_t len);
 DEFINE_IFUNC(, void *, memmove, (void * _Nonnull, const void * _Nonnull,
     size_t))
 {
@@ -2741,16 +2774,13 @@ DEFINE_IFUNC(, void *, memmove, (void * _Nonnull, cons
 	    memmove_erms : memmove_std);
 }
 
-void    *memcpy_std(void * _Nonnull dst, const void * _Nonnull src,
-	    size_t len);
-void    *memcpy_erms(void * _Nonnull dst, const void * _Nonnull src,
-	    size_t len);
 DEFINE_IFUNC(, void *, memcpy, (void * _Nonnull, const void * _Nonnull,size_t))
 {
 
 	return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ?
 	    memcpy_erms : memcpy_std);
 }
+#endif
 
 void	pagezero_std(void *addr);
 void	pagezero_erms(void *addr);

Modified: head/sys/amd64/conf/GENERIC
==============================================================================
--- head/sys/amd64/conf/GENERIC	Thu Nov 21 08:20:05 2019	(r354941)
+++ head/sys/amd64/conf/GENERIC	Thu Nov 21 11:22:08 2019	(r354942)
@@ -106,6 +106,7 @@ options 	VERBOSE_SYSINIT=0	# Support debug.verbose_sys
 #options 	KCOV			# Kernel Coverage Sanitizer
 # Warning: KUBSAN can result in a kernel too large for loader to load
 #options 	KUBSAN			# Kernel Undefined Behavior Sanitizer
+#options 	KCSAN			# Kernel Concurrency Sanitizer
 
 # Kernel dump features.
 options 	EKCD			# Support for encrypted kernel dumps

Added: head/sys/amd64/conf/GENERIC-KCSAN
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ head/sys/amd64/conf/GENERIC-KCSAN	Thu Nov 21 11:22:08 2019	(r354942)
@@ -0,0 +1,33 @@
+#
+# GENERIC-KCSAN -- Kernel Concurrency Sanitizer kernel configuration file
+#		   for FreeBSD/amd64
+#
+# This configuration file removes several debugging options, including
+# WITNESS and INVARIANTS checking, which are known to have significant
+# performance impact on running systems.  When benchmarking new features
+# this kernel should be used instead of the standard GENERIC.
+# This kernel configuration should never appear outside of the HEAD
+# of the FreeBSD tree.
+#
+# For more information on this file, please read the config(5) manual page,
+# and/or the handbook section on Kernel Configuration Files:
+#
+#    https://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html
+#
+# The handbook is also available locally in /usr/share/doc/handbook
+# if you've installed the doc distribution, otherwise always see the
+# FreeBSD World Wide Web server (https://www.FreeBSD.org/) for the
+# latest information.
+#
+# An exhaustive list of options and more detailed explanations of the
+# device lines is also present in the ../../conf/NOTES and NOTES files.
+# If you are in doubt as to the purpose or necessity of a line, check first
+# in NOTES.
+#
+# $FreeBSD$
+
+include GENERIC
+
+ident   GENERIC-KCSAN
+
+options 	KCSAN

Modified: head/sys/amd64/include/atomic.h
==============================================================================
--- head/sys/amd64/include/atomic.h	Thu Nov 21 08:20:05 2019	(r354941)
+++ head/sys/amd64/include/atomic.h	Thu Nov 21 11:22:08 2019	(r354942)
@@ -57,6 +57,20 @@
 #define	wmb()	__asm __volatile("sfence;" : : : "memory")
 #define	rmb()	__asm __volatile("lfence;" : : : "memory")
 
+#ifdef _KERNEL
+/*
+ * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf).
+ *
+ * The open-coded number is used instead of the symbolic expression to
+ * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers.
+ * An assertion in amd64/vm_machdep.c ensures that the value is correct.
+ */
+#define	OFFSETOF_MONITORBUF	0x100
+#endif
+
+#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
+#include <sys/_cscan_atomic.h>
+#else
 #include <sys/atomic_common.h>
 
 /*
@@ -345,15 +359,6 @@ atomic_testandclear_long(volatile u_long *p, u_int v)
 
 #if defined(_KERNEL)
 
-/*
- * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf).
- *
- * The open-coded number is used instead of the symbolic expression to
- * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers.
- * An assertion in amd64/vm_machdep.c ensures that the value is correct.
- */
-#define	OFFSETOF_MONITORBUF	0x100
-
 #if defined(SMP) || defined(KLD_MODULE)
 static __inline void
 __storeload_barrier(void)
@@ -678,5 +683,7 @@ u_long	atomic_swap_long(volatile u_long *p, u_long v);
 #define	atomic_readandclear_ptr	atomic_readandclear_long
 
 #endif /* !WANT_FUNCTIONS */
+
+#endif /* KCSAN && !KCSAN_RUNTIME */
 
 #endif /* !_MACHINE_ATOMIC_H_ */

Added: head/sys/amd64/include/csan.h
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ head/sys/amd64/include/csan.h	Thu Nov 21 11:22:08 2019	(r354942)
@@ -0,0 +1,67 @@
+/*	$NetBSD: csan.h,v 1.2 2019/11/06 06:57:22 maxv Exp $	*/
+
+/*
+ * Copyright (c) 2019 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Maxime Villard.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <machine/cpufunc.h>
+#include <machine/stack.h>
+#include <machine/vmparam.h>
+
+static inline bool
+kcsan_md_is_avail(void)
+{
+	return true;
+}
+
+static inline void
+kcsan_md_disable_intrs(uint64_t *state)
+{
+
+	*state = intr_disable();
+}
+
+static inline void
+kcsan_md_enable_intrs(uint64_t *state)
+{
+
+	intr_restore(*state);
+}
+
+static inline void
+kcsan_md_delay(uint64_t us)
+{
+	DELAY(us);
+}
+
+static void
+kcsan_md_unwind(void)
+{
+}

Modified: head/sys/arm64/arm64/bus_machdep.c
==============================================================================
--- head/sys/arm64/arm64/bus_machdep.c	Thu Nov 21 08:20:05 2019	(r354941)
+++ head/sys/arm64/arm64/bus_machdep.c	Thu Nov 21 11:22:08 2019	(r354942)
@@ -25,6 +25,8 @@
  *
  */
 
+#define	KCSAN_RUNTIME
+
 #include "opt_platform.h"
 
 #include <sys/param.h>

Modified: head/sys/arm64/arm64/copystr.c
==============================================================================
--- head/sys/arm64/arm64/copystr.c	Thu Nov 21 08:20:05 2019	(r354941)
+++ head/sys/arm64/arm64/copystr.c	Thu Nov 21 11:22:08 2019	(r354942)
@@ -32,7 +32,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 
 int
-copystr(const void * __restrict kfaddr, void * __restrict kdaddr, size_t len,
+(copystr)(const void * __restrict kfaddr, void * __restrict kdaddr, size_t len,
     size_t * __restrict lencopied)
 {
 	const char *src;

Modified: head/sys/arm64/arm64/machdep.c
==============================================================================
--- head/sys/arm64/arm64/machdep.c	Thu Nov 21 08:20:05 2019	(r354941)
+++ head/sys/arm64/arm64/machdep.c	Thu Nov 21 11:22:08 2019	(r354942)
@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/bus.h>
 #include <sys/cons.h>
 #include <sys/cpu.h>
+#include <sys/csan.h>
 #include <sys/devmap.h>
 #include <sys/efi.h>
 #include <sys/exec.h>
@@ -1208,6 +1209,8 @@ initarm(struct arm64_bootparams *abp)
 	dbg_init();
 	kdb_init();
 	pan_enable();
+
+	kcsan_cpu_init(0);
 
 	env = kern_getenv("kernelname");
 	if (env != NULL)

Modified: head/sys/arm64/arm64/mp_machdep.c
==============================================================================
--- head/sys/arm64/arm64/mp_machdep.c	Thu Nov 21 08:20:05 2019	(r354941)
+++ head/sys/arm64/arm64/mp_machdep.c	Thu Nov 21 11:22:08 2019	(r354942)
@@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/bus.h>
 #include <sys/cpu.h>
+#include <sys/csan.h>
 #include <sys/kernel.h>
 #include <sys/ktr.h>
 #include <sys/malloc.h>
@@ -252,6 +253,8 @@ init_secondary(uint64_t cpu)
 	}
 
 	mtx_unlock_spin(&ap_boot_mtx);
+
+	kcsan_cpu_init(cpu);
 
 	/* Enter the scheduler */
 	sched_throw(NULL);

Modified: head/sys/arm64/conf/GENERIC
==============================================================================
--- head/sys/arm64/conf/GENERIC	Thu Nov 21 08:20:05 2019	(r354941)
+++ head/sys/arm64/conf/GENERIC	Thu Nov 21 11:22:08 2019	(r354942)
@@ -100,6 +100,7 @@ options 	VERBOSE_SYSINIT=0	# Support debug.verbose_sys
 #options 	KCOV			# Kernel Coverage Sanitizer
 # Warning: KUBSAN can result in a kernel too large for loader to load
 #options 	KUBSAN			# Kernel Undefined Behavior Sanitizer
+#options 	KCSAN			# Kernel Concurrency Sanitizer
 
 # Kernel dump features.
 options 	EKCD			# Support for encrypted kernel dumps

Modified: head/sys/arm64/include/atomic.h
==============================================================================
--- head/sys/arm64/include/atomic.h	Thu Nov 21 08:20:05 2019	(r354941)
+++ head/sys/arm64/include/atomic.h	Thu Nov 21 11:22:08 2019	(r354942)
@@ -29,8 +29,6 @@
 #ifndef	_MACHINE_ATOMIC_H_
 #define	_MACHINE_ATOMIC_H_
 
-#include <sys/atomic_common.h>
-
 #define	isb()		__asm __volatile("isb" : : : "memory")
 
 /*
@@ -55,6 +53,12 @@
 #define	wmb()	dmb(st)	/* Full system memory barrier store */
 #define	rmb()	dmb(ld)	/* Full system memory barrier load */
 
+#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
+#include <sys/_cscan_atomic.h>
+#else
+
+#include <sys/atomic_common.h>
+
 #define	ATOMIC_OP(op, asm_op, bar, a, l)				\
 static __inline void							\
 atomic_##op##_##bar##8(volatile uint8_t *p, uint8_t val)		\
@@ -600,6 +604,8 @@ atomic_thread_fence_seq_cst(void)
 
 	dmb(sy);
 }
+
+#endif /* KCSAN && !KCSAN_RUNTIME */
 
 #endif /* _MACHINE_ATOMIC_H_ */
 

Modified: head/sys/arm64/include/bus.h
==============================================================================
--- head/sys/arm64/include/bus.h	Thu Nov 21 08:20:05 2019	(r354941)
+++ head/sys/arm64/include/bus.h	Thu Nov 21 11:22:08 2019	(r354942)
@@ -89,6 +89,9 @@
 #define	BUS_SPACE_BARRIER_READ	0x01
 #define	BUS_SPACE_BARRIER_WRITE	0x02
 
+#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
+#include <sys/_cscan_bus.h>
+#else
 
 struct bus_space {
 	/* cookie */
@@ -463,6 +466,8 @@ struct bus_space {
 	__bs_copy(4, t, h1, o1, h2, o2, c)
 #define	bus_space_copy_region_8(t, h1, o1, h2, o2, c)				\
 	__bs_copy(8, t, h1, o1, h2, o2, c)
+
+#endif
 
 #include <machine/bus_dma.h>
 

Added: head/sys/arm64/include/csan.h
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ head/sys/arm64/include/csan.h	Thu Nov 21 11:22:08 2019	(r354942)
@@ -0,0 +1,104 @@
+/*	$NetBSD: csan.h,v 1.2 2019/11/06 06:57:22 maxv Exp $	*/
+
+/*
+ * Copyright (c) 2019 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Maxime Villard.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <machine/cpufunc.h>
+#include <machine/stack.h>
+#include <machine/vmparam.h>
+
+static inline bool
+kcsan_md_is_avail(void)
+{
+	return true;
+}
+
+static inline void
+kcsan_md_disable_intrs(uint64_t *state)
+{
+
+	*state = intr_disable();
+}
+
+static inline void
+kcsan_md_enable_intrs(uint64_t *state)
+{
+
+	intr_restore(*state);
+}
+
+static inline void
+kcsan_md_delay(uint64_t us)
+{
+	DELAY(us);
+}
+
+static void
+kcsan_md_unwind(void)
+{
+#ifdef DDB
+	c_db_sym_t sym;
+	db_expr_t offset;
+	const char *symname;
+#endif
+	struct unwind_state frame;
+	uint64_t sp;
+	int nsym;
+
+	__asm __volatile("mov %0, sp" : "=&r" (sp));
+
+	frame.sp = sp;
+	frame.fp = (uint64_t)__builtin_frame_address(0);
+	frame.pc = (uint64_t)kcsan_md_unwind;
+	nsym = 0;
+
+	while (1) {
+		unwind_frame(&frame);
+		if (!INKERNEL((vm_offset_t)frame.fp) ||
+		     !INKERNEL((vm_offset_t)frame.pc))
+			break;
+
+#ifdef DDB
+		sym = db_search_symbol((vm_offset_t)frame.pc, DB_STGY_PROC,
+		    &offset);
+		db_symbol_values(sym, &symname, NULL);
+		printf("#%d %p in %s+%#lx\n", nsym, (void *)frame.pc,
+		    symname, offset);
+#else
+		printf("#%d %p\n", nsym, (void *)frame.pc);
+#endif
+		nsym++;
+
+		if (nsym >= 15) {
+			break;
+		}
+	}
+}

Modified: head/sys/conf/files
==============================================================================
--- head/sys/conf/files	Thu Nov 21 08:20:05 2019	(r354941)
+++ head/sys/conf/files	Thu Nov 21 11:22:08 2019	(r354942)
@@ -3805,6 +3805,8 @@ kern/subr_compressor.c		standard \
 kern/subr_coverage.c		optional coverage \
 	compile-with "${NORMAL_C:N-fsanitize*}"
 kern/subr_counter.c		standard
+kern/subr_csan.c		optional kcsan \
+	compile-with "${NORMAL_C:N-fsanitize*}"
 kern/subr_devstat.c		standard
 kern/subr_disk.c		standard
 kern/subr_early.c		standard

Modified: head/sys/conf/files.arm64
==============================================================================
--- head/sys/conf/files.arm64	Thu Nov 21 08:20:05 2019	(r354941)
+++ head/sys/conf/files.arm64	Thu Nov 21 11:22:08 2019	(r354942)
@@ -286,8 +286,10 @@ kern/pic_if.m			optional	intrng
 kern/subr_devmap.c		standard
 kern/subr_intr.c		optional	intrng
 libkern/bcmp.c			standard
-libkern/memcmp.c		standard
-libkern/memset.c		standard
+libkern/memcmp.c		standard				\
+	compile-with "${NORMAL_C:N-fsanitize*}"
+libkern/memset.c		standard				\
+	compile-with "${NORMAL_C:N-fsanitize*}"
 libkern/arm64/crc32c_armv8.S	standard
 cddl/dev/dtrace/aarch64/dtrace_asm.S			optional dtrace compile-with "${DTRACE_S}"
 cddl/dev/dtrace/aarch64/dtrace_subr.c			optional dtrace compile-with "${DTRACE_C}"

Modified: head/sys/conf/kern.post.mk
==============================================================================
--- head/sys/conf/kern.post.mk	Thu Nov 21 08:20:05 2019	(r354941)
+++ head/sys/conf/kern.post.mk	Thu Nov 21 11:22:08 2019	(r354942)
@@ -38,6 +38,10 @@ MKMODULESENV+=	WITH_CTF="${WITH_CTF}"
 MKMODULESENV+=	WITH_EXTRA_TCP_STACKS="${WITH_EXTRA_TCP_STACKS}"
 .endif
 
+.if defined(KCSAN_ENABLED)
+MKMODULESENV+=	KCSAN_ENABLED="yes"
+.endif
+
 .if defined(SAN_CFLAGS)
 MKMODULESENV+=	SAN_CFLAGS="${SAN_CFLAGS}"
 .endif

Modified: head/sys/conf/kern.pre.mk
==============================================================================
--- head/sys/conf/kern.pre.mk	Thu Nov 21 08:20:05 2019	(r354941)
+++ head/sys/conf/kern.pre.mk	Thu Nov 21 11:22:08 2019	(r354942)
@@ -117,6 +117,11 @@ PROF=		-pg
 .endif
 DEFINED_PROF=	${PROF}
 
+KCSAN_ENABLED!=	grep KCSAN opt_global.h || true ; echo
+.if !empty(KCSAN_ENABLED)
+SAN_CFLAGS+=	-fsanitize=thread
+.endif
+
 KUBSAN_ENABLED!=	grep KUBSAN opt_global.h || true ; echo
 .if !empty(KUBSAN_ENABLED)
 SAN_CFLAGS+=	-fsanitize=undefined

Modified: head/sys/conf/options
==============================================================================
--- head/sys/conf/options	Thu Nov 21 08:20:05 2019	(r354941)
+++ head/sys/conf/options	Thu Nov 21 11:22:08 2019	(r354942)
@@ -230,6 +230,7 @@ ZSTDIO		opt_zstdio.h
 # Sanitizers
 COVERAGE	opt_global.h
 KCOV
+KCSAN		opt_global.h
 KUBSAN		opt_global.h
 
 # POSIX kernel options

Modified: head/sys/kern/subr_csan.c
==============================================================================
--- head/sys/kern/subr_csan.c	Thu Nov 21 08:20:05 2019	(r354941)
+++ head/sys/kern/subr_csan.c	Thu Nov 21 11:22:08 2019	(r354942)
@@ -3,6 +3,7 @@
 /*
  * Copyright (c) 2019 The NetBSD Foundation, Inc.
  * All rights reserved.
+ * Copyright (c) 2019 Andrew Turner
  *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Maxime Villard.
@@ -29,19 +30,26 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
+#define	KCSAN_RUNTIME
+
+#include "opt_ddb.h"
+
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $");
+__FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
-#include <sys/device.h>
 #include <sys/kernel.h>
-#include <sys/param.h>
+#include <sys/bus.h>
 #include <sys/conf.h>
-#include <sys/systm.h>
-#include <sys/types.h>
-#include <sys/csan.h>
 #include <sys/cpu.h>
+#include <sys/csan.h>
+#include <sys/proc.h>
+#include <sys/smp.h>
+#include <sys/systm.h>
 
+#include <ddb/ddb.h>
+#include <ddb/db_sym.h>
+
 #ifdef KCSAN_PANIC
 #define REPORT panic
 #else
@@ -62,7 +70,7 @@ typedef struct {
 	csan_cell_t cell;
 } csan_cpu_t;
 
-static csan_cpu_t kcsan_cpus[MAXCPUS];
+static csan_cpu_t kcsan_cpus[MAXCPU];
 static bool kcsan_enabled __read_mostly;
 
 #define __RET_ADDR	(uintptr_t)__builtin_return_address(0)
@@ -77,34 +85,43 @@ static bool kcsan_enabled __read_mostly;
 
 /* -------------------------------------------------------------------------- */
 
-void
-kcsan_init(void)
+static void
+kcsan_enable(void *dummy __unused)
 {
+
+	printf("Enabling KCSCAN, expect reduced performance.\n");
 	kcsan_enabled = true;
 }
+SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL);
 
 void
-kcsan_cpu_init(struct cpu_info *ci)
+kcsan_cpu_init(u_int cpu)
 {
-	kcsan_cpus[cpu_index(ci)].inited = true;
+	kcsan_cpus[cpu].inited = true;
 }
 
 /* -------------------------------------------------------------------------- */
 
 static inline void
-kcsan_report(csan_cell_t *new, cpuid_t newcpu, csan_cell_t *old, cpuid_t oldcpu)
+kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu)
 {
 	const char *newsym, *oldsym;
+#ifdef DDB
+	c_db_sym_t sym;
+	db_expr_t offset;
 
-	if (ksyms_getname(NULL, &newsym, (vaddr_t)new->pc, KSYMS_PROC) != 0) {
-		newsym = "Unknown";
-	}
-	if (ksyms_getname(NULL, &oldsym, (vaddr_t)old->pc, KSYMS_PROC) != 0) {
-		oldsym = "Unknown";
-	}
+	sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset);
+	db_symbol_values(sym, &newsym, NULL);
+
+	sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset);
+	db_symbol_values(sym, &oldsym, NULL);
+#else
+	newsym = "";
+	oldsym = "";
+#endif
 	REPORT("CSan: Racy Access "
-	    "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>] "
-	    "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>]\n",
+	    "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] "
+	    "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n",
 	    newcpu,
 	    (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"),
 	    (void *)new->addr, new->size, (void *)new->pc, newsym,
@@ -134,8 +151,6 @@ kcsan_access(uintptr_t addr, size_t size, bool write, 
 
 	if (__predict_false(!kcsan_enabled))
 		return;
-	if (__predict_false(kcsan_md_unsupported((vaddr_t)addr)))
-		return;
 
 	new.addr = addr;
 	new.size = size;
@@ -143,7 +158,7 @@ kcsan_access(uintptr_t addr, size_t size, bool write, 
 	new.atomic = atomic;
 	new.pc = pc;
 
-	for (i = 0; i < ncpu; i++) {
+	CPU_FOREACH(i) {
 		__builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old));
 
 		if (old.addr + old.size <= new.addr)
@@ -155,7 +170,7 @@ kcsan_access(uintptr_t addr, size_t size, bool write, 
 		if (__predict_true(kcsan_access_is_atomic(&new, &old)))
 			continue;
 
-		kcsan_report(&new, cpu_number(), &old, i);
+		kcsan_report(&new, PCPU_GET(cpuid), &old, i);
 		break;
 	}
 
@@ -164,7 +179,7 @@ kcsan_access(uintptr_t addr, size_t size, bool write, 
 
 	kcsan_md_disable_intrs(&intr);
 
-	cpu = &kcsan_cpus[cpu_number()];
+	cpu = &kcsan_cpus[PCPU_GET(cpuid)];
 	if (__predict_false(!cpu->inited))
 		goto out;
 	cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES;
@@ -184,6 +199,11 @@ out:
 	void __tsan_read##size(uintptr_t addr)				\
 	{								\
 		kcsan_access(addr, size, false, false, __RET_ADDR);	\
+	}								\
+	void __tsan_unaligned_read##size(uintptr_t);			\
+	void __tsan_unaligned_read##size(uintptr_t addr)		\
+	{								\
+		kcsan_access(addr, size, false, false, __RET_ADDR);	\
 	}
 
 CSAN_READ(1)
@@ -197,6 +217,11 @@ CSAN_READ(16)
 	void __tsan_write##size(uintptr_t addr)				\
 	{								\
 		kcsan_access(addr, size, true, false, __RET_ADDR);	\
+	}								\
+	void __tsan_unaligned_write##size(uintptr_t);			\
+	void __tsan_unaligned_write##size(uintptr_t addr)		\
+	{								\
+		kcsan_access(addr, size, true, false, __RET_ADDR);	\
 	}
 
 CSAN_WRITE(1)
@@ -321,35 +346,14 @@ kcsan_strlen(const char *str)
 	return (s - str);
 }
 
-#undef kcopy
 #undef copystr
-#undef copyinstr
-#undef copyoutstr
 #undef copyin
+#undef copyin_nofault
+#undef copyinstr
 #undef copyout
+#undef copyout_nofault
 
-int	kcsan_kcopy(const void *, void *, size_t);
-int	kcsan_copystr(const void *, void *, size_t, size_t *);
-int	kcsan_copyinstr(const void *, void *, size_t, size_t *);
-int	kcsan_copyoutstr(const void *, void *, size_t, size_t *);
-int	kcsan_copyin(const void *, void *, size_t);
-int	kcsan_copyout(const void *, void *, size_t);
-int	kcopy(const void *, void *, size_t);
-int	copystr(const void *, void *, size_t, size_t *);
-int	copyinstr(const void *, void *, size_t, size_t *);
-int	copyoutstr(const void *, void *, size_t, size_t *);
-int	copyin(const void *, void *, size_t);
-int	copyout(const void *, void *, size_t);
-
 int
-kcsan_kcopy(const void *src, void *dst, size_t len)
-{
-	kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
-	kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
-	return kcopy(src, dst, len);
-}
-
-int
 kcsan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done)
 {
 	kcsan_access((uintptr_t)kdaddr, len, true, false, __RET_ADDR);
@@ -364,13 +368,6 @@ kcsan_copyin(const void *uaddr, void *kaddr, size_t le
 }
 
 int
-kcsan_copyout(const void *kaddr, void *uaddr, size_t len)
-{
-	kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR);
-	return copyout(kaddr, uaddr, len);
-}
-
-int
 kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
 {
 	kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
@@ -378,377 +375,477 @@ kcsan_copyinstr(const void *uaddr, void *kaddr, size_t
 }
 
 int
-kcsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done)
+kcsan_copyout(const void *kaddr, void *uaddr, size_t len)
 {
 	kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR);
-	return copyoutstr(kaddr, uaddr, len, done);
+	return copyout(kaddr, uaddr, len);
 }
 
 /* -------------------------------------------------------------------------- */
 
-#undef atomic_add_32
-#undef atomic_add_int
-#undef atomic_add_long
-#undef atomic_add_ptr
-#undef atomic_add_64
-#undef atomic_add_32_nv
-#undef atomic_add_int_nv
-#undef atomic_add_long_nv
-#undef atomic_add_ptr_nv
-#undef atomic_add_64_nv
-#undef atomic_and_32
-#undef atomic_and_uint
-#undef atomic_and_ulong
-#undef atomic_and_64
-#undef atomic_and_32_nv
-#undef atomic_and_uint_nv
-#undef atomic_and_ulong_nv
-#undef atomic_and_64_nv
-#undef atomic_or_32
-#undef atomic_or_uint
-#undef atomic_or_ulong
-#undef atomic_or_64
-#undef atomic_or_32_nv
-#undef atomic_or_uint_nv
-#undef atomic_or_ulong_nv
-#undef atomic_or_64_nv
-#undef atomic_cas_32
-#undef atomic_cas_uint
-#undef atomic_cas_ulong
-#undef atomic_cas_ptr
-#undef atomic_cas_64
-#undef atomic_cas_32_ni
-#undef atomic_cas_uint_ni
-#undef atomic_cas_ulong_ni
-#undef atomic_cas_ptr_ni
-#undef atomic_cas_64_ni
-#undef atomic_swap_32
-#undef atomic_swap_uint
-#undef atomic_swap_ulong
-#undef atomic_swap_ptr
-#undef atomic_swap_64
-#undef atomic_dec_32
-#undef atomic_dec_uint
-#undef atomic_dec_ulong
-#undef atomic_dec_ptr
-#undef atomic_dec_64
-#undef atomic_dec_32_nv
-#undef atomic_dec_uint_nv
-#undef atomic_dec_ulong_nv
-#undef atomic_dec_ptr_nv
-#undef atomic_dec_64_nv
-#undef atomic_inc_32
-#undef atomic_inc_uint
-#undef atomic_inc_ulong
-#undef atomic_inc_ptr
-#undef atomic_inc_64
-#undef atomic_inc_32_nv
-#undef atomic_inc_uint_nv
-#undef atomic_inc_ulong_nv
-#undef atomic_inc_ptr_nv
-#undef atomic_inc_64_nv
+#include <machine/atomic.h>
+#include <sys/_cscan_atomic.h>
 
-#define CSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \
-	void atomic_add_##name(volatile targ1 *, targ2); \
-	void kcsan_atomic_add_##name(volatile targ1 *, targ2); \
-	void kcsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \
-	{ \
-		kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
-		    __RET_ADDR); \
-		atomic_add_##name(ptr, val); \
-	} \
-	tret atomic_add_##name##_nv(volatile targ1 *, targ2); \
-	tret kcsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \
-	tret kcsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \
-	{ \
-		kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
-		    __RET_ADDR); \
-		return atomic_add_##name##_nv(ptr, val); \
+#define	_CSAN_ATOMIC_FUNC_ADD(name, type)				\
+	void kcsan_atomic_add_##name(volatile type *ptr, type val)	\
+	{								\
+		kcsan_access((uintptr_t)ptr, sizeof(type), true, true,	\
+		    __RET_ADDR);					\
+		atomic_add_##name(ptr, val); 				\
 	}
 
-#define CSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \
-	void atomic_and_##name(volatile targ1 *, targ2); \
-	void kcsan_atomic_and_##name(volatile targ1 *, targ2); \
-	void kcsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \
-	{ \
-		kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
-		    __RET_ADDR); \
-		atomic_and_##name(ptr, val); \
-	} \
-	tret atomic_and_##name##_nv(volatile targ1 *, targ2); \
-	tret kcsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \
-	tret kcsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \
-	{ \
-		kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
-		    __RET_ADDR); \
-		return atomic_and_##name##_nv(ptr, val); \
+#define	CSAN_ATOMIC_FUNC_ADD(name, type)				\
+	_CSAN_ATOMIC_FUNC_ADD(name, type)				\
+	_CSAN_ATOMIC_FUNC_ADD(acq_##name, type)				\
+	_CSAN_ATOMIC_FUNC_ADD(rel_##name, type)
+
+#define	_CSAN_ATOMIC_FUNC_CLEAR(name, type)				\
+	void kcsan_atomic_clear_##name(volatile type *ptr, type val)	\
+	{								\
+		kcsan_access((uintptr_t)ptr, sizeof(type), true, true,	\
+		    __RET_ADDR);					\
+		atomic_clear_##name(ptr, val); 				\
 	}
 
-#define CSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \
-	void atomic_or_##name(volatile targ1 *, targ2); \
-	void kcsan_atomic_or_##name(volatile targ1 *, targ2); \
-	void kcsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \
-	{ \
-		kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
-		    __RET_ADDR); \
-		atomic_or_##name(ptr, val); \
-	} \
-	tret atomic_or_##name##_nv(volatile targ1 *, targ2); \
-	tret kcsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \
-	tret kcsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \
-	{ \
-		kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
-		    __RET_ADDR); \
-		return atomic_or_##name##_nv(ptr, val); \
+#define	CSAN_ATOMIC_FUNC_CLEAR(name, type)				\
+	_CSAN_ATOMIC_FUNC_CLEAR(name, type)				\
+	_CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type)			\
+	_CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
+
+#define	_CSAN_ATOMIC_FUNC_CMPSET(name, type)				\
+	int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1,	\
+	    type val2)							\
+	{								\
+		kcsan_access((uintptr_t)ptr, sizeof(type), true, true,	\
+		    __RET_ADDR);					\
+		return (atomic_cmpset_##name(ptr, val1, val2));		\
 	}
 
-#define CSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \
-	tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \
-	tret kcsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
-	tret kcsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \
-	{ \
-		kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
-		    __RET_ADDR); \
-		return atomic_cas_##name(ptr, exp, new); \
-	} \
-	tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
-	tret kcsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
-	tret kcsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \
-	{ \
-		kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
-		    __RET_ADDR); \
-		return atomic_cas_##name##_ni(ptr, exp, new); \
+#define	CSAN_ATOMIC_FUNC_CMPSET(name, type)				\
+	_CSAN_ATOMIC_FUNC_CMPSET(name, type)				\
+	_CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type)			\
+	_CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
+
+#define	_CSAN_ATOMIC_FUNC_FCMPSET(name, type)				\
+	int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1,	\
+	    type val2)							\
+	{								\
+		kcsan_access((uintptr_t)ptr, sizeof(type), true, true,	\
+		    __RET_ADDR);					\
+		return (atomic_fcmpset_##name(ptr, val1, val2));	\
 	}
 
-#define CSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \
-	tret atomic_swap_##name(volatile targ1 *, targ2); \
-	tret kcsan_atomic_swap_##name(volatile targ1 *, targ2); \
-	tret kcsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \
-	{ \
-		kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
-		    __RET_ADDR); \
-		return atomic_swap_##name(ptr, val); \
+#define	CSAN_ATOMIC_FUNC_FCMPSET(name, type)				\
+	_CSAN_ATOMIC_FUNC_FCMPSET(name, type)				\
+	_CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type)			\
+	_CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
+
+#define	CSAN_ATOMIC_FUNC_FETCHADD(name, type)				\
+	type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val)	\
+	{								\
+		kcsan_access((uintptr_t)ptr, sizeof(type), true, true,	\
+		    __RET_ADDR);					\
+		return (atomic_fetchadd_##name(ptr, val));		\
 	}
 
-#define CSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \
-	void atomic_dec_##name(volatile targ1 *); \
-	void kcsan_atomic_dec_##name(volatile targ1 *); \
-	void kcsan_atomic_dec_##name(volatile targ1 *ptr) \
-	{ \
-		kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
-		    __RET_ADDR); \

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***


More information about the svn-src-head mailing list