git: a422084abbda - main - Add the KMSAN runtime
Mark Johnston
markj at FreeBSD.org
Wed Aug 11 01:31:03 UTC 2021
The branch main has been updated by markj:
URL: https://cgit.FreeBSD.org/src/commit/?id=a422084abbda10edc0b591021536a7c9c6d0c8b4
commit a422084abbda10edc0b591021536a7c9c6d0c8b4
Author: Mark Johnston <markj at FreeBSD.org>
AuthorDate: 2021-08-10 19:52:36 +0000
Commit: Mark Johnston <markj at FreeBSD.org>
CommitDate: 2021-08-11 01:27:53 +0000
Add the KMSAN runtime
KMSAN enables the use of LLVM's MemorySanitizer in the kernel. This
enables precise detection of uses of uninitialized memory. As with
KASAN, this feature has substantial runtime overhead and is intended to
be used as part of some automated testing regime.
The runtime maintains a pair of shadow maps. One is used to track the
state of memory in the kernel map at bit-granularity: a bit in the
kernel map is initialized when the corresponding shadow bit is clear,
and is uninitialized otherwise. The second shadow map stores
information about the origin of uninitialized regions of the kernel map,
simplifying debugging.
KMSAN relies on being able to intercept certain functions which cannot
be instrumented by the compiler. KMSAN thus implements interceptors
which manually update shadow state and in some cases explicitly check
for uninitialized bytes. For instance, all calls to copyout() are
subject to such checks.
The runtime exports several functions which can be used to verify the
shadow map for a given buffer. Helpers provide the same functionality
for a few structures commonly used for I/O, such as CAM CCBs, BIOs and
mbufs. These are handy when debugging a KMSAN report whose
proximate and root causes are far away from each other.
Obtained from: NetBSD
Sponsored by: The FreeBSD Foundation
---
sys/amd64/include/msan.h | 84 +++
sys/conf/files | 2 +
sys/kern/kern_thread.c | 14 +-
sys/kern/subr_msan.c | 1619 ++++++++++++++++++++++++++++++++++++++++++++++
sys/sys/msan.h | 87 +++
sys/sys/proc.h | 2 +
6 files changed, 1801 insertions(+), 7 deletions(-)
diff --git a/sys/amd64/include/msan.h b/sys/amd64/include/msan.h
new file mode 100644
index 000000000000..431fd25ae915
--- /dev/null
+++ b/sys/amd64/include/msan.h
@@ -0,0 +1,84 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2021 The FreeBSD Foundation
+ *
+ * This software was developed by Mark Johnston under sponsorship from the
+ * FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_MSAN_H_
+#define _MACHINE_MSAN_H_
+
+#ifdef KMSAN
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_page.h>
+#include <machine/vmparam.h>
+
+typedef uint32_t msan_orig_t;
+
+/*
+ * Our 32-bit origin cells encode a 2-bit type and 30-bit pointer. The pointer
+ * is compressed by making it a positive offset relative to KERNBASE.
+ */
+#define KMSAN_ORIG_TYPE_SHIFT 30u
+#define KMSAN_ORIG_PTR_MASK ((1u << KMSAN_ORIG_TYPE_SHIFT) - 1)
+
+static inline msan_orig_t
+kmsan_md_orig_encode(int type, uintptr_t ptr)
+{
+ return ((type << KMSAN_ORIG_TYPE_SHIFT) |
+ ((ptr & KMSAN_ORIG_PTR_MASK)));
+}
+
+static inline void
+kmsan_md_orig_decode(msan_orig_t orig, int *type, uintptr_t *ptr)
+{
+ *type = orig >> KMSAN_ORIG_TYPE_SHIFT;
+ *ptr = (orig & KMSAN_ORIG_PTR_MASK) | KERNBASE;
+}
+
+static inline vm_offset_t
+kmsan_md_addr_to_shad(vm_offset_t addr)
+{
+ return (addr - VM_MIN_KERNEL_ADDRESS + KMSAN_SHAD_MIN_ADDRESS);
+}
+
+static inline vm_offset_t
+kmsan_md_addr_to_orig(vm_offset_t addr)
+{
+ return (addr - VM_MIN_KERNEL_ADDRESS + KMSAN_ORIG_MIN_ADDRESS);
+}
+
+static inline bool
+kmsan_md_unsupported(vm_offset_t addr)
+{
+ return (addr < VM_MIN_KERNEL_ADDRESS || addr >= KERNBASE);
+}
+
+#endif /* KMSAN */
+
+#endif /* !_MACHINE_MSAN_H_ */
diff --git a/sys/conf/files b/sys/conf/files
index 44eda0e8bae6..a5690ec1df75 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -3925,6 +3925,8 @@ kern/subr_lock.c standard
kern/subr_log.c standard
kern/subr_mchain.c optional libmchain
kern/subr_module.c standard
+kern/subr_msan.c optional kmsan \
+ compile-with "${NORMAL_C:N-fsanitize*}"
kern/subr_msgbuf.c standard
kern/subr_param.c standard
kern/subr_pcpu.c standard
diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c
index 4cfd502664bd..28a517e9b35e 100644
--- a/sys/kern/kern_thread.c
+++ b/sys/kern/kern_thread.c
@@ -84,11 +84,11 @@ __FBSDID("$FreeBSD$");
* structures.
*/
#ifdef __amd64__
-_Static_assert(offsetof(struct thread, td_flags) == 0xfc,
+_Static_assert(offsetof(struct thread, td_flags) == 0x108,
"struct thread KBI td_flags");
-_Static_assert(offsetof(struct thread, td_pflags) == 0x104,
+_Static_assert(offsetof(struct thread, td_pflags) == 0x110,
"struct thread KBI td_pflags");
-_Static_assert(offsetof(struct thread, td_frame) == 0x4a0,
+_Static_assert(offsetof(struct thread, td_frame) == 0x4a8,
"struct thread KBI td_frame");
_Static_assert(offsetof(struct thread, td_emuldata) == 0x6b0,
"struct thread KBI td_emuldata");
@@ -104,13 +104,13 @@ _Static_assert(offsetof(struct proc, p_emuldata) == 0x4b8,
"struct proc KBI p_emuldata");
#endif
#ifdef __i386__
-_Static_assert(offsetof(struct thread, td_flags) == 0x98,
+_Static_assert(offsetof(struct thread, td_flags) == 0x9c,
"struct thread KBI td_flags");
-_Static_assert(offsetof(struct thread, td_pflags) == 0xa0,
+_Static_assert(offsetof(struct thread, td_pflags) == 0xa4,
"struct thread KBI td_pflags");
-_Static_assert(offsetof(struct thread, td_frame) == 0x304,
+_Static_assert(offsetof(struct thread, td_frame) == 0x308,
"struct thread KBI td_frame");
-_Static_assert(offsetof(struct thread, td_emuldata) == 0x348,
+_Static_assert(offsetof(struct thread, td_emuldata) == 0x34c,
"struct thread KBI td_emuldata");
_Static_assert(offsetof(struct proc, p_flag) == 0x6c,
"struct proc KBI p_flag");
diff --git a/sys/kern/subr_msan.c b/sys/kern/subr_msan.c
new file mode 100644
index 000000000000..2b6a0d585128
--- /dev/null
+++ b/sys/kern/subr_msan.c
@@ -0,0 +1,1619 @@
+/* $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $ */
+
+/*
+ * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net
+ * All rights reserved.
+ * Copyright (c) 2021 The FreeBSD Foundation
+ *
+ * Portions of this software were developed by Mark Johnston under sponsorship
+ * from the FreeBSD Foundation.
+ *
+ * This code is part of the KMSAN subsystem of the NetBSD kernel.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#define SAN_RUNTIME
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+#if 0
+__KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $");
+#endif
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bio.h>
+#include <sys/buf.h>
+#include <sys/conf.h>
+#include <sys/kdb.h>
+#include <sys/kernel.h>
+#include <sys/linker.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/memdesc.h>
+#include <sys/msan.h>
+#include <sys/proc.h>
+#include <sys/stack.h>
+#include <sys/sysctl.h>
+#include <sys/uio.h>
+
+#include <cam/cam.h>
+#include <cam/cam_ccb.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/msan.h>
+#include <machine/stdarg.h>
+
+void kmsan_init_arg(size_t);
+void kmsan_init_ret(size_t);
+
+/* -------------------------------------------------------------------------- */
+
+/*
+ * Part of the compiler ABI.
+ */
+
+typedef struct {
+ uint8_t *shad;
+ msan_orig_t *orig;
+} msan_meta_t;
+
+#define MSAN_PARAM_SIZE 800
+#define MSAN_RETVAL_SIZE 800
+typedef struct {
+ uint8_t param_shadow[MSAN_PARAM_SIZE];
+ uint8_t retval_shadow[MSAN_RETVAL_SIZE];
+ uint8_t va_arg_shadow[MSAN_PARAM_SIZE];
+ uint8_t va_arg_origin[MSAN_PARAM_SIZE];
+ uint64_t va_arg_overflow_size;
+ msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)];
+ msan_orig_t retval_origin;
+} msan_tls_t;
+
+/* -------------------------------------------------------------------------- */
+
+#define MSAN_NCONTEXT 4
+#define MSAN_ORIG_MASK (~0x3)
+
+typedef struct kmsan_td {
+ size_t ctx;
+ msan_tls_t tls[MSAN_NCONTEXT];
+} msan_td_t;
+
+static msan_tls_t dummy_tls;
+
+/*
+ * Use separate dummy regions for loads and stores: stores may mark the region
+ * as uninitialized, and that can trigger false positives.
+ */
+static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
+static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
+static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE);
+static msan_td_t msan_thread0;
+static bool kmsan_enabled __read_mostly;
+
+static bool kmsan_reporting = false;
+
+/*
+ * Avoid clobbering any thread-local state before we panic.
+ */
+#define kmsan_panic(f, ...) do { \
+ kmsan_enabled = false; \
+ panic(f, __VA_ARGS__); \
+} while (0)
+
+#define REPORT(f, ...) do { \
+ if (panic_on_violation) { \
+ kmsan_panic(f, __VA_ARGS__); \
+ } else { \
+ struct stack st; \
+ \
+ stack_save(&st); \
+ printf(f "\n", __VA_ARGS__); \
+ stack_print_ddb(&st); \
+ } \
+} while (0)
+
+FEATURE(kmsan, "Kernel memory sanitizer");
+
+static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
+ "KMSAN options");
+
+static bool panic_on_violation = 1;
+SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN,
+ &panic_on_violation, 0,
+ "Panic if an invalid access is detected");
+
+static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer");
+
+/* -------------------------------------------------------------------------- */
+
+static inline const char *
+kmsan_orig_name(int type)
+{
+ switch (type) {
+ case KMSAN_TYPE_STACK:
+ return ("stack");
+ case KMSAN_TYPE_KMEM:
+ return ("kmem");
+ case KMSAN_TYPE_MALLOC:
+ return ("malloc");
+ case KMSAN_TYPE_UMA:
+ return ("UMA");
+ default:
+ return ("unknown");
+ }
+}
+
+static void
+kmsan_report_hook(const void *addr, size_t size, size_t off, const char *hook)
+{
+ msan_orig_t *orig;
+ const char *typename;
+ char *var, *fn;
+ uintptr_t ptr;
+ long foff;
+ char buf[128];
+ int type;
+
+ if (__predict_false(panicstr != NULL || kdb_active || kmsan_reporting))
+ return;
+
+ kmsan_reporting = true;
+ __compiler_membar();
+
+ orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
+ orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
+
+ if (*orig == 0) {
+ REPORT("MSan: Uninitialized memory in %s, offset %zu",
+ hook, off);
+ goto out;
+ }
+
+ kmsan_md_orig_decode(*orig, &type, &ptr);
+ typename = kmsan_orig_name(type);
+
+ if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
+ sizeof(buf), &foff) == 0) {
+ REPORT("MSan: Uninitialized %s memory in %s, "
+ "offset %zu/%zu, addr %p, from %s+%#lx",
+ typename, hook, off, size, addr, buf, foff);
+ } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
+ /*
+ * The format of the string is: "----var at function". Parse it to
+ * display a nice warning.
+ */
+ var = (char *)ptr + 4;
+ strlcpy(buf, var, sizeof(buf));
+ var = buf;
+ fn = strchr(buf, '@');
+ *fn++ = '\0';
+ REPORT("MSan: Uninitialized %s memory in %s, offset %zu, "
+ "variable '%s' from %s", typename, hook, off, var, fn);
+ } else {
+ REPORT("MSan: Uninitialized %s memory in %s, "
+ "offset %zu/%zu, addr %p, PC %p",
+ typename, hook, off, size, addr, (void *)ptr);
+ }
+
+out:
+ __compiler_membar();
+ kmsan_reporting = false;
+}
+
+static void
+kmsan_report_inline(msan_orig_t orig, unsigned long pc)
+{
+ const char *typename;
+ char *var, *fn;
+ uintptr_t ptr;
+ char buf[128];
+ long foff;
+ int type;
+
+ if (__predict_false(panicstr != NULL || kdb_active || kmsan_reporting))
+ return;
+
+ kmsan_reporting = true;
+ __compiler_membar();
+
+ if (orig == 0) {
+ REPORT("MSan: uninitialized variable in %p", (void *)pc);
+ goto out;
+ }
+
+ kmsan_md_orig_decode(orig, &type, &ptr);
+ typename = kmsan_orig_name(type);
+
+ if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
+ sizeof(buf), &foff) == 0) {
+ REPORT("MSan: Uninitialized %s memory from %s+%#lx",
+ typename, buf, foff);
+ } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
+ /*
+ * The format of the string is: "----var at function". Parse it to
+ * display a nice warning.
+ */
+ var = (char *)ptr + 4;
+ strlcpy(buf, var, sizeof(buf));
+ var = buf;
+ fn = strchr(buf, '@');
+ *fn++ = '\0';
+ REPORT("MSan: Uninitialized variable '%s' from %s", var, fn);
+ } else {
+ REPORT("MSan: Uninitialized %s memory, origin %x",
+ typename, orig);
+ }
+
+out:
+ __compiler_membar();
+ kmsan_reporting = false;
+}
+
+/* -------------------------------------------------------------------------- */
+
+static inline msan_meta_t
+kmsan_meta_get(const void *addr, size_t size, const bool write)
+{
+ msan_meta_t ret;
+
+ if (__predict_false(!kmsan_enabled)) {
+ ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
+ ret.orig = (msan_orig_t *)msan_dummy_orig;
+ } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) {
+ ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
+ ret.orig = (msan_orig_t *)msan_dummy_orig;
+ } else {
+ ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr);
+ ret.orig =
+ (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
+ ret.orig = (msan_orig_t *)((uintptr_t)ret.orig &
+ MSAN_ORIG_MASK);
+ }
+
+ return (ret);
+}
+
+static inline void
+kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size)
+{
+ msan_orig_t *orig;
+ size_t i;
+
+ if (__predict_false(!kmsan_enabled))
+ return;
+ if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr)))
+ return;
+
+ orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
+ size += ((uintptr_t)orig & (sizeof(*orig) - 1));
+ orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
+
+ for (i = 0; i < size; i += 4) {
+ orig[i / 4] = o;
+ }
+}
+
+static inline void
+kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size)
+{
+ uint8_t *shad;
+
+ if (__predict_false(!kmsan_enabled))
+ return;
+ if (__predict_false(kmsan_md_unsupported(addr)))
+ return;
+
+ shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
+ __builtin_memset(shad, c, size);
+}
+
+static inline void
+kmsan_meta_copy(void *dst, const void *src, size_t size)
+{
+ uint8_t *orig_src, *orig_dst;
+ uint8_t *shad_src, *shad_dst;
+ msan_orig_t *_src, *_dst;
+ size_t i;
+
+ if (__predict_false(!kmsan_enabled))
+ return;
+ if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst)))
+ return;
+ if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) {
+ kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size);
+ return;
+ }
+
+ shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src);
+ shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst);
+ __builtin_memmove(shad_dst, shad_src, size);
+
+ orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src);
+ orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst);
+ for (i = 0; i < size; i++) {
+ _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK);
+ _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK);
+ *_dst = *_src;
+ orig_src++;
+ orig_dst++;
+ }
+}
+
+static inline void
+kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook)
+{
+ uint8_t *shad;
+ size_t i;
+
+ if (__predict_false(!kmsan_enabled))
+ return;
+ if (__predict_false(kmsan_md_unsupported(addr)))
+ return;
+
+ shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
+ for (i = 0; i < size; i++) {
+ if (__predict_true(shad[i] == 0))
+ continue;
+ kmsan_report_hook((const char *)addr + i, size, i, hook);
+ break;
+ }
+}
+
+void
+kmsan_init_arg(size_t n)
+{
+ msan_td_t *mtd;
+ uint8_t *arg;
+
+ if (__predict_false(!kmsan_enabled))
+ return;
+ if (__predict_false(curthread == NULL))
+ return;
+ mtd = curthread->td_kmsan;
+ arg = mtd->tls[mtd->ctx].param_shadow;
+ __builtin_memset(arg, 0, n);
+}
+
+void
+kmsan_init_ret(size_t n)
+{
+ msan_td_t *mtd;
+ uint8_t *arg;
+
+ if (__predict_false(!kmsan_enabled))
+ return;
+ if (__predict_false(curthread == NULL))
+ return;
+ mtd = curthread->td_kmsan;
+ arg = mtd->tls[mtd->ctx].retval_shadow;
+ __builtin_memset(arg, 0, n);
+}
+
+static void
+kmsan_check_arg(size_t size, const char *hook)
+{
+ msan_td_t *mtd;
+ uint8_t *arg;
+ size_t i;
+
+ if (__predict_false(!kmsan_enabled))
+ return;
+ if (__predict_false(curthread == NULL))
+ return;
+ mtd = curthread->td_kmsan;
+ arg = mtd->tls[mtd->ctx].param_shadow;
+
+ for (i = 0; i < size; i++) {
+ if (__predict_true(arg[i] == 0))
+ continue;
+ kmsan_report_hook((const char *)arg + i, size, i, hook);
+ break;
+ }
+}
+
+void
+kmsan_thread_alloc(struct thread *td)
+{
+ msan_td_t *mtd;
+
+ if (!kmsan_enabled)
+ return;
+
+ mtd = td->td_kmsan;
+ if (mtd == NULL) {
+ /* We might be recycling a thread. */
+ kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) +
+ sizeof(int));
+ mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK);
+ }
+ kmsan_memset(mtd, 0, sizeof(*mtd));
+ mtd->ctx = 0;
+
+ if (td->td_kstack != 0)
+ kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages),
+ KMSAN_STATE_UNINIT);
+
+ td->td_kmsan = mtd;
+}
+
+void
+kmsan_thread_free(struct thread *td)
+{
+ msan_td_t *mtd;
+
+ if (!kmsan_enabled)
+ return;
+ if (__predict_false(td == curthread))
+ kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__);
+
+ mtd = td->td_kmsan;
+ kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *));
+ free(mtd, M_KMSAN);
+ td->td_kmsan = NULL;
+}
+
+void kmsan_intr_enter(void);
+void kmsan_intr_leave(void);
+
+void
+kmsan_intr_enter(void)
+{
+ msan_td_t *mtd;
+
+ if (__predict_false(!kmsan_enabled))
+ return;
+
+ mtd = curthread->td_kmsan;
+ mtd->ctx++;
+ if (__predict_false(mtd->ctx >= MSAN_NCONTEXT))
+ kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
+}
+
+void
+kmsan_intr_leave(void)
+{
+ msan_td_t *mtd;
+
+ if (__predict_false(!kmsan_enabled))
+ return;
+
+ mtd = curthread->td_kmsan;
+ if (__predict_false(mtd->ctx == 0))
+ kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
+ mtd->ctx--;
+}
+
+/* -------------------------------------------------------------------------- */
+
+void
+kmsan_shadow_map(vm_offset_t addr, size_t size)
+{
+ size_t npages, i;
+ vm_offset_t va;
+
+ MPASS(addr % PAGE_SIZE == 0);
+ MPASS(size % PAGE_SIZE == 0);
+
+ if (!kmsan_enabled)
+ return;
+
+ npages = atop(size);
+
+ va = kmsan_md_addr_to_shad(addr);
+ for (i = 0; i < npages; i++) {
+ pmap_kmsan_enter(va + ptoa(i));
+ }
+
+ va = kmsan_md_addr_to_orig(addr);
+ for (i = 0; i < npages; i++) {
+ pmap_kmsan_enter(va + ptoa(i));
+ }
+}
+
+void
+kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc)
+{
+ msan_orig_t orig;
+
+ orig = kmsan_md_orig_encode(type, pc);
+ kmsan_origin_fill(addr, orig, size);
+}
+
+void
+kmsan_mark(const void *addr, size_t size, uint8_t c)
+{
+ kmsan_shadow_fill((uintptr_t)addr, c, size);
+}
+
+static void
+kmsan_mark_bio(const struct bio *bp, uint8_t c)
+{
+ kmsan_mark(bp->bio_data, bp->bio_length, c);
+}
+
+static void
+kmsan_mark_ccb(const union ccb *ccb, uint8_t c)
+{
+ if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_IN)
+ return;
+ if ((ccb->ccb_h.flags & CAM_DATA_MASK) != CAM_DATA_VADDR)
+ return;
+
+ switch (ccb->ccb_h.func_code) {
+ case XPT_SCSI_IO: {
+ const struct ccb_scsiio *scsiio;
+
+ scsiio = &ccb->ctio;
+ kmsan_mark(scsiio->data_ptr, scsiio->dxfer_len, c);
+ break;
+ }
+ case XPT_ATA_IO: {
+ const struct ccb_ataio *ataio;
+
+ ataio = &ccb->ataio;
+ kmsan_mark(ataio->data_ptr, ataio->dxfer_len, c);
+ break;
+ }
+ case XPT_NVME_IO: {
+ const struct ccb_nvmeio *nvmeio;
+
+ nvmeio = &ccb->nvmeio;
+ kmsan_mark(nvmeio->data_ptr, nvmeio->dxfer_len, c);
+ break;
+ }
+ default:
+ kmsan_panic("%s: unhandled CCB type %d", __func__,
+ ccb->ccb_h.func_code);
+ }
+}
+
+static void
+kmsan_mark_mbuf(const struct mbuf *m, uint8_t c)
+{
+ do {
+ if ((m->m_flags & M_EXTPG) == 0)
+ kmsan_mark(m->m_data, m->m_len, c);
+ m = m->m_next;
+ } while (m != NULL);
+}
+
+void
+kmsan_check(const void *p, size_t sz, const char *descr)
+{
+ kmsan_shadow_check((uintptr_t)p, sz, descr);
+}
+
+void
+kmsan_check_bio(const struct bio *bp, const char *descr)
+{
+ kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr);
+}
+
+void
+kmsan_check_ccb(const union ccb *ccb, const char *descr)
+{
+ if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_OUT)
+ return;
+ switch (ccb->ccb_h.func_code) {
+ case XPT_SCSI_IO: {
+ const struct ccb_scsiio *scsiio;
+
+ scsiio = &ccb->ctio;
+ kmsan_check(scsiio->data_ptr, scsiio->dxfer_len, descr);
+ break;
+ }
+ case XPT_ATA_IO: {
+ const struct ccb_ataio *ataio;
+
+ ataio = &ccb->ataio;
+ kmsan_check(ataio->data_ptr, ataio->dxfer_len, descr);
+ break;
+ }
+ case XPT_NVME_IO: {
+ const struct ccb_nvmeio *nvmeio;
+
+ nvmeio = &ccb->nvmeio;
+ kmsan_check(nvmeio->data_ptr, nvmeio->dxfer_len, descr);
+ break;
+ }
+ default:
+ kmsan_panic("%s: unhandled CCB type %d", __func__,
+ ccb->ccb_h.func_code);
+ }
+}
+
+void
+kmsan_check_mbuf(const struct mbuf *m, const char *descr)
+{
+ do {
+ kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr);
+ } while ((m = m->m_next) != NULL);
+}
+
+void
+kmsan_init(void)
+{
+ int disabled;
+
+ disabled = 0;
+ TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled);
+ if (disabled)
+ return;
+
+ /* Initialize the TLS for curthread. */
+ msan_thread0.ctx = 0;
+ thread0.td_kmsan = &msan_thread0;
+
+ /* Now officially enabled. */
+ kmsan_enabled = true;
+}
+
+/* -------------------------------------------------------------------------- */
+
+msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t);
+msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t);
+
+msan_meta_t
+__msan_metadata_ptr_for_load_n(void *addr, size_t size)
+{
+ return (kmsan_meta_get(addr, size, false));
+}
+
+msan_meta_t
+__msan_metadata_ptr_for_store_n(void *addr, size_t size)
+{
+ return (kmsan_meta_get(addr, size, true));
+}
+
+#define MSAN_META_FUNC(size) \
+ msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \
+ msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \
+ { \
+ return (kmsan_meta_get(addr, size, false)); \
+ } \
+ msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \
+ msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \
+ { \
+ return (kmsan_meta_get(addr, size, true)); \
+ }
+
+MSAN_META_FUNC(1)
+MSAN_META_FUNC(2)
+MSAN_META_FUNC(4)
+MSAN_META_FUNC(8)
+
+void __msan_instrument_asm_store(const void *, size_t);
+msan_orig_t __msan_chain_origin(msan_orig_t);
+void __msan_poison(const void *, size_t);
+void __msan_unpoison(const void *, size_t);
+void __msan_poison_alloca(const void *, uint64_t, const char *);
+void __msan_unpoison_alloca(const void *, uint64_t);
+void __msan_warning(msan_orig_t);
+msan_tls_t *__msan_get_context_state(void);
+
+void
+__msan_instrument_asm_store(const void *addr, size_t size)
+{
+ kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
+}
+
+msan_orig_t
+__msan_chain_origin(msan_orig_t origin)
+{
+ return (origin);
+}
+
+void
+__msan_poison(const void *addr, size_t size)
+{
+ kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
+}
+
+void
+__msan_unpoison(const void *addr, size_t size)
+{
+ kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
+}
+
+void
+__msan_poison_alloca(const void *addr, uint64_t size, const char *descr)
+{
+ msan_orig_t orig;
+
+ orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr);
+ kmsan_origin_fill(addr, orig, size);
+ kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
+}
+
+void
+__msan_unpoison_alloca(const void *addr, uint64_t size)
+{
+ kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
+}
+
+void
+__msan_warning(msan_orig_t origin)
+{
+ if (__predict_false(!kmsan_enabled))
+ return;
+ kmsan_report_inline(origin, KMSAN_RET_ADDR);
+}
+
+msan_tls_t *
+__msan_get_context_state(void)
+{
+ msan_td_t *mtd;
+
+ /*
+ * When APs are started, they execute some C code before curthread is
+ * set. We have to handle that here.
+ */
+ if (__predict_false(!kmsan_enabled || curthread == NULL))
+ return (&dummy_tls);
+ mtd = curthread->td_kmsan;
+ return (&mtd->tls[mtd->ctx]);
+}
+
+/* -------------------------------------------------------------------------- */
+
+/*
+ * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle
+ * initialized areas properly.
+ */
+
+void *
+kmsan_memcpy(void *dst, const void *src, size_t len)
+{
+ /* No kmsan_check_arg, because inlined. */
+ kmsan_init_ret(sizeof(void *));
+ if (__predict_true(len != 0)) {
+ kmsan_meta_copy(dst, src, len);
*** 938 LINES SKIPPED ***
More information about the dev-commits-src-all
mailing list