svn commit: r343550 - in head/sys: amd64/conf arm64/conf conf kern sys
Andrew Turner
andrew at FreeBSD.org
Tue Jan 29 11:04:20 UTC 2019
Author: andrew
Date: Tue Jan 29 11:04:17 2019
New Revision: 343550
URL: https://svnweb.freebsd.org/changeset/base/343550
Log:
Extract the coverage sanitizer KPI to a new file.
This will allow multiple consumers of the coverage data to be compiled
into the kernel together. The only requirement is only one can be
registered at a given point in time, however it is expected they will
only register when the coverage data is needed.
A new kernel conflig option COVERAGE is added. This will allow kcov to
become a module that can be loaded as needed, or compiled into the
kernel.
While here clean up the #include style a little.
Reviewed by: kib
Sponsored by: DARPA, AFRL
Differential Revision: https://reviews.freebsd.org/D18955
Added:
head/sys/kern/subr_coverage.c
- copied, changed from r343547, head/sys/kern/kern_kcov.c
head/sys/sys/coverage.h
- copied, changed from r343547, head/sys/sys/kcov.h
Modified:
head/sys/amd64/conf/GENERIC
head/sys/amd64/conf/GENERIC-NODEBUG
head/sys/arm64/conf/GENERIC
head/sys/arm64/conf/GENERIC-NODEBUG
head/sys/conf/files
head/sys/conf/kern.pre.mk
head/sys/conf/options
head/sys/kern/kern_kcov.c
head/sys/sys/kcov.h
Modified: head/sys/amd64/conf/GENERIC
==============================================================================
--- head/sys/amd64/conf/GENERIC Tue Jan 29 10:28:50 2019 (r343549)
+++ head/sys/amd64/conf/GENERIC Tue Jan 29 11:04:17 2019 (r343550)
@@ -100,9 +100,12 @@ options WITNESS # Enable checks to detect deadlocks
options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed
options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones
options VERBOSE_SYSINIT=0 # Support debug.verbose_sysinit, off by default
+
+# Kernel Sanitizers
+#options COVERAGE # Generic kernel coverage. Used by KCOV
+#options KCOV # Kernel Coverage Sanitizer
# Warning: KUBSAN can result in a kernel too large for loader to load
#options KUBSAN # Kernel Undefined Behavior Sanitizer
-#options KCOV # Kernel Coverage Sanitizer
# Kernel dump features.
options EKCD # Support for encrypted kernel dumps
Modified: head/sys/amd64/conf/GENERIC-NODEBUG
==============================================================================
--- head/sys/amd64/conf/GENERIC-NODEBUG Tue Jan 29 10:28:50 2019 (r343549)
+++ head/sys/amd64/conf/GENERIC-NODEBUG Tue Jan 29 11:04:17 2019 (r343550)
@@ -37,4 +37,5 @@ nooptions WITNESS_SKIPSPIN
nooptions BUF_TRACKING
nooptions DEADLKRES
nooptions FULL_BUF_TRACKING
-
+nooptions COVERAGE
+nooptions KCOV
Modified: head/sys/arm64/conf/GENERIC
==============================================================================
--- head/sys/arm64/conf/GENERIC Tue Jan 29 10:28:50 2019 (r343549)
+++ head/sys/arm64/conf/GENERIC Tue Jan 29 11:04:17 2019 (r343550)
@@ -92,9 +92,12 @@ options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9)
options ALT_BREAK_TO_DEBUGGER # Enter debugger on keyboard escape sequence
options USB_DEBUG # enable debug msgs
options VERBOSE_SYSINIT=0 # Support debug.verbose_sysinit, off by default
+
+# Kernel Sanitizers
+#options COVERAGE # Generic kernel coverage. Used by KCOV
+#options KCOV # Kernel Coverage Sanitizer
# Warning: KUBSAN can result in a kernel too large for loader to load
#options KUBSAN # Kernel Undefined Behavior Sanitizer
-#options KCOV # Kernel Coverage Sanitizer
# Kernel dump features.
options EKCD # Support for encrypted kernel dumps
Modified: head/sys/arm64/conf/GENERIC-NODEBUG
==============================================================================
--- head/sys/arm64/conf/GENERIC-NODEBUG Tue Jan 29 10:28:50 2019 (r343549)
+++ head/sys/arm64/conf/GENERIC-NODEBUG Tue Jan 29 11:04:17 2019 (r343550)
@@ -36,3 +36,5 @@ nooptions WITNESS
nooptions WITNESS_SKIPSPIN
nooptions DEADLKRES
nooptions USB_DEBUG
+nooptions COVERAGE
+nooptions KCOV
Modified: head/sys/conf/files
==============================================================================
--- head/sys/conf/files Tue Jan 29 10:28:50 2019 (r343549)
+++ head/sys/conf/files Tue Jan 29 11:04:17 2019 (r343550)
@@ -3883,6 +3883,8 @@ kern/subr_capability.c standard
kern/subr_clock.c standard
kern/subr_compressor.c standard \
compile-with "${NORMAL_C} -I$S/contrib/zstd/lib/freebsd"
+kern/subr_coverage.c optional coverage \
+ compile-with "${NORMAL_C} -fno-sanitize=all"
kern/subr_counter.c standard
kern/subr_devstat.c standard
kern/subr_disk.c standard
Modified: head/sys/conf/kern.pre.mk
==============================================================================
--- head/sys/conf/kern.pre.mk Tue Jan 29 10:28:50 2019 (r343549)
+++ head/sys/conf/kern.pre.mk Tue Jan 29 11:04:17 2019 (r343550)
@@ -118,8 +118,8 @@ KUBSAN_ENABLED!= grep KUBSAN opt_global.h || true ; ec
SAN_CFLAGS+= -fsanitize=undefined
.endif
-KCOV_ENABLED!= grep KCOV opt_kcov.h || true ; echo
-.if !empty(KCOV_ENABLED)
+COVERAGE_ENABLED!= grep COVERAGE opt_global.h || true ; echo
+.if !empty(COVERAGE_ENABLED)
SAN_CFLAGS+= -fsanitize-coverage=trace-pc,trace-cmp
.endif
Modified: head/sys/conf/options
==============================================================================
--- head/sys/conf/options Tue Jan 29 10:28:50 2019 (r343549)
+++ head/sys/conf/options Tue Jan 29 11:04:17 2019 (r343550)
@@ -57,7 +57,6 @@ DDB_CTF opt_ddb.h
DDB_NUMSYM opt_ddb.h
FULL_BUF_TRACKING opt_global.h
GDB
-KCOV opt_kcov.h
KDB opt_global.h
KDB_TRACE opt_kdb.h
KDB_UNATTENDED opt_kdb.h
@@ -234,6 +233,8 @@ VERBOSE_SYSINIT
ZSTDIO opt_zstdio.h
# Sanitizers
+COVERAGE opt_global.h
+KCOV
KUBSAN opt_global.h
# POSIX kernel options
Modified: head/sys/kern/kern_kcov.c
==============================================================================
--- head/sys/kern/kern_kcov.c Tue Jan 29 10:28:50 2019 (r343549)
+++ head/sys/kern/kern_kcov.c Tue Jan 29 11:04:17 2019 (r343550)
@@ -39,29 +39,26 @@
__FBSDID("$FreeBSD$");
#include <sys/param.h>
+#include <sys/systm.h>
#include <sys/conf.h>
-#include <sys/file.h>
#include <sys/kcov.h>
#include <sys/kernel.h>
+#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mman.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/rwlock.h>
-#include <sys/stat.h>
#include <sys/sysctl.h>
-#include <sys/systm.h>
-#include <sys/types.h>
#include <vm/vm.h>
+#include <vm/pmap.h>
#include <vm/vm_extern.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_pager.h>
-#include <vm/pmap.h>
-
MALLOC_DEFINE(M_KCOV_INFO, "kcovinfo", "KCOV info type");
#define KCOV_ELEMENT_SIZE sizeof(uint64_t)
@@ -138,17 +135,6 @@ static d_close_t kcov_close;
static d_mmap_single_t kcov_mmap_single;
static d_ioctl_t kcov_ioctl;
-void __sanitizer_cov_trace_pc(void);
-void __sanitizer_cov_trace_cmp1(uint8_t, uint8_t);
-void __sanitizer_cov_trace_cmp2(uint16_t, uint16_t);
-void __sanitizer_cov_trace_cmp4(uint32_t, uint32_t);
-void __sanitizer_cov_trace_cmp8(uint64_t, uint64_t);
-void __sanitizer_cov_trace_const_cmp1(uint8_t, uint8_t);
-void __sanitizer_cov_trace_const_cmp2(uint16_t, uint16_t);
-void __sanitizer_cov_trace_const_cmp4(uint32_t, uint32_t);
-void __sanitizer_cov_trace_const_cmp8(uint64_t, uint64_t);
-void __sanitizer_cov_trace_switch(uint64_t, uint64_t *);
-
static int kcov_alloc(struct kcov_info *info, size_t entries);
static void kcov_init(const void *unused);
@@ -169,6 +155,7 @@ SYSCTL_UINT(_kern_kcov, OID_AUTO, max_entries, CTLFLAG
"Maximum number of entries in the kcov buffer");
static struct mtx kcov_lock;
+static int active_count;
static struct kcov_info *
get_kinfo(struct thread *td)
@@ -197,25 +184,13 @@ get_kinfo(struct thread *td)
return (info);
}
-/*
- * Main entry point. A call to this function will be inserted
- * at every edge, and if coverage is enabled for the thread
- * this function will add the PC to the buffer.
- */
-void
-__sanitizer_cov_trace_pc(void)
+static void
+trace_pc(uintptr_t ret)
{
struct thread *td;
struct kcov_info *info;
uint64_t *buf, index;
- /*
- * To guarantee curthread is properly set, we exit early
- * until the driver has been initialized
- */
- if (cold)
- return;
-
td = curthread;
info = get_kinfo(td);
if (info == NULL)
@@ -237,7 +212,7 @@ __sanitizer_cov_trace_pc(void)
if (index + 2 > info->entries)
return;
- buf[index + 1] = (uint64_t)__builtin_return_address(0);
+ buf[index + 1] = ret;
buf[0] = index + 1;
}
@@ -248,13 +223,6 @@ trace_cmp(uint64_t type, uint64_t arg1, uint64_t arg2,
struct kcov_info *info;
uint64_t *buf, index;
- /*
- * To guarantee curthread is properly set, we exit early
- * until the driver has been initialized
- */
- if (cold)
- return (false);
-
td = curthread;
info = get_kinfo(td);
if (info == NULL)
@@ -287,109 +255,7 @@ trace_cmp(uint64_t type, uint64_t arg1, uint64_t arg2,
return (true);
}
-void
-__sanitizer_cov_trace_cmp1(uint8_t arg1, uint8_t arg2)
-{
-
- trace_cmp(KCOV_CMP_SIZE(0), arg1, arg2,
- (uint64_t)__builtin_return_address(0));
-}
-
-void
-__sanitizer_cov_trace_cmp2(uint16_t arg1, uint16_t arg2)
-{
-
- trace_cmp(KCOV_CMP_SIZE(1), arg1, arg2,
- (uint64_t)__builtin_return_address(0));
-}
-
-void
-__sanitizer_cov_trace_cmp4(uint32_t arg1, uint32_t arg2)
-{
-
- trace_cmp(KCOV_CMP_SIZE(2), arg1, arg2,
- (uint64_t)__builtin_return_address(0));
-}
-
-void
-__sanitizer_cov_trace_cmp8(uint64_t arg1, uint64_t arg2)
-{
-
- trace_cmp(KCOV_CMP_SIZE(3), arg1, arg2,
- (uint64_t)__builtin_return_address(0));
-}
-
-void
-__sanitizer_cov_trace_const_cmp1(uint8_t arg1, uint8_t arg2)
-{
-
- trace_cmp(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2,
- (uint64_t)__builtin_return_address(0));
-}
-
-void
-__sanitizer_cov_trace_const_cmp2(uint16_t arg1, uint16_t arg2)
-{
-
- trace_cmp(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2,
- (uint64_t)__builtin_return_address(0));
-}
-
-void
-__sanitizer_cov_trace_const_cmp4(uint32_t arg1, uint32_t arg2)
-{
-
- trace_cmp(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2,
- (uint64_t)__builtin_return_address(0));
-}
-
-void
-__sanitizer_cov_trace_const_cmp8(uint64_t arg1, uint64_t arg2)
-{
-
- trace_cmp(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2,
- (uint64_t)__builtin_return_address(0));
-}
-
/*
- * val is the switch operand
- * cases[0] is the number of case constants
- * cases[1] is the size of val in bits
- * cases[2..n] are the case constants
- */
-void
-__sanitizer_cov_trace_switch(uint64_t val, uint64_t *cases)
-{
- uint64_t i, count, ret, type;
-
- count = cases[0];
- ret = (uint64_t)__builtin_return_address(0);
-
- switch (cases[1]) {
- case 8:
- type = KCOV_CMP_SIZE(0);
- break;
- case 16:
- type = KCOV_CMP_SIZE(1);
- break;
- case 32:
- type = KCOV_CMP_SIZE(2);
- break;
- case 64:
- type = KCOV_CMP_SIZE(3);
- break;
- default:
- return;
- }
-
- val |= KCOV_CMP_CONST;
-
- for (i = 0; i < count; i++)
- if (!trace_cmp(type, val, cases[i + 2], ret))
- return;
-}
-
-/*
* The fd is being closed, cleanup everything we can.
*/
static void
@@ -456,6 +322,7 @@ kcov_close(struct cdev *dev, int fflag, int devtype, s
struct kcov_info *info;
int error;
+
if ((error = devfs_get_cdevpriv((void **)&info)) != 0)
return (error);
@@ -571,6 +438,16 @@ kcov_ioctl(struct cdev *dev, u_long cmd, caddr_t data,
error = EINVAL;
break;
}
+
+ /* Lets hope nobody opens this 2 billion times */
+ KASSERT(active_count < INT_MAX,
+ ("%s: Open too many times", __func__));
+ active_count++;
+ if (active_count == 1) {
+ cov_register_pc(&trace_pc);
+ cov_register_cmp(&trace_cmp);
+ }
+
KASSERT(info->thread == NULL,
("Enabling kcov when already enabled"));
info->thread = td;
@@ -589,6 +466,13 @@ kcov_ioctl(struct cdev *dev, u_long cmd, caddr_t data,
error = EINVAL;
break;
}
+ KASSERT(active_count > 0, ("%s: Open count is zero", __func__));
+ active_count--;
+ if (active_count == 0) {
+ cov_register_pc(&trace_pc);
+ cov_register_cmp(&trace_cmp);
+ }
+
td->td_kcov_info = NULL;
atomic_store_int(&info->state, KCOV_STATE_READY);
/*
@@ -618,6 +502,12 @@ kcov_thread_dtor(void *arg __unused, struct thread *td
return;
mtx_lock_spin(&kcov_lock);
+ KASSERT(active_count > 0, ("%s: Open count is zero", __func__));
+ active_count--;
+ if (active_count == 0) {
+ cov_register_pc(&trace_pc);
+ cov_register_cmp(&trace_cmp);
+ }
td->td_kcov_info = NULL;
if (info->state != KCOV_STATE_DYING) {
/*
@@ -673,4 +563,4 @@ kcov_init(const void *unused)
EVENTHANDLER_PRI_ANY);
}
-SYSINIT(kcovdev, SI_SUB_DEVFS, SI_ORDER_ANY, kcov_init, NULL);
+SYSINIT(kcovdev, SI_SUB_LAST, SI_ORDER_ANY, kcov_init, NULL);
Copied and modified: head/sys/kern/subr_coverage.c (from r343547, head/sys/kern/kern_kcov.c)
==============================================================================
--- head/sys/kern/kern_kcov.c Tue Jan 29 09:22:20 2019 (r343547, copy source)
+++ head/sys/kern/subr_coverage.c Tue Jan 29 11:04:17 2019 (r343550)
@@ -39,105 +39,10 @@
__FBSDID("$FreeBSD$");
#include <sys/param.h>
-#include <sys/conf.h>
-#include <sys/file.h>
-#include <sys/kcov.h>
-#include <sys/kernel.h>
-#include <sys/lock.h>
-#include <sys/malloc.h>
-#include <sys/mman.h>
-#include <sys/mutex.h>
-#include <sys/proc.h>
-#include <sys/rwlock.h>
-#include <sys/stat.h>
-#include <sys/sysctl.h>
-#include <sys/systm.h>
-#include <sys/types.h>
+#include <sys/coverage.h>
-#include <vm/vm.h>
-#include <vm/vm_extern.h>
-#include <vm/vm_object.h>
-#include <vm/vm_page.h>
-#include <vm/vm_pager.h>
+#include <machine/atomic.h>
-#include <vm/pmap.h>
-
-MALLOC_DEFINE(M_KCOV_INFO, "kcovinfo", "KCOV info type");
-
-#define KCOV_ELEMENT_SIZE sizeof(uint64_t)
-
-/*
- * To know what the code can safely perform at any point in time we use a
- * state machine. In the normal case the state transitions are:
- *
- * OPEN -> READY -> RUNNING -> DYING
- * | | ^ | ^ ^
- * | | +--------+ | |
- * | +-------------------+ |
- * +-----------------------------+
- *
- * The states are:
- * OPEN: The kcov fd has been opened, but no buffer is available to store
- * coverage data.
- * READY: The buffer to store coverage data has been allocated. Userspace
- * can set this by using ioctl(fd, KIOSETBUFSIZE, entries);. When
- * this has been set the buffer can be written to by the kernel,
- * and mmaped by userspace.
- * RUNNING: The coverage probes are able to store coverage data in the buffer.
- * This is entered with ioctl(fd, KIOENABLE, mode);. The READY state
- * can be exited by ioctl(fd, KIODISABLE); or exiting the thread to
- * return to the READY state to allow tracing to be reused, or by
- * closing the kcov fd to enter the DYING state.
- * DYING: The fd has been closed. All states can enter into this state when
- * userspace closes the kcov fd.
- *
- * We need to be careful when moving into and out of the RUNNING state. As
- * an interrupt may happen while this is happening the ordering of memory
- * operations is important so struct kcov_info is valid for the tracing
- * functions.
- *
- * When moving into the RUNNING state prior stores to struct kcov_info need
- * to be observed before the state is set. This allows for interrupts that
- * may call into one of the coverage functions to fire at any point while
- * being enabled and see a consistent struct kcov_info.
- *
- * When moving out of the RUNNING state any later stores to struct kcov_info
- * need to be observed after the state is set. As with entering this is to
- * present a consistent struct kcov_info to interrupts.
- */
-typedef enum {
- KCOV_STATE_INVALID,
- KCOV_STATE_OPEN, /* The device is open, but with no buffer */
- KCOV_STATE_READY, /* The buffer has been allocated */
- KCOV_STATE_RUNNING, /* Recording trace data */
- KCOV_STATE_DYING, /* The fd was closed */
-} kcov_state_t;
-
-/*
- * (l) Set while holding the kcov_lock mutex and not in the RUNNING state.
- * (o) Only set once while in the OPEN state. Cleaned up while in the DYING
- * state, and with no thread associated with the struct kcov_info.
- * (s) Set atomically to enter or exit the RUNNING state, non-atomically
- * otherwise. See above for a description of the other constraints while
- * moving into or out of the RUNNING state.
- */
-struct kcov_info {
- struct thread *thread; /* (l) */
- vm_object_t bufobj; /* (o) */
- vm_offset_t kvaddr; /* (o) */
- size_t entries; /* (o) */
- size_t bufsize; /* (o) */
- kcov_state_t state; /* (s) */
- int mode; /* (l) */
- bool mmap;
-};
-
-/* Prototypes */
-static d_open_t kcov_open;
-static d_close_t kcov_close;
-static d_mmap_single_t kcov_mmap_single;
-static d_ioctl_t kcov_ioctl;
-
void __sanitizer_cov_trace_pc(void);
void __sanitizer_cov_trace_cmp1(uint8_t, uint8_t);
void __sanitizer_cov_trace_cmp2(uint16_t, uint16_t);
@@ -149,52 +54,35 @@ void __sanitizer_cov_trace_const_cmp4(uint32_t, uint32
void __sanitizer_cov_trace_const_cmp8(uint64_t, uint64_t);
void __sanitizer_cov_trace_switch(uint64_t, uint64_t *);
-static int kcov_alloc(struct kcov_info *info, size_t entries);
-static void kcov_init(const void *unused);
+static cov_trace_pc_t cov_trace_pc;
+static cov_trace_cmp_t cov_trace_cmp;
-static struct cdevsw kcov_cdevsw = {
- .d_version = D_VERSION,
- .d_open = kcov_open,
- .d_close = kcov_close,
- .d_mmap_single = kcov_mmap_single,
- .d_ioctl = kcov_ioctl,
- .d_name = "kcov",
-};
+void
+cov_register_pc(cov_trace_pc_t trace_pc)
+{
-SYSCTL_NODE(_kern, OID_AUTO, kcov, CTLFLAG_RW, 0, "Kernel coverage");
+ atomic_store_ptr(&cov_trace_pc, trace_pc);
+}
-static u_int kcov_max_entries = KCOV_MAXENTRIES;
-SYSCTL_UINT(_kern_kcov, OID_AUTO, max_entries, CTLFLAG_RW,
- &kcov_max_entries, 0,
- "Maximum number of entries in the kcov buffer");
+void
+cov_unregister_pc(void)
+{
-static struct mtx kcov_lock;
+ atomic_store_ptr(&cov_trace_pc, NULL);
+}
-static struct kcov_info *
-get_kinfo(struct thread *td)
+void
+cov_register_cmp(cov_trace_cmp_t trace_cmp)
{
- struct kcov_info *info;
- /* We might have a NULL thread when releasing the secondary CPUs */
- if (td == NULL)
- return (NULL);
+ atomic_store_ptr(&cov_trace_cmp, trace_cmp);
+}
- /*
- * We are in an interrupt, stop tracing as it is not explicitly
- * part of a syscall.
- */
- if (td->td_intr_nesting_level > 0 || td->td_intr_frame != NULL)
- return (NULL);
+void
+cov_unregister_cmp(void)
+{
- /*
- * If info is NULL or the state is not running we are not tracing.
- */
- info = td->td_kcov_info;
- if (info == NULL ||
- atomic_load_acq_int(&info->state) != KCOV_STATE_RUNNING)
- return (NULL);
-
- return (info);
+ atomic_store_ptr(&cov_trace_cmp, NULL);
}
/*
@@ -205,150 +93,104 @@ get_kinfo(struct thread *td)
void
__sanitizer_cov_trace_pc(void)
{
- struct thread *td;
- struct kcov_info *info;
- uint64_t *buf, index;
+ cov_trace_pc_t trace_pc;
- /*
- * To guarantee curthread is properly set, we exit early
- * until the driver has been initialized
- */
- if (cold)
- return;
-
- td = curthread;
- info = get_kinfo(td);
- if (info == NULL)
- return;
-
- /*
- * Check we are in the PC-trace mode.
- */
- if (info->mode != KCOV_MODE_TRACE_PC)
- return;
-
- KASSERT(info->kvaddr != 0,
- ("__sanitizer_cov_trace_pc: NULL buf while running"));
-
- buf = (uint64_t *)info->kvaddr;
-
- /* The first entry of the buffer holds the index */
- index = buf[0];
- if (index + 2 > info->entries)
- return;
-
- buf[index + 1] = (uint64_t)__builtin_return_address(0);
- buf[0] = index + 1;
+ trace_pc = (cov_trace_pc_t)atomic_load_ptr(&cov_trace_pc);
+ if (trace_pc != NULL)
+ trace_pc((uint64_t)__builtin_return_address(0));
}
-static bool
-trace_cmp(uint64_t type, uint64_t arg1, uint64_t arg2, uint64_t ret)
-{
- struct thread *td;
- struct kcov_info *info;
- uint64_t *buf, index;
-
- /*
- * To guarantee curthread is properly set, we exit early
- * until the driver has been initialized
- */
- if (cold)
- return (false);
-
- td = curthread;
- info = get_kinfo(td);
- if (info == NULL)
- return (false);
-
- /*
- * Check we are in the comparison-trace mode.
- */
- if (info->mode != KCOV_MODE_TRACE_CMP)
- return (false);
-
- KASSERT(info->kvaddr != 0,
- ("__sanitizer_cov_trace_pc: NULL buf while running"));
-
- buf = (uint64_t *)info->kvaddr;
-
- /* The first entry of the buffer holds the index */
- index = buf[0];
-
- /* Check we have space to store all elements */
- if (index * 4 + 4 + 1 > info->entries)
- return (false);
-
- buf[index * 4 + 1] = type;
- buf[index * 4 + 2] = arg1;
- buf[index * 4 + 3] = arg2;
- buf[index * 4 + 4] = ret;
- buf[0] = index + 1;
-
- return (true);
-}
-
+/*
+ * Comparison entry points. When the kernel performs a comparison
+ * operation the compiler inserts a call to one of the following
+ * functions to record the operation.
+ */
void
__sanitizer_cov_trace_cmp1(uint8_t arg1, uint8_t arg2)
{
+ cov_trace_cmp_t trace_cmp;
- trace_cmp(KCOV_CMP_SIZE(0), arg1, arg2,
- (uint64_t)__builtin_return_address(0));
+ trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp);
+ if (trace_cmp != NULL)
+ trace_cmp(COV_CMP_SIZE(0), arg1, arg2,
+ (uint64_t)__builtin_return_address(0));
}
void
__sanitizer_cov_trace_cmp2(uint16_t arg1, uint16_t arg2)
{
+ cov_trace_cmp_t trace_cmp;
- trace_cmp(KCOV_CMP_SIZE(1), arg1, arg2,
- (uint64_t)__builtin_return_address(0));
+ trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp);
+ if (trace_cmp != NULL)
+ trace_cmp(COV_CMP_SIZE(1), arg1, arg2,
+ (uint64_t)__builtin_return_address(0));
}
void
__sanitizer_cov_trace_cmp4(uint32_t arg1, uint32_t arg2)
{
+ cov_trace_cmp_t trace_cmp;
- trace_cmp(KCOV_CMP_SIZE(2), arg1, arg2,
- (uint64_t)__builtin_return_address(0));
+ trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp);
+ if (trace_cmp != NULL)
+ trace_cmp(COV_CMP_SIZE(2), arg1, arg2,
+ (uint64_t)__builtin_return_address(0));
}
void
__sanitizer_cov_trace_cmp8(uint64_t arg1, uint64_t arg2)
{
+ cov_trace_cmp_t trace_cmp;
- trace_cmp(KCOV_CMP_SIZE(3), arg1, arg2,
- (uint64_t)__builtin_return_address(0));
+ trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp);
+ if (trace_cmp != NULL)
+ trace_cmp(COV_CMP_SIZE(3), arg1, arg2,
+ (uint64_t)__builtin_return_address(0));
}
void
__sanitizer_cov_trace_const_cmp1(uint8_t arg1, uint8_t arg2)
{
+ cov_trace_cmp_t trace_cmp;
- trace_cmp(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2,
- (uint64_t)__builtin_return_address(0));
+ trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp);
+ if (trace_cmp != NULL)
+ trace_cmp(COV_CMP_SIZE(0) | COV_CMP_CONST, arg1, arg2,
+ (uint64_t)__builtin_return_address(0));
}
void
__sanitizer_cov_trace_const_cmp2(uint16_t arg1, uint16_t arg2)
{
+ cov_trace_cmp_t trace_cmp;
- trace_cmp(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2,
- (uint64_t)__builtin_return_address(0));
+ trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp);
+ if (trace_cmp != NULL)
+ trace_cmp(COV_CMP_SIZE(1) | COV_CMP_CONST, arg1, arg2,
+ (uint64_t)__builtin_return_address(0));
}
void
__sanitizer_cov_trace_const_cmp4(uint32_t arg1, uint32_t arg2)
{
+ cov_trace_cmp_t trace_cmp;
- trace_cmp(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2,
- (uint64_t)__builtin_return_address(0));
+ trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp);
+ if (trace_cmp != NULL)
+ trace_cmp(COV_CMP_SIZE(2) | COV_CMP_CONST, arg1, arg2,
+ (uint64_t)__builtin_return_address(0));
}
void
__sanitizer_cov_trace_const_cmp8(uint64_t arg1, uint64_t arg2)
{
+ cov_trace_cmp_t trace_cmp;
- trace_cmp(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2,
- (uint64_t)__builtin_return_address(0));
+ trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp);
+ if (trace_cmp != NULL)
+ trace_cmp(COV_CMP_SIZE(3) | COV_CMP_CONST, arg1, arg2,
+ (uint64_t)__builtin_return_address(0));
}
/*
@@ -361,316 +203,35 @@ void
__sanitizer_cov_trace_switch(uint64_t val, uint64_t *cases)
{
uint64_t i, count, ret, type;
+ cov_trace_cmp_t trace_cmp;
+ trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp);
+ if (trace_cmp == NULL)
+ return;
+
count = cases[0];
ret = (uint64_t)__builtin_return_address(0);
switch (cases[1]) {
case 8:
- type = KCOV_CMP_SIZE(0);
+ type = COV_CMP_SIZE(0);
break;
case 16:
- type = KCOV_CMP_SIZE(1);
+ type = COV_CMP_SIZE(1);
break;
case 32:
- type = KCOV_CMP_SIZE(2);
+ type = COV_CMP_SIZE(2);
break;
case 64:
- type = KCOV_CMP_SIZE(3);
+ type = COV_CMP_SIZE(3);
break;
default:
return;
}
- val |= KCOV_CMP_CONST;
+ val |= COV_CMP_CONST;
for (i = 0; i < count; i++)
if (!trace_cmp(type, val, cases[i + 2], ret))
return;
}
-
-/*
- * The fd is being closed, cleanup everything we can.
- */
-static void
-kcov_mmap_cleanup(void *arg)
-{
- struct kcov_info *info = arg;
- struct thread *thread;
-
- mtx_lock_spin(&kcov_lock);
- /*
- * Move to KCOV_STATE_DYING to stop adding new entries.
- *
- * If the thread is running we need to wait until thread exit to
- * clean up as it may currently be adding a new entry. If this is
- * the case being in KCOV_STATE_DYING will signal that the buffer
- * needs to be cleaned up.
- */
- atomic_store_int(&info->state, KCOV_STATE_DYING);
- atomic_thread_fence_seq_cst();
- thread = info->thread;
- mtx_unlock_spin(&kcov_lock);
-
- if (thread != NULL)
- return;
-
- /*
- * We can safely clean up the info struct as it is in the
- * KCOV_STATE_DYING state with no thread associated.
- *
- * The KCOV_STATE_DYING stops new threads from using it.
- * The lack of a thread means nothing is currently using the buffers.
- */
-
- if (info->kvaddr != 0) {
- pmap_qremove(info->kvaddr, info->bufsize / PAGE_SIZE);
- kva_free(info->kvaddr, info->bufsize);
- }
- if (info->bufobj != NULL && !info->mmap)
- vm_object_deallocate(info->bufobj);
- free(info, M_KCOV_INFO);
-}
-
-static int
-kcov_open(struct cdev *dev, int oflags, int devtype, struct thread *td)
-{
- struct kcov_info *info;
- int error;
-
- info = malloc(sizeof(struct kcov_info), M_KCOV_INFO, M_ZERO | M_WAITOK);
- info->state = KCOV_STATE_OPEN;
- info->thread = NULL;
- info->mode = -1;
- info->mmap = false;
-
- if ((error = devfs_set_cdevpriv(info, kcov_mmap_cleanup)) != 0)
- kcov_mmap_cleanup(info);
-
- return (error);
-}
-
-static int
-kcov_close(struct cdev *dev, int fflag, int devtype, struct thread *td)
-{
- struct kcov_info *info;
- int error;
-
- if ((error = devfs_get_cdevpriv((void **)&info)) != 0)
- return (error);
-
- KASSERT(info != NULL, ("kcov_close with no kcov_info structure"));
-
- /* Trying to close, but haven't disabled */
- if (info->state == KCOV_STATE_RUNNING)
- return (EBUSY);
-
- return (0);
-}
-
-static int
-kcov_mmap_single(struct cdev *dev, vm_ooffset_t *offset, vm_size_t size,
- struct vm_object **object, int nprot)
-{
- struct kcov_info *info;
- int error;
-
- if ((nprot & (PROT_EXEC | PROT_READ | PROT_WRITE)) !=
- (PROT_READ | PROT_WRITE))
- return (EINVAL);
-
- if ((error = devfs_get_cdevpriv((void **)&info)) != 0)
- return (error);
-
- if (info->kvaddr == 0 || size / KCOV_ELEMENT_SIZE != info->entries ||
- info->mmap != false)
- return (EINVAL);
-
- info->mmap = true;
- *offset = 0;
- *object = info->bufobj;
- return (0);
-}
-
-static int
-kcov_alloc(struct kcov_info *info, size_t entries)
-{
- size_t n, pages;
- vm_page_t *m;
-
- KASSERT(info->kvaddr == 0, ("kcov_alloc: Already have a buffer"));
- KASSERT(info->state == KCOV_STATE_OPEN,
- ("kcov_alloc: Not in open state (%x)", info->state));
-
- if (entries < 2 || entries > kcov_max_entries)
- return (EINVAL);
-
- /* Align to page size so mmap can't access other kernel memory */
- info->bufsize = roundup2(entries * KCOV_ELEMENT_SIZE, PAGE_SIZE);
- pages = info->bufsize / PAGE_SIZE;
-
- if ((info->kvaddr = kva_alloc(info->bufsize)) == 0)
- return (ENOMEM);
-
- info->bufobj = vm_pager_allocate(OBJT_PHYS, 0, info->bufsize,
- PROT_READ | PROT_WRITE, 0, curthread->td_ucred);
-
- m = malloc(sizeof(*m) * pages, M_TEMP, M_WAITOK);
- VM_OBJECT_WLOCK(info->bufobj);
- for (n = 0; n < pages; n++) {
- m[n] = vm_page_grab(info->bufobj, n,
- VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_WIRED);
- m[n]->valid = VM_PAGE_BITS_ALL;
- }
- VM_OBJECT_WUNLOCK(info->bufobj);
- pmap_qenter(info->kvaddr, m, pages);
- free(m, M_TEMP);
-
- info->entries = entries;
-
- return (0);
-}
-
-static int
-kcov_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag __unused,
- struct thread *td)
-{
- struct kcov_info *info;
- int mode, error;
-
- if ((error = devfs_get_cdevpriv((void **)&info)) != 0)
- return (error);
-
- if (cmd == KIOSETBUFSIZE) {
- /*
- * Set the size of the coverage buffer. Should be called
- * before enabling coverage collection for that thread.
- */
- if (info->state != KCOV_STATE_OPEN) {
- return (EBUSY);
- }
- error = kcov_alloc(info, *(u_int *)data);
- if (error == 0)
- info->state = KCOV_STATE_READY;
- return (error);
- }
-
- mtx_lock_spin(&kcov_lock);
- switch (cmd) {
- case KIOENABLE:
- if (info->state != KCOV_STATE_READY) {
- error = EBUSY;
- break;
- }
- if (td->td_kcov_info != NULL) {
- error = EINVAL;
- break;
- }
- mode = *(int *)data;
- if (mode != KCOV_MODE_TRACE_PC && mode != KCOV_MODE_TRACE_CMP) {
- error = EINVAL;
- break;
- }
- KASSERT(info->thread == NULL,
- ("Enabling kcov when already enabled"));
- info->thread = td;
- info->mode = mode;
- /*
- * Ensure the mode has been set before starting coverage
- * tracing.
- */
- atomic_store_rel_int(&info->state, KCOV_STATE_RUNNING);
- td->td_kcov_info = info;
- break;
- case KIODISABLE:
- /* Only the currently enabled thread may disable itself */
- if (info->state != KCOV_STATE_RUNNING ||
- info != td->td_kcov_info) {
- error = EINVAL;
- break;
- }
- td->td_kcov_info = NULL;
- atomic_store_int(&info->state, KCOV_STATE_READY);
- /*
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-head
mailing list