git: df114daef4c4 - main - Import the Hardware Trace (HWT) framework.

From: Ruslan Bukin <br_at_FreeBSD.org>
Date: Thu, 03 Jul 2025 15:03:05 UTC
The branch main has been updated by br:

URL: https://cgit.FreeBSD.org/src/commit/?id=df114daef4c48548c3c2b86717612761185ae18f

commit df114daef4c48548c3c2b86717612761185ae18f
Author:     Ruslan Bukin <br@FreeBSD.org>
AuthorDate: 2025-07-03 13:10:45 +0000
Commit:     Ruslan Bukin <br@FreeBSD.org>
CommitDate: 2025-07-03 14:48:34 +0000

    Import the Hardware Trace (HWT) framework.
    
    The HWT framework provides infrastructure for hardware-assisted tracing. It
    collects detailed information about software execution and records it as
    "events" in highly compressed format into DRAM. The events cover information
    about control flow changes of a program, whether branches taken or not,
    exceptions taken, timing information, cycles elapsed and more. This allows
    to reconstruct entire program flow of a given application.
    
    This comes with separate machine-dependent tracing backends for trace
    collection, trace decoder libraries and an instrumentation tool.
    
    Reviewed by:    kib (sys/kern bits)
    Sponsored by:   UKRI
    Differential Revision:  https://reviews.freebsd.org/D40466
---
 sys/conf/files                |  13 ++
 sys/conf/options              |   3 +
 sys/dev/hwt/hwt.c             | 242 ++++++++++++++++++++
 sys/dev/hwt/hwt_backend.c     | 289 ++++++++++++++++++++++++
 sys/dev/hwt/hwt_backend.h     |  87 ++++++++
 sys/dev/hwt/hwt_config.c      | 108 +++++++++
 sys/dev/hwt/hwt_config.h      |  36 +++
 sys/dev/hwt/hwt_context.c     | 201 +++++++++++++++++
 sys/dev/hwt/hwt_context.h     |  86 ++++++++
 sys/dev/hwt/hwt_contexthash.c | 133 +++++++++++
 sys/dev/hwt/hwt_contexthash.h |  42 ++++
 sys/dev/hwt/hwt_cpu.c         | 115 ++++++++++
 sys/dev/hwt/hwt_cpu.h         |  45 ++++
 sys/dev/hwt/hwt_hook.c        | 323 +++++++++++++++++++++++++++
 sys/dev/hwt/hwt_hook.h        |  56 +++++
 sys/dev/hwt/hwt_intr.h        |  33 +++
 sys/dev/hwt/hwt_ioctl.c       | 443 +++++++++++++++++++++++++++++++++++++
 sys/dev/hwt/hwt_ioctl.h       |  35 +++
 sys/dev/hwt/hwt_owner.c       | 157 +++++++++++++
 sys/dev/hwt/hwt_owner.h       |  45 ++++
 sys/dev/hwt/hwt_ownerhash.c   | 141 ++++++++++++
 sys/dev/hwt/hwt_ownerhash.h   |  42 ++++
 sys/dev/hwt/hwt_record.c      | 302 +++++++++++++++++++++++++
 sys/dev/hwt/hwt_record.h      |  47 ++++
 sys/dev/hwt/hwt_thread.c      | 162 ++++++++++++++
 sys/dev/hwt/hwt_thread.h      |  64 ++++++
 sys/dev/hwt/hwt_vm.c          | 501 ++++++++++++++++++++++++++++++++++++++++++
 sys/dev/hwt/hwt_vm.h          |  47 ++++
 sys/kern/kern_exec.c          |  19 ++
 sys/kern/kern_linker.c        |   5 +-
 sys/kern/kern_pmc.c           |   4 +
 sys/kern/kern_thr.c           |  12 +-
 sys/kern/kern_thread.c        |   9 +
 sys/kern/sched_4bsd.c         |  22 +-
 sys/kern/sched_ule.c          |  19 ++
 sys/kern/vfs_vnops.c          |  23 ++
 sys/modules/Makefile          |   5 +
 sys/modules/hwt/Makefile      |  21 ++
 sys/sys/hwt.h                 | 129 +++++++++++
 sys/sys/hwt_record.h          |  70 ++++++
 sys/sys/proc.h                |   2 +
 sys/vm/vm_mmap.c              |  16 ++
 42 files changed, 4150 insertions(+), 4 deletions(-)

diff --git a/sys/conf/files b/sys/conf/files
index 75ee10be5896..f6d473b1431b 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -1776,6 +1776,19 @@ dev/hwpmc/hwpmc_soft.c		optional hwpmc
 dev/hwreset/hwreset.c		optional hwreset
 dev/hwreset/hwreset_array.c	optional hwreset
 dev/hwreset/hwreset_if.m	optional hwreset
+dev/hwt/hwt.c			optional hwt
+dev/hwt/hwt_backend.c		optional hwt
+dev/hwt/hwt_config.c		optional hwt
+dev/hwt/hwt_context.c		optional hwt
+dev/hwt/hwt_contexthash.c	optional hwt
+dev/hwt/hwt_cpu.c		optional hwt
+dev/hwt/hwt_hook.c		optional hwt
+dev/hwt/hwt_ioctl.c		optional hwt
+dev/hwt/hwt_owner.c		optional hwt
+dev/hwt/hwt_ownerhash.c		optional hwt
+dev/hwt/hwt_record.c		optional hwt
+dev/hwt/hwt_thread.c		optional hwt
+dev/hwt/hwt_vm.c		optional hwt
 dev/ichiic/ig4_acpi.c		optional ig4 acpi iicbus
 dev/ichiic/ig4_iic.c		optional ig4 iicbus
 dev/ichiic/ig4_pci.c		optional ig4 pci iicbus
diff --git a/sys/conf/options b/sys/conf/options
index 03e8964e965d..a637b0b74a77 100644
--- a/sys/conf/options
+++ b/sys/conf/options
@@ -885,6 +885,9 @@ DCONS_FORCE_GDB		opt_dcons.h
 HWPMC_DEBUG		opt_global.h
 HWPMC_HOOKS
 
+# Hardware Trace (HWT) framework options
+HWT_HOOKS
+
 # 802.11 support layer
 IEEE80211_DEBUG		opt_wlan.h
 IEEE80211_DEBUG_REFCNT	opt_wlan.h
diff --git a/sys/dev/hwt/hwt.c b/sys/dev/hwt/hwt.c
new file mode 100644
index 000000000000..c476e6031ba8
--- /dev/null
+++ b/sys/dev/hwt/hwt.c
@@ -0,0 +1,242 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023-2025 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This work was supported by Innovate UK project 105694, "Digital Security
+ * by Design (DSbD) Technology Platform Prototype".
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Hardware Tracing framework.
+ *
+ *    The framework manages hardware tracing units that collect information
+ * about software execution and store it as events in highly compressed format
+ * into DRAM. The events cover information about control flow changes of a
+ * program, whether branches taken or not, exceptions taken, timing information,
+ * cycles elapsed and more. That allows us to restore entire program flow of a
+ * given application without performance impact.
+ *
+ * Design overview.
+ *
+ *    The framework provides character devices for mmap(2) and ioctl(2) system
+ * calls to allow user to manage CPU (hardware) tracing units.
+ *
+ * /dev/hwt:
+ *    .ioctl:
+ *        hwt_ioctl():
+ *               a) HWT_IOC_ALLOC
+ *                  Allocates kernel tracing context CTX based on requested mode
+ *                  of operation. Verifies the information that comes with the
+ *                  request (pid, cpus), allocates unique ID for the context.
+ *                  Creates a new character device for CTX management.
+ *
+ * /dev/hwt_%d[_%d], ident[, thread_id]
+ *    .mmap
+ *        Maps tracing buffers of the corresponding thread to userspace.
+ *    .ioctl
+ *        hwt_thread_ioctl():
+ *               a) HWT_IOC_START
+ *                  Enables tracing unit for a given context.
+ *               b) HWT_IOC_RECORD_GET
+ *                  Transfers (small) record entries collected during program
+ *                  execution for a given context to userspace, such as mmaping
+ *                  tables of executable and dynamic libraries, interpreter,
+ *                  kernel mappings, tid of threads created, etc.
+ *               c) HWT_IOC_SET_CONFIG
+ *                  Allows to specify backend-specific configuration of the
+ *                  trace unit.
+ *               d) HWT_IOC_WAKEUP
+ *                  Wakes up a thread that is currently sleeping.
+ *               e) HWT_IOC_BUFPTR_GET
+ *                  Transfers current hardware pointer in the filling buffer
+ *                  to the userspace.
+ *               f) HWT_IOC_SVC_BUF
+ *                  To avoid data loss, userspace may notify kernel it has
+ *                  copied out the given buffer, so kernel is ok to overwrite
+ *
+ * HWT context lifecycle in THREAD mode of operation:
+ * 1. User invokes HWT_IOC_ALLOC ioctl with information about pid to trace and
+ *    size of the buffers for the trace data to allocate.
+ *    Some architectures may have different tracing units supported, so user
+ *    also provides backend name to use for this context, e.g. "coresight".
+ * 2. Kernel allocates context, lookups the proc for the given pid. Then it
+ *    creates first hwt_thread in the context and allocates trace buffers for
+ *    it. Immediately, kernel initializes tracing backend.
+ *    Kernel creates character device and returns unique identificator of
+ *    trace context to the user.
+ * 3. To manage the new context, user opens the character device created.
+ *    User invokes HWT_IOC_START ioctl, kernel marks context as RUNNING.
+ *    At this point any HWT hook invocation by scheduler enables/disables
+ *    tracing for threads associated with the context (threads of the proc).
+ *    Any new threads creation (of the target proc) procedures will be invoking
+ *    corresponding hooks in HWT framework, so that new hwt_thread and buffers
+ *    allocated, character device for mmap(2) created on the fly.
+ * 4. User issues HWT_IOC_RECORD_GET ioctl to fetch information about mmaping
+ *    tables and threads created during application startup.
+ * 5. User mmaps tracing buffers of each thread to userspace (using
+ *    /dev/hwt_%d_%d % (ident, thread_id) character devices).
+ * 6. User can repeat 4 if expected thread is not yet created during target
+ *    application execution.
+ * 7. User issues HWT_IOC_BUFPTR_GET ioctl to get current filling level of the
+ *    hardware buffer of a given thread.
+ * 8. User invokes trace decoder library to process available data and see the
+ *    results in human readable form.
+ * 9. User repeats 7 if needed.
+ *
+ * HWT context lifecycle in CPU mode of operation:
+ * 1. User invokes HWT_IOC_ALLOC ioctl providing a set of CPU to trace within
+ *    single CTX.
+ * 2. Kernel verifies the set of CPU and allocates tracing context, creates
+ *    a buffer for each CPU.
+ *    Kernel creates a character device for every CPU provided in the request.
+ *    Kernel initialized tracing backend.
+ * 3. User opens character devices of interest to map the buffers to userspace.
+ *    User can start tracing by invoking HWT_IOC_START on any of character
+ *    device within the context, entire context will be marked as RUNNING.
+ * 4. The rest is similar to the THREAD mode.
+ *
+ */
+
+#include <sys/param.h>
+#include <sys/conf.h>
+#include <sys/eventhandler.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+
+#include <dev/hwt/hwt_context.h>
+#include <dev/hwt/hwt_contexthash.h>
+#include <dev/hwt/hwt_thread.h>
+#include <dev/hwt/hwt_owner.h>
+#include <dev/hwt/hwt_ownerhash.h>
+#include <dev/hwt/hwt_backend.h>
+#include <dev/hwt/hwt_record.h>
+#include <dev/hwt/hwt_ioctl.h>
+#include <dev/hwt/hwt_hook.h>
+
+#define	HWT_DEBUG
+#undef	HWT_DEBUG
+
+#ifdef	HWT_DEBUG
+#define	dprintf(fmt, ...)	printf(fmt, ##__VA_ARGS__)
+#else
+#define	dprintf(fmt, ...)
+#endif
+
+static eventhandler_tag hwt_exit_tag;
+static struct cdev *hwt_cdev;
+static struct cdevsw hwt_cdevsw = {
+	.d_version	= D_VERSION,
+	.d_name		= "hwt",
+	.d_mmap_single	= NULL,
+	.d_ioctl	= hwt_ioctl
+};
+
+static void
+hwt_process_exit(void *arg __unused, struct proc *p)
+{
+	struct hwt_owner *ho;
+
+	/* Stop HWTs associated with exiting owner, if any. */
+	ho = hwt_ownerhash_lookup(p);
+	if (ho)
+		hwt_owner_shutdown(ho);
+}
+
+static int
+hwt_load(void)
+{
+	struct make_dev_args args;
+	int error;
+
+	make_dev_args_init(&args);
+	args.mda_devsw = &hwt_cdevsw;
+	args.mda_flags = MAKEDEV_CHECKNAME | MAKEDEV_WAITOK;
+	args.mda_uid = UID_ROOT;
+	args.mda_gid = GID_WHEEL;
+	args.mda_mode = 0660;
+	args.mda_si_drv1 = NULL;
+
+	hwt_backend_load();
+	hwt_ctx_load();
+	hwt_contexthash_load();
+	hwt_ownerhash_load();
+	hwt_record_load();
+
+	error = make_dev_s(&args, &hwt_cdev, "hwt");
+	if (error != 0)
+		return (error);
+
+	hwt_exit_tag = EVENTHANDLER_REGISTER(process_exit, hwt_process_exit,
+	    NULL, EVENTHANDLER_PRI_ANY);
+
+	hwt_hook_load();
+
+	return (0);
+}
+
+static int
+hwt_unload(void)
+{
+
+	hwt_hook_unload();
+	EVENTHANDLER_DEREGISTER(process_exit, hwt_exit_tag);
+	destroy_dev(hwt_cdev);
+	hwt_record_unload();
+	hwt_ownerhash_unload();
+	hwt_contexthash_unload();
+	hwt_ctx_unload();
+	hwt_backend_unload();
+
+	return (0);
+}
+
+static int
+hwt_modevent(module_t mod, int type, void *data)
+{
+	int error;
+
+	switch (type) {
+	case MOD_LOAD:
+		error = hwt_load();
+		break;
+	case MOD_UNLOAD:
+		error = hwt_unload();
+		break;
+	default:
+		error = 0;
+		break;
+	}
+
+	return (error);
+}
+
+static moduledata_t hwt_mod = {
+	"hwt",
+	hwt_modevent,
+	NULL
+};
+
+DECLARE_MODULE(hwt, hwt_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST);
+MODULE_VERSION(hwt, 1);
diff --git a/sys/dev/hwt/hwt_backend.c b/sys/dev/hwt/hwt_backend.c
new file mode 100644
index 000000000000..1ba5db0d3d09
--- /dev/null
+++ b/sys/dev/hwt/hwt_backend.c
@@ -0,0 +1,289 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023-2025 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This work was supported by Innovate UK project 105694, "Digital Security
+ * by Design (DSbD) Technology Platform Prototype".
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/* Hardware Trace (HWT) framework. */
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/mutex.h>
+#include <sys/hwt.h>
+
+#include <dev/hwt/hwt_hook.h>
+#include <dev/hwt/hwt_context.h>
+#include <dev/hwt/hwt_config.h>
+#include <dev/hwt/hwt_thread.h>
+#include <dev/hwt/hwt_backend.h>
+
+#define	HWT_BACKEND_DEBUG
+#undef	HWT_BACKEND_DEBUG
+
+#ifdef	HWT_BACKEND_DEBUG
+#define	dprintf(fmt, ...)	printf(fmt, ##__VA_ARGS__)
+#else
+#define	dprintf(fmt, ...)
+#endif
+
+static struct mtx hwt_backend_mtx;
+
+struct hwt_backend_entry {
+	struct hwt_backend *backend;
+	LIST_ENTRY(hwt_backend_entry) next;
+};
+
+static LIST_HEAD(, hwt_backend_entry)	hwt_backends;
+
+static MALLOC_DEFINE(M_HWT_BACKEND, "hwt_backend", "HWT backend");
+
+int
+hwt_backend_init(struct hwt_context *ctx)
+{
+	int error;
+
+	dprintf("%s\n", __func__);
+
+	error = ctx->hwt_backend->ops->hwt_backend_init(ctx);
+
+	return (error);
+}
+
+void
+hwt_backend_deinit(struct hwt_context *ctx)
+{
+
+	dprintf("%s\n", __func__);
+
+	ctx->hwt_backend->ops->hwt_backend_deinit(ctx);
+}
+
+int
+hwt_backend_configure(struct hwt_context *ctx, int cpu_id, int thread_id)
+{
+	int error;
+
+	dprintf("%s\n", __func__);
+
+	error = ctx->hwt_backend->ops->hwt_backend_configure(ctx, cpu_id,
+	    thread_id);
+
+	return (error);
+}
+
+void
+hwt_backend_enable(struct hwt_context *ctx, int cpu_id)
+{
+
+	dprintf("%s\n", __func__);
+
+	ctx->hwt_backend->ops->hwt_backend_enable(ctx, cpu_id);
+}
+
+void
+hwt_backend_disable(struct hwt_context *ctx, int cpu_id)
+{
+
+	dprintf("%s\n", __func__);
+
+	ctx->hwt_backend->ops->hwt_backend_disable(ctx, cpu_id);
+}
+
+void
+hwt_backend_enable_smp(struct hwt_context *ctx)
+{
+
+	dprintf("%s\n", __func__);
+
+	ctx->hwt_backend->ops->hwt_backend_enable_smp(ctx);
+}
+
+void
+hwt_backend_disable_smp(struct hwt_context *ctx)
+{
+
+	dprintf("%s\n", __func__);
+
+	ctx->hwt_backend->ops->hwt_backend_disable_smp(ctx);
+}
+
+void __unused
+hwt_backend_dump(struct hwt_context *ctx, int cpu_id)
+{
+
+	dprintf("%s\n", __func__);
+
+	ctx->hwt_backend->ops->hwt_backend_dump(cpu_id);
+}
+
+int
+hwt_backend_read(struct hwt_context *ctx, struct hwt_vm *vm, int *ident,
+    vm_offset_t *offset, uint64_t *data)
+{
+	int error;
+
+	dprintf("%s\n", __func__);
+
+	error = ctx->hwt_backend->ops->hwt_backend_read(vm, ident,
+	    offset, data);
+
+	return (error);
+}
+
+struct hwt_backend *
+hwt_backend_lookup(const char *name)
+{
+	struct hwt_backend_entry *entry;
+	struct hwt_backend *backend;
+
+	HWT_BACKEND_LOCK();
+	LIST_FOREACH(entry, &hwt_backends, next) {
+		backend = entry->backend;
+		if (strcmp(backend->name, name) == 0) {
+			HWT_BACKEND_UNLOCK();
+			return (backend);
+		}
+	}
+	HWT_BACKEND_UNLOCK();
+
+	return (NULL);
+}
+
+int
+hwt_backend_register(struct hwt_backend *backend)
+{
+	struct hwt_backend_entry *entry;
+
+	if (backend == NULL ||
+	    backend->name == NULL ||
+	    backend->ops == NULL)
+		return (EINVAL);
+
+	entry = malloc(sizeof(struct hwt_backend_entry), M_HWT_BACKEND,
+	    M_WAITOK | M_ZERO);
+	entry->backend = backend;
+
+	HWT_BACKEND_LOCK();
+	LIST_INSERT_HEAD(&hwt_backends, entry, next);
+	HWT_BACKEND_UNLOCK();
+
+	return (0);
+}
+
+int
+hwt_backend_unregister(struct hwt_backend *backend)
+{
+	struct hwt_backend_entry *entry, *tmp;
+
+	if (backend == NULL)
+		return (EINVAL);
+
+	/* TODO: check if not in use */
+
+	HWT_BACKEND_LOCK();
+	LIST_FOREACH_SAFE(entry, &hwt_backends, next, tmp) {
+		if (entry->backend == backend) {
+			LIST_REMOVE(entry, next);
+			HWT_BACKEND_UNLOCK();
+			free(entry, M_HWT_BACKEND);
+			return (0);
+		}
+	}
+	HWT_BACKEND_UNLOCK();
+
+	return (ENOENT);
+}
+
+void
+hwt_backend_load(void)
+{
+
+	mtx_init(&hwt_backend_mtx, "hwt backend", NULL, MTX_DEF);
+	LIST_INIT(&hwt_backends);
+}
+
+void
+hwt_backend_unload(void)
+{
+
+	/* TODO: ensure all unregistered */
+
+	mtx_destroy(&hwt_backend_mtx);
+}
+
+void
+hwt_backend_stop(struct hwt_context *ctx)
+{
+	dprintf("%s\n", __func__);
+
+	ctx->hwt_backend->ops->hwt_backend_stop(ctx);
+}
+
+int
+hwt_backend_svc_buf(struct hwt_context *ctx, void *data, size_t data_size,
+    int data_version)
+{
+	int error;
+
+	dprintf("%s\n", __func__);
+
+	error = ctx->hwt_backend->ops->hwt_backend_svc_buf(ctx, data, data_size,
+	    data_version);
+
+	return (error);
+}
+
+int
+hwt_backend_thread_alloc(struct hwt_context *ctx, struct hwt_thread *thr)
+{
+	int error;
+
+	dprintf("%s\n", __func__);
+
+	if (ctx->hwt_backend->ops->hwt_backend_thread_alloc == NULL)
+		return (0);
+	KASSERT(thr->private == NULL,
+		    ("%s: thread private data is not NULL\n", __func__));
+	error = ctx->hwt_backend->ops->hwt_backend_thread_alloc(thr);
+
+	return (error);
+}
+
+void
+hwt_backend_thread_free(struct hwt_thread *thr)
+{
+	dprintf("%s\n", __func__);
+
+	if (thr->backend->ops->hwt_backend_thread_free == NULL)
+		return;
+	KASSERT(thr->private != NULL,
+		    ("%s: thread private data is NULL\n", __func__));
+	thr->backend->ops->hwt_backend_thread_free(thr);
+
+	return;
+}
diff --git a/sys/dev/hwt/hwt_backend.h b/sys/dev/hwt/hwt_backend.h
new file mode 100644
index 000000000000..3b6c9442a7a6
--- /dev/null
+++ b/sys/dev/hwt/hwt_backend.h
@@ -0,0 +1,87 @@
+/*-
+ * Copyright (c) 2023-2025 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This work was supported by Innovate UK project 105694, "Digital Security
+ * by Design (DSbD) Technology Platform Prototype".
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _DEV_HWT_HWT_BACKEND_H_
+#define _DEV_HWT_HWT_BACKEND_H_
+
+struct hwt_backend_ops {
+	int (*hwt_backend_init)(struct hwt_context *);
+	int (*hwt_backend_deinit)(struct hwt_context *);
+	int (*hwt_backend_configure)(struct hwt_context *, int cpu_id,
+	    int thread_id);
+	int (*hwt_backend_svc_buf)(struct hwt_context *, void *data,
+	    size_t data_size, int data_version);
+	void (*hwt_backend_enable)(struct hwt_context *, int cpu_id);
+	void (*hwt_backend_disable)(struct hwt_context *, int cpu_id);
+	int (*hwt_backend_read)(struct hwt_vm *, int *ident,
+	    vm_offset_t *offset, uint64_t *data);
+	void (*hwt_backend_stop)(struct hwt_context *);
+	/* For backends that are tied to local CPU registers */
+	int (*hwt_backend_enable_smp)(struct hwt_context *);
+	int (*hwt_backend_disable_smp)(struct hwt_context *);
+	/* Allocation and initialization of backend-specific thread data. */
+	int (*hwt_backend_thread_alloc)(struct hwt_thread *);
+	void (*hwt_backend_thread_free)(struct hwt_thread *);
+	/* Debugging only. */
+	void (*hwt_backend_dump)(int cpu_id);
+};
+
+struct hwt_backend {
+	const char			*name;
+	struct hwt_backend_ops		*ops;
+	/* buffers require kernel virtual addresses */
+	bool				kva_req;
+};
+
+int hwt_backend_init(struct hwt_context *ctx);
+void hwt_backend_deinit(struct hwt_context *ctx);
+int hwt_backend_configure(struct hwt_context *ctx, int cpu_id, int thread_id);
+void hwt_backend_enable(struct hwt_context *ctx, int cpu_id);
+void hwt_backend_disable(struct hwt_context *ctx, int cpu_id);
+void hwt_backend_enable_smp(struct hwt_context *ctx);
+void hwt_backend_disable_smp(struct hwt_context *ctx);
+void hwt_backend_dump(struct hwt_context *ctx, int cpu_id);
+int hwt_backend_read(struct hwt_context *ctx, struct hwt_vm *vm, int *ident,
+    vm_offset_t *offset, uint64_t *data);
+int hwt_backend_register(struct hwt_backend *);
+int hwt_backend_unregister(struct hwt_backend *);
+void hwt_backend_stop(struct hwt_context *);
+int hwt_backend_svc_buf(struct hwt_context *ctx, void *data, size_t data_size,
+    int data_version);
+struct hwt_backend * hwt_backend_lookup(const char *name);
+int hwt_backend_thread_alloc(struct hwt_context *ctx, struct hwt_thread *);
+void hwt_backend_thread_free(struct hwt_thread *);
+
+void hwt_backend_load(void);
+void hwt_backend_unload(void);
+
+#define	HWT_BACKEND_LOCK()		mtx_lock(&hwt_backend_mtx)
+#define	HWT_BACKEND_UNLOCK()		mtx_unlock(&hwt_backend_mtx)
+
+#endif /* !_DEV_HWT_HWT_BACKEND_H_ */
+
diff --git a/sys/dev/hwt/hwt_config.c b/sys/dev/hwt/hwt_config.c
new file mode 100644
index 000000000000..30688e7fc76b
--- /dev/null
+++ b/sys/dev/hwt/hwt_config.c
@@ -0,0 +1,108 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023-2025 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This work was supported by Innovate UK project 105694, "Digital Security
+ * by Design (DSbD) Technology Platform Prototype".
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/param.h>
+#include <sys/proc.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/mutex.h>
+#include <sys/lock.h>
+#include <sys/hwt.h>
+
+#include <vm/vm.h>
+
+#include <dev/hwt/hwt_hook.h>
+#include <dev/hwt/hwt_context.h>
+#include <dev/hwt/hwt_contexthash.h>
+#include <dev/hwt/hwt_config.h>
+#include <dev/hwt/hwt_thread.h>
+#include <dev/hwt/hwt_record.h>
+
+#define	HWT_MAXCONFIGSIZE	PAGE_SIZE
+
+#define	HWT_CONFIG_DEBUG
+#undef	HWT_CONFIG_DEBUG
+
+#ifdef	HWT_CONFIG_DEBUG
+#define	dprintf(fmt, ...)	printf(fmt, ##__VA_ARGS__)
+#else
+#define	dprintf(fmt, ...)
+#endif
+
+static MALLOC_DEFINE(M_HWT_CONFIG, "hwt_config", "HWT config");
+
+int
+hwt_config_set(struct thread *td, struct hwt_context *ctx,
+    struct hwt_set_config *sconf)
+{
+	size_t config_size;
+	void *old_config;
+	void *config;
+	int error;
+
+	config_size = sconf->config_size;
+	if (config_size == 0)
+		return (0);
+
+	if (config_size > HWT_MAXCONFIGSIZE)
+		return (EFBIG);
+
+	config = malloc(config_size, M_HWT_CONFIG, M_WAITOK | M_ZERO);
+
+	error = copyin(sconf->config, config, config_size);
+	if (error) {
+		free(config, M_HWT_CONFIG);
+		return (error);
+	}
+
+	HWT_CTX_LOCK(ctx);
+	old_config = ctx->config;
+	ctx->config = config;
+	ctx->config_size = sconf->config_size;
+	ctx->config_version = sconf->config_version;
+	HWT_CTX_UNLOCK(ctx);
+
+	if (old_config != NULL)
+		free(old_config, M_HWT_CONFIG);
+
+	return (error);
+}
+
+void
+hwt_config_free(struct hwt_context *ctx)
+{
+
+	if (ctx->config == NULL)
+		return;
+
+	free(ctx->config, M_HWT_CONFIG);
+
+	ctx->config = NULL;
+}
diff --git a/sys/dev/hwt/hwt_config.h b/sys/dev/hwt/hwt_config.h
new file mode 100644
index 000000000000..47485583063c
--- /dev/null
+++ b/sys/dev/hwt/hwt_config.h
@@ -0,0 +1,36 @@
+/*-
+ * Copyright (c) 2023-2025 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This work was supported by Innovate UK project 105694, "Digital Security
+ * by Design (DSbD) Technology Platform Prototype".
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _DEV_HWT_HWT_CONFIG_H_
+#define _DEV_HWT_HWT_CONFIG_H_
+
+int hwt_config_set(struct thread *td, struct hwt_context *ctx,
+    struct hwt_set_config *sconf);
+void hwt_config_free(struct hwt_context *ctx);
+
+#endif /* !_DEV_HWT_HWT_CONFIG_H_ */
diff --git a/sys/dev/hwt/hwt_context.c b/sys/dev/hwt/hwt_context.c
new file mode 100644
index 000000000000..9af76cffc928
--- /dev/null
+++ b/sys/dev/hwt/hwt_context.c
@@ -0,0 +1,201 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023-2025 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This work was supported by Innovate UK project 105694, "Digital Security
+ * by Design (DSbD) Technology Platform Prototype".
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/param.h>
+#include <sys/bitstring.h>
+#include <sys/conf.h>
+#include <sys/proc.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/mman.h>
+#include <sys/mutex.h>
+#include <sys/refcount.h>
+#include <sys/rwlock.h>
+#include <sys/hwt.h>
+
+#include <dev/hwt/hwt_hook.h>
+#include <dev/hwt/hwt_context.h>
+#include <dev/hwt/hwt_config.h>
+#include <dev/hwt/hwt_thread.h>
+#include <dev/hwt/hwt_owner.h>
+#include <dev/hwt/hwt_vm.h>
+#include <dev/hwt/hwt_cpu.h>
+
+#define	HWT_DEBUG
+#undef	HWT_DEBUG
+
+#ifdef	HWT_DEBUG
+#define	dprintf(fmt, ...)	printf(fmt, ##__VA_ARGS__)
+#else
+#define	dprintf(fmt, ...)
+#endif
+
+static MALLOC_DEFINE(M_HWT_CTX, "hwt_ctx", "Hardware Trace");
+
+static bitstr_t *ident_set;
+static int ident_set_size;
+static struct mtx ident_set_mutex;
+
+static int
+hwt_ctx_ident_alloc(int *new_ident)
+{
+
+	mtx_lock(&ident_set_mutex);
+	bit_ffc(ident_set, ident_set_size, new_ident);
+	if (*new_ident == -1) {
+		mtx_unlock(&ident_set_mutex);
+		return (ENOMEM);
+	}
+	bit_set(ident_set, *new_ident);
+	mtx_unlock(&ident_set_mutex);
+
+	return (0);
+}
*** 3705 LINES SKIPPED ***