git: e51ef8ae490f - main - hwpmc: Initial support for AMD IBS
Date: Fri, 27 Feb 2026 22:08:46 UTC
The branch main has been updated by imp:
URL: https://cgit.FreeBSD.org/src/commit/?id=e51ef8ae490fc9f73191f33e7ad388c2511c454a
commit e51ef8ae490fc9f73191f33e7ad388c2511c454a
Author: Ali Mashtizadeh <ali@mashtizadeh.com>
AuthorDate: 2026-01-30 09:12:17 +0000
Commit: Warner Losh <imp@FreeBSD.org>
CommitDate: 2026-02-27 21:22:16 +0000
hwpmc: Initial support for AMD IBS
This patch adds support for AMD IBS. It adds a new class of performance
counter that cotains two events: ibs-fetch and ibs-op events. Unlike
most existing sampled events, IBS events provide a number of values
containing extra information regarding the sample. To support this we
use the existing callchain event, and introduce a new flag for multipart
payloads. The first 8 bytes of the pc_sample contains a header that
defines up to four payloads.
Sponsored by: Netflix
Reviewed by: imp,mhorne
Pull Request: https://github.com/freebsd/freebsd-src/pull/2022
---
lib/libpmc/libpmc.c | 64 ++++-
sys/amd64/include/pmc_mdep.h | 5 +
sys/conf/files.x86 | 1 +
sys/dev/hwpmc/hwpmc_amd.c | 25 +-
sys/dev/hwpmc/hwpmc_ibs.c | 614 +++++++++++++++++++++++++++++++++++++++++
sys/dev/hwpmc/hwpmc_ibs.h | 176 ++++++++++++
sys/dev/hwpmc/hwpmc_mod.c | 96 ++++++-
sys/dev/hwpmc/pmc_events.h | 13 +-
sys/i386/include/pmc_mdep.h | 5 +
sys/modules/hwpmc/Makefile | 4 +-
sys/sys/pmc.h | 19 +-
sys/sys/pmclog.h | 14 +
sys/x86/x86/local_apic.c | 27 +-
usr.sbin/pmcstat/pmcstat_log.c | 99 ++++++-
14 files changed, 1130 insertions(+), 32 deletions(-)
diff --git a/lib/libpmc/libpmc.c b/lib/libpmc/libpmc.c
index 10e357f55935..155da7cf6a7b 100644
--- a/lib/libpmc/libpmc.c
+++ b/lib/libpmc/libpmc.c
@@ -50,8 +50,8 @@
#if defined(__amd64__) || defined(__i386__)
static int k8_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
struct pmc_op_pmcallocate *_pmc_config);
-#endif
-#if defined(__amd64__) || defined(__i386__)
+static int ibs_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
+ struct pmc_op_pmcallocate *_pmc_config);
static int tsc_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
struct pmc_op_pmcallocate *_pmc_config);
#endif
@@ -132,6 +132,7 @@ struct pmc_class_descr {
PMC_CLASSDEP_TABLE(iaf, IAF);
PMC_CLASSDEP_TABLE(k8, K8);
+PMC_CLASSDEP_TABLE(ibs, IBS);
PMC_CLASSDEP_TABLE(armv7, ARMV7);
PMC_CLASSDEP_TABLE(armv8, ARMV8);
PMC_CLASSDEP_TABLE(cmn600_pmu, CMN600_PMU);
@@ -201,8 +202,7 @@ static const struct pmc_class_descr NAME##_class_table_descr = \
#if defined(__i386__) || defined(__amd64__)
PMC_CLASS_TABLE_DESC(k8, K8, k8, k8);
-#endif
-#if defined(__i386__) || defined(__amd64__)
+PMC_CLASS_TABLE_DESC(ibs, IBS, ibs, ibs);
PMC_CLASS_TABLE_DESC(tsc, TSC, tsc, tsc);
#endif
#if defined(__arm__)
@@ -691,9 +691,49 @@ k8_allocate_pmc(enum pmc_event pe, char *ctrspec,
return (0);
}
-#endif
+static int
+ibs_allocate_pmc(enum pmc_event pe, char *ctrspec,
+ struct pmc_op_pmcallocate *pmc_config)
+{
+ char *e, *p, *q;
+ uint64_t ctl;
+
+ pmc_config->pm_caps |=
+ (PMC_CAP_SYSTEM | PMC_CAP_EDGE | PMC_CAP_PRECISE);
+ pmc_config->pm_md.pm_ibs.ibs_ctl = 0;
+
+ /* setup parsing tables */
+ switch (pe) {
+ case PMC_EV_IBS_FETCH:
+ pmc_config->pm_md.pm_ibs.ibs_type = IBS_PMC_FETCH;
+ break;
+ case PMC_EV_IBS_OP:
+ pmc_config->pm_md.pm_ibs.ibs_type = IBS_PMC_OP;
+ break;
+ default:
+ return (-1);
+ }
+
+ /* parse parameters */
+ while ((p = strsep(&ctrspec, ",")) != NULL) {
+ if (KWPREFIXMATCH(p, "ctl=")) {
+ q = strchr(p, '=');
+ if (*++q == '\0') /* skip '=' */
+ return (-1);
+
+ ctl = strtoull(q, &e, 0);
+ if (e == q || *e != '\0')
+ return (-1);
+
+ pmc_config->pm_md.pm_ibs.ibs_ctl |= ctl;
+ } else {
+ return (-1);
+ }
+ }
+
+ return (0);
+}
-#if defined(__i386__) || defined(__amd64__)
static int
tsc_allocate_pmc(enum pmc_event pe, char *ctrspec,
struct pmc_op_pmcallocate *pmc_config)
@@ -1268,6 +1308,10 @@ pmc_event_names_of_class(enum pmc_class cl, const char ***eventnames,
ev = k8_event_table;
count = PMC_EVENT_TABLE_SIZE(k8);
break;
+ case PMC_CLASS_IBS:
+ ev = ibs_event_table;
+ count = PMC_EVENT_TABLE_SIZE(ibs);
+ break;
case PMC_CLASS_ARMV7:
switch (cpu_info.pm_cputype) {
default:
@@ -1471,6 +1515,10 @@ pmc_init(void)
case PMC_CLASS_K8:
pmc_class_table[n++] = &k8_class_table_descr;
break;
+
+ case PMC_CLASS_IBS:
+ pmc_class_table[n++] = &ibs_class_table_descr;
+ break;
#endif
case PMC_CLASS_SOFT:
@@ -1676,7 +1724,9 @@ _pmc_name_of_event(enum pmc_event pe, enum pmc_cputype cpu)
if (pe >= PMC_EV_K8_FIRST && pe <= PMC_EV_K8_LAST) {
ev = k8_event_table;
evfence = k8_event_table + PMC_EVENT_TABLE_SIZE(k8);
-
+ } else if (pe >= PMC_EV_IBS_FIRST && pe <= PMC_EV_IBS_LAST) {
+ ev = ibs_event_table;
+ evfence = ibs_event_table + PMC_EVENT_TABLE_SIZE(ibs);
} else if (pe >= PMC_EV_ARMV7_FIRST && pe <= PMC_EV_ARMV7_LAST) {
switch (cpu) {
case PMC_CPU_ARMV7_CORTEX_A8:
diff --git a/sys/amd64/include/pmc_mdep.h b/sys/amd64/include/pmc_mdep.h
index 5c20d8473855..24b785312a16 100644
--- a/sys/amd64/include/pmc_mdep.h
+++ b/sys/amd64/include/pmc_mdep.h
@@ -41,6 +41,7 @@ struct pmc_mdep;
#include <dev/hwpmc/hwpmc_amd.h>
#include <dev/hwpmc/hwpmc_core.h>
+#include <dev/hwpmc/hwpmc_ibs.h>
#include <dev/hwpmc/hwpmc_tsc.h>
#include <dev/hwpmc/hwpmc_uncore.h>
@@ -51,6 +52,7 @@ struct pmc_mdep;
*/
#define PMC_MDEP_CLASS_INDEX_TSC 1
#define PMC_MDEP_CLASS_INDEX_K8 2
+#define PMC_MDEP_CLASS_INDEX_IBS 3
#define PMC_MDEP_CLASS_INDEX_P4 2
#define PMC_MDEP_CLASS_INDEX_IAP 2
#define PMC_MDEP_CLASS_INDEX_IAF 3
@@ -62,6 +64,7 @@ struct pmc_mdep;
*
* TSC The timestamp counter
* K8 AMD Athlon64 and Opteron PMCs in 64 bit mode.
+ * IBS AMD IBS
* PIV Intel P4/HTT and P4/EMT64
* IAP Intel Core/Core2/Atom CPUs in 64 bits mode.
* IAF Intel fixed-function PMCs in Core2 and later CPUs.
@@ -71,6 +74,7 @@ struct pmc_mdep;
union pmc_md_op_pmcallocate {
struct pmc_md_amd_op_pmcallocate pm_amd;
+ struct pmc_md_ibs_op_pmcallocate pm_ibs;
struct pmc_md_iap_op_pmcallocate pm_iap;
struct pmc_md_ucf_op_pmcallocate pm_ucf;
struct pmc_md_ucp_op_pmcallocate pm_ucp;
@@ -85,6 +89,7 @@ union pmc_md_op_pmcallocate {
union pmc_md_pmc {
struct pmc_md_amd_pmc pm_amd;
+ struct pmc_md_ibs_pmc pm_ibs;
struct pmc_md_iaf_pmc pm_iaf;
struct pmc_md_iap_pmc pm_iap;
struct pmc_md_ucf_pmc pm_ucf;
diff --git a/sys/conf/files.x86 b/sys/conf/files.x86
index fabcd5d9ebe5..8a7e0b78feb4 100644
--- a/sys/conf/files.x86
+++ b/sys/conf/files.x86
@@ -114,6 +114,7 @@ dev/hptrr/hptrr_osm_bsd.c optional hptrr
dev/hptrr/hptrr_config.c optional hptrr
dev/hptrr/$M-elf.hptrr_lib.o optional hptrr
dev/hwpmc/hwpmc_amd.c optional hwpmc
+dev/hwpmc/hwpmc_ibs.c optional hwpmc
dev/hwpmc/hwpmc_intel.c optional hwpmc
dev/hwpmc/hwpmc_core.c optional hwpmc
dev/hwpmc/hwpmc_uncore.c optional hwpmc
diff --git a/sys/dev/hwpmc/hwpmc_amd.c b/sys/dev/hwpmc/hwpmc_amd.c
index b34cbffcffa8..cf44f9362a72 100644
--- a/sys/dev/hwpmc/hwpmc_amd.c
+++ b/sys/dev/hwpmc/hwpmc_amd.c
@@ -543,6 +543,10 @@ amd_intr(struct trapframe *tf)
pac = amd_pcpu[cpu];
+ retval = pmc_ibs_intr(tf);
+ if (retval)
+ goto done;
+
/*
* look for all PMCs that have interrupted:
* - look for a running, sampling PMC which has overflowed
@@ -613,6 +617,7 @@ amd_intr(struct trapframe *tf)
}
}
+done:
if (retval)
counter_u64_add(pmc_stats.pm_intr_processed, 1);
else
@@ -760,7 +765,7 @@ pmc_amd_initialize(void)
struct pmc_classdep *pcd;
struct pmc_mdep *pmc_mdep;
enum pmc_cputype cputype;
- int error, i, ncpus;
+ int error, i, ncpus, nclasses;
int family, model, stepping;
int amd_core_npmcs, amd_l3_npmcs, amd_df_npmcs;
struct amd_descr *d;
@@ -884,10 +889,16 @@ pmc_amd_initialize(void)
M_WAITOK | M_ZERO);
/*
- * These processors have two classes of PMCs: the TSC and
- * programmable PMCs.
+ * These processors have two or three classes of PMCs: the TSC,
+ * programmable PMCs, and AMD IBS.
*/
- pmc_mdep = pmc_mdep_alloc(2);
+ if ((amd_feature2 & AMDID2_IBS) != 0) {
+ nclasses = 3;
+ } else {
+ nclasses = 2;
+ }
+
+ pmc_mdep = pmc_mdep_alloc(nclasses);
ncpus = pmc_cpu_max();
@@ -927,6 +938,12 @@ pmc_amd_initialize(void)
PMCDBG0(MDP, INI, 0, "amd-initialize");
+ if (nclasses >= 3) {
+ error = pmc_ibs_initialize(pmc_mdep, ncpus);
+ if (error != 0)
+ goto error;
+ }
+
return (pmc_mdep);
error:
diff --git a/sys/dev/hwpmc/hwpmc_ibs.c b/sys/dev/hwpmc/hwpmc_ibs.c
new file mode 100644
index 000000000000..66d3260cf040
--- /dev/null
+++ b/sys/dev/hwpmc/hwpmc_ibs.c
@@ -0,0 +1,614 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2026, Ali Jose Mashtizadeh
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/* Support for the AMD IBS */
+
+#include <sys/param.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mutex.h>
+#include <sys/pcpu.h>
+#include <sys/pmc.h>
+#include <sys/pmckern.h>
+#include <sys/pmclog.h>
+#include <sys/smp.h>
+#include <sys/systm.h>
+
+#include <machine/cpu.h>
+#include <machine/cpufunc.h>
+#include <machine/md_var.h>
+#include <machine/specialreg.h>
+
+#define IBS_STOP_ITER 50 /* Stopping iterations */
+
+/* AMD IBS PMCs */
+struct ibs_descr {
+ struct pmc_descr pm_descr; /* "base class" */
+};
+
+/*
+ * Globals
+ */
+static uint64_t ibs_features;
+
+/*
+ * Per-processor information
+ */
+#define IBS_CPU_RUNNING 1
+#define IBS_CPU_STOPPING 2
+#define IBS_CPU_STOPPED 3
+
+struct ibs_cpu {
+ int pc_status;
+ struct pmc_hw pc_ibspmcs[IBS_NPMCS];
+};
+static struct ibs_cpu **ibs_pcpu;
+
+/*
+ * Read a PMC value from the MSR.
+ */
+static int
+ibs_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v)
+{
+
+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
+ ("[ibs,%d] illegal CPU value %d", __LINE__, cpu));
+ KASSERT(ri >= 0 && ri < IBS_NPMCS,
+ ("[ibs,%d] illegal row-index %d", __LINE__, ri));
+ KASSERT(ibs_pcpu[cpu],
+ ("[ibs,%d] null per-cpu, cpu %d", __LINE__, cpu));
+
+ /* read the IBS ctl */
+ switch (ri) {
+ case IBS_PMC_FETCH:
+ *v = rdmsr(IBS_FETCH_CTL);
+ break;
+ case IBS_PMC_OP:
+ *v = rdmsr(IBS_OP_CTL);
+ break;
+ }
+
+ PMCDBG2(MDP, REA, 2, "ibs-read id=%d -> %jd", ri, *v);
+
+ return (0);
+}
+
+/*
+ * Write a PMC MSR.
+ */
+static int
+ibs_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)
+{
+
+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
+ ("[ibs,%d] illegal CPU value %d", __LINE__, cpu));
+ KASSERT(ri >= 0 && ri < IBS_NPMCS,
+ ("[ibs,%d] illegal row-index %d", __LINE__, ri));
+
+ PMCDBG3(MDP, WRI, 1, "ibs-write cpu=%d ri=%d v=%jx", cpu, ri, v);
+
+ return (0);
+}
+
+/*
+ * Configure hardware PMC according to the configuration recorded in 'pm'.
+ */
+static int
+ibs_config_pmc(int cpu, int ri, struct pmc *pm)
+{
+ struct pmc_hw *phw;
+
+ PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
+
+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
+ ("[ibs,%d] illegal CPU value %d", __LINE__, cpu));
+ KASSERT(ri >= 0 && ri < IBS_NPMCS,
+ ("[ibs,%d] illegal row-index %d", __LINE__, ri));
+
+ phw = &ibs_pcpu[cpu]->pc_ibspmcs[ri];
+
+ KASSERT(pm == NULL || phw->phw_pmc == NULL,
+ ("[ibs,%d] pm=%p phw->pm=%p hwpmc not unconfigured",
+ __LINE__, pm, phw->phw_pmc));
+
+ phw->phw_pmc = pm;
+
+ return (0);
+}
+
+/*
+ * Retrieve a configured PMC pointer from hardware state.
+ */
+static int
+ibs_get_config(int cpu, int ri, struct pmc **ppm)
+{
+
+ *ppm = ibs_pcpu[cpu]->pc_ibspmcs[ri].phw_pmc;
+
+ return (0);
+}
+
+/*
+ * Check if a given PMC allocation is feasible.
+ */
+static int
+ibs_allocate_pmc(int cpu __unused, int ri, struct pmc *pm,
+ const struct pmc_op_pmcallocate *a)
+{
+ uint64_t caps, config;
+
+ KASSERT(ri >= 0 && ri < IBS_NPMCS,
+ ("[ibs,%d] illegal row index %d", __LINE__, ri));
+
+ /* check class match */
+ if (a->pm_class != PMC_CLASS_IBS)
+ return (EINVAL);
+ if (a->pm_md.pm_ibs.ibs_type != ri)
+ return (EINVAL);
+
+ caps = pm->pm_caps;
+
+ PMCDBG2(MDP, ALL, 1, "ibs-allocate ri=%d caps=0x%x", ri, caps);
+
+ if ((caps & PMC_CAP_SYSTEM) == 0)
+ return (EINVAL);
+
+ config = a->pm_md.pm_ibs.ibs_ctl;
+ pm->pm_md.pm_ibs.ibs_ctl = config;
+
+ PMCDBG2(MDP, ALL, 2, "ibs-allocate ri=%d -> config=0x%x", ri, config);
+
+ return (0);
+}
+
+/*
+ * Release machine dependent state associated with a PMC. This is a
+ * no-op on this architecture.
+ */
+static int
+ibs_release_pmc(int cpu, int ri, struct pmc *pmc __unused)
+{
+ struct pmc_hw *phw __diagused;
+
+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
+ ("[ibs,%d] illegal CPU value %d", __LINE__, cpu));
+ KASSERT(ri >= 0 && ri < IBS_NPMCS,
+ ("[ibs,%d] illegal row-index %d", __LINE__, ri));
+
+ PMCDBG1(MDP, ALL, 1, "ibs-release ri=%d", ri);
+
+ phw = &ibs_pcpu[cpu]->pc_ibspmcs[ri];
+
+ KASSERT(phw->phw_pmc == NULL,
+ ("[ibs,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
+
+ return (0);
+}
+
+/*
+ * Start a PMC.
+ */
+static int
+ibs_start_pmc(int cpu __diagused, int ri, struct pmc *pm)
+{
+ uint64_t config;
+
+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
+ ("[ibs,%d] illegal CPU value %d", __LINE__, cpu));
+ KASSERT(ri >= 0 && ri < IBS_NPMCS,
+ ("[ibs,%d] illegal row-index %d", __LINE__, ri));
+
+ PMCDBG2(MDP, STA, 1, "ibs-start cpu=%d ri=%d", cpu, ri);
+
+ /*
+ * This is used to handle spurious NMIs. All that matters is that it
+ * is not in the stopping state.
+ */
+ atomic_store_int(&ibs_pcpu[cpu]->pc_status, IBS_CPU_RUNNING);
+
+ /*
+ * Turn on the ENABLE bit. Zeroing out the control register eliminates
+ * stale valid bits from spurious NMIs and it resets the counter.
+ */
+ switch (ri) {
+ case IBS_PMC_FETCH:
+ wrmsr(IBS_FETCH_CTL, 0);
+ config = pm->pm_md.pm_ibs.ibs_ctl | IBS_FETCH_CTL_ENABLE;
+ wrmsr(IBS_FETCH_CTL, config);
+ break;
+ case IBS_PMC_OP:
+ wrmsr(IBS_OP_CTL, 0);
+ config = pm->pm_md.pm_ibs.ibs_ctl | IBS_OP_CTL_ENABLE;
+ wrmsr(IBS_OP_CTL, config);
+ break;
+ }
+
+ return (0);
+}
+
+/*
+ * Stop a PMC.
+ */
+static int
+ibs_stop_pmc(int cpu __diagused, int ri, struct pmc *pm)
+{
+ int i;
+ uint64_t config;
+
+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
+ ("[ibs,%d] illegal CPU value %d", __LINE__, cpu));
+ KASSERT(ri >= 0 && ri < IBS_NPMCS,
+ ("[ibs,%d] illegal row-index %d", __LINE__, ri));
+
+ PMCDBG1(MDP, STO, 1, "ibs-stop ri=%d", ri);
+
+ /*
+ * Turn off the ENABLE bit, but unfortunately there are a few quirks
+ * that generate excess NMIs. Workaround #420 in the Revision Guide
+ * for AMD Family 10h Processors 41322 Rev. 3.92 March 2012. requires
+ * that we clear the count before clearing enable.
+ *
+ * Even after clearing the counter spurious NMIs are still possible so
+ * we use a per-CPU atomic variable to notify the interrupt handler we
+ * are stopping and discard spurious NMIs. We then retry clearing the
+ * control register for 50us. This gives us enough time and ensures
+ * that the valid bit is not accidently stuck after a spurious NMI.
+ */
+ config = pm->pm_md.pm_ibs.ibs_ctl;
+
+ atomic_store_int(&ibs_pcpu[cpu]->pc_status, IBS_CPU_STOPPING);
+
+ switch (ri) {
+ case IBS_PMC_FETCH:
+ wrmsr(IBS_FETCH_CTL, config & ~IBS_FETCH_CTL_MAXCNTMASK);
+ DELAY(1);
+ config &= ~IBS_FETCH_CTL_ENABLE;
+ wrmsr(IBS_FETCH_CTL, config);
+ break;
+ case IBS_PMC_OP:
+ wrmsr(IBS_FETCH_CTL, config & ~IBS_FETCH_CTL_MAXCNTMASK);
+ DELAY(1);
+ config &= ~IBS_OP_CTL_ENABLE;
+ wrmsr(IBS_OP_CTL, config);
+ break;
+ }
+
+ for (i = 0; i < IBS_STOP_ITER; i++) {
+ DELAY(1);
+
+ switch (ri) {
+ case IBS_PMC_FETCH:
+ wrmsr(IBS_FETCH_CTL, 0);
+ break;
+ case IBS_PMC_OP:
+ wrmsr(IBS_OP_CTL, 0);
+ break;
+ }
+ }
+
+ atomic_store_int(&ibs_pcpu[cpu]->pc_status, IBS_CPU_STOPPED);
+
+ return (0);
+}
+
+static void
+pmc_ibs_process_fetch(struct pmc *pm, struct trapframe *tf, uint64_t config)
+{
+ struct pmc_multipart mpd;
+
+ if (pm == NULL)
+ return;
+
+ if (pm->pm_state != PMC_STATE_RUNNING)
+ return;
+
+ memset(&mpd, 0, sizeof(mpd));
+
+ mpd.pl_type = PMC_CC_MULTIPART_IBS_FETCH;
+ mpd.pl_length = 4;
+ mpd.pl_mpdata[PMC_MPIDX_FETCH_CTL] = config;
+ if (ibs_features) {
+ mpd.pl_mpdata[PMC_MPIDX_FETCH_EXTCTL] = rdmsr(IBS_FETCH_EXTCTL);
+ }
+ mpd.pl_mpdata[PMC_MPIDX_FETCH_CTL] = config;
+ mpd.pl_mpdata[PMC_MPIDX_FETCH_LINADDR] = rdmsr(IBS_FETCH_LINADDR);
+ if ((config & IBS_FETCH_CTL_PHYSADDRVALID) != 0) {
+ mpd.pl_mpdata[PMC_MPIDX_FETCH_PHYSADDR] =
+ rdmsr(IBS_FETCH_PHYSADDR);
+ }
+
+ pmc_process_interrupt_mp(PMC_HR, pm, tf, &mpd);
+}
+
+static void
+pmc_ibs_process_op(struct pmc *pm, struct trapframe *tf, uint64_t config)
+{
+ struct pmc_multipart mpd;
+
+ if (pm == NULL)
+ return;
+
+ if (pm->pm_state != PMC_STATE_RUNNING)
+ return;
+
+ memset(&mpd, 0, sizeof(mpd));
+
+ mpd.pl_type = PMC_CC_MULTIPART_IBS_OP;
+ mpd.pl_length = 8;
+ mpd.pl_mpdata[PMC_MPIDX_OP_CTL] = config;
+ mpd.pl_mpdata[PMC_MPIDX_OP_RIP] = rdmsr(IBS_OP_RIP);
+ mpd.pl_mpdata[PMC_MPIDX_OP_DATA] = rdmsr(IBS_OP_DATA);
+ mpd.pl_mpdata[PMC_MPIDX_OP_DATA2] = rdmsr(IBS_OP_DATA2);
+ mpd.pl_mpdata[PMC_MPIDX_OP_DATA3] = rdmsr(IBS_OP_DATA3);
+ mpd.pl_mpdata[PMC_MPIDX_OP_DC_LINADDR] = rdmsr(IBS_OP_DC_LINADDR);
+ mpd.pl_mpdata[PMC_MPIDX_OP_DC_PHYSADDR] = rdmsr(IBS_OP_DC_PHYSADDR);
+
+ pmc_process_interrupt_mp(PMC_HR, pm, tf, &mpd);
+
+ wrmsr(IBS_OP_CTL, pm->pm_md.pm_ibs.ibs_ctl | IBS_OP_CTL_ENABLE);
+}
+
+/*
+ * Interrupt handler. This function needs to return '1' if the
+ * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed
+ * to sleep or do anything a 'fast' interrupt handler is not allowed
+ * to do.
+ */
+int
+pmc_ibs_intr(struct trapframe *tf)
+{
+ struct ibs_cpu *pac;
+ struct pmc *pm;
+ int retval, cpu;
+ uint64_t config;
+
+ cpu = curcpu;
+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
+ ("[ibs,%d] out of range CPU %d", __LINE__, cpu));
+
+ PMCDBG3(MDP, INT, 1, "cpu=%d tf=%p um=%d", cpu, tf, TRAPF_USERMODE(tf));
+
+ retval = 0;
+
+ pac = ibs_pcpu[cpu];
+
+ config = rdmsr(IBS_FETCH_CTL);
+ if ((config & IBS_FETCH_CTL_VALID) != 0) {
+ pm = pac->pc_ibspmcs[IBS_PMC_FETCH].phw_pmc;
+
+ retval = 1;
+
+ pmc_ibs_process_fetch(pm, tf, config);
+ }
+
+ config = rdmsr(IBS_OP_CTL);
+ if ((retval == 0) && ((config & IBS_OP_CTL_VALID) != 0)) {
+ pm = pac->pc_ibspmcs[IBS_PMC_OP].phw_pmc;
+
+ retval = 1;
+
+ pmc_ibs_process_op(pm, tf, config);
+ }
+
+ if (retval == 0) {
+ // Lets check for a stray NMI when stopping
+ if (atomic_load_int(&pac->pc_status) == IBS_CPU_STOPPING) {
+ return (1);
+ }
+ }
+
+
+ if (retval)
+ counter_u64_add(pmc_stats.pm_intr_processed, 1);
+ else
+ counter_u64_add(pmc_stats.pm_intr_ignored, 1);
+
+ PMCDBG1(MDP, INT, 2, "retval=%d", retval);
+
+ return (retval);
+}
+
+/*
+ * Describe a PMC.
+ */
+static int
+ibs_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
+{
+ struct pmc_hw *phw;
+
+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
+ ("[ibs,%d] illegal CPU %d", __LINE__, cpu));
+ KASSERT(ri >= 0 && ri < IBS_NPMCS,
+ ("[ibs,%d] row-index %d out of range", __LINE__, ri));
+
+ phw = &ibs_pcpu[cpu]->pc_ibspmcs[ri];
+
+ if (ri == IBS_PMC_FETCH) {
+ strlcpy(pi->pm_name, "IBS-FETCH", sizeof(pi->pm_name));
+ pi->pm_class = PMC_CLASS_IBS;
+ pi->pm_enabled = true;
+ *ppmc = phw->phw_pmc;
+ } else {
+ strlcpy(pi->pm_name, "IBS-OP", sizeof(pi->pm_name));
+ pi->pm_class = PMC_CLASS_IBS;
+ pi->pm_enabled = true;
+ *ppmc = phw->phw_pmc;
+ }
+
+ return (0);
+}
+
+/*
+ * Processor-dependent initialization.
+ */
+static int
+ibs_pcpu_init(struct pmc_mdep *md, int cpu)
+{
+ struct ibs_cpu *pac;
+ struct pmc_cpu *pc;
+ struct pmc_hw *phw;
+ int first_ri, n;
+
+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
+ ("[ibs,%d] insane cpu number %d", __LINE__, cpu));
+
+ PMCDBG1(MDP, INI, 1, "ibs-init cpu=%d", cpu);
+
+ ibs_pcpu[cpu] = pac = malloc(sizeof(struct ibs_cpu), M_PMC,
+ M_WAITOK | M_ZERO);
+
+ /*
+ * Set the content of the hardware descriptors to a known
+ * state and initialize pointers in the MI per-cpu descriptor.
+ */
+ pc = pmc_pcpu[cpu];
+ first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IBS].pcd_ri;
+
+ KASSERT(pc != NULL, ("[ibs,%d] NULL per-cpu pointer", __LINE__));
+
+ for (n = 0, phw = pac->pc_ibspmcs; n < IBS_NPMCS; n++, phw++) {
+ phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
+ PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
+ phw->phw_pmc = NULL;
+ pc->pc_hwpmcs[n + first_ri] = phw;
+ }
+
+ return (0);
+}
+
+/*
+ * Processor-dependent cleanup prior to the KLD being unloaded.
+ */
+static int
+ibs_pcpu_fini(struct pmc_mdep *md, int cpu)
+{
+ struct ibs_cpu *pac;
+ struct pmc_cpu *pc;
+ int first_ri, i;
+
+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
+ ("[ibs,%d] insane cpu number (%d)", __LINE__, cpu));
+
+ PMCDBG1(MDP, INI, 1, "ibs-cleanup cpu=%d", cpu);
+
+ /*
+ * Turn off IBS.
+ */
+ wrmsr(IBS_FETCH_CTL, 0);
+ wrmsr(IBS_OP_CTL, 0);
+
+ /*
+ * Free up allocated space.
+ */
+ if ((pac = ibs_pcpu[cpu]) == NULL)
+ return (0);
+
+ ibs_pcpu[cpu] = NULL;
+
+ pc = pmc_pcpu[cpu];
+ KASSERT(pc != NULL, ("[ibs,%d] NULL per-cpu state", __LINE__));
+
+ first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IBS].pcd_ri;
+
+ /*
+ * Reset pointers in the MI 'per-cpu' state.
+ */
+ for (i = 0; i < IBS_NPMCS; i++)
+ pc->pc_hwpmcs[i + first_ri] = NULL;
+
+ free(pac, M_PMC);
+
+ return (0);
+}
+
+/*
+ * Initialize ourselves.
+ */
+int
+pmc_ibs_initialize(struct pmc_mdep *pmc_mdep, int ncpus)
+{
+ u_int regs[4];
+ struct pmc_classdep *pcd;
+
+ /*
+ * Allocate space for pointers to PMC HW descriptors and for
+ * the MDEP structure used by MI code.
+ */
+ ibs_pcpu = malloc(sizeof(struct ibs_cpu *) * pmc_cpu_max(), M_PMC,
+ M_WAITOK | M_ZERO);
+
+ /* Initialize AMD IBS handling. */
+ pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_IBS];
+
+ pcd->pcd_caps = IBS_PMC_CAPS;
+ pcd->pcd_class = PMC_CLASS_IBS;
+ pcd->pcd_num = IBS_NPMCS;
+ pcd->pcd_ri = pmc_mdep->pmd_npmc;
+ pcd->pcd_width = 0;
+
+ pcd->pcd_allocate_pmc = ibs_allocate_pmc;
+ pcd->pcd_config_pmc = ibs_config_pmc;
+ pcd->pcd_describe = ibs_describe;
+ pcd->pcd_get_config = ibs_get_config;
+ pcd->pcd_pcpu_fini = ibs_pcpu_fini;
+ pcd->pcd_pcpu_init = ibs_pcpu_init;
+ pcd->pcd_release_pmc = ibs_release_pmc;
+ pcd->pcd_start_pmc = ibs_start_pmc;
+ pcd->pcd_stop_pmc = ibs_stop_pmc;
+ pcd->pcd_read_pmc = ibs_read_pmc;
+ pcd->pcd_write_pmc = ibs_write_pmc;
+
+ pmc_mdep->pmd_npmc += IBS_NPMCS;
+
+ if (cpu_exthigh >= CPUID_IBSID) {
+ do_cpuid(CPUID_IBSID, regs);
+ ibs_features = regs[0];
+ } else {
+ ibs_features = 0;
+ }
+
+ PMCDBG0(MDP, INI, 0, "ibs-initialize");
+
+ return (0);
+}
+
+/*
+ * Finalization code for AMD CPUs.
+ */
+void
+pmc_ibs_finalize(struct pmc_mdep *md)
+{
+ PMCDBG0(MDP, INI, 1, "ibs-finalize");
+
+ for (int i = 0; i < pmc_cpu_max(); i++)
+ KASSERT(ibs_pcpu[i] == NULL,
+ ("[ibs,%d] non-null pcpu cpu %d", __LINE__, i));
+
+ free(ibs_pcpu, M_PMC);
+ ibs_pcpu = NULL;
+}
diff --git a/sys/dev/hwpmc/hwpmc_ibs.h b/sys/dev/hwpmc/hwpmc_ibs.h
new file mode 100644
index 000000000000..4449b44c8368
--- /dev/null
+++ b/sys/dev/hwpmc/hwpmc_ibs.h
@@ -0,0 +1,176 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2026, Ali Jose Mashtizadeh
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _DEV_HWPMC_IBS_H_
+#define _DEV_HWPMC_IBS_H_ 1
+
+#define IBS_NPMCS 2
+#define IBS_PMC_FETCH 0
+#define IBS_PMC_OP 1
+
+/*
+ * All of the CPUID definitions come from AMD PPR Vol 1 for AMD Family 1Ah
+ * Model 02h C1 (57238) 2024-09-29 Revision 0.24.
+ */
+#define CPUID_IBSID 0x8000001B
+#define CPUID_IBSID_IBSFFV 0x00000001 /* IBS Feature Flags Valid */
+#define CPUID_IBSID_FETCHSAM 0x00000002 /* IBS Fetch Sampling */
+#define CPUID_IBSID_OPSAM 0x00000004 /* IBS Execution Sampling */
+#define CPUID_IBSID_RDWROPCNT 0x00000008 /* RdWr Operationg Counter */
+#define CPUID_IBSID_OPCNT 0x00000010 /* Operation Counter */
+#define CPUID_IBSID_BRNTRGT 0x00000020 /* Branch Target Address */
+#define CPUID_IBSID_OPCNTEXT 0x00000040 /* Extend Counter */
+#define CPUID_IBSID_RIPINVALIDCHK 0x00000080 /* Invalid RIP Indication */
+#define CPUID_IBSID_OPFUSE 0x00000010 /* Fused Branch Operation */
+#define CPUID_IBSID_IBSFETCHCTLEXTD 0x00000020 /* IBS Fetch Control Ext */
+#define CPUID_IBSID_IBSOPDATA4 0x00000040 /* IBS OP DATA4 */
+#define CPUID_IBSID_ZEN4IBSEXTENSIONS 0x00000080 /* IBS Zen 4 Extensions */
+#define CPUID_IBSID_IBSLOADLATENCYFILT 0x00000100 /* Load Latency Filtering */
+#define CPUID_IBSID_IBSUPDTDDTLBSTATS 0x00080000 /* Simplified DTLB Stats */
+
+/*
+ * All of these definitions here come from AMD64 Architecture Programmer's
+ * Manual Volume 2: System Programming (24593) 2025-07-02 Version 3.43. with
+ * the following exceptions:
+ *
+ * OpData4 and fields come from the BKDG for AMD Family 15h Model 70-7Fh
+ * (55072) 2018-06-20 Revision 3.09.
+ */
+
+/* IBS MSRs */
+#define IBS_CTL 0xC001103A /* IBS Control */
+#define IBS_CTL_LVTOFFSETVALID (1ULL << 8)
+#define IBS_CTL_LVTOFFSETMASK 0x0000000F
+
+/* IBS Fetch Control */
+#define IBS_FETCH_CTL 0xC0011030 /* IBS Fetch Control */
+#define IBS_FETCH_CTL_L3MISS (1ULL << 61) /* L3 Cache Miss */
+#define IBS_FETCH_CTL_OPCACHEMISS (1ULL << 60) /* Op Cache Miss */
+#define IBS_FETCH_CTL_L3MISSONLY (1ULL << 59) /* L3 Miss Filtering */
+#define IBS_FETCH_CTL_RANDOMIZE (1ULL << 57) /* Randomized Tagging */
+#define IBS_FETCH_CTL_L1TLBMISS (1ULL << 55) /* L1 TLB Miss */
+// Page size 54:53
+#define IBS_FETCH_CTL_PHYSADDRVALID (1ULL << 52) /* PHYSADDR Valid */
+#define IBS_FETCH_CTL_ICMISS (1ULL << 51) /* Inst. Cache Miss */
+#define IBS_FETCH_CTL_COMPLETE (1ULL << 50) /* Complete */
*** 650 LINES SKIPPED ***