git: b9cee186b83e - main - ice: Add a failure injection facility
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Thu, 17 Sep 2026 08:40:57 UTC
The branch main has been updated by kbowling:
URL: https://cgit.FreeBSD.org/src/commit/?id=b9cee186b83ecb531a6d4b4421230acfd5314c16
commit b9cee186b83ecb531a6d4b4421230acfd5314c16
Author: Kevin Bowling <kbowling@FreeBSD.org>
AuthorDate: 2026-08-19 00:33:33 +0000
Commit: Kevin Bowling <kbowling@FreeBSD.org>
CommitDate: 2026-09-17 08:37:51 +0000
ice: Add a failure injection facility
Add compile-time optional, non-sleeping fail points around every VF
creation resource boundary, before VF VSI reconstruction, and in the
GET_STATS validation path.
Provide an ICE-wide wrapper and device selector so other driver
subsystems can add scoped points without duplicating the failpoint
plumbing. Keep the current SR-IOV points and VF selector in an iov
child namespace.
Compile the facility only with options DRIVER_FAILPOINTS. This shared
option avoids a separate kernel option for every driver that provides
test-only injection hooks. Ordinary kernels contain no ICE failpoint
objects or sysctl nodes. Require an exact PF device name and
optionally a VF index before any point can fire. This prevents a stale
test setting from affecting another PF.
The hooks exposed two reset-lifetime defects while validating the
existing SR-IOV review series. A PF reset could discard a firmware VSI
before teardown, and a rebuilt sibling could leave stale switch-filter
state after IOV destroy.
Tested on an E810-XXV with INVARIANTS and WITNESS. All twelve creation
checkpoints rolled back and permitted immediate resource reuse. Forced
reconstruction and malformed GET_STATS failures also preserved sibling
operation and reply cardinality.
Reviewed by: gallatin, nprice, ziaee
MFC after: 2 weeks
Sponsored by: BBOX.io
Differential Revision: https://reviews.freebsd.org/D58940
---
share/man/man4/ice.4 | 46 +++++++++++++++++++++++++++++--
sys/conf/NOTES | 7 +++++
sys/conf/options | 1 +
sys/dev/ice/ice_fault.h | 37 +++++++++++++++++++++++++
sys/dev/ice/ice_iov.c | 71 ++++++++++++++++++++++++++++++++++++++++++++++++
sys/dev/ice/ice_lib.c | 28 +++++++++++++++++++
sys/dev/ice/ice_opts.h | 1 +
sys/modules/ice/Makefile | 1 +
8 files changed, 190 insertions(+), 2 deletions(-)
diff --git a/share/man/man4/ice.4 b/share/man/man4/ice.4
index 51c172e08fcb..31b61136e36b 100644
--- a/share/man/man4/ice.4
+++ b/share/man/man4/ice.4
@@ -32,7 +32,7 @@
.\"
.\" * Other names and brands may be claimed as the property of others.
.\"
-.Dd August 19, 2026
+.Dd August 27, 2026
.Dt ICE 4
.Os
.Sh NAME
@@ -124,6 +124,8 @@ The following topics are covered in this manual:
.It
.Sx IOVCTL OPTIONS
.It
+.Sx FAILURE INJECTION
+.It
.Sx SUPPORT
.It
.Sx SEE ALSO
@@ -1161,6 +1163,45 @@ option.
.Pp
For more information on standard and mandatory parameters, see
.Xr iovctl.conf 5 .
+.Sh FAILURE INJECTION
+The driver provides destructive, test-only failure injection through the
+.Xr fail 9
+interface.
+Kernels compiled with
+.Cd options DRIVER_FAILPOINTS
+expose the driver fail points beneath the
+.Va debug.fail_point.ice
+sysctl node.
+Ordinary kernels omit these controls.
+.Pp
+Set
+.Va debug.fail_point.ice.device
+to the exact PF device name before a fail point can fire.
+An empty device selector disables every ICE fail point even if an individual
+point remains armed.
+For SR-IOV tests,
+.Va debug.fail_point.ice.iov.vf
+selects a PF-local VF index, and a value of -1 selects every VF on the chosen
+PF.
+.Pp
+List the available selectors and fail points with:
+.Bd -literal -offset indent
+sysctl -aN debug.fail_point.ice
+.Ed
+.Pp
+The individual points accept the syntax described by
+.Xr fail 9 .
+For example, the following injects one
+.Er EIO
+after allocating VF 0's VSI on
+.Li ice0 :
+.Bd -literal -offset indent
+sysctl debug.fail_point.ice.device=ice0
+sysctl debug.fail_point.ice.iov.vf=0
+sysctl debug.fail_point.ice.iov.add_after_vsi_alloc='1*return(5)'
+.Ed
+.Pp
+Clear the individual point and device selector after each destructive test.
.Sh SUPPORT
For general information and support, go to the Intel support website at:
.Lk https://www.intel.com/support/ .
@@ -1173,7 +1214,8 @@ email all the specific information related to the issue to
.Xr led 4 ,
.Xr vlan 4 ,
.Xr ifconfig 8 ,
-.Xr sysctl 8
+.Xr sysctl 8 ,
+.Xr fail 9
.Sh HISTORY
The
.Nm
diff --git a/sys/conf/NOTES b/sys/conf/NOTES
index d405de62bed7..a51f471befa7 100644
--- a/sys/conf/NOTES
+++ b/sys/conf/NOTES
@@ -438,6 +438,13 @@ options QUEUE_MACRO_DEBUG_TRASH
#
options SYSCTL_DEBUG
+#
+# Compile test-only failure-injection hooks in supporting device drivers.
+# These expose destructive controls below debug.fail_point and are omitted
+# by default.
+#
+options DRIVER_FAILPOINTS
+
#
# Enable textdump by default, this disables kernel core dumps.
#
diff --git a/sys/conf/options b/sys/conf/options
index 03b9f1472904..516ef52b0caf 100644
--- a/sys/conf/options
+++ b/sys/conf/options
@@ -52,6 +52,7 @@ DDB_CAPTURE_DEFAULTBUFSIZE opt_ddb.h
DDB_CAPTURE_MAXBUFSIZE opt_ddb.h
DDB_CTF opt_ddb.h
DDB_NUMSYM opt_ddb.h
+DRIVER_FAILPOINTS
EARLY_PRINTF opt_global.h
EXTERR_STRINGS opt_global.h
FULL_BUF_TRACKING opt_global.h
diff --git a/sys/dev/ice/ice_fault.h b/sys/dev/ice/ice_fault.h
new file mode 100644
index 000000000000..9206187ff684
--- /dev/null
+++ b/sys/dev/ice/ice_fault.h
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2026 BBOX.io
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#ifndef _ICE_FAULT_H_
+#define _ICE_FAULT_H_
+
+#include "ice_opts.h"
+
+#ifdef DRIVER_FAILPOINTS
+
+#include <sys/fail.h>
+
+struct ice_softc;
+
+SYSCTL_DECL(_debug_fail_point_ice);
+
+bool ice_fail_point_device_matches(struct ice_softc *sc);
+
+#define ICE_FAIL_POINT_CODE_COND(_sc, _parent, _name, _cond, _flags, _code...) \
+ KFAIL_POINT_CODE_COND(_parent, _name, \
+ ice_fail_point_device_matches((_sc)) && (_cond), _flags, _code)
+#define ICE_FAIL_POINT_CODE(_sc, _parent, _name, _flags, _code...) \
+ ICE_FAIL_POINT_CODE_COND(_sc, _parent, _name, true, _flags, _code)
+
+#else /* !DRIVER_FAILPOINTS */
+
+#define ICE_FAIL_POINT_CODE_COND(_sc, _parent, _name, _cond, _flags, _code...) \
+ do { } while (0)
+#define ICE_FAIL_POINT_CODE(_sc, _parent, _name, _flags, _code...) \
+ do { } while (0)
+
+#endif /* DRIVER_FAILPOINTS */
+
+#endif /* _ICE_FAULT_H_ */
diff --git a/sys/dev/ice/ice_iov.c b/sys/dev/ice/ice_iov.c
index 61f53aad89b9..33c4a5f7054a 100644
--- a/sys/dev/ice/ice_iov.c
+++ b/sys/dev/ice/ice_iov.c
@@ -38,7 +38,17 @@
*/
#include "ice_iov.h"
+#include "ice_fault.h"
+#ifdef DRIVER_FAILPOINTS
+static SYSCTL_NODE(_debug_fail_point_ice, OID_AUTO, iov,
+ CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "ice SR-IOV fail points");
+
+static int ice_iov_fail_vf = -1;
+SYSCTL_INT(_debug_fail_point_ice_iov, OID_AUTO, vf,
+ CTLFLAG_RW | CTLFLAG_MPSAFE, &ice_iov_fail_vf, 0,
+ "VF eligible for ice SR-IOV fail points (-1 selects every VF)");
+#endif /* DRIVER_FAILPOINTS */
static struct ice_vf *ice_iov_get_vf(struct ice_softc *sc, int vf_num);
static void ice_iov_ready_vf(struct ice_softc *sc, struct ice_vf *vf);
static void ice_reset_vf(struct ice_softc *sc, struct ice_vf *vf,
@@ -84,6 +94,28 @@ static int ice_vc_select_vlans(struct ice_vf *vf, u16 *vids, u16 count,
static enum virtchnl_status_code ice_iov_err_to_virt_err(int ice_err);
static int ice_vf_validate_mac(struct ice_vf *vf, const uint8_t *addr);
+#ifdef DRIVER_FAILPOINTS
+static bool
+ice_iov_fail_vf_matches(uint16_t vfnum)
+{
+ return (ice_iov_fail_vf == -1 || ice_iov_fail_vf == vfnum);
+}
+#endif
+
+#define ICE_IOV_FAIL_POINT(_sc, _vfnum, _name, _error, _label) do { \
+ ICE_FAIL_POINT_CODE_COND(_sc, _debug_fail_point_ice_iov, _name, \
+ ice_iov_fail_vf_matches((_vfnum)), \
+ FAIL_POINT_NONSLEEPABLE, { \
+ (_error) = RETURN_VALUE; \
+ if ((_error) <= 0) \
+ (_error) = EIO; \
+ device_printf((_sc)->dev, \
+ "injecting VF %u failure at %s: %d\n", \
+ (unsigned int)(_vfnum), #_name, (_error)); \
+ goto _label; \
+ }); \
+} while (0)
+
/**
* ice_iov_attach - Initialize SR-IOV PF host support
* @sc: device softc structure
@@ -237,6 +269,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
return (ENOMEM);
vf->vsi = vsi;
vsi->vf_num = vfnum;
+ ICE_IOV_FAIL_POINT(sc, vfnum, add_after_vsi_alloc, error,
+ release_vsi);
vf_num_queues = nvlist_get_number(params, "num-queues");
/* Validate and clamp value if invalid */
@@ -256,6 +290,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
/* Reserve VF queue allocation from PF queues */
ice_alloc_vsi_qmap(vsi, vf_num_queues, vf_num_queues);
vsi->num_tx_queues = vsi->num_rx_queues = vf_num_queues;
+ ICE_IOV_FAIL_POINT(sc, vfnum, add_after_queue_maps, error,
+ release_vsi);
/* Assign Tx queues from PF space */
error = ice_resmgr_assign_scattered(&sc->tx_qmgr, vsi->tx_qmap,
@@ -265,6 +301,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
ice_err_str(error));
goto release_vsi;
}
+ ICE_IOV_FAIL_POINT(sc, vfnum, add_after_tx_reservation, error,
+ release_vsi);
/* Assign Rx queues from PF space */
error = ice_resmgr_assign_scattered(&sc->rx_qmgr, vsi->rx_qmap,
@@ -274,6 +312,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
ice_err_str(error));
goto release_vsi;
}
+ ICE_IOV_FAIL_POINT(sc, vfnum, add_after_rx_reservation, error,
+ release_vsi);
vsi->max_frame_size = ICE_MAX_FRAME_SIZE;
@@ -291,6 +331,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
txq->me = i;
txq->vsi = vsi;
}
+ ICE_IOV_FAIL_POINT(sc, vfnum, add_after_tx_queue_memory, error,
+ free_txqs);
/* Allocate queue structure memory */
vsi->rx_queues = (struct ice_rx_queue *)
@@ -306,6 +348,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
rxq->me = i;
rxq->vsi = vsi;
}
+ ICE_IOV_FAIL_POINT(sc, vfnum, add_after_rx_queue_memory, error,
+ free_rxqs);
/* Allocate space to store the IRQ vector data */
vf->num_irq_vectors = vf_num_queues + 1;
@@ -319,6 +363,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
error = ENOMEM;
goto free_rxqs;
}
+ ICE_IOV_FAIL_POINT(sc, vfnum, add_after_tx_irq_memory, error,
+ free_txirqvs);
vf->rx_irqvs = (struct ice_irq_vector *)
malloc(sizeof(struct ice_irq_vector) * (vf->num_irq_vectors),
M_ICE, M_NOWAIT);
@@ -329,6 +375,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
error = ENOMEM;
goto free_txirqvs;
}
+ ICE_IOV_FAIL_POINT(sc, vfnum, add_after_rx_irq_memory, error,
+ free_rxirqvs);
/* Assign VF interrupts from PF space */
if (!(vf->vf_imap =
@@ -338,12 +386,16 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
error = ENOMEM;
goto free_rxirqvs;
}
+ ICE_IOV_FAIL_POINT(sc, vfnum, add_after_imap_memory, error,
+ free_imap);
error = ice_resmgr_assign_contiguous(&sc->dev_imgr, vf->vf_imap, vf->num_irq_vectors);
if (error) {
device_printf(dev, "Unable to assign VF-%d interrupt mapping: %s\n",
vfnum, ice_err_str(error));
goto free_imap;
}
+ ICE_IOV_FAIL_POINT(sc, vfnum, add_after_imap_reservation, error,
+ release_imap);
if (nvlist_exists_binary(params, "mac-addr")) {
mac = nvlist_get_binary(params, "mac-addr", &size);
@@ -378,6 +430,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
vfnum, ice_err_str(error));
goto release_imap;
}
+ ICE_IOV_FAIL_POINT(sc, vfnum, add_after_vsi_init, error,
+ release_imap);
/* Add the broadcast address */
error = ice_add_vsi_mac_filter(vsi, broadcastaddr);
@@ -386,6 +440,8 @@ ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params)
vfnum, ice_err_str(error));
goto release_imap;
}
+ ICE_IOV_FAIL_POINT(sc, vfnum, add_after_broadcast_filter, error,
+ release_imap);
atomic_set_32(&vf->vf_flags, VF_FLAG_ENABLED);
ice_iov_ready_vf(sc, vf);
@@ -602,6 +658,8 @@ ice_iov_rebuild_vf(struct ice_softc *sc, struct ice_vsi *vsi)
vf = ice_iov_get_vf(sc, vsi->vf_num);
atomic_clear_32(&vf->vf_flags, VF_FLAG_INITIALIZED);
atomic_set_32(&vf->vf_flags, VF_FLAG_REBUILD_FAILED);
+ ICE_IOV_FAIL_POINT(sc, vf->vf_num, rebuild_before_initialize, error,
+ fail);
/* A new hardware VSI starts a new raw statistics epoch. */
accumulated_stats = vsi->hw_stats.cur;
@@ -626,6 +684,11 @@ ice_iov_rebuild_vf(struct ice_softc *sc, struct ice_vsi *vsi)
atomic_clear_32(&vf->vf_flags, VF_FLAG_REBUILD_FAILED);
ice_iov_ready_vf(sc, vf);
return (0);
+
+#ifdef DRIVER_FAILPOINTS
+fail:
+ return (error);
+#endif /* DRIVER_FAILPOINTS */
}
/**
@@ -1816,6 +1879,14 @@ ice_vc_get_stats_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf)
struct ice_hw *hw = &sc->hw;
vqs = (struct virtchnl_queue_select *)msg_buf;
+ ICE_FAIL_POINT_CODE_COND(sc, _debug_fail_point_ice_iov,
+ get_stats_bad_vsi, ice_iov_fail_vf_matches(vf->vf_num),
+ FAIL_POINT_NONSLEEPABLE, {
+ vqs->vsi_id = vsi->idx + 1;
+ device_printf(sc->dev,
+ "injecting invalid GET_STATS VSI ID for VF %u\n",
+ (unsigned int)vf->vf_num);
+ });
if (vqs->vsi_id != vsi->idx) {
device_printf(sc->dev,
diff --git a/sys/dev/ice/ice_lib.c b/sys/dev/ice/ice_lib.c
index e7dce30222d5..9d634b6a7658 100644
--- a/sys/dev/ice/ice_lib.c
+++ b/sys/dev/ice/ice_lib.c
@@ -42,6 +42,7 @@
#include "ice_lib.h"
#include "ice_iflib.h"
+#include "ice_fault.h"
#ifdef PCI_IOV
#include "ice_iov.h"
#endif
@@ -62,6 +63,33 @@
*/
MALLOC_DEFINE(M_ICE, "ice", "Intel(R) 100Gb Network Driver lib allocations");
+#ifdef DRIVER_FAILPOINTS
+
+/*
+ * ICE fail points are global, but only the selected PF may trigger them. An
+ * empty selector disables every point even if a stale failpoint setting
+ * remains armed.
+ */
+SYSCTL_NODE(_debug_fail_point, OID_AUTO, ice,
+ CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "ice driver fail points");
+
+static char ice_fail_device[32];
+SYSCTL_STRING(_debug_fail_point_ice, OID_AUTO, device,
+ CTLFLAG_RW | CTLFLAG_MPSAFE, ice_fail_device,
+ sizeof(ice_fail_device), "device eligible for ice fail points");
+
+bool
+ice_fail_point_device_matches(struct ice_softc *sc)
+{
+ const char *nameunit;
+
+ nameunit = device_get_nameunit(sc->dev);
+ return (ice_fail_device[0] != '\0' && nameunit != NULL &&
+ strcmp(nameunit, ice_fail_device) == 0);
+}
+
+#endif /* DRIVER_FAILPOINTS */
+
/*
* Helper function prototypes
*/
diff --git a/sys/dev/ice/ice_opts.h b/sys/dev/ice/ice_opts.h
index 479ead0dc41e..2b72f7b4940a 100644
--- a/sys/dev/ice/ice_opts.h
+++ b/sys/dev/ice/ice_opts.h
@@ -43,5 +43,6 @@
#include "opt_inet.h"
#include "opt_inet6.h"
#include "opt_rss.h"
+#include "opt_driver_failpoints.h"
#endif
diff --git a/sys/modules/ice/Makefile b/sys/modules/ice/Makefile
index 9f9c9f602cda..0ee0837715d6 100644
--- a/sys/modules/ice/Makefile
+++ b/sys/modules/ice/Makefile
@@ -8,6 +8,7 @@ SRCS += irdma_di_if.h irdma_if.h
# Option headers
SRCS += opt_inet.h opt_inet6.h opt_rss.h opt_iflib.h
+SRCS += opt_driver_failpoints.h
# Core source
SRCS += ice_lib.c ice_osdep.c ice_resmgr.c ice_strings.c