git: 983e628a0c47 - main - ixl: Rebuild VF resources after a PF reset
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Mon, 10 Aug 2026 09:56:30 UTC
The branch main has been updated by kbowling:
URL: https://cgit.FreeBSD.org/src/commit/?id=983e628a0c47afb47d201ce629e9619fb751254d
commit 983e628a0c47afb47d201ce629e9619fb751254d
Author: Kevin Bowling <kbowling@FreeBSD.org>
AuthorDate: 2026-08-10 03:38:40 +0000
Commit: Kevin Bowling <kbowling@FreeBSD.org>
CommitDate: 2026-08-10 09:55:53 +0000
ixl: Rebuild VF resources after a PF reset
A PF or EMP reset destroys the firmware switch topology, including
every VF VSI. The driver rebuilt only its PF VSI and left configured
VFs with stale switch element and VSI identifiers.
Notify VFs before a driver initiated reset, recreate the IOV VEB, and
rebuild each configured VF VSI and queue mapping after the PF switch
is restored. Keep a VF out of VFACTIVE if its reconstruction fails
so one failure cannot expose incomplete resources or prevent the PF
and other VFs from recovering.
Invalidate cached VF firmware identifiers and runtime state before
recreating the VEB. If VEB creation itself fails, teardown and mailbox
paths can no longer use pre-reset SEIDs or VSI data.
Factor the common VEB setup out of IOV initialization so initial
setup and post-reset reconstruction use the same topology and filter
sequence.
MFC after: 2 weeks
---
sys/dev/ixl/ixl_pf.h | 5 ++
sys/dev/ixl/ixl_pf_iflib.c | 13 ++++
sys/dev/ixl/ixl_pf_iov.c | 150 +++++++++++++++++++++++++++++++++++++--------
sys/dev/ixl/ixl_pf_main.c | 13 ++--
4 files changed, 148 insertions(+), 33 deletions(-)
diff --git a/sys/dev/ixl/ixl_pf.h b/sys/dev/ixl/ixl_pf.h
index 80583b152a1c..8d06738034a4 100644
--- a/sys/dev/ixl/ixl_pf.h
+++ b/sys/dev/ixl/ixl_pf.h
@@ -384,6 +384,11 @@ int ixl_prepare_for_reset(struct ixl_pf *pf, bool is_up);
int ixl_rebuild_hw_structs_after_reset(struct ixl_pf *, bool is_up);
int ixl_pf_reset(struct ixl_pf *);
+#ifdef PCI_IOV
+void ixl_notify_vfs_reset(struct ixl_pf *);
+int ixl_rebuild_vfs_after_reset(struct ixl_pf *);
+#endif
+
void ixl_set_queue_rx_itr(struct ixl_rx_queue *);
void ixl_set_queue_tx_itr(struct ixl_tx_queue *);
diff --git a/sys/dev/ixl/ixl_pf_iflib.c b/sys/dev/ixl/ixl_pf_iflib.c
index 98054a21c291..8289cc72bc64 100644
--- a/sys/dev/ixl/ixl_pf_iflib.c
+++ b/sys/dev/ixl/ixl_pf_iflib.c
@@ -937,6 +937,9 @@ ixl_prepare_for_reset(struct ixl_pf *pf, bool is_up)
device_t dev = pf->dev;
int error = 0;
+#ifdef PCI_IOV
+ ixl_notify_vfs_reset(pf);
+#endif
if (is_up)
ixl_if_stop(pf->vsi.ctx);
@@ -999,6 +1002,7 @@ ixl_rebuild_hw_structs_after_reset(struct ixl_pf *pf, bool is_up)
if (error) {
device_printf(dev, "Failed to reserve queues for PF LAN VSI, error %d\n",
error);
+ goto ixl_rebuild_hw_structs_after_reset_err;
}
error = ixl_switch_config(pf);
@@ -1033,6 +1037,15 @@ ixl_rebuild_hw_structs_after_reset(struct ixl_pf *pf, bool is_up)
/* Receive broadcast Ethernet frames */
i40e_aq_set_vsi_broadcast(&pf->hw, vsi->seid, TRUE, NULL);
+#ifdef PCI_IOV
+ if (pf->num_vfs != 0) {
+ error = ixl_rebuild_vfs_after_reset(pf);
+ if (error != 0)
+ device_printf(dev,
+ "Failed to rebuild one or more VFs: %d\n", error);
+ }
+#endif
+
/* Determine link state */
ixl_attach_get_link_status(pf);
diff --git a/sys/dev/ixl/ixl_pf_iov.c b/sys/dev/ixl/ixl_pf_iov.c
index 8a62876831d6..358210f0cc2f 100644
--- a/sys/dev/ixl/ixl_pf_iov.c
+++ b/sys/dev/ixl/ixl_pf_iov.c
@@ -81,6 +81,7 @@ static void ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf, void
static void ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, uint16_t msg_size);
static int ixl_vf_reserve_queues(struct ixl_pf *pf, struct ixl_vf *vf, int num_queues);
static int ixl_config_pf_vsi_loopback(struct ixl_pf *pf, bool enable);
+static int ixl_setup_iov_switch(struct ixl_pf *pf);
static int ixl_adminq_err_to_errno(enum i40e_admin_queue_err err);
@@ -1979,18 +1980,132 @@ ixl_config_pf_vsi_loopback(struct ixl_pf *pf, bool enable)
return (error);
}
-int
-ixl_if_iov_init(if_ctx_t ctx, uint16_t num_vfs, const nvlist_t *params)
+static int
+ixl_setup_iov_switch(struct ixl_pf *pf)
{
- struct ixl_pf *pf = iflib_get_softc(ctx);
- device_t dev = iflib_get_dev(ctx);
struct i40e_hw *hw;
struct ixl_vsi *pf_vsi;
- enum i40e_status_code ret;
- int error, i;
+ enum i40e_status_code status;
+ int error;
hw = &pf->hw;
pf_vsi = &pf->vsi;
+ status = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid,
+ 1, false, &pf->veb_seid, false, NULL);
+ if (status != I40E_SUCCESS) {
+ error = ixl_adminq_err_to_errno(hw->aq.asq_last_status);
+ device_printf(pf->dev,
+ "i40e_aq_add_veb failed; status %s error %s\n",
+ i40e_stat_str(hw, status),
+ i40e_aq_str(hw, hw->aq.asq_last_status));
+ return (error);
+ }
+ if (pf->enable_vf_loopback) {
+ error = ixl_config_pf_vsi_loopback(pf, true);
+ if (error != 0)
+ goto fail;
+ }
+
+ /* Adding a VEB reinstalls the firmware's default PF filters. */
+ ixl_del_default_hw_filters(pf_vsi);
+ ixl_reconfigure_filters(pf_vsi);
+ return (0);
+
+fail:
+ i40e_aq_delete_element(hw, pf->veb_seid, NULL);
+ pf->veb_seid = 0;
+ return (error);
+}
+
+void
+ixl_notify_vfs_reset(struct ixl_pf *pf)
+{
+ struct virtchnl_pf_event event;
+ struct ixl_vf *vf;
+ int i;
+
+ if (pf->num_vfs == 0 || !i40e_check_asq_alive(&pf->hw))
+ return;
+ bzero(&event, sizeof(event));
+ event.event = VIRTCHNL_EVENT_RESET_IMPENDING;
+ event.severity = PF_EVENT_SEVERITY_CERTAIN_DOOM;
+ for (i = 0; i < pf->num_vfs; i++) {
+ vf = &pf->vfs[i];
+ if (vf->vf_flags & VF_FLAG_ENABLED) {
+ ixl_send_vf_msg(pf, vf, VIRTCHNL_OP_EVENT,
+ I40E_SUCCESS, &event, sizeof(event));
+ }
+ }
+}
+
+int
+ixl_rebuild_vfs_after_reset(struct ixl_pf *pf)
+{
+ struct i40e_hw *hw;
+ struct ixl_vf *vf;
+ uint32_t vfrtrig;
+ int error, first_error, i;
+
+ hw = &pf->hw;
+ pf->veb_seid = 0;
+ for (i = 0; i < pf->num_vfs; i++) {
+ vf = &pf->vfs[i];
+ if (!(vf->vf_flags & VF_FLAG_ENABLED))
+ continue;
+
+ vf->vf_flags &= ~VF_FLAG_INITIALIZED;
+ vf->vsi.seid = 0;
+ vf->vsi.vsi_num = 0;
+ vf->vsi.num_tx_queues = 0;
+ vf->vsi.num_rx_queues = 0;
+ bzero(&vf->vsi.info, sizeof(vf->vsi.info));
+ ixl_pf_qmgr_clear_queue_flags(&vf->qtag);
+ ixl_free_filters(&vf->vsi.ftl);
+ vf->vsi.num_hw_filters = 0;
+ vf->vsi.num_macs = 0;
+ vf->vsi.num_vlans = 0;
+ bit_nclear(vf->vsi.vlans_map, 0,
+ IXL_VLANS_MAP_LEN - 1);
+ vf->num_mac_filters = 0;
+ }
+
+ error = ixl_setup_iov_switch(pf);
+ if (error != 0)
+ return (error);
+
+ first_error = 0;
+ for (i = 0; i < pf->num_vfs; i++) {
+ vf = &pf->vfs[i];
+ if (!(vf->vf_flags & VF_FLAG_ENABLED))
+ continue;
+
+ wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num),
+ VIRTCHNL_VFR_COMPLETED);
+ vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
+ vfrtrig &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK;
+ wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
+
+ error = ixl_vf_setup_vsi(pf, vf);
+ if (error != 0) {
+ device_printf(pf->dev,
+ "Failed to rebuild VF-%d: %d\n", i, error);
+ if (first_error == 0)
+ first_error = error;
+ continue;
+ }
+ ixl_vf_map_queues(pf, vf);
+ wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num),
+ VIRTCHNL_VFR_VFACTIVE);
+ }
+ ixl_flush(hw);
+ return (first_error);
+}
+
+int
+ixl_if_iov_init(if_ctx_t ctx, uint16_t num_vfs, const nvlist_t *params)
+{
+ struct ixl_pf *pf = iflib_get_softc(ctx);
+ int error, i;
pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT |
M_ZERO);
@@ -2001,28 +2116,9 @@ ixl_if_iov_init(if_ctx_t ctx, uint16_t num_vfs, const nvlist_t *params)
for (i = 0; i < num_vfs; i++)
sysctl_ctx_init(&pf->vfs[i].vsi.sysctl_ctx);
- /*
- * Add the VEB and ...
- * - do nothing: VEPA mode
- * - enable loopback mode on connected VSIs: VEB mode
- */
- ret = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid,
- 1, FALSE, &pf->veb_seid, FALSE, NULL);
- if (ret != I40E_SUCCESS) {
- error = hw->aq.asq_last_status;
- device_printf(dev, "i40e_aq_add_veb failed; status %s error %s",
- i40e_stat_str(hw, ret), i40e_aq_str(hw, error));
+ error = ixl_setup_iov_switch(pf);
+ if (error != 0)
goto fail;
- }
- if (pf->enable_vf_loopback)
- ixl_config_pf_vsi_loopback(pf, true);
-
- /*
- * Adding a VEB brings back the default MAC filter(s). Remove them,
- * and let the driver add the proper filters back.
- */
- ixl_del_default_hw_filters(pf_vsi);
- ixl_reconfigure_filters(pf_vsi);
pf->num_vfs = num_vfs;
return (0);
diff --git a/sys/dev/ixl/ixl_pf_main.c b/sys/dev/ixl/ixl_pf_main.c
index 674e45823cc4..cb6f64f66209 100644
--- a/sys/dev/ixl/ixl_pf_main.c
+++ b/sys/dev/ixl/ixl_pf_main.c
@@ -494,7 +494,7 @@ ixl_set_initial_advertised_speeds(struct ixl_pf *pf)
int
ixl_teardown_hw_structs(struct ixl_pf *pf)
{
- enum i40e_status_code status = 0;
+ enum i40e_status_code error, status = I40E_SUCCESS;
struct i40e_hw *hw = &pf->hw;
device_t dev = pf->dev;
@@ -505,20 +505,21 @@ ixl_teardown_hw_structs(struct ixl_pf *pf)
device_printf(dev,
"init: LAN HMC shutdown failure; status %s\n",
i40e_stat_str(hw, status));
- goto err_out;
}
}
/* Shutdown admin queue */
ixl_disable_intr0(hw);
- status = i40e_shutdown_adminq(hw);
- if (status)
+ error = i40e_shutdown_adminq(hw);
+ if (error) {
device_printf(dev,
"init: Admin Queue shutdown failure; status %s\n",
- i40e_stat_str(hw, status));
+ i40e_stat_str(hw, error));
+ if (status == I40E_SUCCESS)
+ status = error;
+ }
ixl_pf_qmgr_release(&pf->qmgr, &pf->qtag);
-err_out:
return (status);
}