git: 57d8db405e68 - main - ice: Restart disabled VF Tx queues from saved configuration
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Sun, 01 Nov 2026 20:52:53 UTC
The branch main has been updated by kbowling:
URL: https://cgit.FreeBSD.org/src/commit/?id=57d8db405e68fa32b98aec803693f0f5880b9b92
commit 57d8db405e68fa32b98aec803693f0f5880b9b92
Author: Kevin Bowling <kbowling@FreeBSD.org>
AuthorDate: 2026-11-01 00:13:34 +0000
Commit: Kevin Bowling <kbowling@FreeBSD.org>
CommitDate: 2026-11-01 20:31:01 +0000
ice: Restart disabled VF Tx queues from saved configuration
DPDK iavf's per-queue stop/start operations send DISABLE_QUEUES followed
by ENABLE_QUEUES without another CONFIG_VSI_QUEUES. Disabling an ice Tx
queue removes its hardware scheduler leaf, so acknowledging ENABLE alone
cannot restart it. Clearing the configured bitmap also prevents the VF
from requesting this sequence.
Track saved configuration separately from active Tx queues. Retain ring
parameters and queue handles on disable, and recreate only the selected
inactive queues on enable. Share the existing Tx context builder between
initial VSI configuration and per-queue restart. Keep the initial
Tx-at-CONFIG behavior and the existing dense-prefix CONFIG validation.
Track cleanup progress per queue and publish valid configuration only
after both Tx and Rx configuration succeed. If an enable fails, undo its
partial setup before permitting a retry. Retain hardware ownership when
cleanup fails and reject another enable of that queue rather than
silently acknowledging it. A failed post-add rollback without a software
scheduler node requires VF reset, not a successful no-op disable.
After a verified VF reset and DMA drain, remove any remaining software
scheduler leaves and invalidate the queue contexts. Treat the initial
queue disable as best effort so a failed rollback cannot permanently
prevent reset recovery. Retain unresolved state if the reset or drain
fails.
Fixes: bc761988b724 ("ice: Add host SR-IOV support")
MFC after: 2 weeks
Sponsored by: BBOX.io
Differential Revision: https://reviews.freebsd.org/D59851
---
sys/dev/ice/ice_iov.c | 116 +++++++++++++++++++++++++++++++++++++-------------
sys/dev/ice/ice_iov.h | 2 +
sys/dev/ice/ice_lib.c | 94 ++++++++++++++++++++++++----------------
sys/dev/ice/ice_lib.h | 1 +
4 files changed, 147 insertions(+), 66 deletions(-)
diff --git a/sys/dev/ice/ice_iov.c b/sys/dev/ice/ice_iov.c
index e5ee1a03936d..392c3e30237c 100644
--- a/sys/dev/ice/ice_iov.c
+++ b/sys/dev/ice/ice_iov.c
@@ -1063,6 +1063,7 @@ ice_iov_clear_vf_queue_state(struct ice_vf *vf)
{
vf->txq_configured = 0;
vf->rxq_configured = 0;
+ vf->txq_active = 0;
vf->rxq_enabled = 0;
}
@@ -1210,6 +1211,46 @@ fail:
#endif /* DRIVER_FAILPOINTS */
}
+/**
+ * ice_iov_cleanup_vf_tx - Discard Tx software state after a completed VF reset
+ * @sc: device softc structure
+ * @vf: VF whose hardware queues have been reset and drained
+ *
+ * A failed queue disable or post-add rollback can leave a scheduler leaf or
+ * queue context behind. The reset-only AQ command drains hardware without
+ * updating these entries. Free any remaining leaves and invalidate the queue
+ * contexts only after the reset and DMA drain have been verified.
+ */
+static void
+ice_iov_cleanup_vf_tx(struct ice_softc *sc, struct ice_vf *vf)
+{
+ struct ice_port_info *pi = sc->hw.port_info;
+ struct ice_vsi *vsi = vf->vsi;
+ struct ice_sched_node *node;
+ struct ice_tx_queue *txq;
+ struct ice_q_ctx *q_ctx;
+ int i;
+
+ ice_acquire_lock(&pi->sched_lock);
+ for (i = 0; i < vsi->num_tx_queues; i++) {
+ txq = &vsi->tx_queues[i];
+ q_ctx = ice_get_lan_q_ctx(&sc->hw, vsi->idx, txq->tc,
+ txq->q_handle);
+ if (q_ctx != NULL) {
+ if (q_ctx->q_handle == txq->q_handle && q_ctx->q_teid != 0) {
+ node = ice_sched_find_node_by_teid(pi->root,
+ q_ctx->q_teid);
+ if (node != NULL)
+ ice_free_sched_node(pi, node);
+ }
+ q_ctx->q_handle = ICE_INVAL_Q_HANDLE;
+ q_ctx->q_teid = 0;
+ }
+ txq->q_teid = 0;
+ }
+ ice_release_lock(&pi->sched_lock);
+}
+
/**
* ice_reset_vf - Perform a hardware reset (VFR) on a VF
* @sc: device softc structure
@@ -1256,15 +1297,12 @@ ice_reset_vf(struct ice_softc *sc, struct ice_vf *vf, bool trigger_reset,
/*
* Remove the tracked queue leaves from the software scheduler before
* issuing the reset-only AQ command. That command drains hardware but
- * does not update the shared scheduler database. Retain unresolved queue
- * state if cleanup fails so a later reset can retry it.
+ * does not update the shared scheduler database. A failed queue disable
+ * need not prevent reset recovery: retain unresolved software state until
+ * the reset and DMA drain have been verified below.
*/
- status = ice_vc_disable_queues(sc, vf, vf->txq_configured,
+ (void)ice_vc_disable_queues(sc, vf, vf->txq_active,
vf->rxq_enabled);
- if (status == 0)
- ice_iov_clear_vf_queue_state(vf);
- else if (error == 0)
- error = status;
/* This zero-queue command is required to complete every VF reset. */
status = ice_dis_vsi_txq(hw->port_info, vf->vsi->idx, 0, 0,
@@ -1356,6 +1394,9 @@ ice_reset_vf(struct ice_softc *sc, struct ice_vf *vf, bool trigger_reset,
return (error);
}
+ ice_iov_cleanup_vf_tx(sc, vf);
+ ice_iov_clear_vf_queue_state(vf);
+
if (!release_vf) {
/* Discard any anti-spoof MDD indication caused by the reset. */
ice_iov_clear_vf_mdd(sc, vf);
@@ -1952,6 +1993,7 @@ ice_vc_disable_tx_queue(struct ice_softc *sc, struct ice_vf *vf, u16 qid)
struct ice_vsi *vsi = vf->vsi;
struct ice_tx_queue *txq = &vsi->tx_queues[qid];
struct ice_hw *hw = &sc->hw;
+ struct ice_q_ctx *q_ctx;
u16 q_handle, q_id;
u32 q_teid;
int status;
@@ -1969,7 +2011,16 @@ ice_vc_disable_tx_queue(struct ice_softc *sc, struct ice_vf *vf, u16 qid)
ice_aq_str(hw->adminq.sq_last_status));
return (EIO);
}
- txq->q_handle = 0;
+ /* A failed post-add rollback may have left no software scheduler node. */
+ q_ctx = ice_get_lan_q_ctx(hw, vsi->idx, txq->tc, q_handle);
+ if (status == ICE_ERR_DOES_NOT_EXIST && q_ctx != NULL &&
+ q_ctx->q_handle == q_handle && q_ctx->q_teid != 0) {
+ device_printf(sc->dev,
+ "VF-%d Tx queue %u still requires reset after failed setup\n",
+ vf->vf_num, qid);
+ return (EIO);
+ }
+
txq->q_teid = 0;
return (0);
@@ -1984,8 +2035,8 @@ ice_vc_disable_tx_queue(struct ice_softc *sc, struct ice_vf *vf, u16 qid)
*
* Queue disable is idempotent. Queues which are not configured or enabled
* have no hardware work to perform and are treated as successfully disabled.
- * Since CONFIG_VSI_QUEUES also enables Tx, disabling a Tx queue removes its
- * tracked configuration and the VF must configure it before enabling it again.
+ * Tx disable removes the hardware queue, but retains its ring parameters and
+ * queue handle so ENABLE_QUEUES can recreate it without CONFIG_VSI_QUEUES.
*/
static int
ice_vc_disable_queues(struct ice_softc *sc, struct ice_vf *vf,
@@ -2009,20 +2060,13 @@ ice_vc_disable_queues(struct ice_softc *sc, struct ice_vf *vf,
queues &= ~BIT(bit);
}
- queues = tx_queues & vf->txq_configured;
- if (queues == vf->txq_configured && queues != 0) {
- error = ice_vsi_disable_tx(vsi);
- if (error != 0)
- return (error);
- vf->txq_configured = 0;
- return (0);
- }
+ queues = tx_queues & vf->txq_active;
while (queues != 0) {
bit = ffs(queues) - 1;
error = ice_vc_disable_tx_queue(sc, vf, bit);
if (error != 0)
return (error);
- vf->txq_configured &= ~BIT(bit);
+ vf->txq_active &= ~BIT(bit);
queues &= ~BIT(bit);
}
@@ -2209,15 +2253,13 @@ ice_vc_cfg_vsi_qs_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf)
goto done;
}
- error = ice_vc_disable_queues(sc, vf, vf->txq_configured,
+ error = ice_vc_disable_queues(sc, vf, vf->txq_active,
vf->rxq_enabled);
if (error != 0) {
status = VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR;
goto done;
}
- vf->txq_configured = 0;
- vf->rxq_configured = 0;
- vf->rxq_enabled = 0;
+ ice_iov_clear_vf_queue_state(vf);
/*
* Clear TX and RX queues config in case VF
@@ -2264,15 +2306,14 @@ ice_vc_cfg_vsi_qs_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf)
* firmware failure remains discoverable and can be cleaned up by the
* next configuration attempt.
*/
- vf->txq_configured = queue_map;
+ vf->txq_active = queue_map;
error = ice_cfg_vsi_for_tx(vsi);
if (error) {
device_printf(dev,
"VF-%d: Unable to configure VSI for Tx: %s\n",
vf->vf_num, ice_err_str(error));
status = VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR;
- if (ice_vsi_disable_tx(vsi) == 0)
- vf->txq_configured = 0;
+ (void)ice_vc_disable_queues(sc, vf, vf->txq_active, 0);
goto done;
}
@@ -2283,9 +2324,10 @@ ice_vc_cfg_vsi_qs_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf)
"VF-%d: Unable to configure VSI for Rx: %s\n",
vf->vf_num, ice_err_str(error));
status = VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR;
- (void)ice_vc_disable_queues(sc, vf, vf->txq_configured, 0);
+ (void)ice_vc_disable_queues(sc, vf, vf->txq_active, 0);
goto done;
}
+ vf->txq_configured = queue_map;
vf->rxq_configured = queue_map;
done:
@@ -2475,8 +2517,8 @@ ice_vc_set_rss_hena_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf)
*
* Enables VF queues selected in msg_buf for Tx/Rx traffic.
*
- * @remark Only actually operates on Rx queues; Tx queues are enabled in
- * CONFIG_VSI_QUEUES message handler.
+ * Tx queues are initially enabled by CONFIG_VSI_QUEUES. Recreate any which
+ * were subsequently disabled using their saved configuration.
*/
static void
ice_vc_enable_queues_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf)
@@ -2505,6 +2547,21 @@ ice_vc_enable_queues_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf)
goto done;
}
+ queues = vqs->tx_queues & ~vf->txq_active;
+ while (queues != 0) {
+ bit = ffs(queues) - 1;
+ vf->txq_active |= BIT(bit);
+ error = ice_cfg_tx_queue(&vsi->tx_queues[bit]);
+ if (error != 0) {
+ /* Do not ACK a retry if the failed add cannot be undone. */
+ if (ice_vc_disable_queues(sc, vf, BIT(bit), 0) != 0)
+ vf->txq_configured &= ~BIT(bit);
+ v_status = VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR;
+ goto done;
+ }
+ queues &= ~BIT(bit);
+ }
+
queues = vqs->rx_queues & ~vf->rxq_enabled;
while (queues != 0) {
bit = ffs(queues) - 1;
@@ -2519,7 +2576,6 @@ ice_vc_enable_queues_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf)
vf->rxq_enabled |= BIT(bit);
queues &= ~BIT(bit);
}
- /* Tx queues were enabled when their contexts were configured. */
done:
ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_ENABLE_QUEUES,
diff --git a/sys/dev/ice/ice_iov.h b/sys/dev/ice/ice_iov.h
index bdb28b4f7e92..a76afe523563 100644
--- a/sys/dev/ice/ice_iov.h
+++ b/sys/dev/ice/ice_iov.h
@@ -111,6 +111,8 @@ struct ice_vf {
/* VF-relative queue state, indexed by virtchnl queue ID. */
u32 txq_configured;
u32 rxq_configured;
+ /* Tx queues which are live or still require cleanup after an error. */
+ u32 txq_active;
u32 rxq_enabled;
u64 mdd_tx_events;
diff --git a/sys/dev/ice/ice_lib.c b/sys/dev/ice/ice_lib.c
index f027e9092b97..96c58d78509d 100644
--- a/sys/dev/ice/ice_lib.c
+++ b/sys/dev/ice/ice_lib.c
@@ -1705,20 +1705,21 @@ ice_setup_tx_ctx(struct ice_tx_queue *txq, struct ice_tlan_ctx *tlan_ctx, u16 pf
}
/**
- * ice_cfg_vsi_for_tx - Configure the hardware for Tx
- * @vsi: the VSI to configure
+ * ice_cfg_tx_queue - Configure and enable one hardware Tx queue
+ * @txq: the Tx queue to configure
*
- * Configure the device Tx queues through firmware AdminQ commands. After
- * this, Tx queues will be ready for transmit.
+ * Use the saved ring parameters to create the queue and its scheduler leaf.
*/
int
-ice_cfg_vsi_for_tx(struct ice_vsi *vsi)
+ice_cfg_tx_queue(struct ice_tx_queue *txq)
{
+ struct ice_vsi *vsi = txq->vsi;
struct ice_aqc_add_tx_qgrp *qg;
struct ice_hw *hw = &vsi->sc->hw;
+ struct ice_tlan_ctx tlan_ctx = { 0 };
+ struct ice_q_ctx *q_ctx;
device_t dev = vsi->sc->dev;
int status;
- int i;
int err = 0;
u16 qg_size, pf_q;
@@ -1729,47 +1730,68 @@ ice_cfg_vsi_for_tx(struct ice_vsi *vsi)
qg->num_txqs = 1;
- for (i = 0; i < vsi->num_tx_queues; i++) {
- struct ice_tlan_ctx tlan_ctx = { 0 };
- struct ice_tx_queue *txq = &vsi->tx_queues[i];
+ pf_q = vsi->tx_qmap[txq->me];
+ qg->txqs[0].txq_id = htole16(pf_q);
- /* Last configured queue */
- if (txq->desc_count == 0)
- break;
-
- pf_q = vsi->tx_qmap[txq->me];
- qg->txqs[0].txq_id = htole16(pf_q);
-
- err = ice_setup_tx_ctx(txq, &tlan_ctx, pf_q);
- if (err)
- goto free_txqg;
-
- ice_set_ctx(hw, (u8 *)&tlan_ctx, qg->txqs[0].txq_ctx,
- ice_tlan_ctx_info);
+ err = ice_setup_tx_ctx(txq, &tlan_ctx, pf_q);
+ if (err)
+ goto free_txqg;
- status = ice_ena_vsi_txq(hw->port_info, vsi->idx, txq->tc,
- txq->q_handle, 1, qg, qg_size, NULL);
- if (status) {
- device_printf(dev,
- "Failed to set LAN Tx queue %d (TC %d, handle %d) context, err %s aq_err %s\n",
- i, txq->tc, txq->q_handle,
- ice_status_str(status),
- ice_aq_str(hw->adminq.sq_last_status));
- err = ENODEV;
- goto free_txqg;
- }
+ ice_set_ctx(hw, (u8 *)&tlan_ctx, qg->txqs[0].txq_ctx,
+ ice_tlan_ctx_info);
- /* Keep track of the Tx queue TEID */
- if (pf_q == le16toh(qg->txqs[0].txq_id))
- txq->q_teid = le32toh(qg->txqs[0].q_teid);
+ status = ice_ena_vsi_txq(hw->port_info, vsi->idx, txq->tc,
+ txq->q_handle, 1, qg, qg_size, NULL);
+ if (status) {
+ /* Preserve ownership if post-add setup and its rollback failed. */
+ q_ctx = ice_get_lan_q_ctx(hw, vsi->idx, txq->tc, txq->q_handle);
+ if (q_ctx != NULL && q_ctx->q_handle == txq->q_handle)
+ txq->q_teid = q_ctx->q_teid;
+ device_printf(dev,
+ "Failed to set LAN Tx queue %u (TC %d, handle %d) context, err %s aq_err %s\n",
+ txq->me, txq->tc, txq->q_handle,
+ ice_status_str(status),
+ ice_aq_str(hw->adminq.sq_last_status));
+ err = ENODEV;
+ goto free_txqg;
}
+ /* Keep track of the Tx queue TEID */
+ if (pf_q == le16toh(qg->txqs[0].txq_id))
+ txq->q_teid = le32toh(qg->txqs[0].q_teid);
+
free_txqg:
free(qg, M_ICE);
return (err);
}
+/**
+ * ice_cfg_vsi_for_tx - Configure the hardware for Tx
+ * @vsi: the VSI to configure
+ *
+ * Configure the device Tx queues through firmware AdminQ commands. After
+ * this, Tx queues will be ready for transmit.
+ */
+int
+ice_cfg_vsi_for_tx(struct ice_vsi *vsi)
+{
+ int error, i;
+
+ for (i = 0; i < vsi->num_tx_queues; i++) {
+ struct ice_tx_queue *txq = &vsi->tx_queues[i];
+
+ /* Last configured queue */
+ if (txq->desc_count == 0)
+ break;
+ error = ice_cfg_tx_queue(txq);
+ if (error != 0)
+ return (error);
+ }
+
+ return (0);
+}
+
/**
* ice_setup_rx_ctx - Setup an Rx context structure for a receive queue
* @rxq: the receive queue to program
diff --git a/sys/dev/ice/ice_lib.h b/sys/dev/ice/ice_lib.h
index 912e6f690a95..a716c5a2903b 100644
--- a/sys/dev/ice/ice_lib.h
+++ b/sys/dev/ice/ice_lib.h
@@ -965,6 +965,7 @@ void ice_configure_txq_interrupt(struct ice_hw *hw, u16 txqid, u16 vector, u8 it
void ice_configure_all_txq_interrupts(struct ice_vsi *vsi);
void ice_flush_rxq_interrupts(struct ice_vsi *vsi);
void ice_flush_txq_interrupts(struct ice_vsi *vsi);
+int ice_cfg_tx_queue(struct ice_tx_queue *txq);
int ice_cfg_vsi_for_tx(struct ice_vsi *vsi);
int ice_cfg_vsi_for_rx(struct ice_vsi *vsi);
int ice_control_rx_queue(struct ice_vsi *vsi, u16 qidx, bool enable);