git: aae5b168820a - main - ice: Defer RDMA critical error notifications

The branch main has been updated by kbowling:

URL: https://cgit.FreeBSD.org/src/commit/?id=aae5b168820a9c39e39c6dff2f31df0522f91901

commit aae5b168820a9c39e39c6dff2f31df0522f91901
Author:     Kevin Bowling <kbowling@FreeBSD.org>
AuthorDate: 2026-09-03 12:01:18 +0000
Commit:     Kevin Bowling <kbowling@FreeBSD.org>
CommitDate: 2026-09-17 00:56:43 +0000

    ice: Defer RDMA critical error notifications
    
    ice_msix_admin() runs as an interrupt filter inside a critical section.
    ice_rdma_notify_pe_intr() acquires the global RDMA sx and invokes the
    client event handler, both of which require sleepable thread context.  A
    PE or HMC critical error could therefore panic under WITNESS or sleep
    from interrupt context.
    
    Accumulate OICR causes atomically in the interrupt filter and mark them
    pending in the driver state.  Deliver the notification from the iflib
    admin task before processing reset events.  This preserves the existing
    ordering, lets an iRDMA-requested reset run in the same admin pass, and
    coalesces causes from multiple interrupts.
    
    MFC after:      2 weeks
    Sponsored by:   BBOX.io
    Differential Revision:  https://reviews.freebsd.org/D59340
---
 sys/dev/ice/ice_iflib.h    |  3 +++
 sys/dev/ice/ice_lib.h      |  1 +
 sys/dev/ice/ice_strings.c  |  2 ++
 sys/dev/ice/if_ice_iflib.c | 29 ++++++++++++++++++++++++++++-
 4 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/sys/dev/ice/ice_iflib.h b/sys/dev/ice/ice_iflib.h
index 3e3d4e5f8d84..fe1f53f4cb68 100644
--- a/sys/dev/ice/ice_iflib.h
+++ b/sys/dev/ice/ice_iflib.h
@@ -253,6 +253,7 @@ struct ice_mirr_if {
  * @pf_imap: interrupt mapping for PF LAN interrupts
  * @lan_vectors: # of vectors used by LAN driver (length of pf_imap)
  * @ldo_tlv: LAN Default Override settings from NVM
+ * @rdma_oicr: pending PE/HMC interrupt causes for the RDMA client
  *
  * ice_iov.c requires the following parameters (when PCI_IOV is defined):
  * @vfs: array of VF context structures
@@ -333,6 +334,8 @@ struct ice_softc {
 
 	/* driver state flags, only access using atomic functions */
 	u32 state;
+	/* pending RDMA OICR causes, only access using atomic functions */
+	u32 rdma_oicr;
 
 	/* NVM link override settings */
 	struct ice_link_default_override_tlv ldo_tlv;
diff --git a/sys/dev/ice/ice_lib.h b/sys/dev/ice/ice_lib.h
index be4cc674556b..2562a9e2476f 100644
--- a/sys/dev/ice/ice_lib.h
+++ b/sys/dev/ice/ice_lib.h
@@ -695,6 +695,7 @@ enum ice_state {
 	ICE_STATE_CONTROLQ_EVENT_PENDING,
 	ICE_STATE_VFLR_PENDING,
 	ICE_STATE_MDD_PENDING,
+	ICE_STATE_RDMA_PE_INTR_PENDING,
 	ICE_STATE_RESET_OICR_RECV,
 	ICE_STATE_RESET_PFR_REQ,
 	ICE_STATE_PREPARED_FOR_RESET,
diff --git a/sys/dev/ice/ice_strings.c b/sys/dev/ice/ice_strings.c
index 09adf47a33f2..f80d8f85d070 100644
--- a/sys/dev/ice/ice_strings.c
+++ b/sys/dev/ice/ice_strings.c
@@ -1012,6 +1012,8 @@ ice_state_to_str(enum ice_state state)
 		return "VFLR_PENDING";
 	case ICE_STATE_MDD_PENDING:
 		return "MDD_PENDING";
+	case ICE_STATE_RDMA_PE_INTR_PENDING:
+		return "RDMA_PE_INTR_PENDING";
 	case ICE_STATE_RESET_OICR_RECV:
 		return "RESET_OICR_RECV";
 	case ICE_STATE_RESET_PFR_REQ:
diff --git a/sys/dev/ice/if_ice_iflib.c b/sys/dev/ice/if_ice_iflib.c
index f6e619b2ebe5..88f2e276da69 100644
--- a/sys/dev/ice/if_ice_iflib.c
+++ b/sys/dev/ice/if_ice_iflib.c
@@ -131,6 +131,7 @@ static void ice_free_pci_mapping(struct ice_softc *sc);
 static void ice_update_link_status(struct ice_softc *sc, bool update_media);
 static void ice_init_device_features(struct ice_softc *sc);
 static void ice_init_tx_tracking(struct ice_vsi *vsi);
+static void ice_handle_rdma_pe_intr(struct ice_softc *sc);
 static void ice_handle_reset_event(struct ice_softc *sc);
 static void ice_handle_pf_reset_request(struct ice_softc *sc);
 static void ice_prepare_for_reset(struct ice_softc *sc);
@@ -1382,7 +1383,8 @@ ice_msix_admin(void *arg)
 		if (oicr & PFINT_OICR_HMC_ERR_M)
 			/* Log the HMC errors */
 			ice_log_hmc_error(hw, dev);
-		ice_rdma_notify_pe_intr(sc, oicr);
+		atomic_set_32(&sc->rdma_oicr, oicr);
+		ice_set_state(&sc->state, ICE_STATE_RDMA_PE_INTR_PENDING);
 	}
 
 	if (oicr & PFINT_OICR_PCI_EXCEPTION_M) {
@@ -2405,6 +2407,28 @@ ice_transition_safe_mode(struct ice_softc *sc)
 	ice_clear_bit(ICE_FEATURE_RSS, sc->feat_en);
 }
 
+/**
+ * ice_handle_rdma_pe_intr - Notify RDMA of deferred PE/HMC errors
+ * @sc: device private softc
+ *
+ * Deliver PE and HMC error notifications from the admin task because the
+ * RDMA notification path takes a sleepable lock. Multiple OICR causes which
+ * arrive before the task runs are accumulated by the interrupt filter.
+ */
+static void
+ice_handle_rdma_pe_intr(struct ice_softc *sc)
+{
+	u32 oicr;
+
+	if (!ice_testandclear_state(&sc->state,
+	    ICE_STATE_RDMA_PE_INTR_PENDING))
+		return;
+
+	oicr = atomic_readandclear_32(&sc->rdma_oicr);
+	if (oicr != 0)
+		ice_rdma_notify_pe_intr(sc, oicr);
+}
+
 /**
  * ice_if_update_admin_status - update admin status
  * @ctx: iflib ctx structure
@@ -2444,6 +2468,9 @@ ice_if_update_admin_status(if_ctx_t ctx)
 		}
 	}
 
+	/* Notify RDMA before handling a reset it may request. */
+	ice_handle_rdma_pe_intr(sc);
+
 	/* Handle global reset events */
 	ice_handle_reset_event(sc);