git: 4993c1052f89 - main - aq: Invalidate the descriptor cache after stopping all rings

The branch main has been updated by kbowling:

URL: https://cgit.FreeBSD.org/src/commit/?id=4993c1052f899b9a971815df5661ca241641ad8e

commit 4993c1052f899b9a971815df5661ca241641ad8e
Author:     Kevin Bowling <kbowling@FreeBSD.org>
AuthorDate: 2026-09-17 02:35:58 +0000
Commit:     Kevin Bowling <kbowling@FreeBSD.org>
CommitDate: 2026-09-25 05:11:23 +0000

    aq: Invalidate the descriptor cache after stopping all rings
    
    Atlantic controllers can retain receive descriptors and their data
    addresses after their rings are disabled. Reusing or releasing those
    mappings without invalidating the device cache has caused observed
    IOMMU and SMMU faults in the referenced Linux reports (7a1bb49461b1,
    ed4d81c4b3f2 and 7526183cfdbe).
    
    Move global cache invalidation out of the per-ring stop routine. Disable
    every ring first, toggle invalidation once, and wait for its completion
    indication. Exclude Atlantic A0, as in the upstream workaround. Report
    a completion timeout rather than silently discarding it.
    
    Reviewed by:    nprice
    MFC after:      2 weeks
    Sponsored by:   BBOX.io
    Differential Revision:  https://reviews.freebsd.org/D59852
---
 sys/dev/aq/aq_hw.c              | 25 +++++++++++++++++++++++++
 sys/dev/aq/aq_hw.h              |  2 ++
 sys/dev/aq/aq_hw_llh.c          |  9 +++++++++
 sys/dev/aq/aq_hw_llh.h          |  3 +++
 sys/dev/aq/aq_hw_llh_internal.h |  5 +++++
 sys/dev/aq/aq_main.c            |  4 ++++
 sys/dev/aq/aq_ring.c            |  4 ----
 7 files changed, 48 insertions(+), 4 deletions(-)

diff --git a/sys/dev/aq/aq_hw.c b/sys/dev/aq/aq_hw.c
index 934543937a57..5c926f2ed960 100644
--- a/sys/dev/aq/aq_hw.c
+++ b/sys/dev/aq/aq_hw.c
@@ -58,6 +58,31 @@ aq_hw_err_from_flags(struct aq_hw *hw)
 	return (0);
 }
 
+int
+aq_hw_invalidate_descriptor_cache(struct aq_hw *hw)
+{
+	int error;
+
+	/* Atlantic A0 does not implement this stop workaround. */
+	if (IS_CHIP_FEATURE(hw, REVISION_A0))
+		return (0);
+
+	/*
+	 * Cached Rx descriptors retain both descriptor and data addresses.
+	 * Toggle the global cache invalidation only after every ring is down.
+	 */
+	rdm_rx_dma_desc_cache_init_tgl(hw);
+	error = aq_hw_err_from_flags(hw);
+	if (error != 0)
+		return (error);
+
+	error = AQ_HW_WAIT_FOR(rdm_rx_dma_desc_cache_init_done_get(hw) != 0,
+	    1000, 10);
+	if (error != 0)
+		return (error);
+	return (aq_hw_err_from_flags(hw));
+}
+
 inline uint32_t
 aq_hw_read_reg(struct aq_hw *hw, uint32_t reg)
 {
diff --git a/sys/dev/aq/aq_hw.h b/sys/dev/aq/aq_hw.h
index bf4925d139f9..67a2ee975888 100644
--- a/sys/dev/aq/aq_hw.h
+++ b/sys/dev/aq/aq_hw.h
@@ -408,6 +408,8 @@ int aq_hw_set_power(struct aq_hw *hw, unsigned int power_state);
 
 int aq_hw_err_from_flags(struct aq_hw *hw);
 
+int aq_hw_invalidate_descriptor_cache(struct aq_hw *hw);
+
 int hw_atl_b0_hw_vlan_promisc_set(struct aq_hw *hw, bool promisc);
 
 int hw_atl_b0_hw_vlan_set(struct aq_hw *hw,
diff --git a/sys/dev/aq/aq_hw_llh.c b/sys/dev/aq/aq_hw_llh.c
index 2369418a3681..d82413a946ea 100644
--- a/sys/dev/aq/aq_hw_llh.c
+++ b/sys/dev/aq/aq_hw_llh.c
@@ -1728,6 +1728,15 @@ rdm_rx_dma_desc_cache_init_tgl(struct aq_hw *aq_hw)
 	);
 }
 
+uint32_t
+rdm_rx_dma_desc_cache_init_done_get(struct aq_hw *aq_hw)
+{
+	return (AQ_READ_REG_BIT(aq_hw,
+	    rdm_rx_dma_desc_cache_init_done_adr,
+	    rdm_rx_dma_desc_cache_init_done_msk,
+	    rdm_rx_dma_desc_cache_init_done_shift));
+}
+
 void
 tpb_tx_pkt_buff_size_per_tc_set(struct aq_hw *aq_hw,
     uint32_t tx_pkt_buff_size_per_tc, uint32_t buffer)
diff --git a/sys/dev/aq/aq_hw_llh.h b/sys/dev/aq/aq_hw_llh.h
index 47011295aa4c..641e53b6da06 100644
--- a/sys/dev/aq/aq_hw_llh.h
+++ b/sys/dev/aq/aq_hw_llh.h
@@ -934,6 +934,9 @@ void tpb_tx_pkt_buff_size_per_tc_set(struct aq_hw *aq_hw,
 /* toggle rdm rx dma descriptor cache init */
 void rdm_rx_dma_desc_cache_init_tgl(struct aq_hw *aq_hw);
 
+/* get rdm rx dma descriptor cache init done */
+uint32_t rdm_rx_dma_desc_cache_init_done_get(struct aq_hw *aq_hw);
+
 /* set tx path pad insert enable */
 void tpb_tx_path_scp_ins_en_set(struct aq_hw *aq_hw,
     uint32_t tx_path_scp_ins_en);
diff --git a/sys/dev/aq/aq_hw_llh_internal.h b/sys/dev/aq/aq_hw_llh_internal.h
index fa1c9a83985b..48f3774216d2 100644
--- a/sys/dev/aq/aq_hw_llh_internal.h
+++ b/sys/dev/aq/aq_hw_llh_internal.h
@@ -395,6 +395,11 @@
 /* default value of bitfield rdm_desc_init_i */
 #define rdm_rx_dma_desc_cache_init_defaulT 0x0
 
+/* rdm_desc_init_done_i bitfield definitions */
+#define rdm_rx_dma_desc_cache_init_done_adr 0x00005a10
+#define rdm_rx_dma_desc_cache_init_done_msk 0x00000001
+#define rdm_rx_dma_desc_cache_init_done_shift 0
+
 /* rx int_desc_wrb_en bitfield definitions
  * preprocessor definitions for the bitfield "int_desc_wrb_en".
  * port="pif_rdm_int_desc_wrb_en_i"
diff --git a/sys/dev/aq/aq_main.c b/sys/dev/aq/aq_main.c
index d5ba995eb493..5aa162e096b0 100644
--- a/sys/dev/aq/aq_main.c
+++ b/sys/dev/aq/aq_main.c
@@ -873,6 +873,10 @@ aq_if_stop(if_ctx_t ctx)
 			    "could not stop RX ring %d\n", i);
 	}
 
+	if (aq_hw_invalidate_descriptor_cache(hw) != 0)
+		device_printf(softc->dev,
+		    "could not invalidate the RX descriptor cache\n");
+
 	if (aq_hw_reset(&softc->hw, true) != 0)
 		device_printf(softc->dev, "could not reset the MAC on stop\n");
 	memset(&softc->last_stats, 0, sizeof(softc->last_stats));
diff --git a/sys/dev/aq/aq_ring.c b/sys/dev/aq/aq_ring.c
index 13b1881d9147..8fef7ae7765d 100644
--- a/sys/dev/aq/aq_ring.c
+++ b/sys/dev/aq/aq_ring.c
@@ -208,10 +208,6 @@ aq_ring_rx_stop(struct aq_hw *hw, struct aq_ring *ring)
 
 	AQ_DBG_ENTERA("[%d]", ring->index);
 	rdm_rx_desc_en_set(hw, 0U, ring->index);
-	/* Invalidate Descriptor Cache to prevent writing to the cached
-	 * descriptors and to the data pointer of those descriptors
-	 */
-	rdm_rx_dma_desc_cache_init_tgl(hw);
 	err = aq_hw_err_from_flags(hw);
 	AQ_DBG_EXIT(err);
 	return (err);