git: 38187938f522 - main - re(4): quiesce RTL8168G+ and reset before freeing buffers in re_stop()

The branch main has been updated by adrian:

URL: https://cgit.FreeBSD.org/src/commit/?id=38187938f52283143308dd219db35d889da55ea4

commit 38187938f52283143308dd219db35d889da55ea4
Author:     Jérémie Jourdin <jeremie@jourdin.org>
AuthorDate: 2026-07-19 18:39:49 +0000
Commit:     Adrian Chadd <adrian@FreeBSD.org>
CommitDate: 2026-07-19 18:44:36 +0000

    re(4): quiesce RTL8168G+ and reset before freeing buffers in re_stop()
    
    The STOPREQ command written by re_stop() is not defined for
    RTL8168G and later; issuing it can wedge the MAC.
    
    Replace it on those parts with the vendor-documented sequence:
    
    * settle delay
    * bounded poll for Tx queue empty
    * clear TE/RE
    * then bounded poll of the MCU command register (0xD3) FIFO-empty bits.
    
    Also reset the controller before the Rx/Tx buffer free: a controller that
    has not quiesced keeps DMAing stale, still-owned descriptors pointing at
    freed mbufs (use-after-free under INVARIANTS, cross-NIC mbuf corruption
    reported in the PR).
    
    Adds the RL_MCU_* register definitions.
    
    All waits are bounded; error paths only.
    
    * iperf3 --bidir at line rate against RTL8168H (XID 0x541);
      previously wedged the controller until power cycle, with the
      quiesce the reset path recovers.
    
    * Deployed in production on an RTL8168H fleet since 2026-07-01.
    
    Reviewed by:    adrian
    Differential Revision:  https://reviews.freebsd.org/D58276
    PR: kern/166724
---
 sys/dev/re/if_re.c    | 41 ++++++++++++++++++++++++++++++++++++++++-
 sys/dev/rl/if_rlreg.h |  8 ++++++++
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/sys/dev/re/if_re.c b/sys/dev/re/if_re.c
index 50e7e1cad16c..cf7ba90c9a5a 100644
--- a/sys/dev/re/if_re.c
+++ b/sys/dev/re/if_re.c
@@ -3630,7 +3630,37 @@ re_stop(struct rl_softc *sc)
 		    0x00080000);
 	}
 
-	if ((sc->rl_flags & RL_FLAG_WAIT_TXPOLL) != 0) {
+	if ((sc->rl_flags & RL_FLAG_8168G_PLUS) != 0) {
+		/*
+		 * RTL8168G and later.  The STOPREQ command is defined only for
+		 * earlier controllers; issuing it on these parts can leave the
+		 * MAC wedged.  With the RXDV gate enabled above, drain the TX
+		 * descriptor queue and the on-chip TX/RX FIFOs and clear the
+		 * TX/RX enable bits so the DMA engine is idle before the reset
+		 * and buffer free below.  All waits are bounded.
+		 */
+		DELAY(2000);
+		for (i = RL_TIMEOUT; i > 0; i--) {
+			if ((CSR_READ_4(sc, RL_TXCFG) &
+			    RL_TXCFG_QUEUE_EMPTY) != 0)
+				break;
+			DELAY(100);
+		}
+		if (i == 0)
+			device_printf(sc->rl_dev, "stopping TXQ timed out!\n");
+		CSR_WRITE_1(sc, RL_COMMAND, CSR_READ_1(sc, RL_COMMAND) &
+		    ~(RL_CMD_TX_ENB | RL_CMD_RX_ENB));
+		for (i = RL_TIMEOUT * 3; i > 0; i--) {
+			if ((CSR_READ_1(sc, RL_MCU_CMD) &
+			    (RL_MCU_TXFIFO_EMPTY | RL_MCU_RXFIFO_EMPTY)) ==
+			    (RL_MCU_TXFIFO_EMPTY | RL_MCU_RXFIFO_EMPTY))
+				break;
+			DELAY(20);
+		}
+		if (i == 0)
+			device_printf(sc->rl_dev,
+			    "TX/RX FIFO drain timed out!\n");
+	} else if ((sc->rl_flags & RL_FLAG_WAIT_TXPOLL) != 0) {
 		for (i = RL_TIMEOUT; i > 0; i--) {
 			if ((CSR_READ_1(sc, sc->rl_txstart) &
 			    RL_TXSTART_START) == 0)
@@ -3661,6 +3691,15 @@ re_stop(struct rl_softc *sc)
 	CSR_WRITE_2(sc, RL_IMR, 0x0000);
 	CSR_WRITE_2(sc, RL_ISR, 0xFFFF);
 
+	/*
+	 * Reset the controller before freeing the DMA buffers below.  A
+	 * controller that has not fully quiesced can keep fetching stale,
+	 * still-owned descriptors that point at about-to-be-freed mbufs.
+	 * re_init_locked() resets again on the reinit path; the extra reset
+	 * is idempotent and cheap.
+	 */
+	re_reset(sc);
+
 	if (sc->rl_head != NULL) {
 		m_freem(sc->rl_head);
 		sc->rl_head = sc->rl_tail = NULL;
diff --git a/sys/dev/rl/if_rlreg.h b/sys/dev/rl/if_rlreg.h
index 10e25fcb5373..0db4b2066ce6 100644
--- a/sys/dev/rl/if_rlreg.h
+++ b/sys/dev/rl/if_rlreg.h
@@ -145,6 +145,14 @@
 #define	RL_INTRMOD		0x00E2	/* 16 bits */
 #define	RL_MISC			0x00F0
 
+/*
+ * MCU command / FIFO status register (RTL8168G and later).  Polled to
+ * confirm the on-chip TX/RX FIFOs have drained before a controller reset.
+ */
+#define	RL_MCU_CMD		0x00D3
+#define	RL_MCU_TXFIFO_EMPTY	0x20	/* bit 5 */
+#define	RL_MCU_RXFIFO_EMPTY	0x10	/* bit 4 */
+
 /*
  * TX config register bits
  */