git: 42f3088bd980 - main - re(4): harden re_watchdog() recovery and log controller state

The branch main has been updated by adrian:

URL: https://cgit.FreeBSD.org/src/commit/?id=42f3088bd980a469dbbc3e6b568edb7ce0ab2bc3

commit 42f3088bd980a469dbbc3e6b568edb7ce0ab2bc3
Author:     Jérémie Jourdin <jeremie@jourdin.org>
AuthorDate: 2026-07-19 18:47:17 +0000
Commit:     Adrian Chadd <adrian@FreeBSD.org>
CommitDate: 2026-07-19 18:47:21 +0000

    re(4): harden re_watchdog() recovery and log controller state
    
    Distinguish the two failure classes from the PR in a single log line
    (ring indices, ISR/IMR, TXCFG, interrupt mode): lost interrupt vs
    genuine DMA stall.
    
    Bail out instead of re-initializing when the controller reads back
    all-ones (fallen off the bus; reinit cannot help).
    
    Re-assert the driver's existing ASPM-disabled policy before reinit,
    since firmware/power transitions re-arming L0s/L1 is a documented
    stall trigger.
    
    Diagnostics-only on the recovered path; no fast-path change.
    
    * Field diagnostics running on an RTL8168H production fleet; the log
      format distinguishes lost-doorbell / DMA-stall / dead-controller
      without a debug build.
    
    Reviewed by:    adrian
    Differential Revision:  https://reviews.freebsd.org/D58279
    PR: kern/166724
---
 sys/dev/re/if_re.c | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 58 insertions(+), 3 deletions(-)

diff --git a/sys/dev/re/if_re.c b/sys/dev/re/if_re.c
index b04dc3cbaf4a..5a3e8e68d26f 100644
--- a/sys/dev/re/if_re.c
+++ b/sys/dev/re/if_re.c
@@ -3600,6 +3600,8 @@ re_watchdog(struct rl_softc *sc)
 {
 	struct epoch_tracker et;
 	if_t ifp;
+	const char *imode;
+	uint32_t isr, imr, txcfg;
 
 	RL_LOCK_ASSERT(sc);
 
@@ -3607,18 +3609,71 @@ re_watchdog(struct rl_softc *sc)
 		return;
 
 	ifp = sc->rl_ifp;
+	/*
+	 * Snapshot the controller state before reclaim for the diagnostics
+	 * below.  Reading RL_ISR is side-effect free: the interrupt status
+	 * is cleared by writing ones, not by reading.
+	 */
+	imode = (sc->rl_flags & RL_FLAG_MSIX) ? "MSI-X" :
+	    (sc->rl_flags & RL_FLAG_MSI) ? "MSI" : "INTx";
+	isr = CSR_READ_2(sc, RL_ISR);
+	imr = CSR_READ_2(sc, RL_IMR);
 	re_txeof(sc);
 	if (sc->rl_ldata.rl_tx_free == sc->rl_ldata.rl_tx_desc_cnt) {
-		if_printf(ifp, "watchdog timeout (missed Tx interrupts) "
-		    "-- recovering\n");
+		/*
+		 * The retry reclaim drained the ring: a Tx completion interrupt
+		 * was lost.  Log the interrupt state so the lost-interrupt path
+		 * can be diagnosed without a debug build.
+		 */
+		if_printf(ifp, "watchdog timeout (missed Tx interrupts) -- "
+		    "recovering (ISR 0x%04x IMR 0x%04x %s)\n",
+		    isr, imr, imode);
 		if (!if_sendq_empty(ifp))
 			re_start_locked(ifp);
 		return;
 	}
 
-	if_printf(ifp, "watchdog timeout\n");
+	txcfg = CSR_READ_4(sc, RL_TXCFG);
+	/*
+	 * A genuine Tx stall.  Log the Tx ring position and controller
+	 * state in a single line -- enough to distinguish a lost doorbell,
+	 * a DMA stall, and a controller that has fallen off the bus.
+	 */
+	if_printf(ifp, "watchdog timeout -- resetting (tx %d/%d cons %d "
+	    "prod %d ISR 0x%04x IMR 0x%04x TXCFG 0x%08x %s)\n",
+	    sc->rl_ldata.rl_tx_free, sc->rl_ldata.rl_tx_desc_cnt,
+	    sc->rl_ldata.rl_tx_considx, sc->rl_ldata.rl_tx_prodidx,
+	    isr, imr, txcfg, imode);
 	if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
 
+	/*
+	 * If the controller reads back all-ones it has fallen off the bus;
+	 * re_init_locked() cannot recover it, so flag it and bail rather
+	 * than spinning through reset after reset.
+	 */
+	if (txcfg == 0xFFFFFFFF) {
+		if_printf(ifp,
+		    "controller not responding; not reinitializing\n");
+		return;
+	}
+
+	/*
+	 * ASPM is a common cause of these Tx timeouts.  Make sure L0s/L1
+	 * and CLKREQ are still disabled on the link before reinitializing;
+	 * firmware or a power transition may have re-armed them.
+	 */
+	if (sc->rl_expcap != 0) {
+		uint16_t ctl;
+
+		ctl = pci_read_config(sc->rl_dev,
+		    sc->rl_expcap + PCIER_LINK_CTL, 2);
+		if ((ctl & (PCIEM_LINK_CTL_ECPM | PCIEM_LINK_CTL_ASPMC)) != 0) {
+			ctl &= ~(PCIEM_LINK_CTL_ECPM | PCIEM_LINK_CTL_ASPMC);
+			pci_write_config(sc->rl_dev,
+			    sc->rl_expcap + PCIER_LINK_CTL, ctl, 2);
+		}
+	}
+
 	NET_EPOCH_ENTER(et);
 	re_rxeof(sc, NULL);
 	NET_EPOCH_EXIT(et);