git: 62866fce4dc9 - main - gve: Fix DQO TX free-list memory ordering and pending_pkt state

From: Mark Johnston <markj_at_FreeBSD.org>
Date: Wed, 18 Nov 2026 19:24:58 UTC
The branch main has been updated by markj:

URL: https://cgit.FreeBSD.org/src/commit/?id=62866fce4dc9c6595ef8c0b5b003f52ebf10067e

commit 62866fce4dc9c6595ef8c0b5b003f52ebf10067e
Author:     Tim Hostetler <thostet@google.com>
AuthorDate: 2026-11-18 14:23:30 +0000
Commit:     Mark Johnston <markj@FreeBSD.org>
CommitDate: 2026-11-18 19:24:35 +0000

    gve: Fix DQO TX free-list memory ordering and pending_pkt state
    
    The DQO TX path uses lockless producer/consumer free lists
    (free_pending_pkts_prd/csm and free_qpl_bufs_prd/csm) shared between
    the completion and transmit taskqueues. On weakly ordered architectures
    such as arm64, the current synchronization has three issues:
    
    1. gve_free_pending_packet() updates pending_pkt state and links
       pending_pkt->next before pushing the entry onto
       free_pending_pkts_prd using a relaxed atomic_cmpset_32(), while
       gve_alloc_pending_packet() steals free_pending_pkts_prd using a
       relaxed atomic_swap_32(). Without release/acquire ordering, the
       transmit thread can observe a stale pending_pkt->next pointer after
       stealing the producer list (either orphaning freed entries or
       re-allocating entries whose completion tags are still in flight), or
       prior completion stores (such as setting state to FREE or clearing
       mbuf) can become visible after the slot is re-allocated.
    2. Similarly, in QPL mode, gve_tx_alloc_qpl_buf() steals
       free_qpl_bufs_prd with a relaxed atomic_swap_32() without an acquire
       fence to pair with atomic_cmpset_rel_32() in gve_reap_qpl_bufs_dqo().
    3. gve_alloc_pending_packet() marks pending_pkt->state as
       GVE_PACKET_STATE_PENDING_DATA_COMPL and sets the enqueue timestamp
       before pkt->mbuf is populated and before fallible checks in the
       transmit path succeed. If a completion arrives for a duplicate tag
       (from issue 1) while the second transmit is still in progress or
       after it aborts, gve_handle_packet_completion() observes a pending
       state while pkt->mbuf is NULL.
    
    This was discovered during multi-stream TCP testing on ARM64 GCP
    instances running FreeBSD 14.3-RELEASE, where it caused spurious
    missing-completion logs followed by a NULL pointer dereference of
    pending_pkt->mbuf (offset 0x38) in gve_handle_packet_completion():
    
      gve0: No pending data completion: 0
      gve0: No pending data completion: 1
      gve0: No pending data completion: 2
      Fatal data abort:
        x0: 0xffffe00009d61e00
        x1: 0xffffe00009ce6580
      ...
       elr: 0xffff0000013b49d8 (gve_tx_cleanup_tq_dqo + 0x120)
       far: 0x0000000000000038
      panic: vm_fault failed: 0xffff0000013b49d8 error 1
      cpuid = 14
      KDB: stack backtrace:
      #0 0xffff000000516174 at kdb_backtrace+0x60
      #1 0xffff0000004bc4c0 at vpanic+0x13c
      #2 0xffff0000004bc380 at panic+0x44
      #3 0xffff0000008d22bc at data_abort+0x314
      #4 0xffff0000008b1878 at handle_el1h_sync+0x18
      #5 0xffff0000013b49d8 at gve_tx_cleanup_tq_dqo+0x120
      #6 0xffff00000052d4f0 at taskqueue_run_locked+0x17c
      #7 0xffff00000052e4f0 at taskqueue_thread_loop+0xc8
    
    Use atomic_cmpset_rel_32() when pushing onto free_pending_pkts_prd, add
    atomic_thread_fence_acq() after stealing free_pending_pkts_prd in
    gve_alloc_pending_packet() and free_qpl_bufs_prd in
    gve_tx_alloc_qpl_buf(), clear pending_pkt->next upon allocation, and
    defer populating pkt->mbuf, pkt->state, and the enqueue timestamp until
    all fallible checks in the transmit path have succeeded.
    
    Additionally, replace atomic_load_acq_32() with a relaxed
    atomic_load_32() when reading old_head in gve_free_pending_packet(), as
    old_head is only used as a scalar index and never dereferenced by the
    producer.
    
    Signed-off-by: Tim Hostetler <thostet@google.com>
    
    Fixes:          d438b4ef0cfc ("gve: Add DQO RDA support")
    Fixes:          2348ac893d10 ("gve: Add DQO QPL support")
    Reviewed by:    markj, Jasper Tran O'Leary <jtranoleary@google.com>
    Differential Revision:  https://reviews.freebsd.org/D61233
---
 sys/dev/gve/gve_tx_dqo.c | 41 ++++++++++++++++++++++++++++++++---------
 1 file changed, 32 insertions(+), 9 deletions(-)

diff --git a/sys/dev/gve/gve_tx_dqo.c b/sys/dev/gve/gve_tx_dqo.c
index e2dcedb43bbb..cac82a7ca75e 100644
--- a/sys/dev/gve/gve_tx_dqo.c
+++ b/sys/dev/gve/gve_tx_dqo.c
@@ -515,6 +515,13 @@ gve_alloc_pending_packet(struct gve_tx_ring *tx)
 	if (__predict_false(index == -1)) {
 		tx->dqo.free_pending_pkts_csm = atomic_swap_32(
 		    &tx->dqo.free_pending_pkts_prd, -1);
+		/*
+		 * Pairs with atomic_cmpset_rel_32() in
+		 * gve_free_pending_packet() so the completion thread's writes
+		 * to all slots in the stolen list are visible before this xmit
+		 * thread walks ->next and reuses them.
+		 */
+		atomic_thread_fence_acq();
 
 		index = tx->dqo.free_pending_pkts_csm;
 		if (__predict_false(index == -1))
@@ -525,9 +532,7 @@ gve_alloc_pending_packet(struct gve_tx_ring *tx)
 
 	/* Remove pending_pkt from the consumer list */
 	tx->dqo.free_pending_pkts_csm = pending_pkt->next;
-	pending_pkt->state = GVE_PACKET_STATE_PENDING_DATA_COMPL;
-
-	gve_set_timestamp(&pending_pkt->enqueue_time_sec);
+	pending_pkt->next = -1;
 
 	return (pending_pkt);
 }
@@ -545,10 +550,16 @@ gve_free_pending_packet(struct gve_tx_ring *tx,
 
 	/* Add pending_pkt to the producer list */
 	while (true) {
-		old_head = atomic_load_acq_32(&tx->dqo.free_pending_pkts_prd);
+		old_head = atomic_load_32(&tx->dqo.free_pending_pkts_prd);
 
 		pending_pkt->next = old_head;
-		if (atomic_cmpset_32(&tx->dqo.free_pending_pkts_prd,
+		/*
+		 * Pairs with atomic_thread_fence_acq() in
+		 * gve_alloc_pending_packet() so this completion thread's
+		 * updates to state and ->next are visible before the xmit
+		 * thread steals and reuses this slot.
+		 */
+		if (atomic_cmpset_rel_32(&tx->dqo.free_pending_pkts_prd,
 		    old_head, index))
 			break;
 	}
@@ -621,6 +632,14 @@ gve_tx_alloc_qpl_buf(struct gve_tx_ring *tx)
 	if (__predict_false(buf == -1)) {
 		tx->dqo.free_qpl_bufs_csm = atomic_swap_32(
 		    &tx->dqo.free_qpl_bufs_prd, -1);
+		/*
+		 * Pairs with atomic_cmpset_rel_32() in
+		 * gve_reap_qpl_bufs_dqo() so the completion thread's
+		 * qpl_bufs[] entries for the entire stolen list are visible
+		 * before this xmit thread walks and allocates them.
+		 */
+		atomic_thread_fence_acq();
+
 		buf = tx->dqo.free_qpl_bufs_csm;
 		if (__predict_false(buf == -1))
 			return (-1);
@@ -748,12 +767,15 @@ gve_xmit_dqo_qpl(struct gve_tx_ring *tx, struct mbuf *mbuf)
 		return (ENOBUFS);
 	}
 	completion_tag = pkt - tx->dqo.pending_pkts;
-	pkt->mbuf = mbuf;
 
 	err = gve_tx_fill_ctx_descs(tx, mbuf, is_tso, &desc_idx);
 	if (err)
 		goto abort;
 
+	pkt->mbuf = mbuf;
+	pkt->state = GVE_PACKET_STATE_PENDING_DATA_COMPL;
+	gve_set_timestamp(&pkt->enqueue_time_sec);
+
 	gve_tx_copy_mbuf_and_write_pkt_descs(tx, mbuf, pkt,
 	    has_csum_flag, completion_tag, &desc_idx);
 
@@ -770,7 +792,6 @@ gve_xmit_dqo_qpl(struct gve_tx_ring *tx, struct mbuf *mbuf)
 	return (0);
 
 abort:
-	pkt->mbuf = NULL;
 	gve_free_pending_packet(tx, pkt);
 	return (err);
 }
@@ -823,7 +844,6 @@ gve_xmit_dqo(struct gve_tx_ring *tx, struct mbuf **mbuf_ptr)
 	if (err)
 		goto abort;
 	mbuf = *mbuf_ptr;  /* gve_map_mbuf_dqo might replace the mbuf chain */
-	pkt->mbuf = mbuf;
 
 	total_descs_needed = 1 + /* general_ctx_desc */
 	    nsegs +              /* pkt_desc */
@@ -838,6 +858,10 @@ gve_xmit_dqo(struct gve_tx_ring *tx, struct mbuf **mbuf_ptr)
 	if (err)
 		goto abort_with_dma;
 
+	pkt->mbuf = mbuf;
+	pkt->state = GVE_PACKET_STATE_PENDING_DATA_COMPL;
+	gve_set_timestamp(&pkt->enqueue_time_sec);
+
 	bus_dmamap_sync(tx->dqo.buf_dmatag, pkt->dmamap, BUS_DMASYNC_PREWRITE);
 	for (i = 0; i < nsegs; i++) {
 		gve_tx_fill_pkt_desc_dqo(tx, &desc_idx,
@@ -861,7 +885,6 @@ gve_xmit_dqo(struct gve_tx_ring *tx, struct mbuf **mbuf_ptr)
 abort_with_dma:
 	gve_unmap_packet(tx, pkt);
 abort:
-	pkt->mbuf = NULL;
 	gve_free_pending_packet(tx, pkt);
 	return (err);
 }