bce0: Error mapping mbuf into TX chain!

Pyun YongHyeon pyunyh at gmail.com
Mon Aug 7 01:54:06 UTC 2006


On Sun, Aug 06, 2006 at 12:07:41AM +0200, Scott Wilson wrote:
 > On 8/5/06, Pyun YongHyeon <pyunyh at gmail.com> wrote:
 > >On Fri, Aug 04, 2006 at 03:52:40PM +0200, Scott Wilson wrote:
 > > > On 7/13/06, Doug Ambrisko <ambrisko at ambrisko.com> wrote:
 > > > >David (Controller AE) Christensen writes:
 > > > >| Sorry, I've been out on vacation and just got back into town.  I'll 
 > > MFC
 > > > >| the patch within the next day or two.
 > > > >
 > > > >I'll let you merge in the down/up fix that I put into -current.
 > > > >
 > > > >Doug A.
 > > >
 > > > Hi, I just had a bce interface lock up with the same problem:
 > > >
 > > > Aug  4 07:00:16 pe3 kernel: bce0: /usr/src/sys/dev/bce/if_bce.c(4644):
 > > > Error mapping mbuf into TX chain!
 > > > Aug  4 07:00:47 pe3 last message repeated 368 times
 > > > ....
 > > >
 > > > running v 1.2.2.5 of if_bce.c from RELENG_6 which has the
 > > > defragmentation patch mentioned in this thread.  Any suggestions on
 > > > how I can help find a fix?
 > > >
 > > > scott
 > >
 > >Hmm... I can see several bus_dma(9) related bugs in bce(4).
 > >For architectures that have IOMMU hardware it may have corrupted DMA
 > >mapping and I'm pretty sure it wouldn't work on sparc64.
 > >When it has to handle many fragmented frame or has insufficient
 > >number of free Tx descriptors it would show unexpected results.
 > >Unfortunately I don't have hardwares supported by bce(4) and
 > >fixing requiries a working hardware. :-(
 > >
 > 
 > I see ... I am running amd64 on some dell poweredge 1950 boxes.
 > They're xeon processors, but have chosen amd64 because they have 8gig
 > of ram each.
 > 
 > Here are the relevant details on the interface....
 > 
 > bce0: <Broadcom NetXtreme II BCM5708 1000Base-T (B1), v0.9.5> mem
 > 0xf4000000-0xf5ffffff irq 16 at device 0.0 on pci9
 > bce0: ASIC ID 0x57081010; Revision (B1); PCI-X 64-bit 133MHz
 > miibus0: <MII bus> on bce0
 > brgphy0: <BCM5708C 10/100/1000baseTX PHY> on miibus0
 > brgphy0:  10baseT, 10baseT-FDX, 100baseTX, 100baseTX-FDX, 1000baseTX,
 > 1000baseTX-FDX, auto
 > 
 > I could make a machine available remotely to someone if it would help.
 > 

Unfortunately it's very hard to debug things like this with remote
access only. Normally it involves big red-button and serial console
and an additional good working system which is directly connected
with cross-cable to write the driver.

 > Any other advice on how I can help move this forward would be greatly
 > appreciated!
 > 

Ok, here is guess work for Tx path. As I said before I don't have
hardware and don't have documentation for the chip so I wouldn't
be surprised if it does not work at all. 

 > thanks,
 > 
 > scott

-- 
Regards,
Pyun YongHyeon
-------------- next part --------------
--- if_bce.c.orig	Fri Jul 21 03:41:00 2006
+++ if_bce.c	Mon Aug  7 10:36:09 2006
@@ -276,7 +276,6 @@
 /*                                                                          */
 /****************************************************************************/
 static void bce_dma_map_addr		(void *, bus_dma_segment_t *, int, int);
-static void bce_dma_map_tx_desc		(void *, bus_dma_segment_t *, int, bus_size_t, int);
 static int  bce_dma_alloc			(device_t);
 static void bce_dma_free			(struct bce_softc *);
 static void bce_release_resources	(struct bce_softc *);
@@ -300,7 +299,7 @@
 static void bce_free_rx_chain		(struct bce_softc *);
 static void bce_free_tx_chain		(struct bce_softc *);
 
-static int  bce_tx_encap			(struct bce_softc *, struct mbuf *, u16 *, u16 *, u32 *);
+static int  bce_tx_encap			(struct bce_softc *, struct mbuf **, u16 *, u16 *, u32 *);
 static void bce_start_locked		(struct ifnet *);
 static void bce_start				(struct ifnet *);
 static int  bce_ioctl				(struct ifnet *, u_long, caddr_t);
@@ -2150,114 +2149,6 @@
 	return;
 }
 
-
-/****************************************************************************/
-/* Map TX buffers into TX buffer descriptors.                               */
-/*                                                                          */
-/* Given a series of DMA memory containting an outgoing frame, map the      */
-/* segments into the tx_bd structure used by the hardware.                  */
-/*                                                                          */
-/* Returns:                                                                 */
-/*   Nothing.                                                               */
-/****************************************************************************/
-static void
-bce_dma_map_tx_desc(void *arg, bus_dma_segment_t *segs,
-	int nseg, bus_size_t mapsize, int error)
-{
-	struct bce_dmamap_arg *map_arg;
-	struct bce_softc *sc;
-	struct tx_bd *txbd = NULL;
-	int i = 0;
-	u16 prod, chain_prod;
-	u32	prod_bseq;
-#ifdef BCE_DEBUG
-	u16 debug_prod;
-#endif
-
-	map_arg = arg;
-	sc = map_arg->sc;
-
-	if (error) {
-		DBPRINT(sc, BCE_WARN, "%s(): Called with error = %d\n",
-			__FUNCTION__, error);
-		return;
-	}
-
-	/* Signal error to caller if there's too many segments */
-	if (nseg > map_arg->maxsegs) {
-		DBPRINT(sc, BCE_WARN,
-			"%s(): Mapped TX descriptors: max segs = %d, "
-			"actual segs = %d\n",
-			__FUNCTION__, map_arg->maxsegs, nseg);
-
-		map_arg->maxsegs = 0;
-		return;
-	}
-
-	/* prod points to an empty tx_bd at this point. */
-	prod       = map_arg->prod;
-	chain_prod = map_arg->chain_prod;
-	prod_bseq  = map_arg->prod_bseq;
-
-#ifdef BCE_DEBUG
-	debug_prod = chain_prod;
-#endif
-
-	DBPRINT(sc, BCE_INFO_SEND,
-		"%s(): Start: prod = 0x%04X, chain_prod = %04X, "
-		"prod_bseq = 0x%08X\n",
-		__FUNCTION__, prod, chain_prod, prod_bseq);
-
-	/*
-	 * Cycle through each mbuf segment that makes up
-	 * the outgoing frame, gathering the mapping info
-	 * for that segment and creating a tx_bd to for
-	 * the mbuf.
-	 */
-
-	txbd = &map_arg->tx_chain[TX_PAGE(chain_prod)][TX_IDX(chain_prod)];
-
-	/* Setup the first tx_bd for the first segment. */
-	txbd->tx_bd_haddr_lo       = htole32(BCE_ADDR_LO(segs[i].ds_addr));
-	txbd->tx_bd_haddr_hi       = htole32(BCE_ADDR_HI(segs[i].ds_addr));
-	txbd->tx_bd_mss_nbytes     = htole16(segs[i].ds_len);
-	txbd->tx_bd_vlan_tag_flags = htole16(map_arg->tx_flags |
-			TX_BD_FLAGS_START);
-	prod_bseq += segs[i].ds_len;
-
-	/* Setup any remaing segments. */
-	for (i = 1; i < nseg; i++) {
-		prod       = NEXT_TX_BD(prod);
-		chain_prod = TX_CHAIN_IDX(prod);
-
-		txbd = &map_arg->tx_chain[TX_PAGE(chain_prod)][TX_IDX(chain_prod)];
-
-		txbd->tx_bd_haddr_lo       = htole32(BCE_ADDR_LO(segs[i].ds_addr));
-		txbd->tx_bd_haddr_hi       = htole32(BCE_ADDR_HI(segs[i].ds_addr));
-		txbd->tx_bd_mss_nbytes     = htole16(segs[i].ds_len);
-		txbd->tx_bd_vlan_tag_flags = htole16(map_arg->tx_flags);
-
-		prod_bseq += segs[i].ds_len;
-	}
-
-	/* Set the END flag on the last TX buffer descriptor. */
-	txbd->tx_bd_vlan_tag_flags |= htole16(TX_BD_FLAGS_END);
-
-	DBRUN(BCE_INFO_SEND, bce_dump_tx_chain(sc, debug_prod, nseg));
-
-	DBPRINT(sc, BCE_INFO_SEND,
-		"%s(): End: prod = 0x%04X, chain_prod = %04X, "
-		"prod_bseq = 0x%08X\n",
-		__FUNCTION__, prod, chain_prod, prod_bseq);
-
-	/* prod points to the last tx_bd at this point. */
-	map_arg->maxsegs    = nseg;
-	map_arg->prod       = prod;
-	map_arg->chain_prod = chain_prod;
-	map_arg->prod_bseq  = prod_bseq;
-}
-
-
 /****************************************************************************/
 /* Allocate any DMA memory needed by the driver.                            */
 /*                                                                          */
@@ -4380,7 +4271,9 @@
 			DBRUN(BCE_INFO_SEND, 
 				BCE_PRINTF(sc, "%s(): Unloading map/freeing mbuf "
 					"from tx_bd[0x%04X]\n", __FUNCTION__, sw_tx_chain_cons));
-
+        		bus_dmamap_sync(sc->tx_mbuf_tag,
+			    sc->tx_mbuf_map[sw_tx_chain_cons],
+			    BUS_DMASYNC_POSTWRITE);
 			/* Unmap the mbuf. */
 			bus_dmamap_unload(sc->tx_mbuf_tag,
 			    sc->tx_mbuf_map[sw_tx_chain_cons]);
@@ -4588,7 +4481,6 @@
 	BCE_UNLOCK(sc);
 }
 
-
 /****************************************************************************/
 /* Encapsultes an mbuf cluster into the tx_bd chain structure and makes the */
 /* memory visible to the controller.                                        */
@@ -4597,76 +4489,90 @@
 /*   0 for success, positive value for failure.                             */
 /****************************************************************************/
 static int
-bce_tx_encap(struct bce_softc *sc, struct mbuf *m_head, u16 *prod,
+bce_tx_encap(struct bce_softc *sc, struct mbuf **m_head, u16 *prod,
 	u16 *chain_prod, u32 *prod_bseq)
 {
-	u32 vlan_tag_flags = 0;
 	struct m_tag *mtag;
-	struct bce_dmamap_arg map_arg;
+	struct mbuf *m, *n;
+	struct tx_bd *txbd;
+	bus_dma_segment_t txsegs[BCE_MAX_SEGMENTS];
 	bus_dmamap_t map;
-	int i, error, rc = 0;
+	u16 sprod, schain_prod;
+	u32 vlan_tag_flags;
+	int i, error, nsegs;
+
+	txbd = NULL;
+	m = *m_head;
+	map = sc->tx_mbuf_map[*chain_prod];
+	/* Map the mbuf into our DMA address space. */
+	error = bus_dmamap_load_mbuf_sg(sc->tx_mbuf_tag, map, m, txsegs,
+	    &nsegs, 0);
+	if (error == EFBIG) {
+		n = m_defrag(m, M_DONTWAIT);
+		if (n == NULL) {
+			m_freem(m);
+			m = NULL;
+			return (ENOBUFS);
+		}
+		m = n;
+		error = bus_dmamap_load_mbuf_sg(sc->tx_mbuf_tag, map, m,
+		    txsegs, &nsegs, 0);
+		if (error != 0) {
+			m_freem(m);
+			m = NULL;
+			return (error);
+		}
+	} else if (error != 0)
+		return (error);
+	if (nsegs == 0) {
+		m_freem(m);
+		m = NULL;
+		return (EIO);
+	}
+	if ((USABLE_TX_BD - sc->used_tx_bd - BCE_TX_SLACK_SPACE) > nsegs) {
+		bus_dmamap_unload(sc->tx_mbuf_tag, map);
+		return (ENOBUFS);
+	}
 
+	vlan_tag_flags = 0;
 	/* Transfer any checksum offload flags to the bd. */
-	if (m_head->m_pkthdr.csum_flags) {
-		if (m_head->m_pkthdr.csum_flags & CSUM_IP)
+	if (m->m_pkthdr.csum_flags) {
+		if (m->m_pkthdr.csum_flags & CSUM_IP)
 			vlan_tag_flags |= TX_BD_FLAGS_IP_CKSUM;
-		if (m_head->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP))
+		if (m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP))
 			vlan_tag_flags |= TX_BD_FLAGS_TCP_UDP_CKSUM;
 	}
 
 	/* Transfer any VLAN tags to the bd. */
-	mtag = VLAN_OUTPUT_TAG(sc->bce_ifp, m_head);
+	mtag = VLAN_OUTPUT_TAG(sc->bce_ifp, m);
 	if (mtag != NULL)
 		vlan_tag_flags |= (TX_BD_FLAGS_VLAN_TAG |
 			(VLAN_TAG_VALUE(mtag) << 16));
 
-	/* Map the mbuf into DMAable memory. */
-	map = sc->tx_mbuf_map[*chain_prod];
-	map_arg.sc         = sc;
-	map_arg.prod       = *prod;
-	map_arg.chain_prod = *chain_prod;
-	map_arg.prod_bseq  = *prod_bseq;
-	map_arg.tx_flags   = vlan_tag_flags;
-	map_arg.maxsegs    = USABLE_TX_BD - sc->used_tx_bd - 
-		BCE_TX_SLACK_SPACE;
-
-	KASSERT(map_arg.maxsegs > 0, ("Invalid TX maxsegs value!"));
-
-	for (i = 0; i < TX_PAGES; i++)
-		map_arg.tx_chain[i] = sc->tx_bd_chain[i];
-
-	/* Map the mbuf into our DMA address space. */
-	error = bus_dmamap_load_mbuf(sc->tx_mbuf_tag, map, m_head,
-	    bce_dma_map_tx_desc, &map_arg, BUS_DMA_NOWAIT);
-
-	if (error || map_arg.maxsegs == 0) {
-            
-            /* Try to defrag the mbuf if there are too many segments. */
-            if (error == EFBIG && map_arg.maxsegs != 0) {
-                struct mbuf *m0;
-
-	        DBPRINT(sc, BCE_WARN, "%s(): fragmented mbuf (%d pieces)\n",
-                    __FUNCTION__, map_arg.maxsegs);
-
-                m0 = m_defrag(m_head, M_DONTWAIT);
-                if (m0 != NULL) {
-                    m_head = m0;
-                    error = bus_dmamap_load_mbuf(sc->tx_mbuf_tag,
-                        map, m_head, bce_dma_map_tx_desc, &map_arg,
-                        BUS_DMA_NOWAIT);
-                }
-            }
-
-            /* Still getting an error after a defrag. */
-            if (error) {
-                BCE_PRINTF(sc,
-                    "%s(%d): Error mapping mbuf into TX chain!\n",
-                    __FILE__, __LINE__);
-                rc = ENOBUFS;
-                goto bce_tx_encap_exit;
-            }
-
+	/*
+	 * Cycle through each mbuf segment that makes up
+	 * the outgoing frame, gathering the mapping info
+	 * for that segment and creating a tx_bd to for
+	 * the mbuf.
+	 */
+	sprod = *prod;
+	schain_prod = *chain_prod;
+	for (i = 0; i < nsegs; i++) {
+		txbd = &sc->tx_bd_chain[TX_PAGE(*chain_prod)][TX_IDX(*chain_prod)];
+		txbd->tx_bd_haddr_lo = htole32(BCE_ADDR_LO(txsegs[i].ds_addr));
+		txbd->tx_bd_haddr_hi = htole32(BCE_ADDR_HI(txsegs[i].ds_addr));
+		txbd->tx_bd_mss_nbytes = htole16(txsegs[i].ds_len);
+		txbd->tx_bd_vlan_tag_flags = htole16(vlan_tag_flags);
+		*prod_bseq += txsegs[i].ds_len;
+		*prod = NEXT_TX_BD(*prod);
+		*chain_prod = TX_CHAIN_IDX(*chain_prod);
 	}
+	/* Set the END flag on the last TX buffer descriptor. */
+	txbd->tx_bd_vlan_tag_flags |= htole16(TX_BD_FLAGS_END);
+
+	/* Set the START flag on the first TX buffer descriptor. */
+	txbd = &sc->tx_bd_chain[TX_PAGE(schain_prod)][TX_IDX(schain_prod)];
+	txbd->tx_bd_vlan_tag_flags |= htole16(TX_BD_FLAGS_START);
 
 	/*
 	 * Ensure that the map for this transmission
@@ -4677,11 +4583,10 @@
 	 * delete the map before all of the segments
 	 * have been freed.
 	 */
-	sc->tx_mbuf_map[*chain_prod] = 
-		sc->tx_mbuf_map[map_arg.chain_prod];
-	sc->tx_mbuf_map[map_arg.chain_prod] = map;
-	sc->tx_mbuf_ptr[map_arg.chain_prod] = m_head;
-	sc->used_tx_bd += map_arg.maxsegs;
+	sc->tx_mbuf_map[schain_prod] = sc->tx_mbuf_map[*chain_prod];
+	sc->tx_mbuf_map[*chain_prod] = map;
+	sc->tx_mbuf_ptr[*chain_prod] = m;
+	sc->used_tx_bd += nsegs;
 
 	DBRUNIF((sc->used_tx_bd > sc->tx_hi_watermark), 
 		sc->tx_hi_watermark = sc->used_tx_bd);
@@ -4691,17 +4596,15 @@
 	DBRUN(BCE_VERBOSE_SEND, bce_dump_tx_mbuf_chain(sc, *chain_prod, 
 		map_arg.maxsegs));
 
-	/* prod still points the last used tx_bd at this point. */
-	*prod       = map_arg.prod;
-	*chain_prod = map_arg.chain_prod;
-	*prod_bseq  = map_arg.prod_bseq;
-
-bce_tx_encap_exit:
+	/* sync descriptors */
+        bus_dmamap_sync(sc->tx_mbuf_tag, map, BUS_DMASYNC_PREWRITE);
+	for (i = 0; i < TX_PAGES; i++)
+		bus_dmamap_sync(sc->tx_bd_chain_tag, sc->tx_bd_chain_map[i],
+		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
-	return(rc);
+        return (0);
 }
 
-
 /****************************************************************************/
 /* Main transmit routine when called from another routine with a lock.      */
 /*                                                                          */
@@ -4748,7 +4651,9 @@
 		 * head of the queue and set the OACTIVE flag
 		 * to wait for the NIC to drain the chain.
 		 */
-		if (bce_tx_encap(sc, m_head, &tx_prod, &tx_chain_prod, &tx_prod_bseq)) {
+		if (bce_tx_encap(sc, &m_head, &tx_prod, &tx_chain_prod, &tx_prod_bseq)) {
+			if (m_head == NULL)
+				break;
 			IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
 			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 			DBPRINT(sc, BCE_INFO_SEND,


More information about the freebsd-stable mailing list