svn commit: r343085 - head/sys/net

Konstantin Belousov kib at FreeBSD.org
Wed Jan 16 05:44:15 UTC 2019


Author: kib
Date: Wed Jan 16 05:44:14 2019
New Revision: 343085
URL: https://svnweb.freebsd.org/changeset/base/343085

Log:
  Improve iflib busdma(9) KPI use.
  
  - Specify BUS_DMA_NOWAIT for bus_dmamap_load() on rx refill, since
    callbacks are not supposed to be used.
  - Match tso/non-tso tags to corresponding tx map operations.  Create
    separate tso maps for tx descriptors.  In particular, do not use
    non-tso tag to load, unload, or destroy a map created with tso tag.
  - Add missed bus_dmamap_sync() calls.
    Submitted by: marius.
  
  Reported and tested by:	pho
  Reviewed by:	marius
  Sponsored by:	The FreeBSD Foundation
  MFC after:	1 week

Modified:
  head/sys/net/iflib.c

Modified: head/sys/net/iflib.c
==============================================================================
--- head/sys/net/iflib.c	Wed Jan 16 05:17:27 2019	(r343084)
+++ head/sys/net/iflib.c	Wed Jan 16 05:44:14 2019	(r343085)
@@ -282,6 +282,7 @@ typedef struct iflib_sw_rx_desc_array {
 
 typedef struct iflib_sw_tx_desc_array {
 	bus_dmamap_t    *ifsd_map;         /* bus_dma maps for packet */
+	bus_dmamap_t	*ifsd_tso_map;     /* bus_dma maps for TSO packet */
 	struct mbuf    **ifsd_m;           /* pkthdr mbufs */
 } if_txsd_vec_t;
 
@@ -1491,6 +1492,8 @@ iflib_fast_intr_rxtx(void *arg)
 
 		ctx = rxq->ifr_ctx;
 
+		bus_dmamap_sync(rxq->ifr_ifdi->idi_tag, rxq->ifr_ifdi->idi_map,
+		    BUS_DMASYNC_POSTREAD);
 		if (!ctx->isc_txd_credits_update(ctx->ifc_softc, txqid, false)) {
 			IFDI_TX_QUEUE_INTR_ENABLE(ctx, txqid);
 			continue;
@@ -1583,6 +1586,7 @@ iflib_txsd_alloc(iflib_txq_t txq)
 	device_t dev = ctx->ifc_dev;
 	bus_size_t tsomaxsize;
 	int err, nsegments, ntsosegments;
+	bool tso;
 
 	nsegments = scctx->isc_tx_nsegments;
 	ntsosegments = scctx->isc_tx_tso_segments_max;
@@ -1617,8 +1621,8 @@ iflib_txsd_alloc(iflib_txq_t txq)
 		    (uintmax_t)sctx->isc_tx_maxsize, nsegments, (uintmax_t)sctx->isc_tx_maxsegsize);
 		goto fail;
 	}
-	if ((if_getcapabilities(ctx->ifc_ifp) & IFCAP_TSO) &&
-	    (err = bus_dma_tag_create(bus_get_dma_tag(dev),
+	tso = (if_getcapabilities(ctx->ifc_ifp) & IFCAP_TSO) != 0;
+	if (tso && (err = bus_dma_tag_create(bus_get_dma_tag(dev),
 			       1, 0,			/* alignment, bounds */
 			       BUS_SPACE_MAXADDR,	/* lowaddr */
 			       BUS_SPACE_MAXADDR,	/* highaddr */
@@ -1631,7 +1635,6 @@ iflib_txsd_alloc(iflib_txq_t txq)
 			       NULL,			/* lockfuncarg */
 			       &txq->ift_tso_desc_tag))) {
 		device_printf(dev,"Unable to allocate TX TSO DMA tag: %d\n", err);
-
 		goto fail;
 	}
 	if (!(txq->ift_sds.ifsd_m =
@@ -1643,19 +1646,38 @@ iflib_txsd_alloc(iflib_txq_t txq)
 	}
 
         /* Create the descriptor buffer dma maps */
-	if (!(txq->ift_sds.ifsd_map =
-	    (bus_dmamap_t *) malloc(sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
+	if ((txq->ift_sds.ifsd_map = (bus_dmamap_t *)malloc(
+	    sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset],
+	    M_IFLIB, M_NOWAIT | M_ZERO)) == NULL) {
 		device_printf(dev, "Unable to allocate tx_buffer map memory\n");
 		err = ENOMEM;
 		goto fail;
 	}
 
+	if (tso && (txq->ift_sds.ifsd_tso_map = (bus_dmamap_t *)malloc(
+	    sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset],
+	    M_IFLIB, M_NOWAIT | M_ZERO)) == NULL) {
+		device_printf(dev, "Unable to allocate TSO tx_buffer "
+		    "map memory\n");
+		err = ENOMEM;
+		goto fail;
+	}
+
 	for (int i = 0; i < scctx->isc_ntxd[txq->ift_br_offset]; i++) {
-		err = bus_dmamap_create(txq->ift_desc_tag, 0, &txq->ift_sds.ifsd_map[i]);
+		err = bus_dmamap_create(txq->ift_desc_tag, 0,
+		    &txq->ift_sds.ifsd_map[i]);
 		if (err != 0) {
 			device_printf(dev, "Unable to create TX DMA map\n");
 			goto fail;
 		}
+		if (!tso)
+			continue;
+		err = bus_dmamap_create(txq->ift_tso_desc_tag, 0,
+		    &txq->ift_sds.ifsd_tso_map[i]);
+		if (err != 0) {
+			device_printf(dev, "Unable to create TSO TX DMA map\n");
+			goto fail;
+		}
 	}
 	return (0);
 fail:
@@ -1673,10 +1695,22 @@ iflib_txsd_destroy(if_ctx_t ctx, iflib_txq_t txq, int 
 	if (txq->ift_sds.ifsd_map != NULL)
 		map = txq->ift_sds.ifsd_map[i];
 	if (map != NULL) {
+		bus_dmamap_sync(txq->ift_desc_tag, map, BUS_DMASYNC_POSTWRITE);
 		bus_dmamap_unload(txq->ift_desc_tag, map);
 		bus_dmamap_destroy(txq->ift_desc_tag, map);
 		txq->ift_sds.ifsd_map[i] = NULL;
 	}
+
+	map = NULL;
+	if (txq->ift_sds.ifsd_tso_map != NULL)
+		map = txq->ift_sds.ifsd_tso_map[i];
+	if (map != NULL) {
+		bus_dmamap_sync(txq->ift_tso_desc_tag, map,
+		    BUS_DMASYNC_POSTWRITE);
+		bus_dmamap_unload(txq->ift_tso_desc_tag, map);
+		bus_dmamap_destroy(txq->ift_tso_desc_tag, map);
+		txq->ift_sds.ifsd_tso_map[i] = NULL;
+	}
 }
 
 static void
@@ -1690,6 +1724,10 @@ iflib_txq_destroy(iflib_txq_t txq)
 		free(txq->ift_sds.ifsd_map, M_IFLIB);
 		txq->ift_sds.ifsd_map = NULL;
 	}
+	if (txq->ift_sds.ifsd_tso_map != NULL) {
+		free(txq->ift_sds.ifsd_tso_map, M_IFLIB);
+		txq->ift_sds.ifsd_tso_map = NULL;
+	}
 	if (txq->ift_sds.ifsd_m != NULL) {
 		free(txq->ift_sds.ifsd_m, M_IFLIB);
 		txq->ift_sds.ifsd_m = NULL;
@@ -1715,11 +1753,15 @@ iflib_txsd_free(if_ctx_t ctx, iflib_txq_t txq, int i)
 
 	if (txq->ift_sds.ifsd_map != NULL) {
 		bus_dmamap_sync(txq->ift_desc_tag,
-				txq->ift_sds.ifsd_map[i],
-				BUS_DMASYNC_POSTWRITE);
-		bus_dmamap_unload(txq->ift_desc_tag,
-				  txq->ift_sds.ifsd_map[i]);
+		    txq->ift_sds.ifsd_map[i], BUS_DMASYNC_POSTWRITE);
+		bus_dmamap_unload(txq->ift_desc_tag, txq->ift_sds.ifsd_map[i]);
 	}
+	if (txq->ift_sds.ifsd_tso_map != NULL) {
+		bus_dmamap_sync(txq->ift_tso_desc_tag,
+		    txq->ift_sds.ifsd_tso_map[i], BUS_DMASYNC_POSTWRITE);
+		bus_dmamap_unload(txq->ift_tso_desc_tag,
+		    txq->ift_sds.ifsd_tso_map[i]);
+	}
 	m_free(*mp);
 	DBG_COUNTER_INC(tx_frees);
 	*mp = NULL;
@@ -1926,10 +1968,8 @@ _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int coun
 			cb_arg.error = 0;
 			MPASS(sd_map != NULL);
 			err = bus_dmamap_load(fl->ifl_desc_tag, sd_map[frag_idx],
-			    cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, 0);
-			bus_dmamap_sync(fl->ifl_desc_tag, sd_map[frag_idx],
-			    BUS_DMASYNC_PREREAD);
-
+			    cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg,
+			    BUS_DMA_NOWAIT);
 			if (err != 0 || cb_arg.error) {
 				/*
 				 * !zone_pack ?
@@ -1939,6 +1979,8 @@ _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int coun
 				break;
 			}
 
+			bus_dmamap_sync(fl->ifl_desc_tag, sd_map[frag_idx],
+			    BUS_DMASYNC_PREREAD);
 			sd_ba[frag_idx] =  bus_addr = cb_arg.seg.ds_addr;
 			sd_cl[frag_idx] = cl;
 #if MEMORY_LOGGING
@@ -2030,6 +2072,7 @@ static void
 iflib_fl_bufs_free(iflib_fl_t fl)
 {
 	iflib_dma_info_t idi = fl->ifl_ifdi;
+	bus_dmamap_t sd_map;
 	uint32_t i;
 
 	for (i = 0; i < fl->ifl_size; i++) {
@@ -2037,7 +2080,9 @@ iflib_fl_bufs_free(iflib_fl_t fl)
 		caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i];
 
 		if (*sd_cl != NULL) {
-			bus_dmamap_t sd_map = fl->ifl_sds.ifsd_map[i];
+			sd_map = fl->ifl_sds.ifsd_map[i];
+			bus_dmamap_sync(fl->ifl_desc_tag, sd_map,
+			    BUS_DMASYNC_POSTREAD);
 			bus_dmamap_unload(fl->ifl_desc_tag, sd_map);
 			if (*sd_cl != NULL)
 				uma_zfree(fl->ifl_zone, *sd_cl);
@@ -2140,19 +2185,32 @@ static void
 iflib_rx_sds_free(iflib_rxq_t rxq)
 {
 	iflib_fl_t fl;
-	int i;
+	int i, j;
 
 	if (rxq->ifr_fl != NULL) {
 		for (i = 0; i < rxq->ifr_nfl; i++) {
 			fl = &rxq->ifr_fl[i];
 			if (fl->ifl_desc_tag != NULL) {
+				if (fl->ifl_sds.ifsd_map != NULL) {
+					for (j = 0; j < fl->ifl_size; i++) {
+						if (fl->ifl_sds.ifsd_map[i] ==
+						    NULL)
+						continue;
+						bus_dmamap_sync(
+						    fl->ifl_desc_tag,
+						    fl->ifl_sds.ifsd_map[i],
+						    BUS_DMASYNC_POSTREAD);
+						bus_dmamap_unload(
+						    fl->ifl_desc_tag,
+						    fl->ifl_sds.ifsd_map[i]);
+					}
+				}
 				bus_dma_tag_destroy(fl->ifl_desc_tag);
 				fl->ifl_desc_tag = NULL;
 			}
 			free(fl->ifl_sds.ifsd_m, M_IFLIB);
 			free(fl->ifl_sds.ifsd_cl, M_IFLIB);
 			free(fl->ifl_sds.ifsd_ba, M_IFLIB);
-			/* XXX destroy maps first */
 			free(fl->ifl_sds.ifsd_map, M_IFLIB);
 			fl->ifl_sds.ifsd_m = NULL;
 			fl->ifl_sds.ifsd_cl = NULL;
@@ -2430,11 +2488,10 @@ rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, int
 	map = fl->ifl_sds.ifsd_map[cidx];
 	di = fl->ifl_ifdi;
 	next = (cidx + CACHE_LINE_SIZE) & (fl->ifl_size-1);
-	bus_dmamap_sync(di->idi_tag, di->idi_map,
-	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 
 	/* not valid assert if bxe really does SGE from non-contiguous elements */
 	MPASS(fl->ifl_cidx == cidx);
+	bus_dmamap_sync(fl->ifl_desc_tag, map, BUS_DMASYNC_POSTREAD);
 	if (unload)
 		bus_dmamap_unload(fl->ifl_desc_tag, map);
 	fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1);
@@ -2442,7 +2499,7 @@ rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, int
 		fl->ifl_gen = 0;
 	bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
-        bit_clear(fl->ifl_rx_bitmap, cidx);
+	bit_clear(fl->ifl_rx_bitmap, cidx);
 }
 
 static struct mbuf *
@@ -2519,6 +2576,9 @@ iflib_rxd_pkt_get(iflib_rxq_t rxq, if_rxd_info_t ri)
 			m->m_data += 2;
 #endif
 		memcpy(m->m_data, *sd.ifsd_cl, ri->iri_len);
+		bus_dmamap_sync(rxq->ifr_fl->ifl_desc_tag,
+		    rxq->ifr_fl->ifl_sds.ifsd_map[ri->iri_frags[0].irf_idx],
+		    BUS_DMASYNC_PREREAD);
 		m->m_len = ri->iri_frags[0].irf_len;
        } else {
 		m = assemble_segments(rxq, ri, &sd);
@@ -2587,6 +2647,7 @@ iflib_rxeof(iflib_rxq_t rxq, qidx_t budget)
 	if_ctx_t ctx = rxq->ifr_ctx;
 	if_shared_ctx_t sctx = ctx->ifc_sctx;
 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
+	iflib_dma_info_t di;
 	int avail, i;
 	qidx_t *cidxp;
 	struct if_rxd_info ri;
@@ -2631,6 +2692,9 @@ iflib_rxeof(iflib_rxq_t rxq, qidx_t budget)
 		ri.iri_cidx = *cidxp;
 		ri.iri_ifp = ifp;
 		ri.iri_frags = rxq->ifr_frags;
+		di = rxq->ifr_fl[rxq->ifr_frags[0].irf_flid].ifl_ifdi;
+		bus_dmamap_sync(di->idi_tag, di->idi_map,
+		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 		err = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri);
 
 		if (err)
@@ -3006,16 +3070,17 @@ iflib_remove_mbuf(iflib_txq_t txq)
 {
 	int ntxd, pidx;
 	struct mbuf *m, **ifsd_m;
-	bus_dmamap_t *ifsd_map;
 
 	ifsd_m = txq->ift_sds.ifsd_m;
 	ntxd = txq->ift_size;
 	pidx = txq->ift_pidx & (ntxd - 1);
 	ifsd_m = txq->ift_sds.ifsd_m;
-	ifsd_map = txq->ift_sds.ifsd_map;
 	m = ifsd_m[pidx];
 	ifsd_m[pidx] = NULL;
-	bus_dmamap_unload(txq->ift_desc_tag, ifsd_map[pidx]);
+	bus_dmamap_unload(txq->ift_desc_tag, txq->ift_sds.ifsd_map[pidx]);
+	if (txq->ift_sds.ifsd_tso_map != NULL)
+		bus_dmamap_unload(txq->ift_tso_desc_tag,
+		    txq->ift_sds.ifsd_tso_map[pidx]);
 #if MEMORY_LOGGING
 	txq->ift_dequeued++;
 #endif
@@ -3131,11 +3196,13 @@ iflib_encap(iflib_txq_t txq, struct mbuf **m_headp)
 	if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
 		desc_tag = txq->ift_tso_desc_tag;
 		max_segs = scctx->isc_tx_tso_segments_max;
+		map = txq->ift_sds.ifsd_tso_map[pidx];
 		MPASS(desc_tag != NULL);
 		MPASS(max_segs > 0);
 	} else {
 		desc_tag = txq->ift_desc_tag;
 		max_segs = scctx->isc_tx_nsegments;
+		map = txq->ift_sds.ifsd_map[pidx];
 	}
 	if ((sctx->isc_flags & IFLIB_NEED_ETHER_PAD) &&
 	    __predict_false(m_head->m_pkthdr.len < scctx->isc_min_frame_size)) {
@@ -3297,7 +3364,6 @@ iflib_tx_desc_free(iflib_txq_t txq, int n)
 {
 	uint32_t qsize, cidx, mask, gen;
 	struct mbuf *m, **ifsd_m;
-	bus_dmamap_t *ifsd_map;
 	bool do_prefetch;
 
 	cidx = txq->ift_cidx;
@@ -3305,7 +3371,6 @@ iflib_tx_desc_free(iflib_txq_t txq, int n)
 	qsize = txq->ift_size;
 	mask = qsize-1;
 	ifsd_m = txq->ift_sds.ifsd_m;
-	ifsd_map = txq->ift_sds.ifsd_map;
 	do_prefetch = (txq->ift_ctx->ifc_flags & IFC_PREFETCH);
 
 	while (n-- > 0) {
@@ -3315,7 +3380,19 @@ iflib_tx_desc_free(iflib_txq_t txq, int n)
 		}
 		if ((m = ifsd_m[cidx]) != NULL) {
 			prefetch(&ifsd_m[(cidx + CACHE_PTR_INCREMENT) & mask]);
-			bus_dmamap_unload(txq->ift_desc_tag, ifsd_map[cidx]);
+			if (m->m_pkthdr.csum_flags & CSUM_TSO) {
+				bus_dmamap_sync(txq->ift_tso_desc_tag,
+				    txq->ift_sds.ifsd_tso_map[cidx],
+				    BUS_DMASYNC_POSTWRITE);
+				bus_dmamap_unload(txq->ift_tso_desc_tag,
+				    txq->ift_sds.ifsd_tso_map[cidx]);
+			} else {
+				bus_dmamap_sync(txq->ift_desc_tag,
+				    txq->ift_sds.ifsd_map[cidx],
+				    BUS_DMASYNC_POSTWRITE);
+				bus_dmamap_unload(txq->ift_desc_tag,
+				    txq->ift_sds.ifsd_map[cidx]);
+			}
 			/* XXX we don't support any drivers that batch packets yet */
 			MPASS(m->m_nextpkt == NULL);
 			m_freem(m);
@@ -3400,6 +3477,8 @@ iflib_txq_can_drain(struct ifmp_ring *r)
 	iflib_txq_t txq = r->cookie;
 	if_ctx_t ctx = txq->ift_ctx;
 
+	bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
+	    BUS_DMASYNC_POSTREAD);
 	return ((TXQ_AVAIL(txq) > MAX_TX_DESC(ctx) + 2) ||
 		ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, false));
 }
@@ -3564,6 +3643,8 @@ _task_fn_tx(void *context)
 	if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
 		return;
 	if (if_getcapenable(ifp) & IFCAP_NETMAP) {
+		bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
+		    BUS_DMASYNC_POSTREAD);
 		if (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, false))
 			netmap_tx_irq(ifp, txq->ift_id);
 		IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id);
@@ -5817,6 +5898,8 @@ iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq)
 	if (ctx->isc_txd_credits_update == NULL)
 		return (0);
 
+	bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
+	    BUS_DMASYNC_POSTREAD);
 	if ((credits = ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, true)) == 0)
 		return (0);
 


More information about the svn-src-all mailing list