PERFORCE change 126712 for review
Kip Macy
kmacy at FreeBSD.org
Sat Sep 22 23:35:51 PDT 2007
http://perforce.freebsd.org/chv.cgi?CH=126712
Change 126712 by kmacy at kmacy_home:ethng on 2007/09/23 06:35:48
switch over to handling buffers in terms of buffer aggregates rather
than mbuf chain - reduces our mbuf working set as well as reducing
the number of cache misses when doing tx cleaning
- some residual fixes remain to be made for compatibility
Affected files ...
.. //depot/projects/ethng/src/sys/dev/cxgb/cxgb_adapter.h#18 edit
.. //depot/projects/ethng/src/sys/dev/cxgb/cxgb_main.c#20 edit
.. //depot/projects/ethng/src/sys/dev/cxgb/cxgb_sge.c#22 edit
.. //depot/projects/ethng/src/sys/dev/cxgb/sys/mvec.h#4 edit
.. //depot/projects/ethng/src/sys/dev/cxgb/sys/uipc_mvec.c#3 edit
Differences ...
==== //depot/projects/ethng/src/sys/dev/cxgb/cxgb_adapter.h#18 (text+ko) ====
@@ -287,6 +287,9 @@
uint32_t txq_enqueued;
unsigned long txq_frees;
struct mtx lock;
+ struct sg_ent txq_sgl[TX_MAX_SEGS / 2 + 1];
+ bus_dma_segment_t txq_segs[TX_MAX_SEGS];
+ struct mbuf *txq_m_vec[TX_WR_COUNT_MAX];
#define TXQ_NAME_LEN 32
char lockbuf[TXQ_NAME_LEN];
};
@@ -535,12 +538,12 @@
void t3b_intr(void *data);
void t3_intr_msi(void *data);
void t3_intr_msix(void *data);
-int t3_encap(struct sge_qset *, struct mbuf **, int, int *free);
+int t3_encap(struct sge_qset *, struct mbuf **, int);
int t3_sge_init_adapter(adapter_t *);
int t3_sge_init_port(struct port_info *);
void t3_sge_deinit_sw(adapter_t *);
-int t3_free_tx_desc(struct sge_txq *q, int n, struct mbuf **m_vec, int m_vec_size, int *desc_reclaimed);
+void t3_free_tx_desc(struct sge_txq *q, int n);
void t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct mbuf *m,
int ethpad, uint32_t rss_hash, uint32_t rss_csum, int lro);
==== //depot/projects/ethng/src/sys/dev/cxgb/cxgb_main.c#20 (text+ko) ====
@@ -51,6 +51,7 @@
#include <sys/sysctl.h>
#include <sys/queue.h>
#include <sys/taskqueue.h>
+#include <sys/proc.h>
#include <net/bpf.h>
#include <net/ethernet.h>
@@ -213,11 +214,7 @@
* The driver uses an auto-queue algorithm by default.
* To disable it and force a single queue-set per port, use singleq = 1.
*/
-#ifdef IFNET_MULTIQUEUE
static int singleq = 0;
-#else
-static int singleq = 1;
-#endif
TUNABLE_INT("hw.cxgb.singleq", &singleq);
SYSCTL_UINT(_hw_cxgb, OID_AUTO, singleq, CTLFLAG_RDTUN, &singleq, 0,
"use a single queue-set per port");
@@ -639,14 +636,14 @@
{
int i;
+
#ifdef IFNET_MULTIQUEUE
cxgb_pcpu_shutdown_threads(sc);
#endif
-
ADAPTER_LOCK(sc);
- /*
- * drops the lock
- */
+/*
+ * drops the lock
+ */
cxgb_down_locked(sc);
#ifdef MSI_SUPPORTED
@@ -671,7 +668,7 @@
* Wait for last callout
*/
- tsleep(&sc, 0, "cxgb unload", 3*hz);
+ DELAY(hz*100);
for (i = 0; i < (sc)->params.nports; ++i) {
if (sc->portdev[i] != NULL)
@@ -705,8 +702,6 @@
MTX_DESTROY(&sc->sge.reg_lock);
MTX_DESTROY(&sc->elmer_lock);
ADAPTER_LOCK_DEINIT(sc);
-
- return;
}
/**
@@ -1027,6 +1022,9 @@
}
ether_ifdetach(p->ifp);
+ printf("waiting for callout to stop ...");
+ DELAY(10000);
+ printf("done\n");
/*
* the lock may be acquired in ifdetach
*/
@@ -1586,11 +1584,6 @@
for (i = 0; i < sc->params.nports; i++)
taskqueue_drain(sc->tq, &sc->port[i].timer_reclaim_task);
}
-#ifdef notyet
-
- if (sc->port[i].tq != NULL)
-#endif
-
}
static int
@@ -1708,8 +1701,7 @@
device_printf(sc->dev, "enabling interrupts on port=%d\n", p->port_id);
t3_port_intr_enable(sc, p->port_id);
- callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz,
- cxgb_tick, sc);
+ callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
ifp->if_drv_flags |= IFF_DRV_RUNNING;
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
@@ -1736,7 +1728,6 @@
ADAPTER_LOCK_ASSERT_NOTOWNED(p->adapter);
ifp = p->ifp;
-
t3_port_intr_disable(p->adapter, p->port_id);
ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
p->phy.ops->power_down(&p->phy, 1);
@@ -1819,8 +1810,7 @@
if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
adapter_t *sc = p->adapter;
- callout_reset(&sc->cxgb_tick_ch,
- sc->params.stats_update_period * hz,
+ callout_reset(&sc->cxgb_tick_ch, hz,
cxgb_tick, sc);
}
PORT_UNLOCK(p);
@@ -1875,42 +1865,24 @@
cxgb_tx_common(struct ifnet *ifp, struct sge_qset *qs, uint32_t txmax)
{
struct sge_txq *txq;
- int err, in_use_init, count, i, free;
- struct mbuf *m_vec[TX_WR_COUNT_MAX];
+ int err, in_use_init, count, i;
+ struct mbuf **m_vec;
txq = &qs->txq[TXQ_ETH];
+ m_vec = txq->txq_m_vec;
in_use_init = txq->in_use;
err = 0;
while ((txq->in_use - in_use_init < txmax) &&
(txq->size > txq->in_use + TX_MAX_DESC)) {
- free = 0;
count = cxgb_dequeue_packet(ifp, txq, m_vec);
+
if (count == 0)
break;
-#ifdef notyet
- /*
- * Convert chain to M_IOVEC
- */
- KASSERT((m->m_flags & M_IOVEC) == 0, ("IOVEC set too early"));
- m0 = m;
- if (collapse_mbufs && m->m_pkthdr.len > MCLBYTES &&
- m_collapse(m, TX_MAX_SEGS, &m0) == EFBIG) {
- if ((m0 = m_defrag(m, M_NOWAIT)) != NULL) {
- m = m0;
- m_collapse(m, TX_MAX_SEGS, &m0);
- } else
- break;
- }
- m = m0;
-#endif
- if ((err = t3_encap(qs, m_vec, count, &free)) != 0)
+ ETHER_BPF_MTAP(ifp, m_vec[0]);
+
+ if ((err = t3_encap(qs, m_vec, count)) != 0)
break;
txq->txq_enqueued += count;
- for (i = 0; i < count; i++)
- BPF_MTAP(ifp, m_vec[i]);
- if (free)
- m_freem(m_vec[0]);
-
}
#ifndef IFNET_MULTIQUEUE
if (__predict_false(err)) {
@@ -2154,12 +2126,26 @@
cxgb_tick(void *arg)
{
adapter_t *sc = (adapter_t *)arg;
+ int i, running = 0;
+
+ for_each_port(sc, i) {
+
+ struct port_info *p = &sc->port[i];
+ struct ifnet *ifp = p->ifp;
+ PORT_LOCK(p);
+
+ if ((ifp->if_drv_flags & IFF_DRV_RUNNING))
+ running = 1;
+ PORT_UNLOCK(p);
+ }
+ if (running == 0)
+ return;
+
taskqueue_enqueue(sc->tq, &sc->tick_task);
if (sc->open_device_map != 0)
- callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz,
- cxgb_tick, sc);
+ callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
}
static void
==== //depot/projects/ethng/src/sys/dev/cxgb/cxgb_sge.c#22 (text+ko) ====
@@ -57,18 +57,17 @@
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
#ifdef CONFIG_DEFINED
#include <cxgb_include.h>
+#include <sys/mvec.h>
#else
#include <dev/cxgb/cxgb_include.h>
+#include <dev/cxgb/sys/mvec.h>
#endif
-#include <vm/vm.h>
-#include <vm/pmap.h>
-
-
-
-
uint32_t collapse_free = 0;
uint32_t mb_free_vec_free = 0;
int txq_fills = 0;
@@ -76,14 +75,15 @@
static int bogus_imm = 0;
#ifndef DISABLE_MBUF_IOVEC
static int recycle_enable = 1;
+#endif
extern int cxgb_txq_mbuf_ring_size;
-#endif
+
#define USE_GTS 0
#define SGE_RX_SM_BUF_SIZE 1536
#define SGE_RX_DROP_THRES 16
-#define SGE_RX_COPY_THRES MHLEN
+#define SGE_RX_COPY_THRES 128
/*
* Period of the Tx buffer reclaim timer. This timer does not need to run
@@ -134,9 +134,7 @@
#define RSPQ_SOP_EOP G_RSPD_SOP_EOP(F_RSPD_SOP|F_RSPD_EOP)
struct tx_sw_desc { /* SW state per Tx descriptor */
- struct mbuf *m[TX_WR_COUNT_MAX];
-
- int count;
+ struct mbuf_iovec mi;
bus_dmamap_t map;
int flags;
};
@@ -211,16 +209,15 @@
static __inline int
reclaim_completed_tx(struct sge_txq *q, int nbufs, struct mbuf **mvec)
{
- int reclaimed, reclaim = desc_reclaimable(q);
- int n = 0;
+ int reclaim = desc_reclaimable(q);
mtx_assert(&q->lock, MA_OWNED);
if (reclaim > 0) {
- n = t3_free_tx_desc(q, reclaim, mvec, nbufs, &reclaimed);
- q->cleaned += reclaimed;
- q->in_use -= reclaimed;
+ t3_free_tx_desc(q, reclaim);
+ q->cleaned += reclaim;
+ q->in_use -= reclaim;
}
- return (n);
+ return (reclaim);
}
/**
@@ -315,7 +312,7 @@
if (sopeop == RSPQ_NSOP_NEOP || sopeop == RSPQ_SOP)
return (0);
- m = m_gethdr(M_NOWAIT, MT_DATA);
+ m = m_gethdr(M_DONTWAIT, MT_DATA);
len = G_RSPD_LEN(ntohl(resp->len_cq));
if (m) {
@@ -423,11 +420,15 @@
q->polling = adap->params.rev > 0;
- if (adap->params.nports > 2)
+ if (adap->params.nports > 2) {
+ q->coalesce_nsecs = 50000;
+ } else {
+#ifdef INVARIANTS
q->coalesce_nsecs = 50000;
- else
+#else
q->coalesce_nsecs = 5000;
-
+#endif
+ }
q->rspq_size = RSPQ_Q_SIZE;
q->fl_size = FL_Q_SIZE;
q->jumbo_size = JUMBO_Q_SIZE;
@@ -832,6 +833,7 @@
callout_init(&sc->sge_timer_ch, CALLOUT_MPSAFE);
callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc);
TASK_INIT(&sc->slow_intr_task, 0, sge_slow_intr_handler, sc);
+ mi_init();
return (0);
}
@@ -853,6 +855,8 @@
for (i = 0; i < sc->params.nports; i++)
if (sc->port[i].tq != NULL)
taskqueue_drain(sc->port[i].tq, &sc->port[i].timer_reclaim_task);
+
+ mi_deinit();
}
/**
@@ -876,7 +880,7 @@
static __inline void
sge_txq_reclaim_(struct sge_txq *txq)
{
- int reclaimable, i, n;
+ int reclaimable, n;
struct mbuf *m_vec[TX_CLEAN_MAX_DESC];
struct port_info *pi;
@@ -890,10 +894,14 @@
}
if (n == 0)
return;
-
+#if 0
+ {
+ int i;
for (i = 0; i < n; i++) {
m_freem_vec(m_vec[i]);
}
+ }
+#endif
if (pi && pi->ifp->if_drv_flags & IFF_DRV_OACTIVE &&
txq->size - txq->in_use >= TX_START_MAX_DESC) {
txq_fills++;
@@ -1018,7 +1026,7 @@
* packet. Ethernet packets require addition of WR and CPL headers.
*/
static __inline unsigned int
-calc_tx_descs(const struct mbuf *m, int nsegs)
+calc_tx_descs(const struct mbuf *m, int nsegs, int tsoinfo)
{
unsigned int flits;
@@ -1027,7 +1035,7 @@
flits = sgl_len(nsegs) + 2;
#ifdef TSO_SUPPORTED
- if (m->m_pkthdr.csum_flags & (CSUM_TSO))
+ if (tsoinfo)
flits++;
#endif
return flits_to_desc(flits);
@@ -1038,25 +1046,24 @@
struct tx_sw_desc *txsd, bus_dma_segment_t *segs, int *nsegs)
{
struct mbuf *m0;
- int err, pktlen;
+ int err, pktlen, pass = 0;
+retry:
+ err = 0;
m0 = *m;
pktlen = m0->m_pkthdr.len;
+#if defined(__i386__) || defined(__amd64__)
+ if (busdma_map_sg_collapse(m, segs, nsegs) == 0) {
+ goto done;
+ } else
+#endif
+ err = bus_dmamap_load_mvec_sg(txq->entry_tag, txsd->map, m0, segs, nsegs, 0);
- err = bus_dmamap_load_mvec_sg(txq->entry_tag, txsd->map, m0, segs, nsegs, 0);
-#ifdef DEBUG
- if (err) {
- int n = 0;
- struct mbuf *mtmp = m0;
- while(mtmp) {
- n++;
- mtmp = mtmp->m_next;
- }
- printf("map_mbufs: bus_dmamap_load_mbuf_sg failed with %d - pkthdr.len==%d nmbufs=%d\n",
- err, m0->m_pkthdr.len, n);
+ if (err == 0) {
+ goto done;
}
-#endif
- if (err == EFBIG) {
+ if (err == EFBIG && pass == 0) {
+ pass = 1;
/* Too many segments, try to defrag */
m0 = m_defrag(m0, M_DONTWAIT);
if (m0 == NULL) {
@@ -1065,22 +1072,20 @@
return (ENOBUFS);
}
*m = m0;
- err = bus_dmamap_load_mbuf_sg(txq->entry_tag, txsd->map, m0, segs, nsegs, 0);
- }
-
- if (err == ENOMEM) {
+ goto retry;
+ } else if (err == ENOMEM) {
return (err);
- }
-
- if (err) {
+ } if (err) {
if (cxgb_debug)
printf("map failure err=%d pktlen=%d\n", err, pktlen);
- m_freem_vec(m0);
+ m_freem(m0);
*m = NULL;
return (err);
}
-
+done:
+#if !defined(__i386__) && !defined(__amd64__)
bus_dmamap_sync(txq->entry_tag, txsd->map, BUS_DMASYNC_PREWRITE);
+#endif
txsd->flags |= TX_SW_DESC_MAPPED;
return (0);
@@ -1101,12 +1106,19 @@
{
int i, idx;
- for (idx = 0, i = 0; i < nsegs; i++, idx ^= 1) {
+ for (idx = 0, i = 0; i < nsegs; i++) {
+ /*
+ * firmware doesn't like empty segments
+ */
+ if (segs[i].ds_len == 0)
+ continue;
if (i && idx == 0)
++sgp;
-
+
sgp->len[idx] = htobe32(segs[i].ds_len);
sgp->addr[idx] = htobe64(segs[i].ds_addr);
+
+ idx ^= 1;
}
if (idx)
@@ -1225,8 +1237,7 @@
* is freed all clusters will be freed
* with it
*/
- txsd->m[0] = NULL;
- txsd->count = 0;
+ txsd->mi.mi_base = NULL;
wrp = (struct work_request_hdr *)txd;
wrp->wr_hi = htonl(V_WR_DATATYPE(1) |
V_WR_SGLSFLT(1)) | wr_hi;
@@ -1243,7 +1254,13 @@
}
}
-
+static void
+dump_mi(struct mbuf_iovec *mi)
+{
+ DPRINTF("mi_flags=0x%08x mi_data=%p mi_len=%d mi_type=%d\n",
+ mi->mi_flags, mi->mi_base + mi->mi_offset, mi->mi_len, mi->mi_type);
+}
+
/* sizeof(*eh) + sizeof(*vhdr) + sizeof(*ip) + sizeof(*tcp) */
#define TCPPKTHDRSIZE (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + 20 + 20)
@@ -1253,12 +1270,19 @@
if ((m)->m_flags & M_VLANTAG) \
cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN((m)->m_pkthdr.ether_vtag); \
} while (0)
+
+#define GET_VTAG_MI(cntrl, mi) \
+do { \
+ if ((mi)->mi_flags & M_VLANTAG) \
+ cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN((mi)->mi_ether_vtag); \
+} while (0)
#else
#define GET_VTAG(cntrl, m)
+#define GET_VTAG_MI(cntrl, m)
#endif
int
-t3_encap(struct sge_qset *qs, struct mbuf **m, int count, int *free)
+t3_encap(struct sge_qset *qs, struct mbuf **m, int count)
{
adapter_t *sc;
struct mbuf *m0;
@@ -1270,16 +1294,14 @@
struct work_request_hdr *wrp;
struct tx_sw_desc *txsd;
- struct sg_ent *sgp, sgl[TX_MAX_SEGS / 2 + 1];
- bus_dma_segment_t segs[TX_MAX_SEGS];
+ struct sg_ent *sgp, *sgl;
+ bus_dma_segment_t *segs;
uint32_t wr_hi, wr_lo, sgl_flits;
struct tx_desc *txd;
-
-
-#if defined(IFNET_MULTIQUEUE) && defined(STRICT_AFFINITY)
- KASSERT(qs->qs_cpuid == curcpu, ("cpu qset mismatch cpuid=%d curcpu=%d", qs->qs_cpuid, curcpu));
-#endif
+ struct mbuf_vec *mv;
+ struct mbuf_iovec *mi;
+
DPRINTF("t3_encap cpu=%d ", curcpu);
pi = qs->port;
@@ -1287,47 +1309,62 @@
txq = &qs->txq[TXQ_ETH];
txsd = &txq->sdesc[txq->pidx];
txd = &txq->desc[txq->pidx];
-
+ sgl = txq->txq_sgl;
+ segs = txq->txq_segs;
+ m0 = *m;
DPRINTF("t3_encap port_id=%d qsidx=%d ", pi->port_id, pi->first_qset);
DPRINTF("mlen=%d txpkt_intf=%d tx_chan=%d\n", m[0]->m_pkthdr.len, pi->txpkt_intf, pi->tx_chan);
- /*
- * XXX handle checksum, TSO, and VLAN here
- *
- */
+
cntrl = V_TXPKT_INTF(pi->txpkt_intf);
-
- /*
- * XXX need to add VLAN support for 6.x
- */
+/*
+ * XXX need to add VLAN support for 6.x
+ */
#ifdef VLAN_SUPPORTED
- if (m[0]->m_pkthdr.csum_flags & (CSUM_TSO))
- tso_info = V_LSO_MSS(m[0]->m_pkthdr.tso_segsz);
+ if (m0->m_pkthdr.csum_flags & (CSUM_TSO))
+ tso_info = V_LSO_MSS(m0->m_pkthdr.tso_segsz);
#endif
- txsd->count = count;
if (count > 1) {
+ if ((err = busdma_map_sg_vec(m, &m0, segs, count)))
+ return (err);
+ nsegs = count;
+ } else if ((err = busdma_map_sg_collapse(&m0, segs, &nsegs))) {
+ printf("failed ... err=%d\n", err);
+ return (err);
+ }
+ if (m0->m_type == MT_DATA) {
+ DPRINTF("mbuf type=%d tags:%d head=%p", m0->m_type, !SLIST_EMPTY(&m0->m_pkthdr.tags),
+ SLIST_FIRST(&m0->m_pkthdr.tags));
+ mi_collapse_mbuf(&txsd->mi, m0);
+ } else {
+ mv = mtomv(m0);
+ txsd->mi.mi_flags = m0->m_flags;
+ txsd->mi.mi_base = (caddr_t)m0;
+ txsd->mi.mi_type = m0->m_type;
+ txsd->mi.mi_len = m0->m_pkthdr.len;
+ }
+ mi = &txsd->mi;
+ dump_mi(&txsd->mi);
+
+ if (count > 1) {
struct cpl_tx_pkt_batch *cpl_batch = (struct cpl_tx_pkt_batch *)txd;
- int i;
-
+ int i, fidx;
+
wrp = (struct work_request_hdr *)txd;
-
+
flits = count*2 + 1;
txq_prod(txq, 1, &txqs);
- for (i = 0; i < count; i++) {
+ for (fidx = 1, i = 0; i < count; i++, mi++, fidx += 2) {
struct cpl_tx_pkt_batch_entry *cbe = &cpl_batch->pkt_entry[i];
-
- cntrl = V_TXPKT_INTF(pi->port_id);
- GET_VTAG(cntrl, m[i]);
- cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT) | (1 << 24);
+
+ cntrl = V_TXPKT_INTF(pi->txpkt_intf);
+ GET_VTAG_MI(cntrl, mi);
+ cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT);
cbe->cntrl = htonl(cntrl);
- cbe->len = htonl(m[i]->m_pkthdr.len | 0x80000000);
- m_set_priority(m[i], txqs.pidx);
- txsd->m[i] = m[i];
- /*
- * XXX - NOT PORTABLE outside of x86
- */
- cbe->addr = htobe64(pmap_kextract(mtod(m[i], vm_offset_t)));
+ cbe->len = htonl(mi->mi_len | 0x80000000);
+ txd->flit[fidx] |= htobe64(1 << 24);
+ cbe->addr = htobe64(segs[i].ds_addr);
}
wrp->wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) |
@@ -1345,24 +1382,27 @@
struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *)txd;
struct ip *ip;
struct tcphdr *tcp;
- char *pkthdr, tmp[TCPPKTHDRSIZE]; /* is this too large for the stack? */
+ char *pkthdr;
txd->flit[2] = 0;
- m0 = m[0];
- GET_VTAG(cntrl, m0);
+ GET_VTAG_MI(cntrl, mi);
cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT_LSO);
hdr->cntrl = htonl(cntrl);
mlen = m0->m_pkthdr.len;
hdr->len = htonl(mlen | 0x80000000);
- if (__predict_false(m0->m_len < TCPPKTHDRSIZE)) {
- pkthdr = &tmp[0];
- m_copydata(m0, 0, TCPPKTHDRSIZE, pkthdr);
- } else {
- pkthdr = mtod(m0, char *);
- }
+ DPRINTF("tso buf len=%d\n", mlen);
+ if (__predict_false(mi->mi_len < TCPPKTHDRSIZE)) {
+ /*
+ * XXX
+ *
+ */
+ pkthdr = NULL;
+ panic("discontig packet - fixxorz");
+ } else
+ pkthdr = m0->m_data;
- if (__predict_false(m0->m_flags & M_VLANTAG)) {
+ if (__predict_false(mi->mi_flags & M_VLANTAG)) {
eth_type = CPL_ETH_II_VLAN;
ip = (struct ip *)(pkthdr + ETHER_HDR_LEN +
ETHER_VLAN_ENCAP_LEN);
@@ -1380,9 +1420,8 @@
flits = 3;
} else {
struct cpl_tx_pkt *cpl = (struct cpl_tx_pkt *)txd;
-
- m0 = m[0];
- GET_VTAG(cntrl, m0);
+
+ GET_VTAG_MI(cntrl, mi);
cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT);
cpl->cntrl = htonl(cntrl);
mlen = m0->m_pkthdr.len;
@@ -1390,13 +1429,23 @@
if (mlen <= WR_LEN - sizeof(*cpl)) {
txq_prod(txq, 1, &txqs);
- txq->sdesc[txqs.pidx].count = 0;
- if (m0->m_len == m0->m_pkthdr.len)
- memcpy(&txd->flit[2], mtod(m0, uint8_t *), mlen);
- else
+ DPRINTF("mlen==%d max=%ld\n", mlen, (WR_LEN - sizeof(*cpl)));
+ if (mi->mi_type != MT_IOVEC &&
+ mi->mi_type != MT_CLIOVEC)
+ memcpy(&txd->flit[2], mi_data(mi), mlen);
+ else {
+ /*
+ * XXX mbuf_iovec
+ */
+#if 0
m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]);
- *free = 1;
+#endif
+ printf("bailing on m_copydata\n");
+ }
+ m_freem_iovec(&txsd->mi);
+ txsd->mi.mi_base = NULL;
+
flits = (mlen + 7) / 8 + 2;
cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) |
V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) |
@@ -1407,17 +1456,24 @@
wr_gen2(txd, txqs.gen);
check_ring_tx_db(sc, txq);
+ DPRINTF("pio buf\n");
return (0);
}
+ DPRINTF("regular buf\n");
flits = 2;
}
wrp = (struct work_request_hdr *)txd;
-
- if ((err = busdma_map_mbufs(m, txq, txsd, segs, &nsegs)) != 0) {
+
+#ifdef nomore
+ /*
+ * XXX need to move into one of the helper routines above
+ *
+ */
+ if ((err = busdma_map_mbufs(m, txq, txsd, segs, &nsegs)) != 0)
return (err);
- }
m0 = *m;
- ndesc = calc_tx_descs(m0, nsegs);
+#endif
+ ndesc = calc_tx_descs(mi, nsegs, tso_info);
sgp = (ndesc == 1) ? (struct sg_ent *)&txd->flit[flits] : sgl;
make_sgl(sgp, segs, nsegs);
@@ -1426,16 +1482,16 @@
DPRINTF("make_sgl success nsegs==%d ndesc==%d\n", nsegs, ndesc);
txq_prod(txq, ndesc, &txqs);
- txsd = &txq->sdesc[txqs.pidx];
wr_hi = htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | txqs.compl);
wr_lo = htonl(V_WR_TID(txq->token));
- txsd->count = count;
- txsd->m[0] = m0;
- m_set_priority(m0, txqs.pidx);
-
write_wr_hdr_sgl(ndesc, txd, &txqs, txq, sgl, flits, sgl_flits, wr_hi, wr_lo);
check_ring_tx_db(pi->adapter, txq);
+ if ((m0->m_type == MT_DATA) && (m0->m_flags & M_EXT)) {
+ m0->m_flags = 0;
+ m_free(m0);
+ }
+
return (0);
}
@@ -1798,13 +1854,11 @@
*
* Returns number of buffers of reclaimed
*/
-int
-t3_free_tx_desc(struct sge_txq *q, int reclaimable, struct mbuf **m_vec,
- int m_vec_size, int *desc_reclaimed)
+void
+t3_free_tx_desc(struct sge_txq *q, int reclaimable)
{
struct tx_sw_desc *txsd;
unsigned int cidx;
- int i, iter, reclaimed, nbufs;
#ifdef T3_TRACE
T3_TRACE2(sc->tb[q->cntxt_id & 7],
@@ -1812,30 +1866,29 @@
#endif
cidx = q->cidx;
txsd = &q->sdesc[cidx];
- prefetch(txsd);
- reclaimed = nbufs = 0;
- for (iter = reclaimed = 0; reclaimed < reclaimable; iter++) {
- if ((iter & 0x1) == 0) {
- prefetch(txsd + 1);
- prefetch(txsd + 2);
- prefetch(txsd + 3);
- prefetch(txsd + 4);
- }
+ DPRINTF("reclaiming %d WR\n", reclaimable);
+ while (reclaimable--) {
DPRINTF("cidx=%d d=%p\n", cidx, txsd);
- if (txsd->count > 0) {
- if (nbufs + txsd->count > m_vec_size)
- break;
+ if (txsd->mi.mi_base != NULL) {
if (txsd->flags & TX_SW_DESC_MAPPED) {
bus_dmamap_unload(q->entry_tag, txsd->map);
txsd->flags &= ~TX_SW_DESC_MAPPED;
}
- for (i = 0; i < txsd->count; i++, nbufs++) {
- prefetch(txsd->m[i]);
- m_vec[nbufs] = txsd->m[i];
- }
- txsd->count = 0;
+#ifdef notyet
+ critical_enter();
+ /*
+ * transfer mbuf_vec contents to cpu local ring
+ * XXX
+ *
+ */
+ critical_exit();
+#else
+ m_freem_iovec(&txsd->mi);
+#endif
-#ifdef DIAGNOSTIC
+ txsd->mi.mi_base = NULL;
+
+#if defined(DIAGNOSTIC) && 0
if (m_get_priority(txsd->m[0]) != cidx)
printf("pri=%d cidx=%d\n", (int)m_get_priority(txsd->m[0]), cidx);
#endif
@@ -1848,12 +1901,9 @@
cidx = 0;
txsd = q->sdesc;
}
- reclaimed++;
}
- *desc_reclaimed = reclaimed;
q->cidx = cidx;
- return (nbufs);
}
/**
@@ -1905,8 +1955,7 @@
struct txq_state txqs;
if (immediate(m)) {
- q->sdesc[pidx].m[0] = NULL;
- q->sdesc[pidx].count = 0;
+ q->sdesc[pidx].mi.mi_base = NULL;
write_imm(d, m, m->m_len, gen);
return;
}
@@ -1970,7 +2019,7 @@
unsigned int pidx, gen;
struct mbuf *m_vec[TX_CLEAN_MAX_DESC];
bus_dma_segment_t segs[TX_MAX_SEGS];
- int i, cleaned;
+ int cleaned;
struct tx_sw_desc *stx = &q->sdesc[q->pidx];
mtx_lock(&q->lock);
@@ -2009,10 +2058,14 @@
write_ofld_wr(adap, m, q, pidx, gen, ndesc, segs, nsegs);
check_ring_tx_db(adap, q);
-
+#if 0
+ {
+ int i;
for (i = 0; i < cleaned; i++) {
m_freem_vec(m_vec[i]);
}
+ }
+#endif
return (0);
}
@@ -2025,15 +2078,14 @@
static void
restart_offloadq(void *data, int npending)
{
-
struct mbuf *m;
struct sge_qset *qs = data;
struct sge_txq *q = &qs->txq[TXQ_OFLD];
adapter_t *adap = qs->port->adapter;
struct mbuf *m_vec[TX_CLEAN_MAX_DESC];
bus_dma_segment_t segs[TX_MAX_SEGS];
- int nsegs, i, cleaned;
struct tx_sw_desc *stx = &q->sdesc[q->pidx];
+ int nsegs, cleaned;
mtx_lock(&q->lock);
again: cleaned = reclaim_completed_tx(q, TX_CLEAN_MAX_DESC, m_vec);
@@ -2076,10 +2128,12 @@
#endif
t3_write_reg(adap, A_SG_KDOORBELL,
F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
+#if 0
for (i = 0; i < cleaned; i++) {
m_freem_vec(m_vec[i]);
}
+#endif
}
/**
@@ -2408,7 +2462,9 @@
m->m_pkthdr.rcvif = ifp;
m->m_pkthdr.header = mtod(m, uint8_t *) + sizeof(*cpl) + ethpad;
+#ifndef DISABLE_MBUF_IOVEC
m_explode(m);
+#endif
/*
* adjust after conversion to mbuf chain
*/
@@ -2677,8 +2733,7 @@
} else if (flags & F_RSPD_IMM_DATA_VALID) {
#ifdef DISABLE_MBUF_IOVEC
- if (cxgb_debug)
- printf("IMM DATA VALID opcode=0x%x rspq->cidx=%d\n", r->rss_hdr.opcode, rspq->cidx);
+ printf("IMM DATA VALID opcode=0x%x rspq->cidx=%d\n", r->rss_hdr.opcode, rspq->cidx);
if(get_imm_packet(adap, r, &rspq->rspq_mh) == 0) {
rspq->next_holdoff = NOMEM_INTR_DELAY;
@@ -2692,7 +2747,7 @@
if (rspq->rspq_mbuf == NULL)
rspq->rspq_mbuf = m_gethdr(M_DONTWAIT, MT_DATA);
- else
+ else
m = m_gethdr(M_DONTWAIT, MT_DATA);
/*
@@ -2713,29 +2768,27 @@
#ifdef DISABLE_MBUF_IOVEC
struct mbuf *m;
- m = m_gethdr(M_NOWAIT, MT_DATA);
+ m = m_gethdr(M_DONTWAIT, MT_DATA);
if (m == NULL) {
log(LOG_WARNING, "failed to get mbuf for packet\n");
break;
+ } else {
+ m->m_next = m->m_nextpkt = NULL;
}
eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mh, r, m);
#else
- if (rspq->rspq_mbuf == NULL)
+ if (rspq->rspq_mbuf == NULL)
rspq->rspq_mbuf = m_gethdr(M_DONTWAIT, MT_DATA);
if (rspq->rspq_mbuf == NULL) {
+ rspq->next_holdoff = NOMEM_INTR_DELAY;
log(LOG_WARNING, "failed to get mbuf for packet\n");
break;
+ } else {
+ rspq->rspq_mbuf->m_pkthdr.rss_hash = rss_hash;
+ rspq->rspq_mbuf->m_next = rspq->rspq_mbuf->m_nextpkt = NULL;
}
- if (rspq->rspq_mbuf == NULL) {
- if ((rspq->rspq_mbuf = m_gethdr(M_DONTWAIT, MT_DATA)) == NULL) {
- DPRINTF("0x%x:%d\n", rss_hash & ((1<<7)-1), curcpu);
- rspq->next_holdoff = NOMEM_INTR_DELAY;
- break;
- } else
- rspq->rspq_mbuf->m_pkthdr.rss_hash = rss_hash;
- }
eop = get_packet(adap, drop_thresh, qs, rspq->rspq_mbuf, r);
#endif
ethpad = 2;
@@ -2743,7 +2796,6 @@
DPRINTF("pure response\n");
rspq->pure_rsps++;
}
-
if (flags & RSPD_CTRL_MASK) {
sleeping |= flags & RSPD_GTS_MASK;
handle_rsp_cntrl_info(qs, flags);
@@ -2757,7 +2809,6 @@
rspq->gen ^= 1;
r = rspq->desc;
}
>>> TRUNCATED FOR MAIL (1000 lines) <<<
More information about the p4-projects
mailing list