PERFORCE change 126532 for review
Kip Macy
kmacy at FreeBSD.org
Mon Sep 17 13:40:32 PDT 2007
http://perforce.freebsd.org/chv.cgi?CH=126532
Change 126532 by kmacy at kmacy:storage:toestack on 2007/09/17 20:39:57
fix compile errors from IFC
Affected files ...
.. //depot/projects/toestack/sys/dev/cxgb/cxgb_main.c#6 edit
.. //depot/projects/toestack/sys/dev/cxgb/cxgb_offload.c#4 edit
.. //depot/projects/toestack/sys/dev/cxgb/cxgb_sge.c#3 edit
.. //depot/projects/toestack/sys/dev/cxgb/ulp/toecore/toedev.c#2 edit
Differences ...
==== //depot/projects/toestack/sys/dev/cxgb/cxgb_main.c#6 (text+ko) ====
@@ -51,6 +51,7 @@
#include <sys/sysctl.h>
#include <sys/queue.h>
#include <sys/taskqueue.h>
+#include <sys/proc.h>
#include <net/bpf.h>
#include <net/ethernet.h>
==== //depot/projects/toestack/sys/dev/cxgb/cxgb_offload.c#4 (text+ko) ====
@@ -109,7 +109,7 @@
TAILQ_INSERT_TAIL(&client_list, client, client_entry);
if (client->add) {
- TAILQ_FOREACH(tdev, &ofld_dev_list, ofld_entry) {
+ TAILQ_FOREACH(tdev, &ofld_dev_list, entry) {
if (offload_activated(tdev))
client->add(tdev);
}
@@ -133,7 +133,7 @@
TAILQ_REMOVE(&client_list, client, client_entry);
if (client->remove) {
- TAILQ_FOREACH(tdev, &ofld_dev_list, ofld_entry) {
+ TAILQ_FOREACH(tdev, &ofld_dev_list, entry) {
if (offload_activated(tdev))
client->remove(tdev);
}
@@ -481,7 +481,7 @@
cxgb_set_dummy_ops(struct toedev *dev)
{
dev->recv = rx_offload_blackhole;
- dev->neigh_update = dummy_neigh_update;
+ dev->arp_update = dummy_neigh_update;
}
/*
@@ -1410,7 +1410,7 @@
TOE_DATA(dev) = t;
dev->recv = process_rx;
- dev->neigh_update = t3_l2t_update;
+ dev->arp_update = t3_l2t_update;
#if 0
offload_proc_dev_setup(dev);
#endif
@@ -1460,7 +1460,7 @@
mtx_lock(&cxgb_db_lock);
snprintf(tdev->name, sizeof(tdev->name), "ofld_dev%d", unit++);
- TAILQ_INSERT_TAIL(&ofld_dev_list, tdev, ofld_entry);
+ TAILQ_INSERT_TAIL(&ofld_dev_list, tdev, entry);
mtx_unlock(&cxgb_db_lock);
}
@@ -1468,7 +1468,7 @@
unregister_tdev(struct toedev *tdev)
{
mtx_lock(&cxgb_db_lock);
- TAILQ_REMOVE(&ofld_dev_list, tdev, ofld_entry);
+ TAILQ_REMOVE(&ofld_dev_list, tdev, entry);
mtx_unlock(&cxgb_db_lock);
}
@@ -1498,7 +1498,7 @@
offload_proc_dev_exit(tdev);
#endif
tdev->recv = NULL;
- tdev->neigh_update = NULL;
+ tdev->arp_update = NULL;
unregister_tdev(tdev);
}
@@ -1620,7 +1620,7 @@
len = sprintf(buf, "Device Interfaces\n");
mtx_lock(&cxgb_db_lock);
- TAILQ_FOREACH(dev, &ofld_dev_list, ofld_entry) {
+ TAILQ_FOREACH(dev, &ofld_dev_list, entry) {
len += sprintf(buf + len, "%-16s", dev->name);
read_lock(&dev_base_lock);
for (ndev = dev_base; ndev; ndev = ndev->next) {
==== //depot/projects/toestack/sys/dev/cxgb/cxgb_sge.c#3 (text+ko) ====
@@ -83,7 +83,7 @@
#define SGE_RX_SM_BUF_SIZE 1536
#define SGE_RX_DROP_THRES 16
-#define SGE_RX_COPY_THRES 128
+#define SGE_RX_COPY_THRES MHLEN
/*
* Period of the Tx buffer reclaim timer. This timer does not need to run
@@ -135,17 +135,25 @@
struct tx_sw_desc { /* SW state per Tx descriptor */
struct mbuf *m[TX_WR_COUNT_MAX];
+
int count;
bus_dmamap_t map;
int flags;
};
struct rx_sw_desc { /* SW state per Rx descriptor */
- void *cl;
- bus_dmamap_t map;
- int flags;
+ union {
+ void *cl;
+ uint32_t *ref;
+ } u;
+ uint8_t *data;
+ bus_dmamap_t map;
+ int flags;
};
+#define rxsd_cl u.cl
+#define rxsd_ref u.ref
+
struct txq_state {
unsigned int compl;
unsigned int gen;
@@ -356,8 +364,12 @@
memcpy(mtod(m, uint8_t *), resp->imm_data, len);
break;
case RSPQ_EOP:
- memcpy(cl, resp->imm_data, len);
- m_iovappend(m, cl, MSIZE, len, 0);
+ memcpy(cl, resp->imm_data, len);
+ /*
+ * XXX
+ */
+ panic("bad append");
+ m_iovappend(m, cl, MSIZE, len, 0, NULL);
break;
default:
bogus_imm++;
@@ -507,6 +519,7 @@
qs->rspq.polling = 0 /* p->polling */;
}
+#if !defined(__i386__) && !defined(__amd64__)
static void
refill_fl_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
@@ -517,7 +530,7 @@
cb_arg->nseg = nseg;
}
-
+#endif
/**
* refill_fl - refill an SGE free-buffer list
* @sc: the controller softc
@@ -534,6 +547,7 @@
struct rx_desc *d = &q->desc[q->pidx];
struct refill_fl_cb_arg cb_arg;
void *cl;
+ uint32_t *ref;
int err;
cb_arg.error = 0;
@@ -541,10 +555,12 @@
/*
* We only allocate a cluster, mbuf allocation happens after rx
*/
- if ((cl = m_cljget(NULL, M_DONTWAIT, q->buf_size)) == NULL) {
+ if ((cl = m_cljget(NULL, M_DONTWAIT, q->zone)) == NULL) {
log(LOG_WARNING, "Failed to allocate cluster\n");
goto done;
}
+ ref = cl;
+
if ((sd->flags & RX_SW_DESC_MAP_CREATED) == 0) {
if ((err = bus_dmamap_create(q->entry_tag, 0, &sd->map))) {
log(LOG_WARNING, "bus_dmamap_create failed %d\n", err);
@@ -553,7 +569,8 @@
}
sd->flags |= RX_SW_DESC_MAP_CREATED;
}
- err = bus_dmamap_load(q->entry_tag, sd->map, cl, q->buf_size,
+#if !defined(__i386__) && !defined(__amd64__)
+ err = bus_dmamap_load(q->entry_tag, sd->map, (uint32_t *)cl + 1, q->buf_size,
refill_fl_cb, &cb_arg, 0);
if (err != 0 || cb_arg.error) {
@@ -563,9 +580,12 @@
*/
return;
}
-
+#else
+ cb_arg.seg.ds_addr = pmap_kextract((vm_offset_t)((uint32_t *)cl + 1));
+#endif
sd->flags |= RX_SW_DESC_INUSE;
- sd->cl = cl;
+ sd->rxsd_cl = cl;
+ sd->data = (uint8_t *)(sd->rxsd_ref + 1);
d->addr_lo = htobe32(cb_arg.seg.ds_addr & 0xffffffff);
d->addr_hi = htobe32(((uint64_t)cb_arg.seg.ds_addr >>32) & 0xffffffff);
d->len_gen = htobe32(V_FLD_GEN1(q->gen));
@@ -607,9 +627,9 @@
if (d->flags & RX_SW_DESC_INUSE) {
bus_dmamap_unload(q->entry_tag, d->map);
bus_dmamap_destroy(q->entry_tag, d->map);
- uma_zfree(q->zone, d->cl);
+ uma_zfree(q->zone, d->rxsd_cl);
}
- d->cl = NULL;
+ d->rxsd_cl = NULL;
if (++cidx == q->size)
cidx = 0;
}
@@ -621,11 +641,17 @@
refill_fl(adap, fl, min(16U, fl->size - fl->credits));
}
+static __inline void
+__refill_fl_lt(adapter_t *adap, struct sge_fl *fl, int max)
+{
+ if ((fl->size - fl->credits) < max)
+ refill_fl(adap, fl, min(max, fl->size - fl->credits));
+}
+
void
refill_fl_service(adapter_t *adap, struct sge_fl *fl)
{
- return; /* XXX no-op for now */
- __refill_fl(adap, fl);
+ __refill_fl_lt(adap, fl, 512);
}
#ifndef DISABLE_MBUF_IOVEC
@@ -759,7 +785,7 @@
{
adapter_t *sc = arg;
#ifndef IFNET_MULTIQUEUE
- struct port_info *p;
+ struct port_info *pi;
struct sge_qset *qs;
struct sge_txq *txq;
int i, j;
@@ -775,7 +801,7 @@
(qs->fl[1].credits < qs->fl[1].size));
if (reclaim_eth || reclaim_ofl || refill_rx) {
p = &sc->port[i];
- taskqueue_enqueue(p->tq, &p->timer_reclaim_task);
+ taskqueue_enqueue(pi->tq, &pi->timer_reclaim_task);
break;
}
}
@@ -810,9 +836,9 @@
}
int
-t3_sge_init_port(struct port_info *p)
+t3_sge_init_port(struct port_info *pi)
{
- TASK_INIT(&p->timer_reclaim_task, 0, sge_timer_reclaim, p);
+ TASK_INIT(&pi->timer_reclaim_task, 0, sge_timer_reclaim, pi);
return (0);
}
@@ -852,9 +878,9 @@
{
int reclaimable, i, n;
struct mbuf *m_vec[TX_CLEAN_MAX_DESC];
- struct port_info *p;
+ struct port_info *pi;
- p = txq->port;
+ pi = txq->port;
reclaim_more:
n = 0;
reclaimable = desc_reclaimable(txq);
@@ -868,11 +894,11 @@
for (i = 0; i < n; i++) {
m_freem_vec(m_vec[i]);
}
- if (p && p->ifp->if_drv_flags & IFF_DRV_OACTIVE &&
+ if (pi && pi->ifp->if_drv_flags & IFF_DRV_OACTIVE &&
txq->size - txq->in_use >= TX_START_MAX_DESC) {
txq_fills++;
- p->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
- taskqueue_enqueue(p->tq, &p->start_task);
+ pi->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
+ taskqueue_enqueue(pi->tq, &pi->start_task);
}
if (n)
@@ -890,9 +916,9 @@
static void
sge_timer_reclaim(void *arg, int ncount)
{
- struct port_info *p = arg;
- int i, nqsets = p->nqsets;
- adapter_t *sc = p->adapter;
+ struct port_info *pi = arg;
+ int i, nqsets = pi->nqsets;
+ adapter_t *sc = pi->adapter;
struct sge_qset *qs;
struct sge_txq *txq;
struct mtx *lock;
@@ -1231,7 +1257,6 @@
#define GET_VTAG(cntrl, m)
#endif
-
int
t3_encap(struct sge_qset *qs, struct mbuf **m, int count, int *free)
{
@@ -1239,7 +1264,7 @@
struct mbuf *m0;
struct sge_txq *txq;
struct txq_state txqs;
- struct port_info *p;
+ struct port_info *pi;
unsigned int ndesc, flits, cntrl, mlen;
int err, nsegs, tso_info = 0;
@@ -1250,25 +1275,26 @@
uint32_t wr_hi, wr_lo, sgl_flits;
struct tx_desc *txd;
- DPRINTF("t3_encap port_id=%d qsidx=%d ", p->port_id, p->first_qset);
+
#if defined(IFNET_MULTIQUEUE) && defined(STRICT_AFFINITY)
KASSERT(qs->qs_cpuid == curcpu, ("cpu qset mismatch cpuid=%d curcpu=%d", qs->qs_cpuid, curcpu));
#endif
DPRINTF("t3_encap cpu=%d ", curcpu);
- p = qs->port;
- sc = p->adapter;
+ pi = qs->port;
+ sc = pi->adapter;
txq = &qs->txq[TXQ_ETH];
txsd = &txq->sdesc[txq->pidx];
txd = &txq->desc[txq->pidx];
- DPRINTF("mlen=%d txpkt_intf=%d tx_chan=%d\n", mlen, p->txpkt_intf, p->tx_chan);
+ DPRINTF("t3_encap port_id=%d qsidx=%d ", pi->port_id, pi->first_qset);
+ DPRINTF("mlen=%d txpkt_intf=%d tx_chan=%d\n", m[0]->m_pkthdr.len, pi->txpkt_intf, pi->tx_chan);
/*
* XXX handle checksum, TSO, and VLAN here
*
*/
- cntrl = V_TXPKT_INTF(p->txpkt_intf);
+ cntrl = V_TXPKT_INTF(pi->txpkt_intf);
/*
* XXX need to add VLAN support for 6.x
@@ -1291,7 +1317,7 @@
for (i = 0; i < count; i++) {
struct cpl_tx_pkt_batch_entry *cbe = &cpl_batch->pkt_entry[i];
- cntrl = V_TXPKT_INTF(p->port_id);
+ cntrl = V_TXPKT_INTF(pi->txpkt_intf);
GET_VTAG(cntrl, m[i]);
cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT) | (1 << 24);
cbe->cntrl = htonl(cntrl);
@@ -1370,7 +1396,6 @@
memcpy(&txd->flit[2], mtod(m0, uint8_t *), mlen);
else
m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]);
-
*free = 1;
flits = (mlen + 7) / 8 + 2;
cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) |
@@ -1409,7 +1434,7 @@
m_set_priority(m0, txqs.pidx);
write_wr_hdr_sgl(ndesc, txd, &txqs, txq, sgl, flits, sgl_flits, wr_hi, wr_lo);
- check_ring_tx_db(p->adapter, txq);
+ check_ring_tx_db(pi->adapter, txq);
return (0);
}
@@ -1779,7 +1804,7 @@
{
struct tx_sw_desc *txsd;
unsigned int cidx;
- int i, reclaimed, nbufs;
+ int i, iter, reclaimed, nbufs;
#ifdef T3_TRACE
T3_TRACE2(sc->tb[q->cntxt_id & 7],
@@ -1787,10 +1812,16 @@
#endif
cidx = q->cidx;
txsd = &q->sdesc[cidx];
+ prefetch(txsd);
reclaimed = nbufs = 0;
-
- for (reclaimed = 0; reclaimed < reclaimable;) {
- DPRINTF("cidx=%d d=%p\n", cidx, d);
+ for (iter = reclaimed = 0; reclaimed < reclaimable; iter++) {
+ if ((iter & 0x1) == 0) {
+ prefetch(txsd + 1);
+ prefetch(txsd + 2);
+ prefetch(txsd + 3);
+ prefetch(txsd + 4);
+ }
+ DPRINTF("cidx=%d d=%p\n", cidx, txsd);
if (txsd->count > 0) {
if (nbufs + txsd->count > m_vec_size)
break;
@@ -1798,13 +1829,17 @@
bus_dmamap_unload(q->entry_tag, txsd->map);
txsd->flags &= ~TX_SW_DESC_MAPPED;
}
- if (m_get_priority(txsd->m[0]) == cidx) {
- for (i = 0; i < txsd->count; i++, nbufs++)
- m_vec[nbufs] = txsd->m[i];
- txsd->count = 0;
- } else {
+ for (i = 0; i < txsd->count; i++, nbufs++) {
+ prefetch(txsd->m[i]);
+ m_vec[nbufs] = txsd->m[i];
+ }
+ txsd->count = 0;
+
+#ifdef DIAGNOSTIC
+ if (m_get_priority(txsd->m[0]) != cidx)
printf("pri=%d cidx=%d\n", (int)m_get_priority(txsd->m[0]), cidx);
- }
+#endif
+
} else
q->txq_skipped++;
@@ -2254,13 +2289,18 @@
q->txq[TXQ_ETH].stop_thres = nports *
flits_to_desc(sgl_len(TX_MAX_SEGS + 1) + 3);
- q->fl[0].buf_size = MCLBYTES;
+ q->fl[0].buf_size = MCLBYTES - sizeof(uint32_t);
q->fl[0].zone = zone_clust;
q->fl[0].type = EXT_CLUSTER;
- q->fl[1].buf_size = MJUMPAGESIZE;
- q->fl[1].zone = zone_jumbop;
- q->fl[1].type = EXT_JUMBOP;
-
+ if (jumbo_phys_contig) {
+ q->fl[1].buf_size = MJUM9BYTES - sizeof(uint32_t);
+ q->fl[1].zone = zone_jumbo9;
+ q->fl[1].type = EXT_JUMBO9;
+ } else {
+ q->fl[1].buf_size = MJUMPAGESIZE - sizeof(uint32_t);
+ q->fl[1].zone = zone_jumbop;
+ q->fl[1].type = EXT_JUMBOP;
+ }
q->lro.enabled = lro_default;
mtx_lock(&sc->sge.reg_lock);
@@ -2372,7 +2412,9 @@
/*
* adjust after conversion to mbuf chain
*/
- m_adj(m, sizeof(*cpl) + ethpad);
+ m->m_pkthdr.len -= (sizeof(*cpl) + ethpad);
+ m->m_len -= (sizeof(*cpl) + ethpad);
+ m->m_data += (sizeof(*cpl) + ethpad);
(*ifp->if_input)(ifp, m);
}
@@ -2406,17 +2448,24 @@
uint32_t len = G_RSPD_LEN(len_cq);
uint32_t flags = ntohl(r->flags);
uint8_t sopeop = G_RSPD_SOP_EOP(flags);
+ uint32_t *ref;
int ret = 0;
- prefetch(sd->cl);
+ prefetch(sd->rxsd_cl);
fl->credits--;
bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(fl->entry_tag, sd->map);
- m_cljset(m, sd->cl, fl->type);
+ ref = sd->rxsd_ref;
+ m_cljset(m, sd->rxsd_cl, fl->type, sd->rxsd_ref);
+ *ref = 1;
m->m_len = len;
-
+ /*
+ * bump past the refcnt address
+ */
+ m->m_data = sd->data;
+
switch(sopeop) {
case RSPQ_SOP_EOP:
DBG(DBG_RX, ("get_packet: SOP-EOP m %p\n", m));
@@ -2474,9 +2523,11 @@
uint32_t flags = ntohl(r->flags);
uint8_t sopeop = G_RSPD_SOP_EOP(flags);
void *cl;
+ uint32_t *ref = NULL;
int ret = 0;
-
- prefetch(sd->cl);
+
+ prefetch((sd + 1)->rxsd_cl);
+ prefetch((sd + 2)->rxsd_cl);
DPRINTF("rx cpu=%d\n", curcpu);
fl->credits--;
@@ -2484,17 +2535,22 @@
if (recycle_enable && len <= SGE_RX_COPY_THRES && sopeop == RSPQ_SOP_EOP) {
cl = mtod(m, void *);
- memcpy(cl, sd->cl, len);
+ memcpy(cl, sd->data, len);
recycle_rx_buf(adap, fl, fl->cidx);
} else {
- cl = sd->cl;
bus_dmamap_unload(fl->entry_tag, sd->map);
+ cl = sd->rxsd_cl;
+ ref = sd->rxsd_ref;
}
+
switch(sopeop) {
case RSPQ_SOP_EOP:
DBG(DBG_RX, ("get_packet: SOP-EOP m %p\n", m));
- if (cl == sd->cl)
- m_cljset(m, cl, fl->type);
+ if (cl == sd->rxsd_cl) {
+ m_cljset(m, cl, fl->type, ref);
+ *ref = 1;
+ m->m_data = sd->data;
+ }
m->m_len = m->m_pkthdr.len = len;
ret = 1;
goto done;
@@ -2513,8 +2569,8 @@
ret = 1;
break;
}
- m_iovappend(m, cl, fl->buf_size, len, 0);
-
+ m_iovappend(m, cl, fl->buf_size, len, sizeof(uint32_t), ref);
+ *ref = 1;
done:
if (++fl->cidx == fl->size)
fl->cidx = 0;
@@ -2710,7 +2766,7 @@
if (eop) {
prefetch(mtod(rspq->rspq_mh.mh_head, uint8_t *));
- prefetch(mtod(rspq->rspq_mh.mh_head, uint8_t *) + L1_CACHE_BYTES);
+ prefetch(mtod(rspq->rspq_mh.mh_head, uint8_t *) + L1_CACHE_BYTES);
if (eth) {
t3_rx_eth_lro(adap, rspq, rspq->rspq_mh.mh_head, ethpad,
@@ -2727,10 +2783,10 @@
ngathered = rx_offload(&adap->tdev, rspq,
rspq->rspq_mh.mh_head, offload_mbufs, ngathered);
}
- __refill_fl(adap, &qs->fl[0]);
- __refill_fl(adap, &qs->fl[1]);
}
+ __refill_fl_lt(adap, &qs->fl[0], 32);
+ __refill_fl_lt(adap, &qs->fl[1], 32);
--budget_left;
}
@@ -2744,6 +2800,8 @@
if (__predict_false(qs->txq_stopped != 0))
restart_tx(qs);
+ __refill_fl_lt(adap, &qs->fl[0], 512);
+ __refill_fl_lt(adap, &qs->fl[1], 512);
budget -= budget_left;
return (budget);
}
==== //depot/projects/toestack/sys/dev/cxgb/ulp/toecore/toedev.c#2 (text+ko) ====
@@ -38,6 +38,7 @@
#include <sys/module.h>
#include <sys/queue.h>
#include <sys/mbuf.h>
+#include <sys/proc.h>
#include <sys/socket.h>
#include <sys/sockio.h>
More information about the p4-projects
mailing list