PERFORCE change 126815 for review
Kip Macy
kmacy at FreeBSD.org
Tue Sep 25 22:15:21 PDT 2007
http://perforce.freebsd.org/chv.cgi?CH=126815
Change 126815 by kmacy at kmacy_home:ethng on 2007/09/26 05:14:21
add rudimentary cluster cache
move buf_stack functions to cxgb_support.c
Affected files ...
.. //depot/projects/ethng/src/sys/dev/cxgb/cxgb_osdep.h#10 edit
.. //depot/projects/ethng/src/sys/dev/cxgb/cxgb_sge.c#24 edit
.. //depot/projects/ethng/src/sys/dev/cxgb/sys/cxgb_support.c#2 edit
.. //depot/projects/ethng/src/sys/dev/cxgb/sys/mvec.h#6 edit
.. //depot/projects/ethng/src/sys/dev/cxgb/sys/uipc_mvec.c#6 edit
Differences ...
==== //depot/projects/ethng/src/sys/dev/cxgb/cxgb_osdep.h#10 (text+ko) ====
@@ -117,37 +117,11 @@
return (m);
}
-struct buf_stack {
- caddr_t *bs_stack;
- volatile int bs_head;
- int bs_size;
-};
-
-static __inline int
-buf_push(struct buf_stack *bs, caddr_t buf)
-{
- if (bs->bs_head + 1 >= bs->bs_size)
- return (1);
-
- bs->bs_stack[++(bs->bs_head)] = buf;
- return (0);
-}
-
-static __inline caddr_t
-buf_pop(struct buf_stack *bs)
-{
- if (bs->bs_head < 0)
- return (NULL);
-
- return (bs->bs_stack[(bs->bs_head)--]);
-}
-
#define PANIC_IF(exp) do { \
if (exp) \
panic("BUG: %s", exp); \
} while (0)
-
#define m_get_priority(m) ((uintptr_t)(m)->m_pkthdr.rcvif)
#define m_set_priority(m, pri) ((m)->m_pkthdr.rcvif = (struct ifnet *)((uintptr_t)pri))
==== //depot/projects/ethng/src/sys/dev/cxgb/cxgb_sge.c#24 (text+ko) ====
@@ -550,7 +550,7 @@
/*
* We only allocate a cluster, mbuf allocation happens after rx
*/
- if ((cl = m_cljget(NULL, M_DONTWAIT, q->zone)) == NULL) {
+ if ((cl = cxgb_cache_get(q->zone)) == NULL) {
log(LOG_WARNING, "Failed to allocate cluster\n");
goto done;
}
@@ -830,6 +830,7 @@
callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc);
TASK_INIT(&sc->slow_intr_task, 0, sge_slow_intr_handler, sc);
mi_init();
+ cxgb_cache_init();
return (0);
}
@@ -1113,7 +1114,6 @@
sgp->len[idx] = htobe32(segs[i].ds_len);
sgp->addr[idx] = htobe64(segs[i].ds_addr);
-
idx ^= 1;
}
@@ -1162,8 +1162,6 @@
#endif
}
-
-
/**
* write_wr_hdr_sgl - write a WR header and, optionally, SGL
* @ndesc: number of Tx descriptors spanned by the SGL
@@ -1181,7 +1179,6 @@
* and we just need to write the WR header. Otherwise we distribute the
* SGL across the number of descriptors it spans.
*/
-
static void
write_wr_hdr_sgl(unsigned int ndesc, struct tx_desc *txd, struct txq_state *txqs,
const struct sge_txq *txq, const struct sg_ent *sgl, unsigned int flits,
@@ -1253,8 +1250,20 @@
static void
dump_mi(struct mbuf_iovec *mi)
{
- DPRINTF("mi_flags=0x%08x mi_data=%p mi_len=%d mi_type=%d\n",
+ int i;
+ struct mbuf_vec *mv;
+
+ printf("mi_flags=0x%08x mi_data=%p mi_len=%d mi_type=%d\n",
mi->mi_flags, mi->mi_base + mi->mi_offset, mi->mi_len, mi->mi_type);
+
+ if (mi->mi_type == EXT_CLIOVEC ||
+ mi->mi_type == EXT_IOVEC) {
+ mv = mtomv((struct mbuf *)mi->mi_base);
+ mi = mv->mv_vec;
+ for (i = 0; i < mv->mv_count; i++, mi++)
+ dump_mi(mi);
+
+ }
}
/* sizeof(*eh) + sizeof(*vhdr) + sizeof(*ip) + sizeof(*tcp) */
@@ -1340,7 +1349,6 @@
txsd->mi.mi_len = m0->m_pkthdr.len;
}
mi = &txsd->mi;
- dump_mi(&txsd->mi);
if (count > 1) {
struct cpl_tx_pkt_batch *cpl_batch = (struct cpl_tx_pkt_batch *)txd;
@@ -1378,7 +1386,12 @@
struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *)txd;
struct ip *ip;
struct tcphdr *tcp;
- char *pkthdr;
+ char *pkthdr, tmp[TCPPKTHDRSIZE];
+ struct mbuf_vec *mv;
+ struct mbuf_iovec *tmpmi;
+
+ mv = mtomv(m0);
+ tmpmi = mv->mv_vec;
txd->flit[2] = 0;
GET_VTAG_MI(cntrl, mi);
@@ -1388,17 +1401,14 @@
hdr->len = htonl(mlen | 0x80000000);
DPRINTF("tso buf len=%d\n", mlen);
- if (__predict_false(mi->mi_len < TCPPKTHDRSIZE)) {
- /*
- * XXX
- *
- */
- pkthdr = NULL;
+ if (__predict_false(tmpmi->mi_len < TCPPKTHDRSIZE)) {
+ pkthdr = tmp;
+ dump_mi(mi);
panic("discontig packet - fixxorz");
} else
pkthdr = m0->m_data;
- if (__predict_false(mi->mi_flags & M_VLANTAG)) {
+ if (__predict_false(m0->m_flags & M_VLANTAG)) {
eth_type = CPL_ETH_II_VLAN;
ip = (struct ip *)(pkthdr + ETHER_HDR_LEN +
ETHER_VLAN_ENCAP_LEN);
@@ -1417,7 +1427,7 @@
} else {
struct cpl_tx_pkt *cpl = (struct cpl_tx_pkt *)txd;
- GET_VTAG_MI(cntrl, mi);
+ GET_VTAG(cntrl, m0);
cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT);
cpl->cntrl = htonl(cntrl);
mlen = m0->m_pkthdr.len;
@@ -1484,7 +1494,7 @@
check_ring_tx_db(pi->adapter, txq);
if ((m0->m_type == MT_DATA) && ((m0->m_flags & (M_EXT|M_NOFREE)) == M_EXT)) {
- m0->m_flags = 0;
+ m0->m_flags &= ~M_EXT ;
m_free(m0);
}
@@ -2267,7 +2277,7 @@
q->txq[i].txq_mr.mr_size = cxgb_txq_mbuf_ring_size;
mtx_init(&q->txq[i].txq_mr.mr_lock, "txq mbuf ring", NULL, MTX_DEF);
}
-
+
init_qset_cntxt(q, id);
if ((ret = alloc_ring(sc, p->fl_size, sizeof(struct rx_desc),
==== //depot/projects/ethng/src/sys/dev/cxgb/sys/cxgb_support.c#2 (text+ko) ====
@@ -52,8 +52,53 @@
#include <dev/cxgb/sys/mvec.h>
#endif
-int
-buf_init(struct buf_stack *bs, int size)
+struct buf_stack {
+ caddr_t *bs_stack;
+ volatile int bs_head;
+ int bs_size;
+};
+
+static __inline int
+buf_stack_push(struct buf_stack *bs, caddr_t buf)
+{
+ if (bs->bs_head + 1 >= bs->bs_size)
+ return (ENOSPC);
+
+ bs->bs_stack[++(bs->bs_head)] = buf;
+ return (0);
+}
+
+static __inline caddr_t
+buf_stack_pop(struct buf_stack *bs)
+{
+ if (bs->bs_head < 0)
+ return (NULL);
+
+ return (bs->bs_stack[(bs->bs_head)--]);
+}
+
+/*
+ * Stack is more than half full
+ * we can free some elements to make room
+ */
+static __inline int
+buf_stack_canfree(struct buf_stack *bs)
+{
+ return (bs->bs_head > (bs->bs_size>>1));
+}
+
+struct cxgb_cache_pcpu {
+ struct buf_stack ccp_jumbo_free;
+ struct buf_stack ccp_cluster_free;
+ uma_zone_t ccp_jumbo_zone;
+};
+
+struct cxgb_cache_system {
+ struct cxgb_cache_pcpu ccs_array[0];
+} *cxgb_caches;
+
+static int
+buf_stack_init(struct buf_stack *bs, int size)
{
bs->bs_size = size;
bs->bs_head = -1;
@@ -63,3 +108,162 @@
return (0);
}
+static void
+buf_stack_deinit(struct buf_stack *bs)
+{
+ if (bs->bs_stack != NULL)
+ free(bs->bs_stack, M_DEVBUF);
+}
+
+static int
+cxgb_cache_pcpu_init(struct cxgb_cache_pcpu *ccp)
+{
+ int err;
+
+ if ((err = buf_stack_init(&ccp->ccp_jumbo_free, 2*JUMBO_Q_SIZE)))
+ return (err);
+
+ if ((err = buf_stack_init(&ccp->ccp_cluster_free, 2*FL_Q_SIZE)))
+ return (err);
+
+ if (jumbo_phys_contig)
+ ccp->ccp_jumbo_zone = zone_jumbo9;
+ else
+ ccp->ccp_jumbo_zone = zone_jumbop;
+
+ return (0);
+}
+
+static void
+cxgb_cache_pcpu_deinit(struct cxgb_cache_pcpu *ccp)
+{
+ /*
+ * XXX free clusters
+ */
+
+ buf_stack_deinit(&ccp->ccp_jumbo_free);
+ buf_stack_deinit(&ccp->ccp_cluster_free);
+
+}
+
+static int inited = 0;
+
+int
+cxgb_cache_init(void)
+{
+ int i, err;
+
+ if (inited++ > 0)
+ return (0);
+
+ if ((cxgb_caches = malloc(sizeof(struct cxgb_cache_pcpu)*mp_ncpus, M_DEVBUF, M_WAITOK|M_ZERO)) == NULL)
+ return (ENOMEM);
+
+ for (i = 0; i < mp_ncpus; i++)
+ if ((err = cxgb_cache_pcpu_init(&cxgb_caches->ccs_array[i])))
+ goto err;
+
+ return (0);
+err:
+ cxgb_cache_flush();
+
+ return (err);
+}
+
+void
+cxgb_cache_flush(void)
+{
+ int i;
+
+ if (--inited > 0)
+ return;
+
+ if (cxgb_caches == NULL)
+ return;
+
+ for (i = 0; i < mp_ncpus; i++)
+ cxgb_cache_pcpu_deinit(&cxgb_caches->ccs_array[i]);
+
+ free(cxgb_caches, M_DEVBUF);
+ cxgb_caches = NULL;
+}
+
+caddr_t
+cxgb_cache_get(uma_zone_t zone)
+{
+ caddr_t cl;
+ struct cxgb_cache_pcpu *ccp;
+
+ critical_enter();
+ ccp = &cxgb_caches->ccs_array[curcpu];
+ if (zone == zone_clust) {
+ cl = buf_stack_pop(&ccp->ccp_cluster_free);
+ } else {
+ cl = buf_stack_pop(&ccp->ccp_jumbo_free);
+ }
+ critical_exit();
+
+ if (cl == NULL)
+ cl = uma_zalloc(zone, M_NOWAIT);
+
+ return (cl);
+}
+
+void
+cxgb_cache_put(uma_zone_t zone, void *cl)
+{
+ struct cxgb_cache_pcpu *ccp;
+ int err = 0;
+
+ critical_enter();
+ ccp = &cxgb_caches->ccs_array[curcpu];
+ if (zone == zone_clust) {
+ err = buf_stack_push(&ccp->ccp_cluster_free, cl);
+ } else if (zone == ccp->ccp_jumbo_zone){
+ err = buf_stack_push(&ccp->ccp_jumbo_free, cl);
+ }
+ critical_exit();
+
+ if (err)
+ uma_zfree(zone, cl);
+}
+
+void
+cxgb_cache_rebalance(void)
+{
+ struct cxgb_cache_pcpu *ccp;
+ caddr_t vec[8];
+ uma_zone_t zone;
+ int i, count;
+
+ return;
+
+
+ critical_enter();
+restart:
+ ccp = &cxgb_caches->ccs_array[curcpu];
+ zone = ccp->ccp_jumbo_zone;
+ for (i = 0; i < 8 && buf_stack_canfree(&ccp->ccp_jumbo_free); i++)
+ vec[i] = buf_stack_pop(&ccp->ccp_jumbo_free);
+ critical_exit();
+ count = i;
+ for (i = 0; i < count; i++)
+ uma_zfree(zone, vec[i]);
+
+ critical_enter();
+ ccp = &cxgb_caches->ccs_array[curcpu];
+ zone = zone_clust;
+ for (i = 0; i < 8 && buf_stack_canfree(&ccp->ccp_cluster_free); i++)
+ vec[i] = buf_stack_pop(&ccp->ccp_cluster_free);
+ critical_exit();
+ count = i;
+ for (i = 0; i < count; i++)
+ uma_zfree(zone, vec[i]);
+
+ critical_enter();
+ ccp = &cxgb_caches->ccs_array[curcpu];
+ if (buf_stack_canfree(&ccp->ccp_cluster_free) || buf_stack_canfree(&ccp->ccp_jumbo_free))
+ goto restart;
+ critical_exit();
+}
+
==== //depot/projects/ethng/src/sys/dev/cxgb/sys/mvec.h#6 (text+ko) ====
@@ -32,6 +32,16 @@
#ifndef _MVEC_H_
#define _MVEC_H_
+int cxgb_cache_init(void);
+
+void cxgb_cache_flush(void);
+
+caddr_t cxgb_cache_get(uma_zone_t zone);
+
+void cxgb_cache_put(uma_zone_t zone, void *cl);
+
+void cxgb_cache_rebalance(void);
+
#define mtomv(m) ((struct mbuf_vec *)((m)->m_pktdat))
#define M_IOVEC 0x100000 /* mbuf immediate data area is used for cluster ptrs */
@@ -164,8 +174,8 @@
case EXT_IOVEC:
uma_zfree(zone_miovec, m);
break;
- case EXT_CLIOVEC:
- uma_zfree(zone_clust, m);
+ case EXT_CLIOVEC:
+ cxgb_cache_put(zone_clust, m);
break;
default:
panic("unexpected type %d\n", type);
==== //depot/projects/ethng/src/sys/dev/cxgb/sys/uipc_mvec.c#6 (text+ko) ====
@@ -202,7 +202,7 @@
return (ENOMEM);
type = EXT_CLIOVEC;
} else {
- if ((m0 = uma_zalloc_arg(zone_miovec, NULL, M_DONTWAIT)) == NULL)
+ if ((m0 = uma_zalloc_arg(zone_miovec, NULL, M_NOWAIT)) == NULL)
return (ENOMEM);
type = EXT_IOVEC;
}
@@ -223,7 +223,8 @@
if (marray[i]->m_flags & M_EXT) {
marray[i]->m_flags = 0;
m_free(marray[i]);
- }
+
+ }
}
*nsegs = seg_count;
*m = m0;
@@ -301,13 +302,13 @@
m_free_fast((struct mbuf *)cl);
break;
case EXT_CLUSTER:
- uma_zfree(zone_clust, cl);
+ cxgb_cache_put(zone_clust, cl);
break;
case EXT_JUMBOP:
- uma_zfree(zone_jumbop, cl);
+ cxgb_cache_put(zone_jumbop, cl);
break;
case EXT_JUMBO9:
- uma_zfree(zone_jumbo9, cl);
+ cxgb_cache_put(zone_jumbo9, cl);
break;
case EXT_JUMBO16:
uma_zfree(zone_jumbo16, cl);
More information about the p4-projects
mailing list