PERFORCE change 135092 for review
Kip Macy
kmacy at FreeBSD.org
Fri Feb 8 22:20:22 PST 2008
http://perforce.freebsd.org/chv.cgi?CH=135092
Change 135092 by kmacy at kmacy:storage:toehead on 2008/02/09 06:19:42
- add logging function for tcb
- ensure tp->rcv_nxt is always protected by the inpcb lock
- ensure that ddp state is always protected by the so_rcv lock
- replace most console logging with ktr entries
- move user_ddp_pending in to ddp_state to avoid queueing an empty mbuf
- ensure that listening context's ULP mode is set
Affected files ...
.. //depot/projects/toehead/sys/dev/cxgb/cxgb_main.c#7 edit
.. //depot/projects/toehead/sys/dev/cxgb/cxgb_osdep.h#4 edit
.. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c#17 edit
.. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_socket.c#18 edit
.. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_ddp.c#8 edit
.. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_defs.h#7 edit
.. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_listen.c#2 edit
.. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_t3_ddp.h#10 edit
.. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_tom.c#8 edit
Differences ...
==== //depot/projects/toehead/sys/dev/cxgb/cxgb_main.c#7 (text+ko) ====
@@ -40,6 +40,7 @@
#include <machine/bus.h>
#include <machine/resource.h>
#include <sys/bus_dma.h>
+#include <sys/ktr.h>
#include <sys/rman.h>
#include <sys/ioccom.h>
#include <sys/mbuf.h>
@@ -281,6 +282,32 @@
static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
+
+void
+cxgb_log_tcb(struct adapter *sc, unsigned int tid)
+{
+ char buf[TCB_SIZE];
+ uint64_t *tcb = (uint64_t *)buf;
+ int i, error;
+ struct mc7 *mem = &sc->cm;
+
+ error = t3_mc7_bd_read(mem, tid*TCB_SIZE/8, TCB_SIZE/8, tcb);
+ if (error)
+ printf("cxgb_tcb_log failed\n");
+
+ CTR1(KTR_CXGB, "TCB tid=%u", tid);
+ for (i = 0; i < TCB_SIZE / 32; i++) {
+ CTR5(KTR_CXGB, "%1d: %08x %08x %08x %08x",
+ i, (uint32_t)tcb[1], (uint32_t)(tcb[1] >> 32),
+ (uint32_t)tcb[0], (uint32_t)(tcb[0] >> 32));
+ tcb += 2;
+ CTR4(KTR_CXGB, " %08x %08x %08x %08x",
+ (uint32_t)tcb[1], (uint32_t)(tcb[1] >> 32),
+ (uint32_t)tcb[0], (uint32_t)(tcb[0] >> 32));
+ tcb += 2;
+ }
+}
+
static __inline char
t3rev2char(struct adapter *adapter)
{
==== //depot/projects/toehead/sys/dev/cxgb/cxgb_osdep.h#4 (text+ko) ====
@@ -83,6 +83,9 @@
#define m_set_socket(m, a) ((m)->m_pkthdr.header = (a))
#define m_get_socket(m) ((m)->m_pkthdr.header)
+#define KTR_CXGB KTR_SPARE2
+void cxgb_log_tcb(struct adapter *sc, unsigned int tid);
+
#define MT_DONTFREE 128
#if __FreeBSD_version > 700030
==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c#17 (text+ko) ====
@@ -66,7 +66,6 @@
#include <netinet/tcp_syncache.h>
#include <net/route.h>
-
#include <dev/cxgb/t3cdev.h>
#include <dev/cxgb/common/cxgb_firmware_exports.h>
#include <dev/cxgb/common/cxgb_t3_cpl.h>
@@ -775,7 +774,7 @@
{
struct cpl_set_tcb_field *req;
- CTR4(KTR_TOM, "__set_tcb_field_ulp(tid=%u word=0x%x mask=%jx val=%jx",
+ CTR4(KTR_TCB, "__set_tcb_field_ulp(tid=%u word=0x%x mask=%jx val=%jx",
toep->tp_tid, word, mask, val);
req = mtod(m, struct cpl_set_tcb_field *);
@@ -1052,8 +1051,6 @@
l2t_release(L2DATA(cdev), toep->tp_l2t);
toep->tp_l2t = NULL;
}
- printf("setting toep->tp_tp to NULL\n");
-
toep->tp_tp = NULL;
if (tp) {
INP_LOCK_ASSERT(tp->t_inpcb);
@@ -1264,7 +1261,6 @@
req->peer_port = inp->inp_fport;
memcpy(&req->local_ip, &inp->inp_laddr, 4);
memcpy(&req->peer_ip, &inp->inp_faddr, 4);
- DPRINTF("connect smt_idx=%d\n", e->smt_idx);
req->opt0h = htonl(calc_opt0h(so, toep->tp_mtu_idx) | V_L2T_IDX(e->idx) |
V_TX_CHANNEL(e->smt_idx));
req->opt0l = htonl(calc_opt0l(so, toep->tp_ulp_mode));
@@ -1449,8 +1445,6 @@
toep = tp->t_toe;
m_set_toep(m, tp->t_toe);
- printf("sending off request\n");
-
toep->tp_state = TCPS_SYN_SENT;
l2t_send(d->cdev, (struct mbuf *)m, e);
@@ -1670,6 +1664,8 @@
so = toeptoso(toep);
tp = toep->tp_tp;
+
+ INP_LOCK_ASSERT(tp->t_inpcb);
SOCKBUF_LOCK(&so->so_rcv);
/* Note that we only accout for CPL_GET_TCB issued by the DDP code. We
@@ -1704,14 +1700,21 @@
ddp_offset = t >> S_TCB_RX_DDP_BUF1_OFFSET;
}
ddp_offset &= M_TCB_RX_DDP_BUF0_OFFSET;
+ m->m_cur_offset = bsp->cur_offset;
+ bsp->cur_offset = ddp_offset;
+ m->m_len = m->m_pkthdr.len = ddp_offset - m->m_cur_offset;
+ CTR5(KTR_TOM,
+ "tcb_rpl_as_ddp_complete: idx=%d seq=0x%x hwbuf=%u ddp_offset=%u cur_offset=%u",
+ q->cur_buf, tp->rcv_nxt, q->cur_buf, ddp_offset, m->m_cur_offset);
+ KASSERT(ddp_offset >= m->m_cur_offset, ("ddp_offset=%u less than cur_offset=%u",
+ ddp_offset, m->m_cur_offset));
+
#ifdef T3_TRACE
T3_TRACE3(TIDTB(so),
"tcb_rpl_as_ddp_complete: seq 0x%x hwbuf %u ddp_offset %u",
tp->rcv_nxt, q->cur_buf, ddp_offset);
#endif
- CTR3(KTR_TOM, "tcb_rpl_as_ddp_complete(seq=0x%x hwbuf=%u ddp_offset=%u",
- tp->rcv_nxt, q->cur_buf, ddp_offset);
#if 0
{
@@ -1743,10 +1746,6 @@
}
#endif
- m->m_cur_offset = bsp->cur_offset;
- bsp->cur_offset = ddp_offset;
- m->m_len = m->m_pkthdr.len = ddp_offset - m->m_cur_offset;
-
if (__predict_false(so_no_receive(so) && m->m_pkthdr.len)) {
handle_excess_rx(toep, m);
return;
@@ -1802,16 +1801,11 @@
return;
}
- KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len));
m->m_ddp_gl = (unsigned char *)bsp->gl;
m->m_flags |= M_DDP;
m->m_seq = tp->rcv_nxt;
tp->rcv_nxt += m->m_pkthdr.len;
tp->t_rcvtime = ticks;
-
-#if 0
- skb->h.th = tcphdr_skb->h.th;
-#endif
#ifdef T3_TRACE
T3_TRACE3(TB(q),
"tcb_rpl_as_ddp_complete: seq 0x%x hwbuf %u lskb->len %u",
@@ -1819,7 +1813,10 @@
#endif
CTR3(KTR_TOM, "tcb_rpl_as_ddp_complete: seq 0x%x hwbuf %u m->m_pktlen %u",
m->m_seq, q->cur_buf, m->m_pkthdr.len);
- SBAPPEND(&so->so_rcv, m);
+ if (m->m_pkthdr.len == 0)
+ q->user_ddp_pending = 0;
+ else
+ SBAPPEND(&so->so_rcv, m);
if (__predict_true((so->so_state & SS_NOFDREF) == 0))
sorwakeup_locked(so);
else
@@ -1840,8 +1837,11 @@
printf("null toep in do_get_tcb_rpl\n");
return (CPL_RET_BUF_DONE);
}
+
+ INP_LOCK(toep->tp_tp->t_inpcb);
tcb_rpl_as_ddp_complete(toep, m);
-
+ INP_UNLOCK(toep->tp_tp->t_inpcb);
+
return (0);
}
@@ -1858,13 +1858,15 @@
if (tp->rcv_nxt == rcv_nxt)
return;
- TRACE_ENTER;
+ INP_LOCK_ASSERT(tp->t_inpcb);
SOCKBUF_LOCK(&so->so_rcv);
q = &toep->tp_ddp_state;
bsp = &q->buf_state[q->cur_buf];
+ KASSERT(SEQ_GT(rcv_nxt, tp->rcv_nxt), ("tp->rcv_nxt=0x%08x decreased rcv_nxt=0x08%x",
+ rcv_nxt, tp->rcv_nxt));
m->m_len = m->m_pkthdr.len = rcv_nxt - tp->rcv_nxt;
KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len));
- printf("rcv_nxt=0x%x tp->rcv_next=0x%x len=%d\n",
+ CTR3(KTR_TOM, "rcv_nxt=0x%x tp->rcv_nxt=0x%x len=%d\n",
rcv_nxt, tp->rcv_nxt, m->m_pkthdr.len);
#ifdef T3_TRACE
@@ -1880,8 +1882,6 @@
if (bsp->flags & DDP_BF_NOCOPY)
bsp->flags &= ~DDP_BF_NOCOPY;
- printf("ddp flags=0x%x\n", m->m_ddp_flags);
-
m->m_seq = tp->rcv_nxt;
tp->rcv_nxt = rcv_nxt;
bsp->cur_offset += m->m_pkthdr.len;
@@ -1893,7 +1893,6 @@
*/
q->ubuf_ddp_ready = 0;
SOCKBUF_UNLOCK(&so->so_rcv);
- TRACE_EXIT;
}
/*
@@ -2017,17 +2016,17 @@
unsigned int ddp_len, rcv_nxt, ddp_report, end_offset, buf_idx;
struct socket *so = toeptoso(toep);
int nomoredata = 0;
+
+ tp = sototcpcb(so);
+ INP_LOCK(tp->t_inpcb);
if (__predict_false(so_no_receive(so))) {
- struct inpcb *inp = sotoinpcb(so);
- INP_LOCK(inp);
handle_excess_rx(toep, m);
- INP_UNLOCK(inp);
+ INP_UNLOCK(tp->t_inpcb);
return;
}
- tp = sototcpcb(so);
q = &toep->tp_ddp_state;
hdr = cplhdr(m);
ddp_report = ntohl(hdr->u.ddp_report);
@@ -2044,33 +2043,32 @@
"new_rx_data_ddp: ddp_report 0x%x",
ddp_report);
#endif
-
+ CTR4(KTR_TOM,
+ "new_rx_data_ddp: tp->rcv_nxt 0x%x cur_offset %u "
+ "hdr seq 0x%x len %u",
+ tp->rcv_nxt, bsp->cur_offset, ntohl(hdr->seq),
+ ntohs(hdr->len));
+ CTR3(KTR_TOM,
+ "new_rx_data_ddp: offset %u ddp_report 0x%x buf_idx=%d",
+ G_DDP_OFFSET(ddp_report), ddp_report, buf_idx);
+
ddp_len = ntohs(hdr->len);
rcv_nxt = ntohl(hdr->seq) + ddp_len;
- /*
- * Overload to store old RCV_NXT
- */
m->m_seq = tp->rcv_nxt;
tp->rcv_nxt = rcv_nxt;
+ tp->t_rcvtime = ticks;
/*
* Store the length in m->m_len. We are changing the meaning of
* m->m_len here, we need to be very careful that nothing from now on
* interprets ->len of this packet the usual way.
*/
- m->m_len = m->m_pkthdr.len = tp->rcv_nxt - m->m_seq;
- /*
- * Length is only meaningful for kbuf
- */
- if (!(bsp->flags & DDP_BF_NOCOPY))
- KASSERT(m->m_len <= bsp->gl->dgl_length,
- ("length received exceeds ddp pages: len=%d dgl_length=%d",
- m->m_len, bsp->gl->dgl_length));
-
- KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len));
- KASSERT(m->m_next == NULL, ("m_len=%p", m->m_next));
-
+ m->m_len = m->m_pkthdr.len = rcv_nxt - m->m_seq;
+ INP_UNLOCK(tp->t_inpcb);
+ CTR3(KTR_TOM,
+ "new_rx_data_ddp: m_len=%u rcv_next 0x%08x rcv_nxt_prev=0x%08x ",
+ m->m_len, rcv_nxt, m->m_seq);
/*
* Figure out where the new data was placed in the buffer and store it
* in when. Assumes the buffer offset starts at 0, consumer needs to
@@ -2078,11 +2076,26 @@
*/
end_offset = G_DDP_OFFSET(ddp_report) + ddp_len;
m->m_cur_offset = end_offset - m->m_pkthdr.len;
+
+ SOCKBUF_LOCK(&so->so_rcv);
m->m_ddp_gl = (unsigned char *)bsp->gl;
m->m_flags |= M_DDP;
bsp->cur_offset = end_offset;
toep->tp_enqueued_bytes += m->m_pkthdr.len;
+
/*
+ * Length is only meaningful for kbuf
+ */
+ if (!(bsp->flags & DDP_BF_NOCOPY))
+ KASSERT(m->m_len <= bsp->gl->dgl_length,
+ ("length received exceeds ddp pages: len=%d dgl_length=%d",
+ m->m_len, bsp->gl->dgl_length));
+
+ KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len));
+ KASSERT(m->m_next == NULL, ("m_len=%p", m->m_next));
+
+
+ /*
* Bit 0 of flags stores whether the DDP buffer is completed.
* Note that other parts of the code depend on this being in bit 0.
*/
@@ -2104,9 +2117,7 @@
if (nomoredata)
m->m_ddp_flags |= DDP_BF_NODATA;
- tp->t_rcvtime = ticks;
- SOCKBUF_LOCK(&so->so_rcv);
SBAPPEND(&so->so_rcv, m);
if ((so->so_state & SS_NOFDREF) == 0)
@@ -2154,24 +2165,29 @@
unsigned int ddp_report, buf_idx, when;
int nomoredata = 0;
+ INP_LOCK(tp->t_inpcb);
if (__predict_false(so_no_receive(so))) {
struct inpcb *inp = sotoinpcb(so);
- INP_LOCK(inp);
handle_excess_rx(toep, m);
INP_UNLOCK(inp);
return;
}
- TRACE_ENTER;
q = &toep->tp_ddp_state;
hdr = cplhdr(m);
ddp_report = ntohl(hdr->ddp_report);
buf_idx = (ddp_report >> S_DDP_BUF_IDX) & 1;
- bsp = &q->buf_state[buf_idx];
+ m->m_pkthdr.csum_data = tp->rcv_nxt;
+
SOCKBUF_LOCK(&so->so_rcv);
+ bsp = &q->buf_state[buf_idx];
when = bsp->cur_offset;
m->m_len = m->m_pkthdr.len = G_DDP_OFFSET(ddp_report) - when;
+ tp->rcv_nxt += m->m_len;
+ tp->t_rcvtime = ticks;
+ INP_UNLOCK(tp->t_inpcb);
+
KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len));
#ifdef T3_TRACE
T3_TRACE5(TIDTB(sk),
@@ -2215,17 +2231,12 @@
if (nomoredata)
m->m_ddp_flags |= DDP_BF_NODATA;
- m->m_pkthdr.csum_data = tp->rcv_nxt;
- tp->rcv_nxt += m->m_len;
-
- tp->t_rcvtime = ticks;
SBAPPEND(&so->so_rcv, m);
if ((so->so_state & SS_NOFDREF) == 0)
sorwakeup_locked(so);
else
SOCKBUF_UNLOCK(&so->so_rcv);
- TRACE_EXIT;
}
/*
@@ -2302,6 +2313,7 @@
return (1);
}
+ INP_LOCK_ASSERT(tp->t_inpcb);
q = &toep->tp_ddp_state;
SOCKBUF_LOCK(&so->so_rcv);
bsp = &q->buf_state[q->cur_buf];
@@ -3215,8 +3227,6 @@
*/
toepcb_hold(newtoep);
syncache_add_accept_req(req, so, newtoep);
-
-
rpl = cplhdr(reply_mbuf);
reply_mbuf->m_pkthdr.len = reply_mbuf->m_len = sizeof(*rpl);
@@ -3241,7 +3251,6 @@
l2t_send(cdev, reply_mbuf, e);
m_free(m);
if (newtoep->tp_ulp_mode) {
- printf("setting ulp mode to DDP\n");
__set_tcb_field(newtoep, ddp_mbuf, W_TCB_RX_DDP_FLAGS,
V_TF_DDP_OFF(1) |
TP_DDP_TIMER_WORKAROUND_MASK,
@@ -3353,8 +3362,6 @@
#if 0
inet_sk(sk)->id = tp->write_seq ^ jiffies;
#endif
-
-
/*
* XXX not clear what rcv_wup maps to
*/
@@ -3485,8 +3492,6 @@
toep->tp_wr_max = toep->tp_wr_avail = TOM_TUNABLE(tdev, max_wrs);
toep->tp_wr_unacked = 0;
toep->tp_qset = G_QNUM(ntohl(m->m_pkthdr.csum_data));
- toep->tp_ulp_mode = TOM_TUNABLE(tdev, ddp) && !(so->so_options & SO_NO_DDP) &&
- tp->rcv_wnd >= MIN_DDP_RCV_WIN ? ULP_MODE_TCPDDP : 0;
toep->tp_qset_idx = 0;
toep->tp_mtu_idx = select_mss(td, tp, toep->tp_l2t->neigh->rt_ifp->if_mtu);
@@ -3499,6 +3504,8 @@
INP_UNLOCK(tp->t_inpcb);
soisconnected(so);
+ CTR1(KTR_TOM, "do_pass_establish tid=%u", toep->tp_tid);
+ cxgb_log_tcb(cdev->adapter, toep->tp_tid);
#ifdef notyet
/*
* XXX not sure how these checks map to us
@@ -3657,6 +3664,9 @@
socket_act_establish(so, m);
INP_UNLOCK(tp->t_inpcb);
+ CTR1(KTR_TOM, "do_act_establish tid=%u", toep->tp_tid);
+ cxgb_log_tcb(cdev->adapter, toep->tp_tid);
+
return (0);
}
@@ -3932,7 +3942,7 @@
{
struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req;
- CTR4(KTR_TOM, "mk_set_tcb_field_ulp(tid=%u word=0x%x mask=%jx val=%jx",
+ CTR4(KTR_TCB, "mk_set_tcb_field_ulp(tid=%u word=0x%x mask=%jx val=%jx",
tid, word, mask, val);
txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
@@ -3987,8 +3997,7 @@
req = (struct cpl_set_tcb_field *)(lock + 1);
- CTR1(KTR_TOM, "t3_cancel_ddpbuf(bufidx=%u)",
- bufidx);
+ CTR1(KTR_TCB, "t3_cancel_ddpbuf(bufidx=%u)", bufidx);
/* Hmmm, not sure if this actually a good thing: reactivating
* the other buffer might be an issue if it has been completed
@@ -4048,7 +4057,7 @@
struct cpl_set_tcb_field *req;
struct ddp_state *p = &toep->tp_ddp_state;
- CTR4(KTR_TOM, "t3_setup_ppods(bufidx=%u tag0=%u tag1=%u len=%u)",
+ CTR4(KTR_TCB, "t3_setup_ppods(bufidx=%u tag0=%u tag1=%u len=%u)",
bufidx, tag0, tag1, len);
SOCKBUF_LOCK_ASSERT(&toeptoso(toep)->so_rcv);
wrlen = sizeof(*wr) + 3 * sizeof(*req) + sizeof(*getreq);
@@ -4123,7 +4132,7 @@
struct work_request_hdr *wr;
struct cpl_set_tcb_field *req;
- CTR6(KTR_TOM, "t3_setup_ddpbufs(len0=%u offset0=%u len1=%u offset1=%u ddp_flags=0x%08x%08x ",
+ CTR6(KTR_TCB, "t3_setup_ddpbufs(len0=%u offset0=%u len1=%u offset1=%u ddp_flags=0x%08x%08x ",
len0, offset0, len1, offset1, ddp_flags >> 32, ddp_flags & 0xffffffff);
SOCKBUF_LOCK_ASSERT(&toeptoso(toep)->so_rcv);
==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_socket.c#18 (text+ko) ====
@@ -575,37 +575,41 @@
uint32_t offset;
int err, flags, avail, len, copied, copied_unacked;
int target; /* Read at least this many bytes */
- int user_ddp_ok, user_ddp_pending = 0;
+ int user_ddp_ok;
struct ddp_state *p;
struct inpcb *inp = sotoinpcb(so);
avail = offset = copied = copied_unacked = 0;
flags = flagsp ? (*flagsp &~ MSG_EOR) : 0;
-
err = sblock(&so->so_rcv, SBLOCKWAIT(flags));
+ p = &toep->tp_ddp_state;
if (err)
return (err);
SOCKBUF_LOCK(&so->so_rcv);
+ p->user_ddp_pending = 0;
restart:
len = uio->uio_resid;
m = so->so_rcv.sb_mb;
target = (flags & MSG_WAITALL) ? len : so->so_rcv.sb_lowat;
- p = &toep->tp_ddp_state;
user_ddp_ok = p->ubuf_ddp_ready;
p->cancel_ubuf = 0;
+ if (len == 0)
+ goto done;
+#if 0
while (m && m->m_len == 0) {
so->so_rcv.sb_mb = m_free(m);
m = so->so_rcv.sb_mb;
}
+#endif
if (m)
goto got_mbuf;
/* empty receive queue */
if (copied >= target && (so->so_rcv.sb_mb == NULL) &&
- !user_ddp_pending)
+ !p->user_ddp_pending)
goto done;
if (copied) {
@@ -629,7 +633,7 @@
goto done;
}
}
- if (so->so_rcv.sb_mb && !user_ddp_pending) {
+ if (so->so_rcv.sb_mb && !p->user_ddp_pending) {
SOCKBUF_UNLOCK(&so->so_rcv);
INP_LOCK(inp);
t3_cleanup_rbuf(tp, copied_unacked);
@@ -638,22 +642,21 @@
copied_unacked = 0;
goto restart;
}
- if (p->kbuf[0] && user_ddp_ok && !user_ddp_pending &&
+ if (p->kbuf[0] && user_ddp_ok && !p->user_ddp_pending &&
uio->uio_iov->iov_len > p->kbuf[0]->dgl_length &&
p->ubuf_ddp_ready) {
- user_ddp_pending =
+ p->user_ddp_pending =
!t3_overlay_ubuf(so, uio, IS_NONBLOCKING(so), flags, 1, 1);
- if (user_ddp_pending) {
+ if (p->user_ddp_pending) {
p->kbuf_posted++;
user_ddp_ok = 0;
}
- printf("user_ddp_pending=%d\n", user_ddp_pending);
}
if (p->kbuf[0] && (p->kbuf_posted == 0)) {
t3_post_kbuf(so, 1);
p->kbuf_posted++;
}
- if (user_ddp_pending) {
+ if (p->user_ddp_pending) {
/* One shot at DDP if we already have enough data */
if (copied >= target)
user_ddp_ok = 0;
@@ -694,8 +697,8 @@
if (m->m_pkthdr.len == 0) {
if ((m->m_ddp_flags & DDP_BF_NOCOPY) == 0)
panic("empty mbuf and NOCOPY not set\n");
- printf("dropping empty mbuf\n");
- user_ddp_pending = 0;
+ CTR0(KTR_TOM, "ddp done notification");
+ p->user_ddp_pending = 0;
sbdroprecord_locked(&so->so_rcv);
goto done;
}
@@ -712,9 +715,11 @@
avail = m->m_pkthdr.len - offset;
if (len < avail) {
if (is_ddp(m) && (m->m_ddp_flags & DDP_BF_NOCOPY))
- panic("bad state in t3_soreceive\n");
+ panic("bad state in t3_soreceive len=%d avail=%d offset=%d\n", len, avail, offset);
avail = len;
}
+ CTR4(KTR_TOM, "t3_soreceive: m_len=%u offset=%u len=%u m_seq=0%08x", m->m_pkthdr.len, offset, len, m->m_seq);
+
#ifdef URGENT_DATA_SUPPORTED
/*
* Check if the data we are preparing to copy contains urgent
@@ -746,25 +751,26 @@
#endif
}
- if (user_ddp_ok && !user_ddp_pending &&
+ if (user_ddp_ok && !p->user_ddp_pending &&
uio->uio_iov->iov_len > p->kbuf[0]->dgl_length &&
p->ubuf_ddp_ready) {
- user_ddp_pending =
+ p->user_ddp_pending =
!t3_overlay_ubuf(so, uio, IS_NONBLOCKING(so), flags, 1, 1);
- if (user_ddp_pending) {
+ if (p->user_ddp_pending) {
p->kbuf_posted++;
user_ddp_ok = 0;
}
- DPRINTF("user_ddp_pending=%d\n", user_ddp_pending);
+ DPRINTF("user_ddp_pending=%d\n", p->user_ddp_pending);
} else
DPRINTF("user_ddp_ok=%d user_ddp_pending=%d iov_len=%ld dgl_length=%d ubuf_ddp_ready=%d ulp_mode=%d is_ddp(m)=%d flags=0x%x ubuf=%p kbuf_posted=%d\n",
- user_ddp_ok, user_ddp_pending, uio->uio_iov->iov_len, p->kbuf[0] ? p->kbuf[0]->dgl_length : 0,
+ user_ddp_ok, p->user_ddp_pending, uio->uio_iov->iov_len, p->kbuf[0] ? p->kbuf[0]->dgl_length : 0,
p->ubuf_ddp_ready, toep->tp_ulp_mode, !!is_ddp(m), m->m_ddp_flags, p->ubuf, p->kbuf_posted);
/*
* If MSG_TRUNC is specified the data is discarded.
* XXX need to check pr_atomic
*/
+ KASSERT(avail > 0, ("avail=%d resid=%d offset=%d", avail, uio->uio_resid, offset));
if (__predict_true(!(flags & MSG_TRUNC))) {
int resid = uio->uio_resid;
@@ -800,11 +806,11 @@
struct mbuf *nextrecord;
if (p->kbuf[0] != NULL && is_ddp(m) && (fl & 1)) {
- if (is_ddp_psh(m) && user_ddp_pending)
+ if (is_ddp_psh(m) && p->user_ddp_pending)
got_psh = 1;
if (fl & DDP_BF_NOCOPY)
- user_ddp_pending = 0;
+ p->user_ddp_pending = 0;
else if ((fl & DDP_BF_NODATA) && IS_NONBLOCKING(so)) {
p->kbuf_posted--;
nomoredata = 1;
@@ -828,10 +834,8 @@
sbdrop_locked(&so->so_rcv, m->m_pkthdr.len);
#endif
exitnow = got_psh || nomoredata;
- if ((so->so_rcv.sb_mb == NULL) && exitnow) {
- printf("exiting\n");
+ if ((so->so_rcv.sb_mb == NULL) && exitnow)
goto done;
- }
if (copied_unacked > (so->so_rcv.sb_hiwat >> 2)) {
SOCKBUF_UNLOCK(&so->so_rcv);
INP_LOCK(inp);
@@ -841,19 +845,17 @@
SOCKBUF_LOCK(&so->so_rcv);
}
}
- if (len > 0)
+ if (len > 0)
goto restart;
-
done:
-
/*
* If we can still receive decide what to do in preparation for the
* next receive. Note that RCV_SHUTDOWN is set if the connection
* transitioned to CLOSE but not if it was in that state to begin with.
*/
if (__predict_true((so->so_state & (SS_ISDISCONNECTING|SS_ISDISCONNECTED)) == 0)) {
- if (user_ddp_pending) {
+ if (p->user_ddp_pending) {
SOCKBUF_UNLOCK(&so->so_rcv);
SOCKBUF_LOCK(&so->so_rcv);
user_ddp_ok = 0;
@@ -861,9 +863,10 @@
if (so->so_rcv.sb_mb) {
if (copied < 0)
copied = 0;
- goto restart;
+ if (len > 0)
+ goto restart;
}
- user_ddp_pending = 0;
+ p->user_ddp_pending = 0;
}
if ((p->kbuf[0] != NULL) && (p->kbuf_posted == 0)) {
#ifdef T3_TRACE
@@ -886,7 +889,7 @@
"chelsio_recvmsg <-: copied %d len %d buffers_freed %d "
"kbuf_posted %d user_ddp_pending %u",
copied, len, buffers_freed, p ? p->kbuf_posted : -1,
- user_ddp_pending);
+ p->user_ddp_pending);
#endif
SOCKBUF_UNLOCK(&so->so_rcv);
done_unlocked:
==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_ddp.c#8 (text+ko) ====
@@ -156,7 +156,6 @@
p->dgl_length = len;
p->dgl_offset = pg_off;
p->dgl_nelem = npages;
- printf("hold %jx -> %jx --- count=%d i=%d\n", addr, addr + npages*PAGE_SIZE, npages, i);
#ifdef NEED_BUSDMA
p->phys_addr[0] = pci_map_page(pdev, p->pages[0], pg_off,
@@ -414,7 +413,7 @@
so->so_rcv.sb_timeo = 30*hz;
while (p->get_tcb_count && !(so->so_state & (SS_ISDISCONNECTING|SS_ISDISCONNECTED))) {
if (count & 0xffffff)
- printf("waiting err=%d get_tcb_count=%d timeo=%d so=%p\n",
+ CTR4(KTR_TCB, "waiting err=%d get_tcb_count=%d timeo=%d so=%p\n",
err, p->get_tcb_count, so->so_rcv.sb_timeo, so);
count++;
err = sbwait(&so->so_rcv);
@@ -496,10 +495,13 @@
" kbuf_idx %d",
p->ubuf_tag, flags, OVERLAY_MASK, ubuf_idx, p->kbuf_idx);
#endif
- CTR5(KTR_TOM,
- "t3_overlay_ubuf: tag %u flags 0x%x mask 0x%x ubuf_idx %d "
- " kbuf_idx %d",
- p->ubuf_tag, flags, OVERLAY_MASK, ubuf_idx, p->kbuf_idx);
+ CTR3(KTR_TOM,
+ "t3_overlay_ubuf: tag %u flags 0x%x mask 0x%x",
+ p->ubuf_tag, flags, OVERLAY_MASK);
+ CTR3(KTR_TOM,
+ "t3_overlay_ubuf: ubuf_idx %d kbuf_idx %d post_kbuf %d",
+ ubuf_idx, p->kbuf_idx, post_kbuf);
+
return (0);
}
@@ -561,11 +563,9 @@
struct toepcb *toep = sototcpcb(so)->t_toe;
struct ddp_state *p = &toep->tp_ddp_state;
- TRACE_ENTER;
t3_set_ddp_tag(so, p->cur_buf, p->kbuf_tag[p->cur_buf] << 6);
t3_set_ddp_buf(so, p->cur_buf, 0, p->kbuf[p->cur_buf]->dgl_length);
t3_repost_kbuf(so, p->cur_buf, modulate, 1, (so->so_state & SS_NBIO));
- TRACE_EXIT;
#ifdef T3_TRACE
T3_TRACE1(TIDTB(so),
"t3_post_kbuf: cur_buf = kbuf_idx = %u ", p->cur_buf);
@@ -640,6 +640,8 @@
t3_setup_ppods(so, p->kbuf[idx], nppods, p->kbuf_tag[idx],
p->kbuf[idx]->dgl_length, 0, 0);
}
+ cxgb_log_tcb(TOEP_T3C_DEV(toep)->adapter, toep->tp_tid);
+
t3_set_ddp_tag(so, 0, p->kbuf_tag[0] << 6);
t3_set_ddp_buf(so, 0, 0, p->kbuf[0]->dgl_length);
t3_repost_kbuf(so, 0, 0, 1, nonblock);
@@ -655,7 +657,8 @@
CTR4(KTR_TOM,
"t3_enter_ddp: kbuf_size %u waitall %u tag0 %d tag1 %d",
kbuf_size, waitall, p->kbuf_tag[0], p->kbuf_tag[1]);
-
+ DELAY(100000);
+ cxgb_log_tcb(TOEP_T3C_DEV(toep)->adapter, toep->tp_tid);
return (0);
err:
==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_defs.h#7 (text+ko) ====
@@ -44,6 +44,7 @@
#define TRACE_EXIT printf("%s:%s:%d exited\n", __FUNCTION__, __FILE__, __LINE__)
#define KTR_TOM KTR_SPARE2
+#define KTR_TCB KTR_SPARE3
struct toepcb;
struct listen_ctx;
==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_listen.c#2 (text+ko) ====
@@ -246,26 +246,20 @@
printf("start listen\n");
- ctx = malloc(sizeof(*ctx), M_CXGB, M_NOWAIT);
+ ctx = malloc(sizeof(*ctx), M_CXGB, M_NOWAIT|M_ZERO);
if (!ctx)
return;
ctx->tom_data = d;
ctx->lso = so;
- ctx->ulp_mode = 0; /* DDP if the default */
+ ctx->ulp_mode = TOM_TUNABLE(dev, ddp) && !(so->so_options & SO_NO_DDP) ? ULP_MODE_TCPDDP : 0;
LIST_INIT(&ctx->synq_head);
stid = cxgb_alloc_stid(d->cdev, d->client, ctx);
if (stid < 0)
goto free_ctx;
-#ifdef notyet
- /*
- * XXX need to mark inpcb as referenced
- */
- sock_hold(sk);
-#endif
m = m_gethdr(M_NOWAIT, MT_DATA);
if (m == NULL)
goto free_stid;
==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_t3_ddp.h#10 (text+ko) ====
@@ -115,6 +115,7 @@
int get_tcb_count;
unsigned int kbuf_posted;
int cancel_ubuf;
+ int user_ddp_pending;
unsigned int kbuf_nppods[NUM_DDP_KBUF];
unsigned int kbuf_tag[NUM_DDP_KBUF];
struct ddp_gather_list *kbuf[NUM_DDP_KBUF]; /* kernel buffer for DDP prefetch */
==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_tom.c#8 (text+ko) ====
@@ -167,12 +167,9 @@
toepcb_release(struct toepcb *toep)
{
if (toep->tp_refcount == 1) {
- printf("doing final toepcb free\n");
-
free(toep, M_DEVBUF);
return;
}
-
atomic_add_acq_int(&toep->tp_refcount, -1);
}
More information about the p4-projects
mailing list