PERFORCE change 134631 for review
Kip Macy
kmacy at FreeBSD.org
Fri Feb 1 21:34:29 PST 2008
http://perforce.freebsd.org/chv.cgi?CH=134631
Change 134631 by kmacy at kmacy:storage:toehead on 2008/02/02 05:33:56
remove most tracing
fix ddp buffer calculations
dramatically reduce debug noise
add DDP_BF_DATA handling
Affected files ...
.. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c#13 edit
.. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_socket.c#14 edit
.. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_ddp.c#6 edit
.. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_t3_ddp.h#9 edit
.. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_tom.c#4 edit
Differences ...
==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c#13 (text+ko) ====
@@ -1616,7 +1616,6 @@
uint64_t t;
__be64 *tcb;
- TRACE_ENTER;
/* Note that we only accout for CPL_GET_TCB issued by the DDP code. We
* really need a cookie in order to dispatch the RPLs.
*/
@@ -1633,7 +1632,6 @@
m_freem(m);
if (__predict_true((so->so_state & SS_NOFDREF) == 0))
sorwakeup(so);
- TRACE_EXIT;
return;
}
@@ -1692,7 +1690,6 @@
if (__predict_false(so_no_receive(so) && m->m_pkthdr.len)) {
handle_excess_rx(toep, m);
- TRACE_EXIT;
return;
}
@@ -1712,7 +1709,7 @@
}
#endif
m->m_ddp_flags = DDP_BF_PSH | DDP_BF_NOCOPY | 1;
- bsp->flags &= ~DDP_BF_NOCOPY;
+ bsp->flags &= ~(DDP_BF_NOCOPY|DDP_BF_NODATA);
q->cur_buf ^= 1;
} else if (bsp->flags & DDP_BF_NOFLIP) {
@@ -1728,11 +1725,11 @@
* and we need to decrement the posted count.
*/
if (m->m_pkthdr.len == 0) {
- if (ddp_offset == 0)
+ if (ddp_offset == 0) {
q->kbuf_posted--;
- panic("length not set");
+ bsp->flags |= DDP_BF_NODATA;
+ }
m_free(m);
- TRACE_EXIT;
return;
}
} else {
@@ -1740,7 +1737,6 @@
* but it got here way late and nobody cares anymore.
*/
m_free(m);
- TRACE_EXIT;
return;
}
@@ -1762,7 +1758,6 @@
sbappend(&so->so_rcv, m);
if (__predict_true((so->so_state & SS_NOFDREF) == 0))
sorwakeup(so);
- TRACE_EXIT;
}
/*
@@ -1775,9 +1770,10 @@
struct toepcb *toep = (struct toepcb *)ctx;
/* OK if socket doesn't exist */
- if (toep == NULL)
+ if (toep == NULL) {
+ printf("null toep in do_get_tcb_rpl\n");
return (CPL_RET_BUF_DONE);
-
+ }
tcb_rpl_as_ddp_complete(toep, m);
return (0);
@@ -1950,7 +1946,8 @@
struct cpl_rx_data_ddp *hdr;
unsigned int ddp_len, rcv_nxt, ddp_report, end_offset, buf_idx;
struct socket *so = toeptoso(toep);
-
+ int nomoredata = 0;
+
if (__predict_false(so_no_receive(so))) {
struct inpcb *inp = sotoinpcb(so);
@@ -1960,7 +1957,6 @@
return;
}
- TRACE_ENTER;
tp = sototcpcb(so);
q = &toep->tp_ddp_state;
hdr = cplhdr(m);
@@ -1994,6 +1990,13 @@
* interprets ->len of this packet the usual way.
*/
m->m_len = m->m_pkthdr.len = tp->rcv_nxt - m->m_seq;
+ /*
+ * Length is only meaningful for kbuf
+ */
+ if (!(bsp->flags & DDP_BF_NOCOPY))
+ KASSERT(m->m_len <= bsp->gl->dgl_length,
+ ("length received exceeds ddp pages: len=%d dgl_length=%d",
+ m->m_len, bsp->gl->dgl_length));
/*
* Figure out where the new data was placed in the buffer and store it
@@ -2014,7 +2017,7 @@
panic("spurious ddp completion");
} else {
m->m_ddp_flags = !!(ddp_report & F_DDP_BUF_COMPLETE);
- if (m->m_ddp_flags && !(bsp->flags & DDP_BF_NOFLIP))
+ if (m->m_ddp_flags && !(bsp->flags & DDP_BF_NOFLIP))
q->cur_buf ^= 1; /* flip buffers */
}
@@ -2025,11 +2028,11 @@
if (ddp_report & F_DDP_PSH)
m->m_ddp_flags |= DDP_BF_PSH;
-
+ if (nomoredata)
+ m->m_ddp_flags |= DDP_BF_NODATA;
+
tp->t_rcvtime = ticks;
- printf("ddp set and ddp_flags=0x%x len=%d m_seq=0x%x rcv_nxt=0x%x\n", m->m_ddp_flags, m->m_len, m->m_seq, rcv_nxt);
-
SOCKBUF_LOCK(&so->so_rcv);
sbappendstream_locked(&so->so_rcv, m);
@@ -2037,7 +2040,6 @@
sorwakeup_locked(so);
else
SOCKBUF_UNLOCK(&so->so_rcv);
- TRACE_EXIT;
}
#define DDP_ERR (F_DDP_PPOD_MISMATCH | F_DDP_LLIMIT_ERR | F_DDP_ULIMIT_ERR |\
@@ -2077,6 +2079,7 @@
struct ddp_buf_state *bsp;
struct cpl_rx_ddp_complete *hdr;
unsigned int ddp_report, buf_idx, when;
+ int nomoredata = 0;
if (__predict_false(so_no_receive(so))) {
struct inpcb *inp = sotoinpcb(so);
@@ -2094,7 +2097,7 @@
bsp = &q->buf_state[buf_idx];
when = bsp->cur_offset;
- m->m_pkthdr.len = G_DDP_OFFSET(ddp_report) - when;
+ m->m_len = m->m_pkthdr.len = G_DDP_OFFSET(ddp_report) - when;
#ifdef T3_TRACE
T3_TRACE5(TIDTB(sk),
@@ -2106,9 +2109,12 @@
bsp->cur_offset += m->m_len;
- if (!(bsp->flags & DDP_BF_NOFLIP))
+ if (!(bsp->flags & DDP_BF_NOFLIP)) {
q->cur_buf ^= 1; /* flip buffers */
-
+ if (G_DDP_OFFSET(ddp_report) < q->kbuf[0]->dgl_length)
+ nomoredata=1;
+ }
+
#ifdef T3_TRACE
T3_TRACE4(TIDTB(sk),
"process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u "
@@ -2118,9 +2124,12 @@
#endif
m->m_ddp_gl = (unsigned char *)bsp->gl;
m->m_flags |= M_DDP;
- m->m_pkthdr.csum_flags = (bsp->flags & DDP_BF_NOCOPY) | 1;
+ m->m_ddp_flags = (bsp->flags & DDP_BF_NOCOPY) | 1;
if (bsp->flags & DDP_BF_NOCOPY)
bsp->flags &= ~DDP_BF_NOCOPY;
+ if (nomoredata)
+ m->m_ddp_flags |= DDP_BF_NODATA;
+
m->m_pkthdr.csum_data = tp->rcv_nxt;
tp->rcv_nxt += m->m_len;
@@ -2209,7 +2218,7 @@
q = &toep->tp_ddp_state;
bsp = &q->buf_state[q->cur_buf];
- m->m_pkthdr.len = rcv_nxt - tp->rcv_nxt;
+ m->m_len = m->m_pkthdr.len = rcv_nxt - tp->rcv_nxt;
m->m_ddp_gl = (unsigned char *)bsp->gl;
m->m_flags |= M_DDP;
m->m_cur_offset = bsp->cur_offset;
@@ -3946,7 +3955,6 @@
struct cpl_set_tcb_field *req;
struct ddp_state *p = &toep->tp_ddp_state;
- TRACE_ENTER;
wrlen = sizeof(*wr) + 3 * sizeof(*req) + sizeof(*getreq);
m = m_gethdr_nofail(wrlen);
m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep));
@@ -4001,7 +4009,6 @@
bufidx, tag0, tag1, len);
#endif
cxgb_ofld_send(TOEP_T3C_DEV(toep), m);
- TRACE_EXIT;
}
/*
==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_socket.c#14 (text+ko) ====
@@ -256,7 +256,7 @@
so_should_ddp(const struct toepcb *toep, int last_recv_len)
{
- printf("ulp_mode=%d last_recv_len=%d ddp_thresh=%d rcv_wnd=%ld ddp_copy_limit=%d\n",
+ DPRINTF("ulp_mode=%d last_recv_len=%d ddp_thresh=%d rcv_wnd=%ld ddp_copy_limit=%d\n",
toep->tp_ulp_mode, last_recv_len, TOM_TUNABLE(toep->tp_toedev, ddp_thres),
toep->tp_tp->rcv_wnd, (TOM_TUNABLE(toep->tp_toedev, ddp_copy_limit) + DDP_RSVD_WIN));
@@ -330,17 +330,13 @@
if (__predict_true(!is_ddp(m))) { /* RX_DATA */
return m_uiomove(m, offset, len, uio);
} if (__predict_true(m->m_ddp_flags & DDP_BF_NOCOPY)) { /* user DDP */
- TRACE_ENTER;
to->iov_len -= len;
to->iov_base = ((caddr_t)to->iov_base) + len;
uio->uio_iov = to;
uio->uio_resid -= len;
- TRACE_EXIT;
return (0);
}
- TRACE_ENTER;
err = t3_ddp_copy(m, offset, uio, len); /* kernel DDP */
- TRACE_EXIT;
return (err);
}
@@ -607,7 +603,7 @@
uio->uio_iov->iov_len > p->kbuf[0]->dgl_length &&
p->ubuf_ddp_ready) {
user_ddp_pending =
- !t3_overlay_ubuf(so, uio, 1, 1);
+ !t3_overlay_ubuf(so, uio, IS_NONBLOCKING(so), flags, 1, 1);
if (user_ddp_pending) {
p->kbuf_posted++;
user_ddp_ok = 0;
@@ -635,7 +631,9 @@
INP_UNLOCK(inp);
SOCKBUF_LOCK(&so->so_rcv);
copied_unacked = 0;
- printf("sbwaiting 2\n");
+ if (so->so_rcv.sb_mb)
+ goto restart;
+ printf("sbwaiting 2 copied=%d target=%d\n", copied, target);
if ((err = sbwait(&so->so_rcv)) != 0)
goto done;
}
@@ -650,8 +648,8 @@
goto done;
}
offset = toep->tp_copied_seq + copied_unacked - m->m_seq;
- printf("m=%p copied_seq=0x%x copied_unacked=%d m_seq=0x%x offset=%d pktlen=%d is_ddp(m)=%d\n",
- m, toep->tp_copied_seq, copied_unacked, m->m_seq, offset, m->m_pkthdr.len, is_ddp(m));
+ DPRINTF("m=%p copied_seq=0x%x copied_unacked=%d m_seq=0x%x offset=%d pktlen=%d is_ddp(m)=%d\n",
+ m, toep->tp_copied_seq, copied_unacked, m->m_seq, offset, m->m_pkthdr.len, !!is_ddp(m));
if (offset >= m->m_pkthdr.len)
panic("t3_soreceive: OFFSET >= LEN offset %d copied_seq 0x%x seq 0x%x "
"pktlen %d ddp flags 0x%x", offset, toep->tp_copied_seq + copied_unacked, m->m_seq,
@@ -697,14 +695,14 @@
uio->uio_iov->iov_len > p->kbuf[0]->dgl_length &&
p->ubuf_ddp_ready) {
user_ddp_pending =
- !t3_overlay_ubuf(so, uio, 1, 1);
+ !t3_overlay_ubuf(so, uio, IS_NONBLOCKING(so), flags, 1, 1);
if (user_ddp_pending) {
p->kbuf_posted++;
user_ddp_ok = 0;
}
- printf("user_ddp_pending=%d\n", user_ddp_pending);
+ DPRINTF("user_ddp_pending=%d\n", user_ddp_pending);
} else
- printf("user_ddp_ok=%d user_ddp_pending=%d iov_len=%ld dgl_length=%d ubuf_ddp_ready=%d ulp_mode=%d is_ddp(m)=%d flags=0x%x ubuf=%p kbuf_posted=%d\n",
+ DPRINTF("user_ddp_ok=%d user_ddp_pending=%d iov_len=%ld dgl_length=%d ubuf_ddp_ready=%d ulp_mode=%d is_ddp(m)=%d flags=0x%x ubuf=%p kbuf_posted=%d\n",
user_ddp_ok, user_ddp_pending, uio->uio_iov->iov_len, p->kbuf[0] ? p->kbuf[0]->dgl_length : 0,
p->ubuf_ddp_ready, toep->tp_ulp_mode, !!is_ddp(m), m->m_ddp_flags, p->ubuf, p->kbuf_posted);
@@ -714,6 +712,7 @@
*/
if (__predict_true(!(flags & MSG_TRUNC))) {
int resid = uio->uio_resid;
+
SOCKBUF_UNLOCK(&so->so_rcv);
if ((err = copy_data(m, offset, avail, uio))) {
if (err)
@@ -742,7 +741,7 @@
*/
if (avail + offset >= m->m_pkthdr.len) {
unsigned int fl = m->m_ddp_flags;
- int got_psh = 0;
+ int exitnow, got_psh = 0, nomoredata = 0;
if (p->kbuf[0] != NULL && is_ddp(m) && (fl & 1)) {
if (is_ddp_psh(m) && user_ddp_pending)
@@ -750,14 +749,17 @@
if (fl & DDP_BF_NOCOPY)
user_ddp_pending = 0;
- else {
+ else if ((fl & DDP_BF_NODATA) && IS_NONBLOCKING(so)) {
+ p->kbuf_posted--;
+ nomoredata = 1;
+ } else {
p->kbuf_posted--;
p->ubuf_ddp_ready = 1;
- printf("ubuf ddp ready\n");
}
}
- if ((so->so_rcv.sb_mb == NULL) && got_psh)
+ exitnow = got_psh || nomoredata;
+ if ((so->so_rcv.sb_mb == NULL) && exitnow)
goto done;
}
if (len > 0)
@@ -771,6 +773,8 @@
* transitioned to CLOSE but not if it was in that state to begin with.
*/
if (__predict_true((so->so_state & (SS_ISDISCONNECTING|SS_ISDISCONNECTED)) == 0)) {
+ SOCKBUF_UNLOCK(&so->so_rcv);
+ SOCKBUF_LOCK(&so->so_rcv);
if (user_ddp_pending) {
user_ddp_ok = 0;
t3_cancel_ubuf(toep);
@@ -787,14 +791,10 @@
"chelsio_recvmsg: about to exit, repost kbuf");
#endif
- printf("posting kbuf\n");
t3_post_kbuf(so, 1);
p->kbuf_posted++;
} else if (so_should_ddp(toep, copied)
-#ifdef notyet
- && !IS_NONBLOCKING(so)
-#endif
- ) {
+ && !IS_NONBLOCKING(so)) {
printf("entering ddp\n");
t3_enter_ddp(so, TOM_TUNABLE(TOE_DEV(so),
ddp_copy_limit), 0);
==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_ddp.c#6 (text+ko) ====
@@ -116,11 +116,11 @@
* a new gather list was allocated it is returned in @newgl.
*/
static int
-t3_pin_pages(bus_dma_tag_t tag, bus_dmamap_t map, unsigned long addr,
+t3_pin_pages(bus_dma_tag_t tag, bus_dmamap_t map, vm_offset_t addr,
size_t len, struct ddp_gather_list **newgl,
const struct ddp_gather_list *gl)
{
- int i, err;
+ int i = 0, err;
size_t pg_off;
unsigned int npages;
struct ddp_gather_list *p;
@@ -131,7 +131,7 @@
if (addr + len > VM_MAXUSER_ADDRESS)
return (EFAULT);
- pg_off = addr & ~PAGE_MASK;
+ pg_off = addr & PAGE_MASK;
npages = (pg_off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
p = malloc(sizeof(struct ddp_gather_list) + npages * sizeof(vm_page_t *),
M_DEVBUF, M_NOWAIT);
@@ -139,13 +139,12 @@
return (ENOMEM);
err = vm_fault_hold_user_pages(addr, p->dgl_pages, npages, VM_HOLD_WRITEABLE);
- printf("held pages\n");
if (err)
goto free_gl;
if (gl && gl->dgl_offset == pg_off && gl->dgl_nelem >= npages &&
gl->dgl_length >= len) {
- for (i = 0; i < npages; ++i)
+ for (i = 0; i < npages; i++)
if (p->dgl_pages[i] != gl->dgl_pages[i])
goto different_gl;
err = 0;
@@ -156,6 +155,8 @@
p->dgl_length = len;
p->dgl_offset = pg_off;
p->dgl_nelem = npages;
+ printf("hold %jx -> %jx --- count=%d i=%d\n", addr, addr + npages*PAGE_SIZE, npages, i);
+
#ifdef NEED_BUSDMA
p->phys_addr[0] = pci_map_page(pdev, p->pages[0], pg_off,
PAGE_SIZE - pg_off,
@@ -165,14 +166,15 @@
PCI_DMA_FROMDEVICE);
#endif
*newgl = p;
- return 0;
+ return (0);
unpin:
vm_fault_unhold_pages(p->dgl_pages, npages);
free_gl:
+
free(p, M_DEVBUF);
*newgl = NULL;
- return err;
+ return (err);
}
static void
@@ -197,7 +199,7 @@
ddp_gl_free_pages(struct ddp_gather_list *gl, int dirty)
{
/*
- * XXX need to be able to
+ * XXX mark pages as dirty before unholding
*/
vm_fault_unhold_pages(gl->dgl_pages, gl->dgl_nelem);
}
@@ -227,7 +229,7 @@
int err, tag, npages, nppods;
struct tom_data *d = TOM_DATA(TOE_DEV(so));
- npages = ((addr & ~PAGE_MASK) + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
+ npages = ((addr & PAGE_MASK) + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
nppods = min(pages2ppods(npages), MAX_PPODS);
nppods = roundup2(nppods, PPOD_CLUSTER_SIZE);
err = t3_alloc_ppods(d, nppods, &tag);
@@ -256,11 +258,10 @@
*/
static void
t3_repost_kbuf(struct socket *so, unsigned int bufidx, int modulate,
- int activate)
+ int activate, int nonblock)
{
struct toepcb *toep = sototcpcb(so)->t_toe;
struct ddp_state *p = &toep->tp_ddp_state;
- TRACE_ENTER;
p->buf_state[bufidx].cur_offset = p->kbuf[bufidx]->dgl_offset;
p->buf_state[bufidx].flags = p->kbuf_noinval ? DDP_BF_NOINVAL : 0;
@@ -283,7 +284,6 @@
V_TF_DDP_BUF1_VALID(1) | V_TF_DDP_ACTIVE_BUF(1),
modulate);
- TRACE_EXIT;
}
/*
@@ -294,7 +294,7 @@
static __inline unsigned long
select_ddp_flags(const struct socket *so, int buf_idx,
- int nonblock, int rcv_flags)
+ int nonblock, int rcv_flags)
{
if (buf_idx == 1) {
if (__predict_false(rcv_flags & MSG_WAITALL))
@@ -338,24 +338,26 @@
struct toepcb *toep = sototcpcb(so)->t_toe;
struct ddp_state *p = &toep->tp_ddp_state;
struct iovec *iov = uio->uio_iov;
- unsigned long addr = (unsigned long)iov->iov_base - oft;
+ vm_offset_t addr = (vm_offset_t)iov->iov_base - oft;
- if (__predict_false(!p->ubuf_nppods)) {
+ if (__predict_false(p->ubuf_nppods == 0)) {
err = alloc_buf1_ppods(so, p, addr, iov->iov_len + oft);
if (err)
return (err);
}
len = (p->ubuf_nppods - NUM_SENTINEL_PPODS) * PPOD_PAGES * PAGE_SIZE;
- len -= addr & ~PAGE_MASK;
+ len -= addr & PAGE_MASK;
if (len > M_TCB_RX_DDP_BUF0_LEN)
len = M_TCB_RX_DDP_BUF0_LEN;
len = min(len, sototcpcb(so)->rcv_wnd - 32768);
len = min(len, iov->iov_len + oft);
- if (len <= p->kbuf[0]->dgl_length)
+ if (len <= p->kbuf[0]->dgl_length) {
+ printf("length too short\n");
return (EINVAL);
-
+ }
+
err = t3_pin_pages(toep->tp_rx_dmat, toep->tp_dmamap, addr, len, &gl, p->ubuf);
if (err)
return (err);
@@ -377,7 +379,6 @@
t3_cancel_ubuf(struct toepcb *toep)
{
struct ddp_state *p = &toep->tp_ddp_state;
- long timeo = MAX_SCHEDULE_TIMEOUT;
int ubuf_pending = t3_ddp_ubuf_pending(toep);
struct socket *so = toeptoso(toep);
@@ -393,12 +394,24 @@
p->buf_state[1].flags & (DDP_BF_NOFLIP | DDP_BF_NOCOPY),
p->get_tcb_count);
#endif
- if (!p->get_tcb_count)
+ if (p->get_tcb_count == 0)
t3_cancel_ddpbuf(toep, p->cur_buf);
- do {
+ else {
+ int err = 0, timeo, flags, count=0;
+ timeo = so->so_rcv.sb_timeo;
+ flags = so->so_rcv.sb_flags;
- cv_timedwait(&toep->tp_cv, &toep->tp_tp->t_inpcb->inp_mtx, timeo);
- } while (p->get_tcb_count && !(so->so_state & (SS_ISDISCONNECTING|SS_ISDISCONNECTED)));
+ so->so_rcv.sb_timeo = 3*hz;
+ while (p->get_tcb_count && !(so->so_state & (SS_ISDISCONNECTING|SS_ISDISCONNECTED))) {
+ if (count & 0xff)
+ printf("waiting err=%d get_tcb_count=%d timeo=%d so=%p\n",
+ err, p->get_tcb_count, so->so_rcv.sb_timeo, so);
+ count++;
+ err = sbwait(&so->so_rcv);
+ }
+ so->so_rcv.sb_timeo = timeo;
+ so->so_rcv.sb_flags = flags;
+ }
ubuf_pending = t3_ddp_ubuf_pending(toep);
}
@@ -417,24 +430,19 @@
*/
int
t3_overlay_ubuf(struct socket *so, const struct uio *uio,
- int nonblock, int rcv_flags)
+ int nonblock, int rcv_flags, int modulate, int post_kbuf)
{
int err, len, ubuf_idx;
unsigned long flags;
struct toepcb *toep = sototcpcb(so)->t_toe;
struct ddp_state *p = &toep->tp_ddp_state;
- struct ddp_buf_state *dbs;
-
- TRACE_ENTER;
if (p->kbuf[0] == NULL) {
- TRACE_EXIT;
return (EINVAL);
}
err = setup_uio_ppods(so, uio, 0, &len);
if (err) {
- TRACE_EXIT;
return (err);
}
@@ -446,16 +454,18 @@
flags = select_ddp_flags(so, ubuf_idx, nonblock, rcv_flags);
- dbs = &p->buf_state[ubuf_idx ^ 1];
-
- dbs->cur_offset = 0;
- dbs->flags = 0;
- dbs->gl = p->kbuf[ubuf_idx ^ 1];
- p->kbuf_idx ^= 1;
- flags |= p->kbuf_idx ?
- V_TF_DDP_BUF1_VALID(1) | V_TF_DDP_PUSH_DISABLE_1(0) :
- V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_PUSH_DISABLE_0(0);
-
+ if (post_kbuf) {
+ struct ddp_buf_state *dbs = &p->buf_state[ubuf_idx ^ 1];
+
+ dbs->cur_offset = 0;
+ dbs->flags = 0;
+ dbs->gl = p->kbuf[ubuf_idx ^ 1];
+ p->kbuf_idx ^= 1;
+ flags |= p->kbuf_idx ?
+ V_TF_DDP_BUF1_VALID(1) | V_TF_DDP_PUSH_DISABLE_1(0) :
+ V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_PUSH_DISABLE_0(0);
+ }
+
if (ubuf_idx == 0) {
t3_overlay_ddpbuf(toep, 0, p->ubuf_tag << 6, p->kbuf_tag[1] << 6,
len);
@@ -475,7 +485,6 @@
" kbuf_idx %d",
p->ubuf_tag, flags, OVERLAY_MASK, ubuf_idx, p->kbuf_idx);
#endif
- TRACE_EXIT;
return (0);
}
@@ -540,7 +549,7 @@
TRACE_ENTER;
t3_set_ddp_tag(so, p->cur_buf, p->kbuf_tag[p->cur_buf] << 6);
t3_set_ddp_buf(so, p->cur_buf, 0, p->kbuf[p->cur_buf]->dgl_length);
- t3_repost_kbuf(so, p->cur_buf, modulate, (so->so_state & SS_NBIO));
+ t3_repost_kbuf(so, p->cur_buf, modulate, 1, (so->so_state & SS_NBIO));
TRACE_EXIT;
#ifdef T3_TRACE
T3_TRACE1(TIDTB(so),
@@ -614,7 +623,7 @@
}
t3_set_ddp_tag(so, 0, p->kbuf_tag[0] << 6);
t3_set_ddp_buf(so, 0, 0, p->kbuf[0]->dgl_length);
- t3_repost_kbuf(so, 0, 0, nonblock);
+ t3_repost_kbuf(so, 0, 0, 1, nonblock);
t3_set_rcv_coalesce_enable(so,
TOM_TUNABLE(TOE_DEV(so), ddp_rcvcoalesce));
@@ -639,20 +648,17 @@
int page_off, resid_init, err;
struct ddp_gather_list *gl = (struct ddp_gather_list *)m->m_ddp_gl;
- TRACE_ENTER;
resid_init = uio->uio_resid;
if (!gl->dgl_pages)
panic("pages not set\n");
offset += gl->dgl_offset + m->m_cur_offset;
- page_off = offset & ~PAGE_MASK;
+ page_off = offset & PAGE_MASK;
+ KASSERT(len <= gl->dgl_length,
+ ("len=%d > dgl_length=%d in ddp_copy\n", len, gl->dgl_length));
err = uiomove_fromphys(gl->dgl_pages, page_off, len, uio);
- printf("err=%d resid_init=%d uio_resid=%d offset=%d len=%d\n",
- err, resid_init, uio->uio_resid, offset, len);
-
- TRACE_EXIT;
return (err);
}
==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_t3_ddp.h#9 (text+ko) ====
@@ -128,6 +128,7 @@
DDP_BF_PSH = 1 << 3, /* set in skb->flags if the a DDP was
completed with a segment having the
PSH flag set */
+ DDP_BF_NODATA = 1 << 4, /* buffer completed before filling */
};
#include <dev/cxgb/ulp/tom/cxgb_toepcb.h>
@@ -135,7 +136,8 @@
/*
* Returns 1 if a UBUF DMA buffer might be active.
*/
-static inline int t3_ddp_ubuf_pending(struct toepcb *toep)
+static inline int
+t3_ddp_ubuf_pending(struct toepcb *toep)
{
struct ddp_state *p = &toep->tp_ddp_state;
@@ -143,6 +145,9 @@
* but DDP_STATE() is only valid if the connection actually enabled
* DDP.
*/
+ if (p->kbuf[0] == NULL)
+ return (0);
+
return (p->buf_state[0].flags & (DDP_BF_NOFLIP | DDP_BF_NOCOPY)) ||
(p->buf_state[1].flags & (DDP_BF_NOFLIP | DDP_BF_NOCOPY));
}
@@ -160,7 +165,7 @@
int rcv_flags, int modulate, int post_kbuf);
void t3_cancel_ubuf(struct toepcb *toep);
int t3_overlay_ubuf(struct socket *so, const struct uio *uio, int nonblock,
- int rcv_flags);
+ int rcv_flags, int modulate, int post_kbuf);
int t3_enter_ddp(struct socket *so, unsigned int kbuf_size, unsigned int waitall);
void t3_cleanup_ddp(struct toepcb *toep);
void t3_release_ddp_resources(struct toepcb *toep);
==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_tom.c#4 (text+ko) ====
@@ -264,14 +264,18 @@
return;
}
+extern void kdb_backetrace(void);
+
/*
* Process a received packet with an unknown/unexpected CPL opcode.
*/
static int
do_bad_cpl(struct t3cdev *cdev, struct mbuf *m, void *ctx)
{
+ kdb_backtrace();
+
log(LOG_ERR, "%s: received bad CPL command %u\n", cdev->name,
- *mtod(m, unsigned int *));
+ 0xFF & *mtod(m, unsigned int *));
return (CPL_RET_BUF_DONE | CPL_RET_BAD_MSG);
}
More information about the p4-projects
mailing list