PERFORCE change 134954 for review
Kip Macy
kmacy at FreeBSD.org
Wed Feb 6 20:36:56 PST 2008
http://perforce.freebsd.org/chv.cgi?CH=134954
Change 134954 by kmacy at kmacy:storage:toehead on 2008/02/07 04:36:06
add inline SBAPPEND for validating mbufs and socket buffers
move back to sbappend
acquire socket buffer lock earlier to serialize access to DDP state
Affected files ...
.. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c#15 edit
Differences ...
==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c#15 (text+ko) ====
@@ -152,6 +152,38 @@
#endif
+static inline void
+SBAPPEND(struct sockbuf *sb, struct mbuf *n)
+{
+ struct mbuf * m;
+
+ m = sb->sb_mb;
+ while (m) {
+ KASSERT(((m->m_flags & M_EXT) && (m->m_ext.ext_type == EXT_EXTREF)) ||
+ !(m->m_flags & M_EXT), ("unexpected type M_EXT=%d ext_type=%d m_len=%d\n",
+ !!(m->m_flags & M_EXT), m->m_ext.ext_type, m->m_len));
+ KASSERT(m->m_next != (struct mbuf *)0xffffffff, ("bad next value m_next=%p m_nextpkt=%p m_flags=0x%x",
+ m->m_next, m->m_nextpkt, m->m_flags));
+ m = m->m_next;
+ }
+ m = n;
+ while (m) {
+ KASSERT(((m->m_flags & M_EXT) && (m->m_ext.ext_type == EXT_EXTREF)) ||
+ !(m->m_flags & M_EXT), ("unexpected type M_EXT=%d ext_type=%d m_len=%d\n",
+ !!(m->m_flags & M_EXT), m->m_ext.ext_type, m->m_len));
+ KASSERT(m->m_next != (struct mbuf *)0xffffffff, ("bad next value m_next=%p m_nextpkt=%p m_flags=0x%x",
+ m->m_next, m->m_nextpkt, m->m_flags));
+ m = m->m_next;
+ }
+ sbappend_locked(sb, n);
+ m = sb->sb_mb;
+ while (m) {
+ KASSERT(m->m_next != (struct mbuf *)0xffffffff, ("bad next value m_next=%p m_nextpkt=%p m_flags=0x%x",
+ m->m_next, m->m_nextpkt, m->m_flags));
+ m = m->m_next;
+ }
+}
+
static inline int
is_t3a(const struct toedev *dev)
{
@@ -221,6 +253,7 @@
m->m_pkthdr.len = m->m_len = sizeof(*req);
req = mtod(m, struct cpl_tid_release *);
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+ req->wr.wr_lo = 0;
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
}
@@ -470,6 +503,7 @@
req = mtod(m, struct cpl_rx_data_ack *);
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+ req->wr.wr_lo = 0;
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tp_tid));
req->credit_dack = htonl(dack | V_RX_CREDITS(credits));
m_set_priority(m, mkprio(CPL_PRIORITY_ACK, toep));
@@ -492,6 +526,7 @@
req = mtod(m, struct cpl_rx_data_ack *);
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+ req->wr.wr_lo = 0;
m->m_pkthdr.len = m->m_len = sizeof(*req);
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tp_tid));
@@ -743,6 +778,7 @@
req = mtod(m, struct cpl_set_tcb_field *);
m->m_pkthdr.len = m->m_len = sizeof(*req);
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+ req->wr.wr_lo = 0;
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, toep->tp_tid));
req->reply = V_NO_REPLY(no_reply);
req->cpu_idx = 0;
@@ -907,8 +943,10 @@
req = mtod(m, struct cpl_get_tcb *);
m->m_pkthdr.len = m->m_len = sizeof(*req);
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+ req->wr.wr_lo = 0;
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_GET_TCB, toep->tp_tid));
req->cpuno = htons(toep->tp_qset);
+ req->rsvd = 0;
if (sototcpcb(so)->t_state == TCPS_SYN_SENT)
mbufq_tail(&toep->out_of_order_queue, m); // defer
else
@@ -1214,6 +1252,7 @@
m->m_pkthdr.len = m->m_len = sizeof(*req);
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+ req->wr.wr_lo = 0;
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, atid));
req->local_port = inp->inp_lport;
req->peer_port = inp->inp_fport;
@@ -1623,6 +1662,9 @@
uint64_t t;
__be64 *tcb;
+ so = toeptoso(toep);
+ SOCKBUF_LOCK(&so->so_rcv);
+
/* Note that we only accout for CPL_GET_TCB issued by the DDP code. We
* really need a cookie in order to dispatch the RPLs.
*/
@@ -1638,7 +1680,9 @@
m_freem(m);
if (__predict_true((so->so_state & SS_NOFDREF) == 0))
- sorwakeup(so);
+ sorwakeup_locked(so);
+ else
+ SOCKBUF_UNLOCK(&so->so_rcv);
return;
}
@@ -1692,9 +1736,8 @@
#endif
m->m_cur_offset = bsp->cur_offset;
bsp->cur_offset = ddp_offset;
- m->m_pkthdr.len = ddp_offset - m->m_cur_offset;
- so = toeptoso(toep);
-
+ m->m_len = m->m_pkthdr.len = ddp_offset - m->m_cur_offset;
+
if (__predict_false(so_no_receive(so) && m->m_pkthdr.len)) {
handle_excess_rx(toep, m);
return;
@@ -1736,10 +1779,13 @@
q->kbuf_posted--;
bsp->flags |= DDP_BF_NODATA;
}
+ SOCKBUF_UNLOCK(&so->so_rcv);
+
m_free(m);
return;
}
} else {
+ SOCKBUF_UNLOCK(&so->so_rcv);
/* This reply is for a CPL_GET_TCB_RPL to cancel the UBUF DDP,
* but it got here way late and nobody cares anymore.
*/
@@ -1747,6 +1793,7 @@
return;
}
+ KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len));
tp = toep->tp_tp;
m->m_ddp_gl = (unsigned char *)bsp->gl;
m->m_flags |= M_DDP;
@@ -1762,8 +1809,7 @@
"tcb_rpl_as_ddp_complete: seq 0x%x hwbuf %u lskb->len %u",
m->m_seq, q->cur_buf, m->m_pkthdr.len);
#endif
- SOCKBUF_LOCK(&so->so_rcv);
- sbappendstream_locked(&so->so_rcv, m);
+ SBAPPEND(&so->so_rcv, m);
if (__predict_true((so->so_state & SS_NOFDREF) == 0))
sorwakeup_locked(so);
else
@@ -1793,6 +1839,7 @@
handle_ddp_data(struct toepcb *toep, struct mbuf *m)
{
struct tcpcb *tp = toep->tp_tp;
+ struct socket *so = toeptoso(toep);
struct ddp_state *q;
struct ddp_buf_state *bsp;
struct cpl_rx_data *hdr = cplhdr(m);
@@ -1802,10 +1849,11 @@
return;
TRACE_ENTER;
+ SOCKBUF_LOCK(&so->so_rcv);
q = &toep->tp_ddp_state;
bsp = &q->buf_state[q->cur_buf];
m->m_len = m->m_pkthdr.len = rcv_nxt - tp->rcv_nxt;
-
+ KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len));
printf("rcv_nxt=0x%x tp->rcv_next=0x%x len=%d\n",
rcv_nxt, tp->rcv_nxt, m->m_pkthdr.len);
@@ -1834,6 +1882,7 @@
* mode.
*/
q->ubuf_ddp_ready = 0;
+ SOCKBUF_UNLOCK(&so->so_rcv);
TRACE_EXIT;
}
@@ -1910,7 +1959,7 @@
if (sb_notify(&so->so_rcv))
DPRINTF("rx_data so=%p flags=0x%x len=%d\n", so, so->so_rcv.sb_flags, m->m_pkthdr.len);
- sbappendstream_locked(&so->so_rcv, m);
+ SBAPPEND(&so->so_rcv, m);
#ifdef notyet
/*
@@ -2009,6 +2058,9 @@
("length received exceeds ddp pages: len=%d dgl_length=%d",
m->m_len, bsp->gl->dgl_length));
+ KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len));
+ KASSERT(m->m_next == NULL, ("m_len=%p", m->m_next));
+
/*
* Figure out where the new data was placed in the buffer and store it
* in when. Assumes the buffer offset starts at 0, consumer needs to
@@ -2045,7 +2097,7 @@
tp->t_rcvtime = ticks;
SOCKBUF_LOCK(&so->so_rcv);
- sbappendstream_locked(&so->so_rcv, m);
+ SBAPPEND(&so->so_rcv, m);
if ((so->so_state & SS_NOFDREF) == 0)
sorwakeup_locked(so);
@@ -2072,7 +2124,7 @@
if (__predict_false(ntohl(hdr->ddpvld_status) & DDP_ERR)) {
log(LOG_ERR, "RX_DATA_DDP for TID %u reported error 0x%x\n",
GET_TID(hdr), G_DDP_VALID(ntohl(hdr->ddpvld_status)));
- return CPL_RET_BUF_DONE;
+ return (CPL_RET_BUF_DONE);
}
#if 0
skb->h.th = tcphdr_skb->h.th;
@@ -2107,9 +2159,10 @@
buf_idx = (ddp_report >> S_DDP_BUF_IDX) & 1;
bsp = &q->buf_state[buf_idx];
+ SOCKBUF_LOCK(&so->so_rcv);
when = bsp->cur_offset;
m->m_len = m->m_pkthdr.len = G_DDP_OFFSET(ddp_report) - when;
-
+ KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len));
#ifdef T3_TRACE
T3_TRACE5(TIDTB(sk),
"process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u "
@@ -2145,12 +2198,12 @@
tp->rcv_nxt += m->m_len;
tp->t_rcvtime = ticks;
- SOCKBUF_LOCK(&so->so_rcv);
- sbappendstream_locked(&so->so_rcv, m);
+ SBAPPEND(&so->so_rcv, m);
if ((so->so_state & SS_NOFDREF) == 0)
sorwakeup_locked(so);
- SOCKBUF_UNLOCK(&so->so_rcv);
+ else
+ SOCKBUF_UNLOCK(&so->so_rcv);
TRACE_EXIT;
}
@@ -2229,8 +2282,10 @@
}
q = &toep->tp_ddp_state;
+ SOCKBUF_LOCK(&so->so_rcv);
bsp = &q->buf_state[q->cur_buf];
m->m_len = m->m_pkthdr.len = rcv_nxt - tp->rcv_nxt;
+ KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len));
m->m_ddp_gl = (unsigned char *)bsp->gl;
m->m_flags |= M_DDP;
m->m_cur_offset = bsp->cur_offset;
@@ -2242,9 +2297,11 @@
if (!(bsp->flags & DDP_BF_NOFLIP))
q->cur_buf ^= 1;
tp->t_rcvtime = ticks;
- sbappendstream(&so->so_rcv, m);
+ SBAPPEND(&so->so_rcv, m);
if (__predict_true((so->so_state & SS_NOFDREF) == 0))
- sorwakeup(so);
+ sorwakeup_locked(so);
+ else
+ SOCKBUF_UNLOCK(&so->so_rcv);
return (1);
}
@@ -3799,6 +3856,7 @@
req = mtod(m, struct ulp_mem_io *);
m->m_pkthdr.len = m->m_len = sizeof(*req) + PPOD_SIZE;
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
+ req->wr.wr_lo = 0;
req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(ppod_addr >> 5) |
V_ULPTX_CMD(ULP_MEM_WRITE));
req->len = htonl(V_ULP_MEMIO_DATA_LEN(PPOD_SIZE / 32) |
@@ -3896,13 +3954,16 @@
struct cpl_get_tcb *getreq;
struct ddp_state *p = &toep->tp_ddp_state;
+ SOCKBUF_LOCK_ASSERT(&toeptoso(toep)->so_rcv);
wrlen = sizeof(*wr) + sizeof(*req) + 2 * sizeof(*lock) +
sizeof(*getreq);
m = m_gethdr_nofail(wrlen);
m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep));
wr = mtod(m, struct work_request_hdr *);
+ bzero(wr, wrlen);
+
wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
- m->m_pkthdr.len = m->m_len = sizeof(wrlen);
+ m->m_pkthdr.len = m->m_len = wrlen;
lock = (struct cpl_barrier *)(wr + 1);
mk_cpl_barrier_ulp(lock);
@@ -3932,7 +3993,7 @@
/* Keep track of the number of oustanding CPL_GET_TCB requests
*/
p->get_tcb_count++;
-
+
#ifdef T3_TRACE
T3_TRACE1(TIDTB(so),
"t3_cancel_ddpbuf: bufidx %u", bufidx);
@@ -3967,17 +4028,19 @@
struct cpl_set_tcb_field *req;
struct ddp_state *p = &toep->tp_ddp_state;
+ SOCKBUF_LOCK_ASSERT(&toeptoso(toep)->so_rcv);
wrlen = sizeof(*wr) + 3 * sizeof(*req) + sizeof(*getreq);
m = m_gethdr_nofail(wrlen);
m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep));
wr = mtod(m, struct work_request_hdr *);
m->m_pkthdr.len = m->m_len = wrlen;
+ bzero(wr, wrlen);
+
/* Set the ATOMIC flag to make sure that TP processes the following
* CPLs in an atomic manner and no wire segments can be interleaved.
*/
wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC);
-
req = (struct cpl_set_tcb_field *)(wr + 1);
mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF0_TAG,
V_TCB_RX_DDP_BUF0_TAG(M_TCB_RX_DDP_BUF0_TAG) |
@@ -4038,12 +4101,15 @@
struct work_request_hdr *wr;
struct cpl_set_tcb_field *req;
+ SOCKBUF_LOCK_ASSERT(&toeptoso(toep)->so_rcv);
wrlen = sizeof(*wr) + sizeof(*req) + (len0 ? sizeof(*req) : 0) +
(len1 ? sizeof(*req) : 0) +
(modulate ? sizeof(struct cpl_rx_data_ack) : 0);
m = m_gethdr_nofail(wrlen);
m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep));
wr = mtod(m, struct work_request_hdr *);
+ bzero(wr, wrlen);
+
wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
m->m_pkthdr.len = m->m_len = wrlen;
@@ -4118,7 +4184,6 @@
tcphdr_skb->h.raw = tcphdr_skb->data;
memset(tcphdr_skb->data, 0, tcphdr_skb->len);
#endif
-
t3tom_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish);
t3tom_register_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl);
More information about the p4-projects
mailing list