svn commit: r236035 - in user/np/toe_iwarp/sys: contrib/rdma/krping
dev/cxgb dev/cxgb/ulp/iw_cxgb dev/cxgb/ulp/tom
Navdeep Parhar
np at FreeBSD.org
Sat May 26 01:33:08 UTC 2012
Author: np
Date: Sat May 26 01:33:07 2012
New Revision: 236035
URL: http://svn.freebsd.org/changeset/base/236035
Log:
Some bugfixes and cleanup of cxgb TOE and iWARP drivers.
Enable TOE multiq.
Modified:
user/np/toe_iwarp/sys/contrib/rdma/krping/krping.c
user/np/toe_iwarp/sys/dev/cxgb/cxgb_offload.h
user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c
user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.h
user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c
user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_listen.c
user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_toepcb.h
user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tom.c
user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tom.h
Modified: user/np/toe_iwarp/sys/contrib/rdma/krping/krping.c
==============================================================================
--- user/np/toe_iwarp/sys/contrib/rdma/krping/krping.c Sat May 26 01:24:39 2012 (r236034)
+++ user/np/toe_iwarp/sys/contrib/rdma/krping/krping.c Sat May 26 01:33:07 2012 (r236035)
@@ -1737,8 +1737,8 @@ int krping_doit(char *cmd)
case 'm':
cb->memlimit = optint;
if (cb->memlimit < 1) {
- log(LOG_ERR, "Invalid memory limit %lu\n"
- ,cb->memlimit);
+ log(LOG_ERR, "Invalid memory limit %ju\n",
+ cb->memlimit);
ret = EINVAL;
} else
DEBUG_LOG(PFX "memory limit %d\n", (int)optint);
Modified: user/np/toe_iwarp/sys/dev/cxgb/cxgb_offload.h
==============================================================================
--- user/np/toe_iwarp/sys/dev/cxgb/cxgb_offload.h Sat May 26 01:24:39 2012 (r236034)
+++ user/np/toe_iwarp/sys/dev/cxgb/cxgb_offload.h Sat May 26 01:33:07 2012 (r236035)
@@ -58,12 +58,6 @@ struct tom_tunables {
enum {
CPL_PRIORITY_DATA = 0, /* data messages */
CPL_PRIORITY_CONTROL = 1 /* offload control messages */
-#ifdef notyet
- CPL_PRIORITY_SETUP = 1, /* connection setup messages */
- CPL_PRIORITY_TEARDOWN = 0, /* connection teardown messages */
- CPL_PRIORITY_LISTEN = 1, /* listen start/stop messages */
- CPL_PRIORITY_ACK = 1, /* RX ACK messages */
-#endif
};
#define S_HDR_NDESC 0
Modified: user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c
==============================================================================
--- user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c Sat May 26 01:24:39 2012 (r236034)
+++ user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c Sat May 26 01:33:07 2012 (r236035)
@@ -1033,10 +1033,9 @@ process_close_complete(struct iwch_ep *e
IWCH_QP_ATTR_NEXT_STATE,
&attrs, 1);
}
- if (ep->parent_ep) {
- ((struct iwch_listen_ep *)ep->parent_ep)->child_ep = NULL;
+ if (ep->parent_ep)
close_socket(&ep->com, 1);
- } else
+ else
close_socket(&ep->com, 0);
close_complete_upcall(ep);
__state_set(&ep->com, DEAD);
@@ -1072,11 +1071,11 @@ terminate(struct sge_qset *qs, struct rs
{
struct adapter *sc = qs->adap;
struct tom_data *td = sc->tom_softc;
- struct cpl_rdma_ec_status *rep = (void *)r;
- unsigned int tid = GET_TID(rep);
- struct toepcb *toep = lookup_tid(&td->tid_maps, tid);
+ uint32_t hash = *((uint32_t *)r + 1);
+ unsigned int tid = ntohl(hash) >> 8 & 0xfffff;
+ struct toepcb *toep = lookup_tid(&td->tid_maps, tid);
struct socket *so = toep->tp_inp->inp_socket;
- struct iwch_ep *ep = so->so_rcv.sb_upcallarg;
+ struct iwch_ep *ep = so->so_rcv.sb_upcallarg;
if (state_read(&ep->com) != FPDU_MODE)
goto done;
@@ -1418,11 +1417,6 @@ iwch_destroy_listen(struct iw_cm_id *cm_
CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep);
- /* lets clean any connected child if there are */
- if (ep->child_ep) {
- if (ep->child_ep->com.so->so_state & SS_ISCONNECTED)
- close_socket(&ep->child_ep->com, 1);
- }
state_set(&ep->com, DEAD);
close_socket(&ep->com, 0);
cm_id->rem_ref(cm_id);
@@ -1607,8 +1601,6 @@ process_newconn(struct iwch_ep *parent_e
child_ep->com.cm_id = NULL;
child_ep->com.thread = parent_ep->com.thread;
child_ep->parent_ep = parent_ep;
- /* Save child reference in parent_ep which could be used for cleanup */
- ((struct iwch_listen_ep *)parent_ep)->child_ep = child_ep;
free(remote, M_SONAME);
get_ep(&parent_ep->com);
Modified: user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.h
==============================================================================
--- user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.h Sat May 26 01:24:39 2012 (r236034)
+++ user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.h Sat May 26 01:33:07 2012 (r236035)
@@ -165,7 +165,6 @@ struct iwch_listen_ep {
struct iwch_ep_common com;
unsigned int stid;
int backlog;
- struct iwch_ep *child_ep;
};
struct iwch_ep {
Modified: user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c
==============================================================================
--- user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c Sat May 26 01:24:39 2012 (r236034)
+++ user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c Sat May 26 01:33:07 2012 (r236035)
@@ -175,27 +175,9 @@ toepcb_detach(struct inpcb *inp)
toep, inp, tp);
tp->t_toe = NULL;
+ tp->t_flags &= ~TF_TOE;
toep->tp_flags &= ~TP_ATTACHED;
- /*
- * Verify that the tcpcb really is going away. tcp_input shouldn't be
- * able to find this inp, or should see it in TIME_WAIT. Otherwise
- * there's a risk of tod_input with no way for the TOE driver to reach
- * its toepcb (we just cleared it).
- *
- * The inp should be DROPPED already or about to go to TIME_WAIT (it
- * can't be in TIME_WAIT already). There is no way to be certain that
- * code that runs after this routine will actually call twstart as it
- * should (while holding the inp's lock throughout), so we do the best
- * we can by asserting that the current TCP state is something from
- * which we can transition to TIME_WAIT.
- */
- KASSERT(inp->inp_flags & INP_DROPPED || /* dropped already */
- (!(inp->inp_flags & INP_TIMEWAIT) && /* not in TIME_WAIT */
- (tp->t_state == TCPS_FIN_WAIT_1 || tp->t_state == TCPS_FIN_WAIT_2 ||
- tp->t_state == TCPS_CLOSING)),
- ("%s: inp %p detached too early?", __func__, inp));
-
if (toep->tp_flags & TP_CPL_DONE)
t3_release_offload_resources(toep);
}
@@ -341,6 +323,28 @@ t3_process_tid_release_list(void *data,
mtx_unlock(&td->tid_release_lock);
}
+static void
+close_conn(struct adapter *sc, struct toepcb *toep)
+{
+ struct mbuf *m;
+ struct cpl_close_con_req *req;
+
+ if (toep->tp_flags & TP_FIN_SENT)
+ return;
+
+ m = M_GETHDR_OFLD(toep->tp_qset, CPL_PRIORITY_DATA, req);
+ if (m == NULL)
+ CXGB_UNIMPLEMENTED();
+
+ req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
+ req->wr.wrh_lo = htonl(V_WR_TID(toep->tp_tid));
+ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, toep->tp_tid));
+ req->rsvd = 0;
+
+ toep->tp_flags |= TP_FIN_SENT;
+ t3_offload_tx(sc, m);
+}
+
static inline void
make_tx_data_wr(struct socket *so, struct tx_data_wr *req, int len,
struct mbuf *tail)
@@ -432,11 +436,6 @@ t3_push_frames(struct socket *so, int re
inp_lock_assert(tp->t_inpcb);
- /* TCP state or socket state not suitable for sending data */
- if (tp->t_state == TCPS_SYN_SENT || tp->t_state == TCPS_CLOSED ||
- (so_state_get(so) & (SS_ISDISCONNECTING | SS_ISDISCONNECTED)))
- return (0);
-
snd = so_sockbuf_snd(so);
SOCKBUF_LOCK(snd);
@@ -557,16 +556,18 @@ t3_push_frames(struct socket *so, int re
}
out:
SOCKBUF_UNLOCK(snd);
+
+ if (sndptr == NULL && (toep->tp_flags & TP_SEND_FIN))
+ close_conn(sc, toep);
+
return (total_bytes);
}
static int
-send_rx_credits(struct toepcb *toep, int credits)
+send_rx_credits(struct adapter *sc, struct toepcb *toep, int credits)
{
struct mbuf *m;
struct cpl_rx_data_ack *req;
- struct toedev *tod = toep->tp_tod;
- struct adapter *sc = tod->tod_softc;
uint32_t dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
m = M_GETHDR_OFLD(toep->tp_qset, CPL_PRIORITY_CONTROL, req);
@@ -584,6 +585,7 @@ send_rx_credits(struct toepcb *toep, int
void
t3_rcvd(struct toedev *tod, struct tcpcb *tp)
{
+ struct adapter *sc = tod->tod_softc;
struct inpcb *inp = tp->t_inpcb;
struct socket *so = inp->inp_socket;
struct sockbuf *so_rcv = &so->so_rcv;
@@ -603,7 +605,7 @@ t3_rcvd(struct toedev *tod, struct tcpcb
if (must_send || toep->tp_rx_credits >= 15 * 1024) {
int credits;
- credits = send_rx_credits(toep, toep->tp_rx_credits);
+ credits = send_rx_credits(sc, toep, toep->tp_rx_credits);
toep->tp_rx_credits -= credits;
tp->rcv_wnd += credits;
tp->rcv_adv += credits;
@@ -628,13 +630,12 @@ do_rx_urg_notify(struct sge_qset *qs, st
int
t3_send_fin(struct toedev *tod, struct tcpcb *tp)
{
- struct mbuf *m;
struct toepcb *toep = tp->t_toe;
- struct adapter *sc = tod->tod_softc;
- struct cpl_close_con_req *req;
struct inpcb *inp = tp->t_inpcb;
struct socket *so = inp_inpcbtosocket(inp);
+#if defined(KTR)
unsigned int tid = toep->tp_tid;
+#endif
INP_INFO_WLOCK_ASSERT(&V_tcbinfo);
INP_WLOCK_ASSERT(inp);
@@ -642,23 +643,8 @@ t3_send_fin(struct toedev *tod, struct t
CTR4(KTR_CXGB, "%s: tid %d, toep %p, flags %x", __func__, tid, toep,
toep->tp_flags);
- if (tp->t_state != TCPS_SYN_SENT)
- t3_push_frames(so, 1);
-
- if (toep->tp_flags & TP_FIN_SENT)
- return (0);
-
- m = M_GETHDR_OFLD(toep->tp_qset, CPL_PRIORITY_DATA, req);
- if (m == NULL)
- CXGB_UNIMPLEMENTED();
-
- req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
- req->wr.wrh_lo = htonl(V_WR_TID(tid));
- OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
- req->rsvd = 0;
-
- toep->tp_flags |= TP_FIN_SENT;
- t3_offload_tx(sc, m);
+ toep->tp_flags |= TP_SEND_FIN;
+ t3_push_frames(so, 1);
return (0);
}
@@ -787,8 +773,12 @@ offload_socket(struct socket *so, struct
INP_WLOCK_ASSERT(inp);
/* Update socket */
+ SOCKBUF_LOCK(&so->so_snd);
so_sockbuf_snd(so)->sb_flags |= SB_NOCOALESCE;
+ SOCKBUF_UNLOCK(&so->so_snd);
+ SOCKBUF_LOCK(&so->so_rcv);
so_sockbuf_rcv(so)->sb_flags |= SB_NOCOALESCE;
+ SOCKBUF_UNLOCK(&so->so_rcv);
/* Update TCP PCB */
tp->tod = toep->tp_tod;
@@ -921,26 +911,28 @@ do_act_open_rpl(struct sge_qset *qs, str
unsigned int atid = G_TID(ntohl(rpl->atid));
struct toepcb *toep = lookup_atid(&td->tid_maps, atid);
struct inpcb *inp = toep->tp_inp;
- struct tcpcb *tp;
+ struct tcpcb *tp = intotcpcb(inp);
+ int s = rpl->status;
- CTR3(KTR_CXGB, "%s: atid %u, status %u ", __func__, atid, rpl->status);
+ CTR3(KTR_CXGB, "%s: atid %u, status %u ", __func__, atid, s);
free_atid(&td->tid_maps, atid);
toep->tp_tid = -1;
- if (act_open_has_tid(rpl->status))
+ if (act_open_has_tid(s))
queue_tid_release(tod, GET_TID(rpl));
- INP_INFO_WLOCK(&V_tcbinfo); /* for tcp_drop */
- INP_WLOCK(inp);
- if (!(inp->inp_flags & INP_DROPPED)) {
- tp = intotcpcb(inp);
- tp = tcp_drop(tp, act_open_rpl_status_to_errno(rpl->status));
- if (tp == NULL)
- INP_WLOCK(inp); /* re-acquire */
+ if (s == CPL_ERR_TCAM_FULL || s == CPL_ERR_CONN_EXIST) {
+ INP_WLOCK(inp);
+ toe_connect_failed(tod, tp, EAGAIN);
+ toepcb_release(toep); /* unlocks inp */
+ } else {
+ INP_INFO_WLOCK(&V_tcbinfo);
+ INP_WLOCK(inp);
+ toe_connect_failed(tod, tp, act_open_rpl_status_to_errno(s));
+ toepcb_release(toep); /* unlocks inp */
+ INP_INFO_WUNLOCK(&V_tcbinfo);
}
- toepcb_release(toep);
- INP_INFO_WUNLOCK(&V_tcbinfo);
m_freem(m);
return (0);
@@ -983,11 +975,7 @@ t3_connect(struct toedev *tod, struct so
if (atid < 0)
goto failed;
-#ifdef notyet
qset = pi->first_qset + (arc4random() % pi->nqsets);
-#else
- qset = 0;
-#endif
m = M_GETHDR_OFLD(qset, CPL_PRIORITY_CONTROL, cpl);
if (m == NULL)
@@ -1241,7 +1229,7 @@ do_peer_close(struct sge_qset *qs, struc
tp = intotcpcb(inp);
CTR5(KTR_CXGB, "%s: tid %u (%s), toep_flags 0x%x, inp %p", __func__,
- tid, tcpstates[tp->t_state], toep->tp_flags, toep->tp_inp);
+ tid, tp ? tcpstates[tp->t_state] : "no tp" , toep->tp_flags, inp);
if (toep->tp_flags & TP_ABORT_RPL_PENDING)
goto done;
@@ -1303,16 +1291,15 @@ do_close_con_rpl(struct sge_qset *qs, st
INP_WLOCK(inp);
tp = intotcpcb(inp);
- so = inp_inpcbtosocket(tp->t_inpcb);
-
CTR4(KTR_CXGB, "%s: tid %u (%s), toep_flags 0x%x", __func__, tid,
- tcpstates[tp->t_state], toep->tp_flags);
-
- tp->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */
+ tp ? tcpstates[tp->t_state] : "no tp", toep->tp_flags);
if ((toep->tp_flags & TP_ABORT_RPL_PENDING))
goto done;
+ so = inp_inpcbtosocket(inp);
+ tp->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */
+
switch (tp->t_state) {
case TCPS_CLOSING:
tcp_twstart(tp);
@@ -1331,16 +1318,9 @@ release:
goto release;
case TCPS_FIN_WAIT_1:
+ if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
+ soisdisconnected(so);
tp->t_state = TCPS_FIN_WAIT_2;
- if ((so_options_get(so) & SO_LINGER) &&
- so_linger_get(so) == 0) {
-
- if (tcp_close(tp))
- inp_wunlock(tp->t_inpcb);
- INP_INFO_WUNLOCK(&V_tcbinfo);
- m_freem(m);
- return (0);
- }
break;
default:
log(LOG_ERR,
Modified: user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_listen.c
==============================================================================
--- user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_listen.c Sat May 26 01:24:39 2012 (r236034)
+++ user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_listen.c Sat May 26 01:33:07 2012 (r236035)
@@ -413,6 +413,21 @@ pass_accept_req_to_protohdrs(const struc
to->to_flags |= TOF_SACKPERM;
}
+static inline void
+hold_synqe(struct synq_entry *synqe)
+{
+
+ refcount_acquire(&synqe->refcnt);
+}
+
+static inline void
+release_synqe(struct synq_entry *synqe)
+{
+
+ if (refcount_release(&synqe->refcnt))
+ m_freem(synqe->m);
+}
+
/*
* Use the trailing space in the mbuf in which the PASS_ACCEPT_REQ arrived to
* store some state temporarily. There will be enough room in the mbuf's
@@ -428,7 +443,7 @@ mbuf_to_synq_entry(struct mbuf *m)
int buflen;
if (__predict_false(M_TRAILINGSPACE(m) < len)) {
- panic("%s: no room for synq_entry (%ld, %d)\n", __func__,
+ panic("%s: no room for synq_entry (%td, %d)\n", __func__,
M_TRAILINGSPACE(m), len);
}
@@ -538,47 +553,26 @@ do_pass_accept_req(struct sge_qset *qs,
INP_INFO_WLOCK(&V_tcbinfo);
- /* Check if the 4-tuple is already in use by the host stack. */
- inp = in_pcblookup(&V_tcbinfo, inc.inc_faddr, inc.inc_fport,
- inc.inc_laddr, inc.inc_lport, INPLOOKUP_WLOCKPCB, ifp);
- if (inp) {
- INP_WLOCK_ASSERT(inp);
- if ((inp->inp_flags & INP_TIMEWAIT) == 0) {
- INP_WUNLOCK(inp);
- INP_INFO_WUNLOCK(&V_tcbinfo);
- REJECT_PASS_ACCEPT();
- }
-
- /*
- * In TIME_WAIT, try to reuse the 4-tuple if possible.
- * tcp_twcheck always unlocks the inp.
- */
- if (!tcp_twcheck(inp, &to, &th, NULL, 0)) {
- INP_UNLOCK_ASSERT(inp);
- INP_INFO_WUNLOCK(&V_tcbinfo);
- REJECT_PASS_ACCEPT(); /* let the kernel deal with it. */
- }
-
- INP_UNLOCK_ASSERT(inp);
- INP_INFO_WLOCK_ASSERT(&V_tcbinfo);
+ /* Don't offload if the 4-tuple is already in use */
+ if (toe_4tuple_check(&inc, &th, ifp) != 0) {
+ INP_INFO_WUNLOCK(&V_tcbinfo);
+ REJECT_PASS_ACCEPT();
}
inp = lctx->inp; /* listening socket (not owned by the TOE) */
INP_WLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
+ if (__predict_false(inp->inp_flags & INP_DROPPED)) {
/*
* The listening socket has closed. The reply from the TOE to
* our CPL_CLOSE_LISTSRV_REQ will ultimately release all
* resources tied to this listen context.
*/
INP_WUNLOCK(inp);
+ INP_INFO_WUNLOCK(&V_tcbinfo);
REJECT_PASS_ACCEPT();
}
so = inp->inp_socket;
- /* Will tell the hardware to respond with a SYN/ACK */
- CTR3(KTR_CXGB, "%s: stid %u, tid %u, OK", __func__, stid, tid);
-
/* Reuse the mbuf that delivered the CPL to us */
synqe = mbuf_to_synq_entry(m);
synqe->flags = TP_IS_A_SYNQ_ENTRY;
@@ -587,17 +581,18 @@ do_pass_accept_req(struct sge_qset *qs,
synqe->tid = tid;
synqe->e = e;
synqe->opt0h = calc_opt0h(so, 0, 0, e);
-#ifdef notyet
synqe->qset = pi->first_qset + (arc4random() % pi->nqsets);
-#else
- synqe->qset = 0;
-#endif
SOCKBUF_LOCK(&so->so_rcv);
synqe->rx_credits = min(select_rcv_wnd(so) >> 10, M_RCV_BUFSIZ);
SOCKBUF_UNLOCK(&so->so_rcv);
refcount_init(&synqe->refcnt, 1);
atomic_store_rel_int(&synqe->reply, RPL_OK);
+ insert_tid(td, synqe, tid);
+ TAILQ_INSERT_TAIL(&lctx->synq, synqe, link);
+ hold_synqe(synqe);
+ hold_lctx(lctx);
+
/* syncache_add releases both pcbinfo and pcb locks */
toe_syncache_add(&inc, &to, &th, inp, tod, synqe);
INP_UNLOCK_ASSERT(inp);
@@ -610,28 +605,52 @@ do_pass_accept_req(struct sge_qset *qs,
* The kernel is free to retry syncache_respond but we'll ignore it due
* to RPL_DONT.
*/
- if (atomic_cmpset_int(&synqe->reply, RPL_OK, RPL_DONT))
+ if (atomic_cmpset_int(&synqe->reply, RPL_OK, RPL_DONT)) {
+
+ INP_WLOCK(inp);
+ if (__predict_false(inp->inp_flags & INP_DROPPED)) {
+ /* listener closed. synqe must have been aborted. */
+ KASSERT(synqe->flags & TP_ABORT_SHUTDOWN,
+ ("%s: listener %p closed but synqe %p not aborted",
+ __func__, inp, synqe));
+
+ CTR5(KTR_CXGB,
+ "%s: stid %u, tid %u, lctx %p, synqe %p, ABORTED",
+ __func__, stid, tid, lctx, synqe);
+ INP_WUNLOCK(inp);
+ release_synqe(synqe);
+ return (__LINE__);
+ }
+
+ KASSERT(!(synqe->flags & TP_ABORT_SHUTDOWN),
+ ("%s: synqe %p aborted, but listener %p not dropped.",
+ __func__, synqe, inp));
+
+ TAILQ_REMOVE(&lctx->synq, synqe, link);
+ release_synqe(synqe); /* removed from synq list */
+ inp = release_lctx(td, lctx);
+ if (inp)
+ INP_WUNLOCK(inp);
+
+ release_synqe(synqe); /* about to exit function */
REJECT_PASS_ACCEPT();
+ }
KASSERT(synqe->reply == RPL_DONE,
("%s: reply %d", __func__, synqe->reply));
- TAILQ_INSERT_HEAD(&lctx->synq, synqe, link);
- hold_lctx(lctx); /* Each synq entry has a ref on its lctx */
- insert_tid(td, synqe, tid);
-
+ CTR3(KTR_CXGB, "%s: stid %u, tid %u, OK", __func__, stid, tid);
+ release_synqe(synqe);
return (0);
reject:
CTR4(KTR_CXGB, "%s: stid %u, tid %u, REJECT (%d)", __func__, stid, tid,
reject_reason);
- if (synqe == NULL || refcount_release(&synqe->refcnt))
+ if (synqe == NULL)
m_freem(m);
-
if (e)
l2t_release(td->l2t, e);
-
queue_tid_release(tod, tid);
return (0);
@@ -688,10 +707,13 @@ do_pass_establish(struct sge_qset *qs, s
CTR5(KTR_CXGB, "%s: stid %u, tid %u, lctx %p, inp_flags 0x%x",
__func__, stid, tid, lctx, inp->inp_flags);
+ KASSERT(qs->idx == synqe->qset,
+ ("%s qset mismatch %d %d", __func__, qs->idx, synqe->qset));
+
INP_INFO_WLOCK(&V_tcbinfo); /* for syncache_expand */
INP_WLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
+ if (__predict_false(inp->inp_flags & INP_DROPPED)) {
/*
* The listening socket has closed. The TOM must have aborted
* all the embryonic connections (including this one) that were
@@ -713,8 +735,8 @@ do_pass_establish(struct sge_qset *qs, s
to.to_tsecr = synqe->ts;
th.th_ack = synqe->iss + 1;
- so = inp->inp_socket;
- if (!toe_syncache_expand(&inc, &to, &th, &so) || so == NULL) {
+ toep = toepcb_alloc(tod);
+ if (toep == NULL) {
reset:
t3_send_reset_synqe(tod, synqe);
INP_WUNLOCK(inp);
@@ -722,36 +744,28 @@ reset:
m_freem(m);
return (0);
}
+ toep->tp_qset = qs->idx;
+ toep->tp_l2t = synqe->e;
+ toep->tp_tid = tid;
+ toep->tp_rx_credits = synqe->rx_credits;
- toep = toepcb_alloc(tod);
- if (toep == NULL)
+ synqe->toep = toep;
+ synqe->cpl = cpl;
+
+ so = inp->inp_socket;
+ if (!toe_syncache_expand(&inc, &to, &th, &so) || so == NULL) {
+ toepcb_free(toep);
goto reset;
+ }
/* Remove the synq entry and release its reference on the lctx */
TAILQ_REMOVE(&lctx->synq, synqe, link);
inp = release_lctx(td, lctx);
if (inp)
INP_WUNLOCK(inp);
-
- KASSERT(qs->idx == synqe->qset,
- ("%s qset mismatch %d %d", __func__, qs->idx, synqe->qset));
-
- toep->tp_qset = qs->idx;
- toep->tp_l2t = synqe->e;
- toep->tp_tid = tid;
- toep->tp_rx_credits = synqe->rx_credits;
- update_tid(td, toep, tid);
-
- inp = sotoinpcb(so); /* Brand new socket, not the listening socket */
- INP_WLOCK(inp);
-
- offload_socket(so, toep);
- make_established(so, cpl->snd_isn, cpl->rcv_isn, cpl->tcp_opt);
-
- INP_WUNLOCK(inp);
INP_INFO_WUNLOCK(&V_tcbinfo);
+ release_synqe(synqe);
- m_freem(synqe->m);
m_freem(m);
return (0);
}
@@ -821,11 +835,7 @@ t3_listen_start(struct toedev *tod, stru
if (listen_hash_find(td, inp) != NULL)
goto done; /* already setup */
-#ifdef notyet
lctx = alloc_lctx(td, inp, pi->first_qset);
-#else
- lctx = alloc_lctx(td, inp, 0);
-#endif
if (lctx == NULL) {
log(LOG_ERR,
"%s: listen request ignored, %s couldn't allocate lctx\n",
@@ -905,7 +915,7 @@ t3_syncache_added(struct toedev *tod __u
{
struct synq_entry *synqe = arg;
- refcount_acquire(&synqe->refcnt);
+ hold_synqe(synqe);
}
void
@@ -913,8 +923,7 @@ t3_syncache_removed(struct toedev *tod _
{
struct synq_entry *synqe = arg;
- if (refcount_release(&synqe->refcnt))
- m_freem(synqe->m);
+ release_synqe(synqe);
}
/* XXX */
@@ -1025,7 +1034,7 @@ do_abort_req_synqe(struct sge_qset *qs,
INP_WUNLOCK(inp);
release_tid(tod, tid, qs->idx);
l2t_release(td->l2t, synqe->e);
- m_freem(synqe->m);
+ release_synqe(synqe);
}
send_abort_rpl(tod, tid, qs->idx);
@@ -1064,7 +1073,7 @@ do_abort_rpl_synqe(struct sge_qset *qs,
INP_WUNLOCK(inp);
release_tid(tod, tid, qs->idx);
l2t_release(td->l2t, synqe->e);
- m_freem(synqe->m);
+ release_synqe(synqe);
}
}
@@ -1084,7 +1093,6 @@ t3_send_reset_synqe(struct toedev *tod,
struct inpcb *inp = lctx->inp;
#endif
- INP_INFO_WLOCK_ASSERT(&V_tcbinfo);
INP_WLOCK_ASSERT(inp);
CTR4(KTR_CXGB, "%s: tid %d, synqe %p (%x)", __func__, tid, synqe,
@@ -1108,3 +1116,23 @@ t3_send_reset_synqe(struct toedev *tod,
l2t_send(sc, m, synqe->e);
}
+
+void
+t3_offload_socket(struct toedev *tod, void *arg, struct socket *so)
+{
+ struct adapter *sc = tod->tod_softc;
+ struct tom_data *td = sc->tom_softc;
+ struct synq_entry *synqe = arg;
+#ifdef INVARIANTS
+ struct inpcb *inp = sotoinpcb(so);
+#endif
+ struct cpl_pass_establish *cpl = synqe->cpl;
+ struct toepcb *toep = synqe->toep;
+
+ INP_INFO_LOCK_ASSERT(&V_tcbinfo); /* prevents bad race with accept() */
+ INP_WLOCK_ASSERT(inp);
+
+ offload_socket(so, toep);
+ make_established(so, cpl->snd_isn, cpl->rcv_isn, cpl->tcp_opt);
+ update_tid(td, toep, synqe->tid);
+}
Modified: user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_toepcb.h
==============================================================================
--- user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_toepcb.h Sat May 26 01:24:39 2012 (r236034)
+++ user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_toepcb.h Sat May 26 01:33:07 2012 (r236035)
@@ -43,6 +43,7 @@
#define TP_CPL_DONE (1 << 8)
#define TP_IS_A_SYNQ_ENTRY (1 << 9)
#define TP_ABORT_RPL_SENT (1 << 10)
+#define TP_SEND_FIN (1 << 11)
struct toepcb {
TAILQ_ENTRY(toepcb) link; /* toep_list */
Modified: user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tom.c
==============================================================================
--- user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tom.c Sat May 26 01:24:39 2012 (r236034)
+++ user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tom.c Sat May 26 01:33:07 2012 (r236035)
@@ -260,6 +260,7 @@ t3_tom_activate(struct adapter *sc)
tod->tod_syncache_added = t3_syncache_added;
tod->tod_syncache_removed = t3_syncache_removed;
tod->tod_syncache_respond = t3_syncache_respond;
+ tod->tod_offload_socket = t3_offload_socket;
/* port MTUs */
mtus = sc->port[0].ifp->if_mtu;
Modified: user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tom.h
==============================================================================
--- user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tom.h Sat May 26 01:24:39 2012 (r236034)
+++ user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tom.h Sat May 26 01:33:07 2012 (r236035)
@@ -109,6 +109,8 @@ struct synq_entry {
int tid;
struct mbuf *m; /* backpointer to containing mbuf */
struct listen_ctx *lctx; /* backpointer to listen ctx */
+ struct cpl_pass_establish *cpl;
+ struct toepcb *toep;
struct l2t_entry *e;
uint32_t iss;
uint32_t ts;
@@ -273,4 +275,5 @@ void t3_syncache_removed(struct toedev *
int t3_syncache_respond(struct toedev *, void *, struct mbuf *);
int do_abort_req_synqe(struct sge_qset *, struct rsp_desc *, struct mbuf *);
int do_abort_rpl_synqe(struct sge_qset *, struct rsp_desc *, struct mbuf *);
+void t3_offload_socket(struct toedev *, void *, struct socket *);
#endif
More information about the svn-src-user
mailing list