PERFORCE change 135280 for review
Steve Wise
swise at FreeBSD.org
Tue Feb 12 13:03:51 PST 2008
http://perforce.freebsd.org/chv.cgi?CH=135280
Change 135280 by swise at swise:vic10:iwarp on 2008/02/12 21:03:35
krping server side works over toe socket.
- added more PDBG() stuff
- only do a shutdown() to initiate a normal close, and when we get
the ec_status() we either close the socket or abort the connection
based on the ec_status.
- connection aborts don't need any upcalls after doing a soclose(linger==0).
- fixed a bug in iwch_modify_qp()
- changed ep lock to a spinlock to handle upcall int context
Affected files ...
.. //depot/projects/iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c#3 edit
.. //depot/projects/iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_qp.c#2 edit
Differences ...
==== //depot/projects/iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c#3 (text+ko) ====
@@ -223,9 +223,9 @@
{
enum iwch_ep_state state;
- mtx_lock(&epc->lock);
+ mtx_lock_spin(&epc->lock);
state = epc->state;
- mtx_unlock(&epc->lock);
+ mtx_unlock_spin(&epc->lock);
return state;
}
@@ -239,10 +239,10 @@
state_set(struct iwch_ep_common *epc, enum iwch_ep_state new)
{
- mtx_lock(&epc->lock);
+ mtx_lock_spin(&epc->lock);
PDBG("%s - %s -> %s\n", __FUNCTION__, states[epc->state], states[new]);
__state_set(epc, new);
- mtx_unlock(&epc->lock);
+ mtx_unlock_spin(&epc->lock);
return;
}
@@ -255,7 +255,7 @@
if (epc) {
memset(epc, 0, size);
refcount_init(&epc->refcount, 1);
- mtx_init(&epc->lock, "iwch_epc lock", NULL, MTX_DEF);
+ mtx_init(&epc->lock, "iwch_epc lock", NULL, MTX_SPIN|MTX_DUPOK);
cv_init(&epc->waitq, "iwch_epc cv");
}
PDBG("%s alloc ep %p\n", __FUNCTION__, epc);
@@ -339,6 +339,7 @@
static void
close_socket(struct iwch_ep_common *epc)
{
+ PDBG("%s ep %p so %p state %s\n", __FUNCTION__, epc, epc->so, states[epc->state]);
SOCK_LOCK(epc->so);
epc->so->so_upcall = NULL;
epc->so->so_upcallarg = NULL;
@@ -349,12 +350,20 @@
}
static void
+shutdown_socket(struct iwch_ep_common *epc)
+{
+ PDBG("%s ep %p so %p state %s\n", __FUNCTION__, epc, epc->so, states[epc->state]);
+ soshutdown(epc->so, SHUT_WR);
+}
+
+static void
abort_socket(struct iwch_ep *ep)
{
struct sockopt sopt;
int err;
struct linger l;
+ PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
l.l_onoff = 1;
l.l_linger = 0;
@@ -368,7 +377,6 @@
err = sosetopt(ep->com.so, &sopt);
if (err)
printf("%s can't set linger to 0, no RST! err %d\n", __FUNCTION__, err);
- close_socket(&ep->com);
}
static void
@@ -477,19 +485,11 @@
}
static void
-abort_connection(struct iwch_ep *ep)
-{
- PDBG("%s ep %p\n", __FILE__, ep);
- state_set(&ep->com, ABORTING);
- abort_socket(ep);
-}
-
-static void
close_complete_upcall(struct iwch_ep *ep)
{
struct iw_cm_event event;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
memset(&event, 0, sizeof(event));
event.event = IW_CM_EVENT_CLOSE;
if (ep->com.cm_id) {
@@ -503,11 +503,23 @@
}
static void
+abort_connection(struct iwch_ep *ep)
+{
+ PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
+ state_set(&ep->com, ABORTING);
+ abort_socket(ep);
+ close_socket(&ep->com);
+ close_complete_upcall(ep);
+ state_set(&ep->com, DEAD);
+ put_ep(&ep->com);
+}
+
+static void
peer_close_upcall(struct iwch_ep *ep)
{
struct iw_cm_event event;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
memset(&event, 0, sizeof(event));
event.event = IW_CM_EVENT_DISCONNECT;
if (ep->com.cm_id) {
@@ -522,7 +534,7 @@
{
struct iw_cm_event event;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
memset(&event, 0, sizeof(event));
event.event = IW_CM_EVENT_CLOSE;
event.status = ECONNRESET;
@@ -541,7 +553,7 @@
{
struct iw_cm_event event;
- PDBG("%s ep %p status %d\n", __FUNCTION__, ep, status);
+ PDBG("%s ep %p so %p state %s status %d\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state], status);
memset(&event, 0, sizeof(event));
event.event = IW_CM_EVENT_CONNECT_REPLY;
event.status = status;
@@ -569,7 +581,7 @@
{
struct iw_cm_event event;
- PDBG("%s ep %p tid %d\n", __FUNCTION__, ep, ep->hwtid);
+ PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
memset(&event, 0, sizeof(event));
event.event = IW_CM_EVENT_CONNECT_REQUEST;
event.local_addr = ep->com.local_addr;
@@ -590,7 +602,7 @@
{
struct iw_cm_event event;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
memset(&event, 0, sizeof(event));
event.event = IW_CM_EVENT_ESTABLISHED;
if (ep->com.cm_id) {
@@ -612,7 +624,7 @@
struct uio uio;
int len;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
/*
* Stop mpa timer. If it expired, then the state has
@@ -765,7 +777,7 @@
struct uio uio;
int len;
- PDBG("%s ep %p so %p\n", __FUNCTION__, ep, ep->com.so);
+ PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
/*
* Stop mpa timer. If it expired, then the state has
@@ -905,9 +917,9 @@
int disconnect = 1;
int release = 0;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
- mtx_lock(&ep->com.lock);
+ mtx_lock_spin(&ep->com.lock);
switch (ep->com.state) {
case MPA_REQ_WAIT:
__state_set(&ep->com, CLOSING);
@@ -962,7 +974,7 @@
default:
BUG_ON(1);
}
- mtx_unlock(&ep->com.lock);
+ mtx_unlock_spin(&ep->com.lock);
if (disconnect)
iwch_ep_disconnect(ep, 0, M_NOWAIT);
if (release)
@@ -978,7 +990,7 @@
int state;
state = state_read(&ep->com);
- PDBG("%s ep %p state %u\n", __FUNCTION__, ep, state);
+ PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
switch (state) {
case MPA_REQ_WAIT:
stop_ep_timer(ep);
@@ -1041,11 +1053,11 @@
struct iwch_qp_attributes attrs;
int release = 0;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
BUG_ON(!ep);
/* The cm_id may be null if we failed to connect */
- mtx_lock(&ep->com.lock);
+ mtx_lock_spin(&ep->com.lock);
switch (ep->com.state) {
case CLOSING:
__state_set(&ep->com, MORIBUND);
@@ -1070,7 +1082,7 @@
BUG_ON(1);
break;
}
- mtx_unlock(&ep->com.lock);
+ mtx_unlock_spin(&ep->com.lock);
if (release)
put_ep(&ep->com);
return;
@@ -1091,7 +1103,9 @@
static int
terminate(struct t3cdev *tdev, struct mbuf *m, void *ctx)
{
- struct iwch_ep *ep = ctx;
+ struct toepcb *toep = (struct toepcb *)ctx;
+ struct socket *so = toeptoso(toep);
+ struct iwch_ep *ep = so->so_upcallarg;
PDBG("%s ep %p\n", __FUNCTION__, ep);
m_adj(m, sizeof(struct cpl_rdma_terminate));
@@ -1105,23 +1119,40 @@
static int
ec_status(struct t3cdev *tdev, struct mbuf *m, void *ctx)
{
+ struct toepcb *toep = (struct toepcb *)ctx;
+ struct socket *so = toeptoso(toep);
struct cpl_rdma_ec_status *rep = cplhdr(m);
- struct iwch_ep *ep = ctx;
+ struct iwch_ep *ep = so->so_upcallarg;
+ struct iwch_qp_attributes attrs;
+ int release;
- PDBG("%s ep %p tid %u status %d\n", __FUNCTION__, ep, ep->hwtid,
- rep->status);
+ PDBG("%s ep %p so %p state %s ec_status %d\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state], rep->status);
+ mtx_lock_spin(&ep->com.lock);
+ stop_ep_timer(ep);
if (rep->status) {
- struct iwch_qp_attributes attrs;
log(LOG_ERR, "%s BAD CLOSE - Aborting tid %u\n",
__FUNCTION__, ep->hwtid);
- stop_ep_timer(ep);
attrs.next_state = IWCH_QP_STATE_ERROR;
iwch_modify_qp(ep->com.qp->rhp,
ep->com.qp, IWCH_QP_ATTR_NEXT_STATE,
&attrs, 1);
- abort_connection(ep);
+ } else {
+ if ((ep->com.cm_id) && (ep->com.qp)) {
+ attrs.next_state = IWCH_QP_STATE_IDLE;
+ iwch_modify_qp(ep->com.qp->rhp,
+ ep->com.qp,
+ IWCH_QP_ATTR_NEXT_STATE,
+ &attrs, 1);
+ }
+ close_complete_upcall(ep);
+ close_socket(&ep->com);
+ __state_set(&ep->com, DEAD);
+ release = 1;
}
+ mtx_unlock_spin(&ep->com.lock);
+ if (release)
+ put_ep(&ep->com);
return CPL_RET_BUF_DONE;
}
@@ -1131,9 +1162,8 @@
struct iwch_ep *ep = (struct iwch_ep *)arg;
struct iwch_qp_attributes attrs;
- mtx_lock(&ep->com.lock);
- PDBG("%s ep %p tid %u state %d\n", __FUNCTION__, ep, ep->hwtid,
- ep->com.state);
+ mtx_lock_spin(&ep->com.lock);
+ PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
switch (ep->com.state) {
case MPA_REQ_SENT:
connect_reply_upcall(ep, -ETIMEDOUT);
@@ -1153,7 +1183,7 @@
panic("unknown state: %d\n", ep->com.state);
}
__state_set(&ep->com, CLOSING);
- mtx_unlock(&ep->com.lock);
+ mtx_unlock_spin(&ep->com.lock);
abort_connection(ep);
put_ep(&ep->com);
}
@@ -1163,7 +1193,7 @@
{
int err;
struct iwch_ep *ep = to_ep(cm_id);
- PDBG("%s ep %p tid %u\n", __FUNCTION__, ep, ep->hwtid);
+ PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
if (state_read(&ep->com) == DEAD) {
put_ep(&ep->com);
@@ -1189,7 +1219,7 @@
struct iwch_dev *h = to_iwch_dev(cm_id->device);
struct iwch_qp *qp = get_qhp(h, conn_param->qpn);
- PDBG("%s ep %p so %p\n", __FUNCTION__, ep, ep->com.so);
+ PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
if (state_read(&ep->com) == DEAD)
return (-ECONNRESET);
@@ -1411,7 +1441,10 @@
{
int close = 0;
- mtx_lock(&ep->com.lock);
+ mtx_lock_spin(&ep->com.lock);
+
+ BUG_ON(!ep);
+ BUG_ON(!ep->com.so);
PDBG("%s ep %p so %p state %s, abrupt %d\n", __FUNCTION__, ep,
ep->com.so, states[ep->com.state], abrupt);
@@ -1450,12 +1483,12 @@
break;
}
out:
- mtx_unlock(&ep->com.lock);
+ mtx_unlock_spin(&ep->com.lock);
if (close) {
if (abrupt)
- abort_socket(ep);
+ abort_connection(ep);
else
- close_socket(&ep->com);
+ shutdown_socket(&ep->com);
}
return 0;
}
@@ -1465,7 +1498,7 @@
{
struct sockaddr_in *local, *remote;
- PDBG("%s ep %p so %p\n", __FUNCTION__, ep, ep->com.so);
+ PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
switch (state_read(&ep->com)) {
case MPA_REQ_SENT:
@@ -1490,11 +1523,11 @@
free(remote, M_SONAME);
process_mpa_request(ep);
break;
- case MPA_REP_SENT:
default:
- printf("%s Unexpected streaming data."
- " ep %p state %d so %p\n",
- __FUNCTION__, ep, state_read(&ep->com), ep->com.so);
+ if (ep->com.so->so_rcv.sb_cc)
+ printf("%s Unexpected streaming data."
+ " ep %p state %d so %p\n",
+ __FUNCTION__, ep, state_read(&ep->com), ep->com.so);
break;
}
return;
@@ -1503,7 +1536,7 @@
static void
process_connected(struct iwch_ep *ep)
{
- PDBG("%s ep %p so %p\n", __FUNCTION__, ep, ep->com.so);
+ PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]);
if ((ep->com.so->so_state & SS_ISCONNECTED) && !ep->com.so->so_error) {
send_mpa_req(ep);
} else {
@@ -1615,7 +1648,7 @@
process_close_complete(ep);
return;
}
-
+
/* rx data */
process_data(ep);
return;
==== //depot/projects/iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_qp.c#2 (text+ko) ====
@@ -936,6 +936,7 @@
wakeup(qhp);
break;
case IWCH_QP_STATE_ERROR:
+ disconnect=1;
goto err;
default:
ret = -EINVAL;
@@ -996,7 +997,7 @@
* an abnormal close (RTS/CLOSING->ERROR).
*/
if (disconnect)
- iwch_ep_disconnect(ep, abort, M_WAITOK);
+ iwch_ep_disconnect(ep, abort, M_NOWAIT);
/*
* If free is 1, then we've disassociated the EP from the QP
More information about the p4-projects
mailing list