git: 40dbb06fa73c - main - inpcb: retire INP_DROPPED and in_pcbdrop()
Date: Sun, 12 Apr 2026 18:35:35 UTC
The branch main has been updated by glebius:
URL: https://cgit.FreeBSD.org/src/commit/?id=40dbb06fa73cac37d57563c07e55efd0cabbd488
commit 40dbb06fa73cac37d57563c07e55efd0cabbd488
Author: Gleb Smirnoff <glebius@FreeBSD.org>
AuthorDate: 2026-04-12 18:33:07 +0000
Commit: Gleb Smirnoff <glebius@FreeBSD.org>
CommitDate: 2026-04-12 18:33:07 +0000
inpcb: retire INP_DROPPED and in_pcbdrop()
The inpcb flag INP_DROPPED served two purposes.
It was used by TCP and subsystems running on top of TCP as a flag that
marks a connection that is now in TCPS_CLOSED, but was in some other state
before (not a new-born connection). Create a new TCP flag TF_DISCONNECTED
for this purpose.
The in_pcbdrop() was a TCP's version of in_pcbdisconnect() that also sets
INP_DROPPED. Use in_pcbdisconnect() instead.
Second purpose of INP_DROPPED was a negative lookup mask in
inp_smr_lock(), as SMR-protected lookup may see inpcbs that had been
removed from the hash. We already have had INP_INHASHLIST that marks
inpcb that is in hash. Convert it into INP_UNCONNECTED with the opposite
meaning. This allows to combine it with INP_FREED for the negative lookup
mask.
The Chelsio/ToE and kTLS changes are done with some style refactoring,
like moving inp/tp assignments up and using macros for that. However, no
deep thinking was taken to check if those checks are really needed, it
could be that some are not.
Reviewed by: rrs
Differential Revision: https://reviews.freebsd.org/D56186
---
sys/dev/cxgbe/crypto/t6_kern_tls.c | 6 +-
sys/dev/cxgbe/crypto/t7_kern_tls.c | 2 +-
sys/dev/cxgbe/cxgbei/cxgbei.c | 17 ++--
sys/dev/cxgbe/cxgbei/icl_cxgbei.c | 13 ++-
sys/dev/cxgbe/iw_cxgbe/qp.c | 24 +++--
sys/dev/cxgbe/nvmf/nvmf_che.c | 23 +++--
sys/dev/cxgbe/tom/t4_connect.c | 3 +-
sys/dev/cxgbe/tom/t4_cpl_io.c | 44 ++++-----
sys/dev/cxgbe/tom/t4_ddp.c | 18 ++--
sys/dev/cxgbe/tom/t4_listen.c | 14 ++-
sys/dev/cxgbe/tom/t4_tls.c | 18 ++--
sys/dev/cxgbe/tom/t4_tom.c | 10 +-
sys/kern/uipc_ktls.c | 71 ++++---------
sys/netinet/in_pcb.c | 74 +++++---------
sys/netinet/in_pcb.h | 9 +-
sys/netinet/in_pcb_var.h | 2 +
sys/netinet/tcp_hpts.c | 12 ++-
sys/netinet/tcp_hpts_test.c | 3 +-
sys/netinet/tcp_log_buf.c | 17 ++--
sys/netinet/tcp_output.c | 1 +
sys/netinet/tcp_stacks/bbr.c | 2 +-
sys/netinet/tcp_stacks/rack.c | 4 +-
sys/netinet/tcp_subr.c | 3 +-
sys/netinet/tcp_timewait.c | 5 +-
sys/netinet/tcp_usrreq.c | 198 ++++++++++++++-----------------------
sys/netinet/tcp_var.h | 2 +-
sys/netinet/toecore.c | 10 +-
sys/netinet6/in6_pcb.c | 8 +-
sys/netipsec/xform_tcp.c | 8 +-
29 files changed, 258 insertions(+), 363 deletions(-)
diff --git a/sys/dev/cxgbe/crypto/t6_kern_tls.c b/sys/dev/cxgbe/crypto/t6_kern_tls.c
index 454b2e264a0e..584e5015acfa 100644
--- a/sys/dev/cxgbe/crypto/t6_kern_tls.c
+++ b/sys/dev/cxgbe/crypto/t6_kern_tls.c
@@ -458,15 +458,15 @@ t6_tls_tag_alloc(if_t ifp, union if_snd_tag_alloc_params *params,
}
inp = params->tls.inp;
+ tp = intotcpcb(inp);
INP_RLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
+ if (tp->t_flags & TF_DISCONNECTED) {
INP_RUNLOCK(inp);
error = ECONNRESET;
goto failed;
}
tlsp->inp = inp;
- tp = intotcpcb(inp);
if (tp->t_flags & TF_REQ_TSTMP) {
tlsp->using_timestamps = true;
if ((tp->ts_offset & 0xfffffff) != 0) {
@@ -501,7 +501,7 @@ t6_tls_tag_alloc(if_t ifp, union if_snd_tag_alloc_params *params,
goto failed;
}
- if (inp->inp_flags & INP_DROPPED) {
+ if (tp->t_flags & TF_DISCONNECTED) {
INP_RUNLOCK(inp);
error = ECONNRESET;
goto failed;
diff --git a/sys/dev/cxgbe/crypto/t7_kern_tls.c b/sys/dev/cxgbe/crypto/t7_kern_tls.c
index d9710b5bd13f..b6078b9b53b6 100644
--- a/sys/dev/cxgbe/crypto/t7_kern_tls.c
+++ b/sys/dev/cxgbe/crypto/t7_kern_tls.c
@@ -246,7 +246,7 @@ t7_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
inp = params->tls.inp;
INP_RLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
+ if (intotcpcb(inp)->t_flags & TF_DISCONNECTED) {
INP_RUNLOCK(inp);
error = ECONNRESET;
goto failed;
diff --git a/sys/dev/cxgbe/cxgbei/cxgbei.c b/sys/dev/cxgbe/cxgbei/cxgbei.c
index ccca45f5f761..4b341c9d37b2 100644
--- a/sys/dev/cxgbe/cxgbei/cxgbei.c
+++ b/sys/dev/cxgbe/cxgbei/cxgbei.c
@@ -499,10 +499,11 @@ do_rx_iscsi_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
toep->ofld_rxq->rx_iscsi_ddp_octets += ip->ip_data_len;
}
+ tp = intotcpcb(inp);
INP_WLOCK(inp);
- if (__predict_false(inp->inp_flags & INP_DROPPED)) {
- CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
- __func__, tid, pdu_len, inp->inp_flags);
+ if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
+ CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), t_flags 0x%x",
+ __func__, tid, pdu_len, tp->t_flags);
INP_WUNLOCK(inp);
icl_cxgbei_conn_pdu_free(NULL, ip);
toep->ulpcb2 = NULL;
@@ -513,7 +514,6 @@ do_rx_iscsi_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
* T6+ does not report data PDUs received via DDP without F
* set. This can result in gaps in the TCP sequence space.
*/
- tp = intotcpcb(inp);
MPASS(chip_id(sc) >= CHELSIO_T6 || icp->icp_seq == tp->rcv_nxt);
tp->rcv_nxt = icp->icp_seq + pdu_len;
tp->t_rcvtime = ticks;
@@ -652,10 +652,11 @@ do_rx_iscsi_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
toep->ofld_rxq->rx_iscsi_data_digest_errors++;
}
+ tp = intotcpcb(inp);
INP_WLOCK(inp);
- if (__predict_false(inp->inp_flags & INP_DROPPED)) {
- CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
- __func__, tid, pdu_len, inp->inp_flags);
+ if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
+ CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), t_flags 0x%x",
+ __func__, tid, pdu_len, tp->t_flags);
INP_WUNLOCK(inp);
icl_cxgbei_conn_pdu_free(NULL, ip);
toep->ulpcb2 = NULL;
@@ -663,8 +664,6 @@ do_rx_iscsi_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
return (0);
}
- tp = intotcpcb(inp);
-
/*
* If icc is NULL, the connection is being closed in
* icl_cxgbei_conn_close(), just drop this data.
diff --git a/sys/dev/cxgbe/cxgbei/icl_cxgbei.c b/sys/dev/cxgbe/cxgbei/icl_cxgbei.c
index d90d7904a8ae..2e7767a0fc27 100644
--- a/sys/dev/cxgbe/cxgbei/icl_cxgbei.c
+++ b/sys/dev/cxgbe/cxgbei/icl_cxgbei.c
@@ -434,6 +434,7 @@ icl_cxgbei_tx_main(void *arg)
struct toepcb *toep = icc->toep;
struct socket *so = ic->ic_socket;
struct inpcb *inp = sotoinpcb(so);
+ struct tcpcb *tp = intotcpcb(inp);
struct icl_pdu *ip;
struct mbuf *m;
struct mbufq mq;
@@ -476,7 +477,7 @@ icl_cxgbei_tx_main(void *arg)
INP_WLOCK(inp);
ICL_CONN_UNLOCK(ic);
- if (__predict_false(inp->inp_flags & INP_DROPPED) ||
+ if (__predict_false(tp->t_flags & TF_DISCONNECTED) ||
__predict_false((toep->flags & TPF_ATTACHED) == 0)) {
mbufq_drain(&mq);
} else {
@@ -1080,7 +1081,7 @@ icl_cxgbei_conn_handoff(struct icl_conn *ic, int fd)
inp = sotoinpcb(so);
INP_WLOCK(inp);
tp = intotcpcb(inp);
- if (inp->inp_flags & INP_DROPPED) {
+ if (tp->t_flags & TF_DISCONNECTED) {
INP_WUNLOCK(inp);
error = ENOTCONN;
goto out;
@@ -1334,6 +1335,7 @@ icl_cxgbei_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip,
struct cxgbei_ddp_state *ddp;
struct ppod_reservation *prsv;
struct inpcb *inp;
+ struct tcpcb *tp;
struct mbufq mq;
uint32_t itt;
int rc = 0;
@@ -1421,8 +1423,9 @@ no_ddp:
* detached already.
*/
inp = sotoinpcb(ic->ic_socket);
+ tp = intotcpcb(inp);
INP_WLOCK(inp);
- if ((inp->inp_flags & INP_DROPPED) != 0) {
+ if ((tp->t_flags & TF_DISCONNECTED) != 0) {
INP_WUNLOCK(inp);
mbufq_drain(&mq);
t4_free_page_pods(prsv);
@@ -1497,6 +1500,7 @@ icl_cxgbei_conn_transfer_setup(struct icl_conn *ic, struct icl_pdu *ip,
struct ppod_reservation *prsv;
struct ctl_sg_entry *sgl, sg_entry;
struct inpcb *inp;
+ struct tcpcb *tp;
struct mbufq mq;
int sg_entries = ctsio->kern_sg_entries;
uint32_t ttt;
@@ -1597,9 +1601,10 @@ no_ddp:
return (ECONNRESET);
}
inp = sotoinpcb(ic->ic_socket);
+ tp = intotcpcb(inp);
INP_WLOCK(inp);
ICL_CONN_UNLOCK(ic);
- if ((inp->inp_flags & INP_DROPPED) != 0) {
+ if ((tp->t_flags & TF_DISCONNECTED) != 0) {
INP_WUNLOCK(inp);
mbufq_drain(&mq);
t4_free_page_pods(prsv);
diff --git a/sys/dev/cxgbe/iw_cxgbe/qp.c b/sys/dev/cxgbe/iw_cxgbe/qp.c
index cbf4bae00a60..372fc5418b91 100644
--- a/sys/dev/cxgbe/iw_cxgbe/qp.c
+++ b/sys/dev/cxgbe/iw_cxgbe/qp.c
@@ -64,7 +64,7 @@ struct cpl_set_tcb_rpl;
#include "iw_cxgbe.h"
#include "user.h"
-static int creds(struct toepcb *toep, struct inpcb *inp, size_t wrsize);
+static int creds(struct toepcb *toep, struct tcpcb *tp, size_t wrsize);
static int max_fr_immd = T4_MAX_FR_IMMD;//SYSCTL parameter later...
static int alloc_ird(struct c4iw_dev *dev, u32 ird)
@@ -1149,7 +1149,7 @@ static void post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe,
term->ecode = qhp->attr.ecode;
} else
build_term_codes(err_cqe, &term->layer_etype, &term->ecode);
- ret = creds(toep, inp, sizeof(*wqe));
+ ret = creds(toep, tp, sizeof(*wqe));
if (ret) {
free_wrqe(wr);
return;
@@ -1253,8 +1253,7 @@ rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp, struct c4iw_ep *ep)
int ret;
struct wrqe *wr;
struct socket *so = ep->com.so;
- struct inpcb *inp = sotoinpcb(so);
- struct tcpcb *tp = intotcpcb(inp);
+ struct tcpcb *tp = intotcpcb(sotoinpcb(so));
struct toepcb *toep = tp->t_toe;
KASSERT(rhp == qhp->rhp && ep == qhp->ep, ("%s: EDOOFUS", __func__));
@@ -1277,7 +1276,7 @@ rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp, struct c4iw_ep *ep)
c4iw_init_wr_wait(&ep->com.wr_wait);
- ret = creds(toep, inp, sizeof(*wqe));
+ ret = creds(toep, tp, sizeof(*wqe));
if (ret) {
free_wrqe(wr);
return ret;
@@ -1315,14 +1314,14 @@ static void build_rtr_msg(u8 p2p_type, struct fw_ri_init *init)
}
static int
-creds(struct toepcb *toep, struct inpcb *inp, size_t wrsize)
+creds(struct toepcb *toep, struct tcpcb *tp, size_t wrsize)
{
struct ofld_tx_sdesc *txsd;
CTR3(KTR_IW_CXGBE, "%s:creB %p %u", __func__, toep , wrsize);
- INP_WLOCK(inp);
- if ((inp->inp_flags & INP_DROPPED) != 0) {
- INP_WUNLOCK(inp);
+ INP_WLOCK(tptoinpcb(tp));
+ if (tp->t_flags & TF_DISCONNECTED) {
+ INP_WUNLOCK(tptoinpcb(tp));
return (EINVAL);
}
txsd = &toep->txsd[toep->txsd_pidx];
@@ -1336,7 +1335,7 @@ creds(struct toepcb *toep, struct inpcb *inp, size_t wrsize)
if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
toep->txsd_pidx = 0;
toep->txsd_avail--;
- INP_WUNLOCK(inp);
+ INP_WUNLOCK(tptoinpcb(tp));
CTR5(KTR_IW_CXGBE, "%s:creE %p %u %u %u", __func__, toep ,
txsd->tx_credits, toep->tx_credits, toep->txsd_pidx);
return (0);
@@ -1351,8 +1350,7 @@ static int rdma_init(struct c4iw_dev *rhp, struct c4iw_qp *qhp)
struct c4iw_rdev *rdev = &qhp->rhp->rdev;
struct adapter *sc = rdev->adap;
struct socket *so = ep->com.so;
- struct inpcb *inp = sotoinpcb(so);
- struct tcpcb *tp = intotcpcb(inp);
+ struct tcpcb *tp = intotcpcb(sotoinpcb(so));
struct toepcb *toep = tp->t_toe;
CTR5(KTR_IW_CXGBE, "%s qhp %p qid 0x%x ep %p tid %u", __func__, qhp,
@@ -1416,7 +1414,7 @@ static int rdma_init(struct c4iw_dev *rhp, struct c4iw_qp *qhp)
c4iw_init_wr_wait(&ep->com.wr_wait);
- ret = creds(toep, inp, sizeof(*wqe));
+ ret = creds(toep, tp, sizeof(*wqe));
if (ret) {
free_wrqe(wr);
free_ird(rhp, qhp->attr.max_ird);
diff --git a/sys/dev/cxgbe/nvmf/nvmf_che.c b/sys/dev/cxgbe/nvmf/nvmf_che.c
index 5c2174b8a40b..afdfc2f1b758 100644
--- a/sys/dev/cxgbe/nvmf/nvmf_che.c
+++ b/sys/dev/cxgbe/nvmf/nvmf_che.c
@@ -555,6 +555,7 @@ che_write_adapter_mem(struct nvmf_che_qpair *qp, uint32_t addr, uint32_t len,
struct toepcb *toep = qp->toep;
struct socket *so = qp->so;
struct inpcb *inp = sotoinpcb(so);
+ struct tcpcb *tp = intotcpcb(inp);
struct mbufq mq;
int error;
@@ -568,7 +569,7 @@ che_write_adapter_mem(struct nvmf_che_qpair *qp, uint32_t addr, uint32_t len,
goto error;
INP_WLOCK(inp);
- if ((inp->inp_flags & INP_DROPPED) != 0) {
+ if ((tp->t_flags & TF_DISCONNECTED) != 0) {
INP_WUNLOCK(inp);
error = ECONNRESET;
goto error;
@@ -862,12 +863,13 @@ nvmf_che_write_pdu(struct nvmf_che_qpair *qp, struct mbuf *m)
struct epoch_tracker et;
struct socket *so = qp->so;
struct inpcb *inp = sotoinpcb(so);
+ struct tcpcb *tp = intotcpcb(inp);
struct toepcb *toep = qp->toep;
CURVNET_SET(so->so_vnet);
NET_EPOCH_ENTER(et);
INP_WLOCK(inp);
- if (__predict_false(inp->inp_flags & INP_DROPPED) ||
+ if (__predict_false(tp->t_flags & TF_DISCONNECTED) ||
__predict_false((toep->flags & TPF_ATTACHED) == 0)) {
m_freem(m);
} else {
@@ -2052,10 +2054,11 @@ do_nvmt_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
("%s: payload length mismatch", __func__));
inp = toep->inp;
+ tp = intotcpcb(inp);
INP_WLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
- CTR(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
- __func__, tid, len, inp->inp_flags);
+ if (tp->t_flags & TF_DISCONNECTED) {
+ CTR(KTR_CXGBE, "%s: tid %u, rx (%d bytes), t_flags 0x%x",
+ __func__, tid, len, tp->t_flags);
INP_WUNLOCK(inp);
m_freem(m);
return (0);
@@ -2070,7 +2073,6 @@ do_nvmt_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
mbufq_enqueue(&qp->rx_data, m);
SOCKBUF_UNLOCK(&so->so_rcv);
- tp = intotcpcb(inp);
tp->t_rcvtime = ticks;
#ifdef VERBOSE_TRACES
@@ -2092,6 +2094,7 @@ do_nvmt_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
struct nvmf_che_qpair *qp = toep->ulpcb;
struct socket *so = qp->so;
struct inpcb *inp = toep->inp;
+ struct tcpcb *tp = intotcpcb(inp);
u_int hlen __diagused;
bool empty;
@@ -2107,9 +2110,9 @@ do_nvmt_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
("%s: payload length mismatch", __func__));
INP_WLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
- CTR(KTR_CXGBE, "%s: tid %u, rx (hlen %u), inp_flags 0x%x",
- __func__, tid, hlen, inp->inp_flags);
+ if (tp->t_flags & TF_DISCONNECTED) {
+ CTR(KTR_CXGBE, "%s: tid %u, rx (hlen %u), t_flags 0x%x",
+ __func__, tid, hlen, tp->t_flags);
INP_WUNLOCK(inp);
m_freem(m);
return (0);
@@ -2505,7 +2508,7 @@ che_allocate_qpair(bool controller, const nvlist_t *nvl)
inp = sotoinpcb(so);
INP_WLOCK(inp);
tp = intotcpcb(inp);
- if (inp->inp_flags & INP_DROPPED) {
+ if (tp->t_flags & TF_DISCONNECTED) {
INP_WUNLOCK(inp);
free(qp->fl_cid_set, M_NVMF_CHE);
free(qp->fl_cids, M_NVMF_CHE);
diff --git a/sys/dev/cxgbe/tom/t4_connect.c b/sys/dev/cxgbe/tom/t4_connect.c
index c236ee060bc2..e5f6053e2cb6 100644
--- a/sys/dev/cxgbe/tom/t4_connect.c
+++ b/sys/dev/cxgbe/tom/t4_connect.c
@@ -78,6 +78,7 @@ do_act_establish(struct sge_iq *iq, const struct rss_header *rss,
u_int atid = G_TID_TID(ntohl(cpl->tos_atid));
struct toepcb *toep = lookup_atid(sc, atid);
struct inpcb *inp = toep->inp;
+ struct tcpcb *tp = intotcpcb(inp);
KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
KASSERT(toep->tid == atid, ("%s: toep tid/atid mismatch", __func__));
@@ -95,7 +96,7 @@ do_act_establish(struct sge_iq *iq, const struct rss_header *rss,
toep->ctrlq = &sc->sge.ctrlq[toep->params.ctrlq_idx];
}
- if (inp->inp_flags & INP_DROPPED) {
+ if (tp->t_flags & TF_DISCONNECTED) {
/* socket closed by the kernel before hw told us it connected */
diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c
index 7e1c497240c2..6e34d5f54897 100644
--- a/sys/dev/cxgbe/tom/t4_cpl_io.c
+++ b/sys/dev/cxgbe/tom/t4_cpl_io.c
@@ -245,13 +245,13 @@ send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt)
struct cpl_abort_req *req;
int tid = toep->tid;
struct inpcb *inp = toep->inp;
- struct tcpcb *tp = intotcpcb(inp); /* don't use if INP_DROPPED */
+ struct tcpcb *tp = intotcpcb(inp);
INP_WLOCK_ASSERT(inp);
CTR6(KTR_CXGBE, "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x%s",
__func__, toep->tid,
- inp->inp_flags & INP_DROPPED ? "inp dropped" :
+ tp->t_flags & TF_DISCONNECTED ? "TCP disconnected" :
tcpstates[tp->t_state],
toep->flags, inp->inp_flags,
toep->flags & TPF_ABORT_SHUTDOWN ?
@@ -273,7 +273,7 @@ send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt)
req = wrtod(wr);
INIT_TP_WR_MIT_CPL(req, CPL_ABORT_REQ, tid);
- if (inp->inp_flags & INP_DROPPED)
+ if (tp->t_flags & TF_DISCONNECTED)
req->rsvd0 = htobe32(snd_nxt);
else
req->rsvd0 = htobe32(tp->snd_nxt);
@@ -284,7 +284,7 @@ send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt)
* XXX: What's the correct way to tell that the inp hasn't been detached
* from its socket? Should I even be flushing the snd buffer here?
*/
- if ((inp->inp_flags & INP_DROPPED) == 0) {
+ if ((tp->t_flags & TF_DISCONNECTED) == 0) {
struct socket *so = inp->inp_socket;
if (so != NULL) /* because I'm not sure. See comment above */
@@ -1588,8 +1588,8 @@ t4_tod_output(struct toedev *tod, struct tcpcb *tp)
struct toepcb *toep = tp->t_toe;
INP_WLOCK_ASSERT(inp);
- KASSERT((inp->inp_flags & INP_DROPPED) == 0,
- ("%s: inp %p dropped.", __func__, inp));
+ KASSERT((tp->t_flags & TF_DISCONNECTED) == 0,
+ ("%s: tcpcb %p disconnected", __func__, tp));
KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
t4_push_data(sc, toep, 0);
@@ -1607,8 +1607,8 @@ t4_send_fin(struct toedev *tod, struct tcpcb *tp)
struct toepcb *toep = tp->t_toe;
INP_WLOCK_ASSERT(inp);
- KASSERT((inp->inp_flags & INP_DROPPED) == 0,
- ("%s: inp %p dropped.", __func__, inp));
+ KASSERT((tp->t_flags & TF_DISCONNECTED) == 0,
+ ("%s: tcpcb %p disconnected", __func__, tp));
KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
toep->flags |= TPF_SEND_FIN;
@@ -1628,8 +1628,8 @@ t4_send_rst(struct toedev *tod, struct tcpcb *tp)
struct toepcb *toep = tp->t_toe;
INP_WLOCK_ASSERT(inp);
- KASSERT((inp->inp_flags & INP_DROPPED) == 0,
- ("%s: inp %p dropped.", __func__, inp));
+ KASSERT((tp->t_flags & TF_DISCONNECTED) == 0,
+ ("%s: tcpcb %p disconnected", __func__, tp));
KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
/* hmmmm */
@@ -1921,7 +1921,7 @@ do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
}
toep->flags |= TPF_ABORT_SHUTDOWN;
- if ((inp->inp_flags & INP_DROPPED) == 0) {
+ if ((tp->t_flags & TF_DISCONNECTED) == 0) {
struct socket *so = inp->inp_socket;
if (so != NULL)
@@ -2010,17 +2010,16 @@ do_rx_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
m_adj(m, sizeof(*cpl));
len = m->m_pkthdr.len;
+ tp = intotcpcb(inp);
INP_WLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
- CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
- __func__, tid, len, inp->inp_flags);
+ if (tp->t_flags & TF_DISCONNECTED) {
+ CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), t_flags 0x%x",
+ __func__, tid, len, tp->t_flags);
INP_WUNLOCK(inp);
m_freem(m);
return (0);
}
- tp = intotcpcb(inp);
-
if (__predict_false(ulp_mode(toep) == ULP_MODE_TLS &&
toep->flags & TPF_TLS_RECEIVE)) {
/* Received "raw" data on a TLS socket. */
@@ -2170,6 +2169,7 @@ do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
}
inp = toep->inp;
+ tp = intotcpcb(inp);
KASSERT(opcode == CPL_FW4_ACK,
("%s: unexpected opcode 0x%x", __func__, opcode));
@@ -2183,10 +2183,8 @@ do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
return (0);
}
- KASSERT((inp->inp_flags & INP_DROPPED) == 0,
- ("%s: inp_flags 0x%x", __func__, inp->inp_flags));
-
- tp = intotcpcb(inp);
+ KASSERT((tp->t_flags & TF_DISCONNECTED) == 0,
+ ("%s: t_flags 0x%x", __func__, tp->t_flags));
if (cpl->flags & CPL_FW4_ACK_FLAGS_SEQVAL) {
tcp_seq snd_una = be32toh(cpl->snd_una);
@@ -2627,8 +2625,9 @@ sendanother:
/* Inlined tcp_usr_send(). */
inp = toep->inp;
+ tp = intotcpcb(inp);
INP_WLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
+ if (tp->t_flags & TF_DISCONNECTED) {
INP_WUNLOCK(inp);
SOCK_IO_SEND_UNLOCK(so);
error = ECONNRESET;
@@ -2642,8 +2641,7 @@ sendanother:
sbappendstream(sb, m, 0);
m = NULL;
- if (!(inp->inp_flags & INP_DROPPED)) {
- tp = intotcpcb(inp);
+ if (!(tp->t_flags & TF_DISCONNECTED)) {
if (moretocome)
tp->t_flags |= TF_MORETOCOME;
error = tcp_output(tp);
diff --git a/sys/dev/cxgbe/tom/t4_ddp.c b/sys/dev/cxgbe/tom/t4_ddp.c
index 35fb1061d867..9d422c2b793e 100644
--- a/sys/dev/cxgbe/tom/t4_ddp.c
+++ b/sys/dev/cxgbe/tom/t4_ddp.c
@@ -641,8 +641,8 @@ handle_ddp_data_aio(struct toepcb *toep, __be32 ddp_report, __be32 rcv_nxt,
uint32_t report = be32toh(ddp_report);
unsigned int db_idx;
struct inpcb *inp = toep->inp;
+ struct tcpcb *tp = intotcpcb(inp);
struct ddp_buffer *db;
- struct tcpcb *tp;
struct socket *so;
struct sockbuf *sb;
struct kaiocb *job;
@@ -664,13 +664,13 @@ handle_ddp_data_aio(struct toepcb *toep, __be32 ddp_report, __be32 rcv_nxt,
db = &toep->ddp.db[db_idx];
job = db->job;
- if (__predict_false(inp->inp_flags & INP_DROPPED)) {
+ if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
/*
* This can happen due to an administrative tcpdrop(8).
* Just fail the request with ECONNRESET.
*/
- CTR5(KTR_CXGBE, "%s: tid %u, seq 0x%x, len %d, inp_flags 0x%x",
- __func__, toep->tid, be32toh(rcv_nxt), len, inp->inp_flags);
+ CTR5(KTR_CXGBE, "%s: tid %u, seq 0x%x, len %d, t_flags 0x%x",
+ __func__, toep->tid, be32toh(rcv_nxt), len, tp->t_flags);
if (aio_clear_cancel_function(job))
ddp_complete_one(job, ECONNRESET);
goto completed;
@@ -859,7 +859,7 @@ handle_ddp_data_rcvbuf(struct toepcb *toep, __be32 ddp_report, __be32 rcv_nxt,
{
uint32_t report = be32toh(ddp_report);
struct inpcb *inp = toep->inp;
- struct tcpcb *tp;
+ struct tcpcb *tp = intotcpcb(inp);
struct socket *so;
struct sockbuf *sb;
struct ddp_buffer *db;
@@ -881,20 +881,18 @@ handle_ddp_data_rcvbuf(struct toepcb *toep, __be32 ddp_report, __be32 rcv_nxt,
toep->ddp.active_id, toep->tid));
db = &toep->ddp.db[db_idx];
- if (__predict_false(inp->inp_flags & INP_DROPPED)) {
+ if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
/*
* This can happen due to an administrative tcpdrop(8).
* Just ignore the received data.
*/
- CTR5(KTR_CXGBE, "%s: tid %u, seq 0x%x, len %d, inp_flags 0x%x",
- __func__, toep->tid, be32toh(rcv_nxt), len, inp->inp_flags);
+ CTR5(KTR_CXGBE, "%s: tid %u, seq 0x%x, len %d, t_flags 0x%x",
+ __func__, toep->tid, be32toh(rcv_nxt), len, tp->t_flags);
if (invalidated)
complete_ddp_buffer(toep, db, db_idx);
goto out;
}
- tp = intotcpcb(inp);
-
/*
* For RX_DDP_COMPLETE, len will be zero and rcv_nxt is the
* sequence number of the next byte to receive. The length of
diff --git a/sys/dev/cxgbe/tom/t4_listen.c b/sys/dev/cxgbe/tom/t4_listen.c
index b879f6883f25..359267b7db90 100644
--- a/sys/dev/cxgbe/tom/t4_listen.c
+++ b/sys/dev/cxgbe/tom/t4_listen.c
@@ -886,6 +886,7 @@ do_pass_open_rpl(struct sge_iq *iq, const struct rss_header *rss,
unsigned int status = cpl->status;
struct listen_ctx *lctx = lookup_stid(sc, stid);
struct inpcb *inp = lctx->inp;
+ struct tcpcb *tp = intotcpcb(inp);
#ifdef INVARIANTS
unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
#endif
@@ -911,13 +912,13 @@ do_pass_open_rpl(struct sge_iq *iq, const struct rss_header *rss,
* If the inp has been dropped (listening socket closed) then
* listen_stop must have run and taken the inp out of the hash.
*/
- if (inp->inp_flags & INP_DROPPED) {
+ if (tp->t_flags & TF_DISCONNECTED) {
KASSERT(listen_hash_del(sc, inp) == NULL,
("%s: inp %p still in listen hash", __func__, inp));
}
#endif
- if (inp->inp_flags & INP_DROPPED && status != CPL_ERR_NONE) {
+ if (tp->t_flags & TF_DISCONNECTED && status != CPL_ERR_NONE) {
if (release_lctx(sc, lctx) != NULL)
INP_WUNLOCK(inp);
return (status);
@@ -928,7 +929,7 @@ do_pass_open_rpl(struct sge_iq *iq, const struct rss_header *rss,
* it has started the hardware listener. Stop it; the lctx will be
* released in do_close_server_rpl.
*/
- if (inp->inp_flags & INP_DROPPED) {
+ if (tp->t_flags & TF_DISCONNECTED) {
destroy_server(sc, lctx);
INP_WUNLOCK(inp);
return (status);
@@ -1336,6 +1337,7 @@ do_pass_accept_req(struct sge_iq *iq, const struct rss_header *rss,
unsigned int tid = GET_TID(cpl);
struct listen_ctx *lctx = lookup_stid(sc, stid);
struct inpcb *inp;
+ struct tcpcb *tp;
struct socket *so;
struct in_conninfo inc;
struct tcphdr th;
@@ -1477,10 +1479,11 @@ found:
}
inp = lctx->inp; /* listening socket, not owned by TOE */
+ tp = intotcpcb(inp);
INP_RLOCK(inp);
/* Don't offload if the listening socket has closed */
- if (__predict_false(inp->inp_flags & INP_DROPPED)) {
+ if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
INP_RUNLOCK(inp);
NET_EPOCH_EXIT(et);
REJECT_PASS_ACCEPT_REQ(false);
@@ -1622,6 +1625,7 @@ do_pass_establish(struct sge_iq *iq, const struct rss_header *rss,
struct synq_entry *synqe = lookup_tid(sc, tid);
struct listen_ctx *lctx = synqe->lctx;
struct inpcb *inp = lctx->inp, *new_inp;
+ struct tcpcb *tp = intotcpcb(inp);
struct socket *so;
struct tcphdr th;
struct tcpopt to;
@@ -1653,7 +1657,7 @@ do_pass_establish(struct sge_iq *iq, const struct rss_header *rss,
KASSERT(vi->adapter == sc,
("%s: vi %p, sc %p mismatch", __func__, vi, sc));
- if (__predict_false(inp->inp_flags & INP_DROPPED)) {
+ if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
reset:
send_abort_rpl_synqe(TOEDEV(ifp), synqe, CPL_ABORT_SEND_RST);
INP_WUNLOCK(inp);
diff --git a/sys/dev/cxgbe/tom/t4_tls.c b/sys/dev/cxgbe/tom/t4_tls.c
index bbcc1c88c3db..0616279ba15e 100644
--- a/sys/dev/cxgbe/tom/t4_tls.c
+++ b/sys/dev/cxgbe/tom/t4_tls.c
@@ -762,7 +762,7 @@ do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
unsigned int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct inpcb *inp = toep->inp;
- struct tcpcb *tp;
+ struct tcpcb *tp = intotcpcb(inp);
int len;
/* XXX: Should this match do_rx_data instead? */
@@ -781,9 +781,9 @@ do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
("%s: payload length mismatch", __func__));
INP_WLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
- CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
- __func__, tid, len, inp->inp_flags);
+ if (tp->t_flags & TF_DISCONNECTED) {
+ CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), t_flags 0x%x",
+ __func__, tid, len, tp->t_flags);
INP_WUNLOCK(inp);
m_freem(m);
return (0);
@@ -803,7 +803,6 @@ do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
#endif
}
- tp = intotcpcb(inp);
tp->t_rcvtime = ticks;
#ifdef VERBOSE_TRACES
@@ -824,7 +823,7 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
unsigned int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct inpcb *inp = toep->inp;
- struct tcpcb *tp;
+ struct tcpcb *tp = intotcpcb(inp);
struct socket *so;
struct sockbuf *sb;
struct mbuf *tls_data;
@@ -851,9 +850,9 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
("%s: payload length mismatch", __func__));
INP_WLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
- CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
- __func__, tid, len, inp->inp_flags);
+ if (tp->t_flags & TF_DISCONNECTED) {
+ CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), t_flags 0x%x",
+ __func__, tid, len, tp->t_flags);
INP_WUNLOCK(inp);
m_freem(m);
return (0);
@@ -862,7 +861,6 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
pdu_length = G_CPL_RX_TLS_CMP_PDULENGTH(be32toh(cpl->pdulength_length));
so = inp_inpcbtosocket(inp);
- tp = intotcpcb(inp);
#ifdef VERBOSE_TRACES
CTR6(KTR_CXGBE, "%s: tid %u PDU len %d len %d seq %u, rcv_nxt %u",
diff --git a/sys/dev/cxgbe/tom/t4_tom.c b/sys/dev/cxgbe/tom/t4_tom.c
index 8dfffd465345..950608053be7 100644
--- a/sys/dev/cxgbe/tom/t4_tom.c
+++ b/sys/dev/cxgbe/tom/t4_tom.c
@@ -1830,7 +1830,7 @@ live_tid_failure_cleanup(struct adapter *sc, struct toepcb *toep, u_int status)
INP_WLOCK(inp);
tp = intotcpcb(inp);
toep->flags |= TPF_ABORT_SHUTDOWN;
- if ((inp->inp_flags & INP_DROPPED) == 0) {
+ if ((tp->t_flags & TF_DISCONNECTED) == 0) {
struct socket *so = inp->inp_socket;
if (so != NULL)
@@ -2283,8 +2283,8 @@ find_offload_adapter_cb(struct adapter *sc, void *arg)
struct find_offload_adapter_data *fa = arg;
struct socket *so = fa->so;
struct tom_data *td = sc->tom_softc;
- struct tcpcb *tp;
- struct inpcb *inp;
+ struct inpcb *inp = sotoinpcb(so);
+ struct tcpcb *tp = intotcpcb(inp);
/* Non-TCP were filtered out earlier. */
MPASS(so->so_proto->pr_protocol == IPPROTO_TCP);
@@ -2295,10 +2295,8 @@ find_offload_adapter_cb(struct adapter *sc, void *arg)
if (td == NULL)
return; /* TOE not enabled on this adapter. */
- inp = sotoinpcb(so);
INP_WLOCK(inp);
- if ((inp->inp_flags & INP_DROPPED) == 0) {
- tp = intotcpcb(inp);
+ if ((tp->t_flags & TF_DISCONNECTED) == 0) {
if (tp->t_flags & TF_TOE && tp->tod == &td->tod)
fa->sc = sc; /* Found. */
}
diff --git a/sys/kern/uipc_ktls.c b/sys/kern/uipc_ktls.c
index 35009ad77722..0112cf8b6941 100644
--- a/sys/kern/uipc_ktls.c
+++ b/sys/kern/uipc_ktls.c
@@ -870,21 +870,15 @@ ktls_clone_session(struct ktls_session *tls, int direction)
static int
ktls_try_toe(struct socket *so, struct ktls_session *tls, int direction)
{
- struct inpcb *inp;
- struct tcpcb *tp;
+ struct inpcb *inp = sotoinpcb(so);
+ struct tcpcb *tp = intotcpcb(inp);
int error;
- inp = so->so_pcb;
INP_WLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
- INP_WUNLOCK(inp);
- return (ECONNRESET);
- }
- if (inp->inp_socket == NULL) {
+ if (tp->t_flags & TF_DISCONNECTED) {
INP_WUNLOCK(inp);
return (ECONNRESET);
}
- tp = intotcpcb(inp);
if (!(tp->t_flags & TF_TOE)) {
INP_WUNLOCK(inp);
return (EOPNOTSUPP);
@@ -923,19 +917,14 @@ ktls_alloc_snd_tag(struct inpcb *inp, struct ktls_session *tls, bool force,
union if_snd_tag_alloc_params params;
struct ifnet *ifp;
struct nhop_object *nh;
- struct tcpcb *tp;
+ struct tcpcb *tp = intotcpcb(inp);
int error;
INP_RLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
+ if (tp->t_flags & TF_DISCONNECTED) {
INP_RUNLOCK(inp);
return (ECONNRESET);
}
- if (inp->inp_socket == NULL) {
- INP_RUNLOCK(inp);
- return (ECONNRESET);
- }
- tp = intotcpcb(inp);
/*
* Check administrative controls on ifnet TLS to determine if
@@ -1027,11 +1016,7 @@ ktls_alloc_rcv_tag(struct inpcb *inp, struct ktls_session *tls,
return (ENXIO);
INP_RLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
- INP_RUNLOCK(inp);
- return (ECONNRESET);
- }
- if (inp->inp_socket == NULL) {
+ if (intotcpcb(inp)->t_flags & TF_DISCONNECTED) {
INP_RUNLOCK(inp);
return (ECONNRESET);
}
@@ -1506,23 +1491,15 @@ ktls_get_rx_mode(struct socket *so, int *modep)
int
ktls_get_rx_sequence(struct inpcb *inp, uint32_t *tcpseq, uint64_t *tlsseq)
{
- struct socket *so;
- struct tcpcb *tp;
+ struct socket *so = inp->inp_socket;
+ struct tcpcb *tp = intotcpcb(inp);
INP_RLOCK(inp);
- so = inp->inp_socket;
- if (__predict_false(so == NULL)) {
- INP_RUNLOCK(inp);
- return (EINVAL);
- }
- if (inp->inp_flags & INP_DROPPED) {
+ if (tp->t_flags & TF_DISCONNECTED) {
INP_RUNLOCK(inp);
return (ECONNRESET);
}
- tp = intotcpcb(inp);
- MPASS(tp != NULL);
-
SOCKBUF_LOCK(&so->so_rcv);
*tcpseq = tp->rcv_nxt - so->so_rcv.sb_tlscc;
*tlsseq = so->so_rcv.sb_tls_seqno;
@@ -1697,7 +1674,7 @@ ktls_reset_receive_tag(void *context, int pending)
ifp = NULL;
INP_RLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
+ if (intotcpcb(inp)->t_flags & TF_DISCONNECTED) {
INP_RUNLOCK(inp);
goto out;
}
@@ -1818,8 +1795,8 @@ ktls_reset_send_tag(void *context, int pending)
} else {
NET_EPOCH_ENTER(et);
INP_WLOCK(inp);
- if (!(inp->inp_flags & INP_DROPPED)) {
- tp = intotcpcb(inp);
+ tp = intotcpcb(inp);
+ if (!(tp->t_flags & TF_DISCONNECTED)) {
CURVNET_SET(inp->inp_vnet);
tp = tcp_drop(tp, ECONNABORTED);
CURVNET_RESTORE();
@@ -2461,26 +2438,19 @@ ktls_resync_ifnet(struct socket *so, uint32_t tls_len, uint64_t tls_rcd_num)
{
union if_snd_tag_modify_params params;
struct m_snd_tag *mst;
- struct inpcb *inp;
- struct tcpcb *tp;
+ struct inpcb *inp = sotoinpcb(so);
+ struct tcpcb *tp = intotcpcb(inp);
mst = so->so_rcv.sb_tls_info->snd_tag;
if (__predict_false(mst == NULL))
return (EINVAL);
- inp = sotoinpcb(so);
- if (__predict_false(inp == NULL))
- return (EINVAL);
-
INP_RLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
+ if (tp->t_flags & TF_DISCONNECTED) {
INP_RUNLOCK(inp);
return (ECONNRESET);
}
- tp = intotcpcb(inp);
- MPASS(tp != NULL);
-
/* Get the TCP sequence number of the next valid TLS header. */
SOCKBUF_LOCK(&so->so_rcv);
params.tls_rx.tls_hdr_tcp_sn =
@@ -2500,12 +2470,11 @@ ktls_drop(struct socket *so, int error)
{
struct epoch_tracker et;
struct inpcb *inp = sotoinpcb(so);
- struct tcpcb *tp;
+ struct tcpcb *tp = intotcpcb(inp);
NET_EPOCH_ENTER(et);
INP_WLOCK(inp);
- if (!(inp->inp_flags & INP_DROPPED)) {
- tp = intotcpcb(inp);
+ if (!(tp->t_flags & TF_DISCONNECTED)) {
CURVNET_SET(inp->inp_vnet);
tp = tcp_drop(tp, error);
CURVNET_RESTORE();
@@ -3372,7 +3341,8 @@ ktls_disable_ifnet_help(void *context, int pending __unused)
INP_WLOCK(inp);
so = inp->inp_socket;
MPASS(so != NULL);
- if (inp->inp_flags & INP_DROPPED) {
+ tp = intotcpcb(inp);
+ if (tp->t_flags & TF_DISCONNECTED) {
goto out;
}
@@ -3383,8 +3353,7 @@ ktls_disable_ifnet_help(void *context, int pending __unused)
if (err == 0) {
counter_u64_add(ktls_ifnet_disable_ok, 1);
/* ktls_set_tx_mode() drops inp wlock, so recheck flags */
- if ((inp->inp_flags & INP_DROPPED) == 0 &&
- (tp = intotcpcb(inp)) != NULL &&
+ if ((tp->t_flags & TF_DISCONNECTED) == 0 &&
tp->t_fb->tfb_hwtls_change != NULL)
*** 1071 LINES SKIPPED ***