git: c3c20de3b2d1 - main - tcp: move HPTS/LRO flags out of inpcb to tcpcb
Date: Tue, 25 Apr 2023 19:21:54 UTC
The branch main has been updated by glebius:
URL: https://cgit.FreeBSD.org/src/commit/?id=c3c20de3b2d1357d081ce6c41f15f23cd1020e35
commit c3c20de3b2d1357d081ce6c41f15f23cd1020e35
Author: Gleb Smirnoff <glebius@FreeBSD.org>
AuthorDate: 2023-04-25 19:19:48 +0000
Commit: Gleb Smirnoff <glebius@FreeBSD.org>
CommitDate: 2023-04-25 19:19:48 +0000
tcp: move HPTS/LRO flags out of inpcb to tcpcb
These flags are TCP specific. While here, make also several LRO
internal functions to pass tcpcb pointer instead of inpcb one.
Reviewed by: rrs
Differential Revision: https://reviews.freebsd.org/D39698
---
sys/netinet/in_pcb.h | 12 +++---
sys/netinet/tcp_hpts.c | 2 +-
sys/netinet/tcp_lro.c | 69 ++++++++++++++------------------
sys/netinet/tcp_stacks/bbr.c | 35 ++++++++--------
sys/netinet/tcp_stacks/rack.c | 68 +++++++++++++++----------------
sys/netinet/tcp_stacks/rack_bbr_common.c | 4 +-
sys/netinet/tcp_subr.c | 14 +++----
sys/netinet/tcp_syncache.c | 2 +-
sys/netinet/tcp_var.h | 25 +++++++-----
9 files changed, 112 insertions(+), 119 deletions(-)
diff --git a/sys/netinet/in_pcb.h b/sys/netinet/in_pcb.h
index 62c5758268a7..574d575de8f0 100644
--- a/sys/netinet/in_pcb.h
+++ b/sys/netinet/in_pcb.h
@@ -593,8 +593,8 @@ int inp_so_options(const struct inpcb *inp);
/*
* Flags for inp_flags2.
*/
-#define INP_MBUF_L_ACKS 0x00000001 /* We need large mbufs for ack compression */
-#define INP_MBUF_ACKCMP 0x00000002 /* TCP mbuf ack compression ok */
+/* 0x00000001 */
+/* 0x00000002 */
/* 0x00000004 */
#define INP_REUSEPORT 0x00000008 /* SO_REUSEPORT option is set */
/* 0x00000010 */
@@ -605,11 +605,11 @@ int inp_so_options(const struct inpcb *inp);
#define INP_RECVRSSBUCKETID 0x00000200 /* populate recv datagram with bucket id */
#define INP_RATE_LIMIT_CHANGED 0x00000400 /* rate limit needs attention */
#define INP_ORIGDSTADDR 0x00000800 /* receive IP dst address/port */
-#define INP_CANNOT_DO_ECN 0x00001000 /* The stack does not do ECN */
+/* 0x00001000 */
#define INP_REUSEPORT_LB 0x00002000 /* SO_REUSEPORT_LB option is set */
-#define INP_SUPPORTS_MBUFQ 0x00004000 /* Supports the mbuf queue method of LRO */
-#define INP_MBUF_QUEUE_READY 0x00008000 /* The transport is pacing, inputs can be queued */
-#define INP_DONT_SACK_QUEUE 0x00010000 /* If a sack arrives do not wake me */
+/* 0x00004000 */
+/* 0x00008000 */
+/* 0x00010000 */
#define INP_2PCP_SET 0x00020000 /* If the Eth PCP should be set explicitly */
#define INP_2PCP_BIT0 0x00040000 /* Eth PCP Bit 0 */
#define INP_2PCP_BIT1 0x00080000 /* Eth PCP Bit 1 */
diff --git a/sys/netinet/tcp_hpts.c b/sys/netinet/tcp_hpts.c
index 59122bb242b9..a10355a38b6f 100644
--- a/sys/netinet/tcp_hpts.c
+++ b/sys/netinet/tcp_hpts.c
@@ -1363,7 +1363,7 @@ again:
* cause a call to output if it is needed.
*/
tp->t_flags2 |= TF2_HPTS_CALLS;
- if ((inp->inp_flags2 & INP_SUPPORTS_MBUFQ) &&
+ if ((tp->t_flags2 & TF2_SUPPORTS_MBUFQ) &&
!STAILQ_EMPTY(&tp->t_inqueue)) {
error = (*tp->t_fb->tfb_do_queued_segments)(tp, 0);
if (error) {
diff --git a/sys/netinet/tcp_lro.c b/sys/netinet/tcp_lro.c
index 76c345add1f8..d76e1e09edd7 100644
--- a/sys/netinet/tcp_lro.c
+++ b/sys/netinet/tcp_lro.c
@@ -91,7 +91,7 @@ static int tcp_lro_rx_common(struct lro_ctrl *lc, struct mbuf *m,
uint32_t csum, bool use_hash);
#ifdef TCPHPTS
-static bool do_bpf_strip_and_compress(struct inpcb *, struct lro_ctrl *,
+static bool do_bpf_strip_and_compress(struct tcpcb *, struct lro_ctrl *,
struct lro_entry *, struct mbuf **, struct mbuf **, struct mbuf **,
bool *, bool, bool, struct ifnet *, bool);
@@ -1192,13 +1192,9 @@ tcp_queue_pkts(struct tcpcb *tp, struct lro_entry *le)
}
static bool
-tcp_lro_check_wake_status(struct inpcb *inp)
+tcp_lro_check_wake_status(struct tcpcb *tp)
{
- struct tcpcb *tp;
- tp = intotcpcb(inp);
- if (__predict_false(tp == NULL))
- return (true);
if (tp->t_fb->tfb_early_wake_check != NULL)
return ((tp->t_fb->tfb_early_wake_check)(tp));
return (false);
@@ -1206,15 +1202,10 @@ tcp_lro_check_wake_status(struct inpcb *inp)
static struct mbuf *
tcp_lro_get_last_if_ackcmp(struct lro_ctrl *lc, struct lro_entry *le,
- struct inpcb *inp, int32_t *new_m, bool can_append_old_cmp)
+ struct tcpcb *tp, int32_t *new_m, bool can_append_old_cmp)
{
- struct tcpcb *tp;
struct mbuf *m;
- tp = intotcpcb(inp);
- if (__predict_false(tp == NULL))
- return (NULL);
-
/* Look at the last mbuf if any in queue */
if (can_append_old_cmp) {
m = STAILQ_LAST(&tp->t_inqueue, mbuf, m_stailqpkt);
@@ -1226,13 +1217,13 @@ tcp_lro_get_last_if_ackcmp(struct lro_ctrl *lc, struct lro_entry *le,
return (m);
} else {
/* Mark we ran out of space */
- inp->inp_flags2 |= INP_MBUF_L_ACKS;
+ tp->t_flags2 |= TF2_MBUF_L_ACKS;
}
}
}
/* Decide mbuf size. */
tcp_lro_log(tp, lc, le, NULL, 21, 0, 0, 0, 0);
- if (inp->inp_flags2 & INP_MBUF_L_ACKS)
+ if (tp->t_flags2 & TF2_MBUF_L_ACKS)
m = m_getcl(M_NOWAIT, MT_DATA, M_ACKCMP | M_PKTHDR);
else
m = m_gethdr(M_NOWAIT, MT_DATA);
@@ -1248,7 +1239,7 @@ tcp_lro_get_last_if_ackcmp(struct lro_ctrl *lc, struct lro_entry *le,
return (m);
}
-static struct inpcb *
+static struct tcpcb *
tcp_lro_lookup(struct ifnet *ifp, struct lro_parser *pa)
{
struct inpcb *inp;
@@ -1277,10 +1268,10 @@ tcp_lro_lookup(struct ifnet *ifp, struct lro_parser *pa)
break;
#endif
default:
- inp = NULL;
- break;
+ return (NULL);
}
- return (inp);
+
+ return (intotcpcb(inp));
}
static inline bool
@@ -1335,7 +1326,6 @@ tcp_lro_ack_valid(struct mbuf *m, struct tcphdr *th, uint32_t **ppts, bool *othe
static int
tcp_lro_flush_tcphpts(struct lro_ctrl *lc, struct lro_entry *le)
{
- struct inpcb *inp;
struct tcpcb *tp;
struct mbuf **pp, *cmp, *mv_to;
struct ifnet *lagg_ifp;
@@ -1364,31 +1354,28 @@ tcp_lro_flush_tcphpts(struct lro_ctrl *lc, struct lro_entry *le)
IN6_IS_ADDR_UNSPECIFIED(&le->inner.data.s_addr.v6)))
return (TCP_LRO_CANNOT);
#endif
- /* Lookup inp, if any. */
- inp = tcp_lro_lookup(lc->ifp,
+ /* Lookup inp, if any. Returns locked TCP inpcb. */
+ tp = tcp_lro_lookup(lc->ifp,
(le->inner.data.lro_type == LRO_TYPE_NONE) ? &le->outer : &le->inner);
- if (inp == NULL)
+ if (tp == NULL)
return (TCP_LRO_CANNOT);
counter_u64_add(tcp_inp_lro_locks_taken, 1);
- /* Get TCP control structure. */
- tp = intotcpcb(inp);
-
/* Check if the inp is dead, Jim. */
if (tp->t_state == TCPS_TIME_WAIT) {
- INP_WUNLOCK(inp);
+ INP_WUNLOCK(tptoinpcb(tp));
return (TCP_LRO_CANNOT);
}
if (tp->t_lro_cpu == HPTS_CPU_NONE && lc->lro_cpu_is_set == 1)
tp->t_lro_cpu = lc->lro_last_cpu;
/* Check if the transport doesn't support the needed optimizations. */
- if ((inp->inp_flags2 & (INP_SUPPORTS_MBUFQ | INP_MBUF_ACKCMP)) == 0) {
- INP_WUNLOCK(inp);
+ if ((tp->t_flags2 & (TF2_SUPPORTS_MBUFQ | TF2_MBUF_ACKCMP)) == 0) {
+ INP_WUNLOCK(tptoinpcb(tp));
return (TCP_LRO_CANNOT);
}
- if (inp->inp_flags2 & INP_MBUF_QUEUE_READY)
+ if (tp->t_flags2 & TF2_MBUF_QUEUE_READY)
should_wake = false;
else
should_wake = true;
@@ -1411,7 +1398,7 @@ tcp_lro_flush_tcphpts(struct lro_ctrl *lc, struct lro_entry *le)
cmp = NULL;
for (pp = &le->m_head; *pp != NULL; ) {
mv_to = NULL;
- if (do_bpf_strip_and_compress(inp, lc, le, pp,
+ if (do_bpf_strip_and_compress(tp, lc, le, pp,
&cmp, &mv_to, &should_wake, bpf_req,
lagg_bpf_req, lagg_ifp, can_append_old_cmp) == false) {
/* Advance to next mbuf. */
@@ -1444,17 +1431,18 @@ tcp_lro_flush_tcphpts(struct lro_ctrl *lc, struct lro_entry *le)
/* Check if any data mbufs left. */
if (le->m_head != NULL) {
counter_u64_add(tcp_inp_lro_direct_queue, 1);
- tcp_lro_log(tp, lc, le, NULL, 22, 1, inp->inp_flags2, 0, 1);
+ tcp_lro_log(tp, lc, le, NULL, 22, 1, tp->t_flags2, 0, 1);
tcp_queue_pkts(tp, le);
}
if (should_wake) {
/* Wakeup */
counter_u64_add(tcp_inp_lro_wokeup_queue, 1);
if ((*tp->t_fb->tfb_do_queued_segments)(tp, 0))
- inp = NULL;
+ /* TCP cb gone and unlocked. */
+ return (0);
}
- if (inp != NULL)
- INP_WUNLOCK(inp);
+ INP_WUNLOCK(tptoinpcb(tp));
+
return (0); /* Success. */
}
#endif
@@ -1674,7 +1662,7 @@ build_ack_entry(struct tcp_ackent *ae, struct tcphdr *th, struct mbuf *m,
* and strip all, but the IPv4/IPv6 header.
*/
static bool
-do_bpf_strip_and_compress(struct inpcb *inp, struct lro_ctrl *lc,
+do_bpf_strip_and_compress(struct tcpcb *tp, struct lro_ctrl *lc,
struct lro_entry *le, struct mbuf **pp, struct mbuf **cmp, struct mbuf **mv_to,
bool *should_wake, bool bpf_req, bool lagg_bpf_req, struct ifnet *lagg_ifp, bool can_append_old_cmp)
{
@@ -1751,7 +1739,7 @@ do_bpf_strip_and_compress(struct inpcb *inp, struct lro_ctrl *lc,
/* Now lets look at the should wake states */
if ((other_opts == true) &&
- ((inp->inp_flags2 & INP_DONT_SACK_QUEUE) == 0)) {
+ ((tp->t_flags2 & TF2_DONT_SACK_QUEUE) == 0)) {
/*
* If there are other options (SACK?) and the
* tcp endpoint has not expressly told us it does
@@ -1760,13 +1748,13 @@ do_bpf_strip_and_compress(struct inpcb *inp, struct lro_ctrl *lc,
*should_wake = true;
} else if (*should_wake == false) {
/* Wakeup override check if we are false here */
- *should_wake = tcp_lro_check_wake_status(inp);
+ *should_wake = tcp_lro_check_wake_status(tp);
}
/* Is the ack compressable? */
if (can_compress == false)
goto done;
/* Does the TCP endpoint support ACK compression? */
- if ((inp->inp_flags2 & INP_MBUF_ACKCMP) == 0)
+ if ((tp->t_flags2 & TF2_MBUF_ACKCMP) == 0)
goto done;
/* Lets get the TOS/traffic class field */
@@ -1785,7 +1773,8 @@ do_bpf_strip_and_compress(struct inpcb *inp, struct lro_ctrl *lc,
/* Now lets get space if we don't have some already */
if (*cmp == NULL) {
new_one:
- nm = tcp_lro_get_last_if_ackcmp(lc, le, inp, &n_mbuf, can_append_old_cmp);
+ nm = tcp_lro_get_last_if_ackcmp(lc, le, tp, &n_mbuf,
+ can_append_old_cmp);
if (__predict_false(nm == NULL))
goto done;
*cmp = nm;
@@ -1812,7 +1801,7 @@ new_one:
nm = *cmp;
if (M_TRAILINGSPACE(nm) < sizeof(struct tcp_ackent)) {
/* We ran out of space */
- inp->inp_flags2 |= INP_MBUF_L_ACKS;
+ tp->t_flags2 |= TF2_MBUF_L_ACKS;
goto new_one;
}
}
diff --git a/sys/netinet/tcp_stacks/bbr.c b/sys/netinet/tcp_stacks/bbr.c
index f8c7557150dd..5ecb558dadb3 100644
--- a/sys/netinet/tcp_stacks/bbr.c
+++ b/sys/netinet/tcp_stacks/bbr.c
@@ -891,7 +891,7 @@ bbr_start_hpts_timer(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts, int32_
* Tell LRO that it can queue packets while
* we pace.
*/
- bbr->rc_inp->inp_flags2 |= INP_MBUF_QUEUE_READY;
+ bbr->rc_tp->t_flags2 |= TF2_MBUF_QUEUE_READY;
if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) &&
(bbr->rc_cwnd_limited == 0)) {
/*
@@ -899,9 +899,9 @@ bbr_start_hpts_timer(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts, int32_
* are running a rack timer we put on
* the do not disturbe even for sack.
*/
- inp->inp_flags2 |= INP_DONT_SACK_QUEUE;
+ tp->t_flags2 |= TF2_DONT_SACK_QUEUE;
} else
- inp->inp_flags2 &= ~INP_DONT_SACK_QUEUE;
+ tp->t_flags2 &= ~TF2_DONT_SACK_QUEUE;
bbr->rc_pacer_started = cts;
(void)tcp_hpts_insert_diag(tp, HPTS_USEC_TO_SLOTS(slot),
@@ -932,12 +932,12 @@ bbr_start_hpts_timer(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts, int32_
* if a sack arrives as long as we are
* not cwnd limited.
*/
- bbr->rc_inp->inp_flags2 |= INP_MBUF_QUEUE_READY;
- inp->inp_flags2 |= INP_DONT_SACK_QUEUE;
+ tp->t_flags2 |= (TF2_MBUF_QUEUE_READY |
+ TF2_DONT_SACK_QUEUE);
} else {
/* All other timers wake us up */
- bbr->rc_inp->inp_flags2 &= ~INP_MBUF_QUEUE_READY;
- inp->inp_flags2 &= ~INP_DONT_SACK_QUEUE;
+ tp->t_flags2 &= ~(TF2_MBUF_QUEUE_READY |
+ TF2_DONT_SACK_QUEUE);
}
bbr->bbr_timer_src = frm;
bbr_log_to_start(bbr, cts, hpts_timeout, slot, 0);
@@ -2498,7 +2498,7 @@ bbr_log_to_start(struct tcp_bbr *bbr, uint32_t cts, uint32_t to, int32_t slot, u
log.u_bbr.flex4 = slot;
log.u_bbr.flex5 = bbr->rc_tp->t_hpts_slot;
log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
- log.u_bbr.pkts_out = bbr->rc_inp->inp_flags2;
+ log.u_bbr.pkts_out = bbr->rc_tp->t_flags2;
log.u_bbr.flex8 = which;
TCP_LOG_EVENTP(bbr->rc_tp, NULL,
&bbr->rc_inp->inp_socket->so_rcv,
@@ -9940,13 +9940,13 @@ bbr_init(struct tcpcb *tp, void **ptr)
}
bbr = (struct tcp_bbr *)*ptr;
bbr->rtt_valid = 0;
- inp->inp_flags2 |= INP_CANNOT_DO_ECN;
- inp->inp_flags2 |= INP_SUPPORTS_MBUFQ;
+ tp->t_flags2 |= TF2_CANNOT_DO_ECN;
+ tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
/* Take off any undesired flags */
- inp->inp_flags2 &= ~INP_MBUF_QUEUE_READY;
- inp->inp_flags2 &= ~INP_DONT_SACK_QUEUE;
- inp->inp_flags2 &= ~INP_MBUF_ACKCMP;
- inp->inp_flags2 &= ~INP_MBUF_L_ACKS;
+ tp->t_flags2 &= ~TF2_MBUF_QUEUE_READY;
+ tp->t_flags2 &= ~TF2_DONT_SACK_QUEUE;
+ tp->t_flags2 &= ~TF2_MBUF_ACKCMP;
+ tp->t_flags2 &= ~TF2_MBUF_L_ACKS;
TAILQ_INIT(&bbr->r_ctl.rc_map);
TAILQ_INIT(&bbr->r_ctl.rc_free);
@@ -12046,7 +12046,7 @@ bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv)
return (retval < 0 ? retval : 0);
}
}
- bbr->rc_inp->inp_flags2 &= ~INP_MBUF_QUEUE_READY;
+ bbr->rc_tp->t_flags2 &= ~TF2_MBUF_QUEUE_READY;
if (hpts_calling &&
(bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
bbr->r_ctl.rc_last_delay_val = 0;
@@ -14078,15 +14078,14 @@ bbr_switch_failed(struct tcpcb *tp)
* pacer (if our flags are up) if so we are good, if
* not we need to get back into the pacer.
*/
- struct inpcb *inp = tptoinpcb(tp);
struct timeval tv;
uint32_t cts;
uint32_t toval;
struct tcp_bbr *bbr;
struct hpts_diag diag;
- inp->inp_flags2 |= INP_CANNOT_DO_ECN;
- inp->inp_flags2 |= INP_SUPPORTS_MBUFQ;
+ tp->t_flags2 |= TF2_CANNOT_DO_ECN;
+ tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
tcp_change_time_units(tp, TCP_TMR_GRANULARITY_TICKS);
if (tp->t_in_hpts > IHPTS_NONE) {
return;
diff --git a/sys/netinet/tcp_stacks/rack.c b/sys/netinet/tcp_stacks/rack.c
index 9e531a1d3182..44606c287f25 100644
--- a/sys/netinet/tcp_stacks/rack.c
+++ b/sys/netinet/tcp_stacks/rack.c
@@ -6822,12 +6822,12 @@ rack_start_hpts_timer(struct tcp_rack *rack, struct tcpcb *tp, uint32_t cts,
* are not on then these flags won't have any effect since it
* won't go through the queuing LRO path).
*
- * INP_MBUF_QUEUE_READY - This flags says that I am busy
+ * TF2_MBUF_QUEUE_READY - This flags says that I am busy
* pacing output, so don't disturb. But
* it also means LRO can wake me if there
* is a SACK arrival.
*
- * INP_DONT_SACK_QUEUE - This flag is used in conjunction
+ * TF2_DONT_SACK_QUEUE - This flag is used in conjunction
* with the above flag (QUEUE_READY) and
* when present it says don't even wake me
* if a SACK arrives.
@@ -6842,7 +6842,7 @@ rack_start_hpts_timer(struct tcp_rack *rack, struct tcpcb *tp, uint32_t cts,
* Other cases should usually have none of the flags set
* so LRO can call into us.
*/
- inp->inp_flags2 &= ~(INP_DONT_SACK_QUEUE|INP_MBUF_QUEUE_READY);
+ tp->t_flags2 &= ~(TF2_DONT_SACK_QUEUE|TF2_MBUF_QUEUE_READY);
if (slot) {
rack->r_ctl.rc_hpts_flags |= PACE_PKT_OUTPUT;
rack->r_ctl.rc_last_output_to = us_cts + slot;
@@ -6854,7 +6854,7 @@ rack_start_hpts_timer(struct tcp_rack *rack, struct tcpcb *tp, uint32_t cts,
* will be effective if mbuf queueing is on or
* compressed acks are being processed.
*/
- inp->inp_flags2 |= INP_MBUF_QUEUE_READY;
+ tp->t_flags2 |= TF2_MBUF_QUEUE_READY;
/*
* But wait if we have a Rack timer running
* even a SACK should not disturb us (with
@@ -6862,7 +6862,7 @@ rack_start_hpts_timer(struct tcp_rack *rack, struct tcpcb *tp, uint32_t cts,
*/
if (rack->r_ctl.rc_hpts_flags & PACE_TMR_RACK) {
if (rack->r_rr_config != 3)
- inp->inp_flags2 |= INP_DONT_SACK_QUEUE;
+ tp->t_flags2 |= TF2_DONT_SACK_QUEUE;
else if (rack->rc_pace_dnd) {
if (IN_RECOVERY(tp->t_flags)) {
/*
@@ -6873,13 +6873,14 @@ rack_start_hpts_timer(struct tcp_rack *rack, struct tcpcb *tp, uint32_t cts,
* and let all sacks wake us up.
*
*/
- inp->inp_flags2 |= INP_DONT_SACK_QUEUE;
+ tp->t_flags2 |= TF2_DONT_SACK_QUEUE;
}
}
}
/* For sack attackers we want to ignore sack */
if (rack->sack_attack_disable == 1) {
- inp->inp_flags2 |= (INP_DONT_SACK_QUEUE|INP_MBUF_QUEUE_READY);
+ tp->t_flags2 |= (TF2_DONT_SACK_QUEUE |
+ TF2_MBUF_QUEUE_READY);
} else if (rack->rc_ack_can_sendout_data) {
/*
* Ahh but wait, this is that special case
@@ -6887,7 +6888,8 @@ rack_start_hpts_timer(struct tcp_rack *rack, struct tcpcb *tp, uint32_t cts,
* backout the changes (used for non-paced
* burst limiting).
*/
- inp->inp_flags2 &= ~(INP_DONT_SACK_QUEUE|INP_MBUF_QUEUE_READY);
+ tp->t_flags2 &= ~(TF2_DONT_SACK_QUEUE |
+ TF2_MBUF_QUEUE_READY);
}
if ((rack->use_rack_rr) &&
(rack->r_rr_config < 2) &&
@@ -6908,7 +6910,7 @@ rack_start_hpts_timer(struct tcp_rack *rack, struct tcpcb *tp, uint32_t cts,
}
} else if (hpts_timeout) {
/*
- * With respect to inp_flags2 here, lets let any new acks wake
+ * With respect to t_flags2(?) here, lets let any new acks wake
* us up here. Since we are not pacing (no pacing timer), output
* can happen so we should let it. If its a Rack timer, then any inbound
* packet probably won't change the sending (we will be blocked)
@@ -8036,7 +8038,7 @@ rack_process_timers(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, uint8
* no-sack wakeup on since we no longer have a PKT_OUTPUT
* flag in place.
*/
- rack->rc_inp->inp_flags2 &= ~INP_DONT_SACK_QUEUE;
+ rack->rc_tp->t_flags2 &= ~TF2_DONT_SACK_QUEUE;
ret = -3;
left = rack->r_ctl.rc_timer_exp - cts;
tcp_hpts_insert(tp, HPTS_MS_TO_SLOTS(left));
@@ -14566,9 +14568,8 @@ rack_switch_failed(struct tcpcb *tp)
* This method gets called if a stack switch was
* attempted and it failed. We are left
* but our hpts timers were stopped and we
- * need to validate time units and inp_flags2.
+ * need to validate time units and t_flags2.
*/
- struct inpcb *inp = tptoinpcb(tp);
struct tcp_rack *rack;
struct timeval tv;
uint32_t cts;
@@ -14578,11 +14579,11 @@ rack_switch_failed(struct tcpcb *tp)
rack = (struct tcp_rack *)tp->t_fb_ptr;
tcp_change_time_units(tp, TCP_TMR_GRANULARITY_USEC);
if (rack->r_mbuf_queue || rack->rc_always_pace || rack->r_use_cmp_ack)
- inp->inp_flags2 |= INP_SUPPORTS_MBUFQ;
+ tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
else
- inp->inp_flags2 &= ~INP_SUPPORTS_MBUFQ;
+ tp->t_flags2 &= ~TF2_SUPPORTS_MBUFQ;
if (rack->r_use_cmp_ack && TCPS_HAVEESTABLISHED(tp->t_state))
- rack->rc_inp->inp_flags2 |= INP_MBUF_ACKCMP;
+ tp->t_flags2 |= TF2_MBUF_ACKCMP;
if (tp->t_in_hpts > IHPTS_NONE) {
/* Strange */
return;
@@ -15089,13 +15090,13 @@ rack_init(struct tcpcb *tp, void **ptr)
}
}
rack_stop_all_timers(tp, rack);
- /* Setup all the inp_flags2 */
+ /* Setup all the t_flags2 */
if (rack->r_mbuf_queue || rack->rc_always_pace || rack->r_use_cmp_ack)
- tptoinpcb(tp)->inp_flags2 |= INP_SUPPORTS_MBUFQ;
+ tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
else
- tptoinpcb(tp)->inp_flags2 &= ~INP_SUPPORTS_MBUFQ;
+ tp->t_flags2 &= ~TF2_SUPPORTS_MBUFQ;
if (rack->r_use_cmp_ack && TCPS_HAVEESTABLISHED(tp->t_state))
- rack->rc_inp->inp_flags2 |= INP_MBUF_ACKCMP;
+ tp->t_flags2 |= TF2_MBUF_ACKCMP;
/*
* Timers in Rack are kept in microseconds so lets
* convert any initial incoming variables
@@ -15417,7 +15418,7 @@ rack_set_state(struct tcpcb *tp, struct tcp_rack *rack)
break;
};
if (rack->r_use_cmp_ack && TCPS_HAVEESTABLISHED(tp->t_state))
- rack->rc_inp->inp_flags2 |= INP_MBUF_ACKCMP;
+ rack->rc_tp->t_flags2 |= TF2_MBUF_ACKCMP;
}
@@ -16528,7 +16529,7 @@ rack_do_segment_nounlock(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th,
* so should process the packets.
*/
slot_remaining = rack->r_ctl.rc_last_output_to - us_cts;
- if (rack->rc_inp->inp_flags2 & INP_DONT_SACK_QUEUE) {
+ if (rack->rc_tp->t_flags2 & TF2_DONT_SACK_QUEUE) {
no_output = 1;
} else {
/*
@@ -22410,7 +22411,7 @@ rack_set_dgp(struct tcp_rack *rack)
rack->use_fixed_rate = 0;
if (rack->gp_ready)
rack_set_cc_pacing(rack);
- rack->rc_inp->inp_flags2 |= INP_SUPPORTS_MBUFQ;
+ rack->rc_tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
rack->rack_attempt_hdwr_pace = 0;
/* rxt settings */
rack->full_size_rxt = 1;
@@ -22419,7 +22420,7 @@ rack_set_dgp(struct tcp_rack *rack)
rack->r_use_cmp_ack = 1;
if (TCPS_HAVEESTABLISHED(rack->rc_tp->t_state) &&
rack->r_use_cmp_ack)
- rack->rc_inp->inp_flags2 |= INP_MBUF_ACKCMP;
+ rack->rc_tp->t_flags2 |= TF2_MBUF_ACKCMP;
/* scwnd=1 */
rack->rack_enable_scwnd = 1;
/* dynamic=100 */
@@ -22536,11 +22537,11 @@ rack_set_profile(struct tcp_rack *rack, int prof)
if (rack_enable_mqueue_for_nonpaced || rack->r_use_cmp_ack) {
rack->r_mbuf_queue = 1;
if (TCPS_HAVEESTABLISHED(rack->rc_tp->t_state))
- rack->rc_inp->inp_flags2 |= INP_MBUF_ACKCMP;
- rack->rc_inp->inp_flags2 |= INP_SUPPORTS_MBUFQ;
+ rack->rc_tp->t_flags2 |= TF2_MBUF_ACKCMP;
+ rack->rc_tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
} else {
rack->r_mbuf_queue = 0;
- rack->rc_inp->inp_flags2 &= ~INP_SUPPORTS_MBUFQ;
+ rack->rc_tp->t_flags2 &= ~TF2_SUPPORTS_MBUFQ;
}
if (rack_enable_shared_cwnd)
rack->rack_enable_scwnd = 1;
@@ -22687,7 +22688,6 @@ rack_process_option(struct tcpcb *tp, struct tcp_rack *rack, int sopt_name,
struct epoch_tracker et;
struct sockopt sopt;
struct cc_newreno_opts opt;
- struct inpcb *inp = tptoinpcb(tp);
uint64_t val;
int error = 0;
uint16_t ca, ss;
@@ -22865,16 +22865,16 @@ rack_process_option(struct tcpcb *tp, struct tcp_rack *rack, int sopt_name,
break;
case TCP_USE_CMP_ACKS:
RACK_OPTS_INC(tcp_use_cmp_acks);
- if ((optval == 0) && (rack->rc_inp->inp_flags2 & INP_MBUF_ACKCMP)) {
+ if ((optval == 0) && (tp->t_flags2 & TF2_MBUF_ACKCMP)) {
/* You can't turn it off once its on! */
error = EINVAL;
} else if ((optval == 1) && (rack->r_use_cmp_ack == 0)) {
rack->r_use_cmp_ack = 1;
rack->r_mbuf_queue = 1;
- inp->inp_flags2 |= INP_SUPPORTS_MBUFQ;
+ tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
}
if (rack->r_use_cmp_ack && TCPS_HAVEESTABLISHED(tp->t_state))
- inp->inp_flags2 |= INP_MBUF_ACKCMP;
+ tp->t_flags2 |= TF2_MBUF_ACKCMP;
break;
case TCP_SHARED_CWND_TIME_LIMIT:
RACK_OPTS_INC(tcp_lscwnd);
@@ -22937,9 +22937,9 @@ rack_process_option(struct tcpcb *tp, struct tcp_rack *rack, int sopt_name,
else
rack->r_mbuf_queue = 0;
if (rack->r_mbuf_queue || rack->rc_always_pace || rack->r_use_cmp_ack)
- inp->inp_flags2 |= INP_SUPPORTS_MBUFQ;
+ tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
else
- inp->inp_flags2 &= ~INP_SUPPORTS_MBUFQ;
+ tp->t_flags2 &= ~TF2_SUPPORTS_MBUFQ;
break;
case TCP_RACK_NONRXT_CFG_RATE:
RACK_OPTS_INC(tcp_rack_cfg_rate);
@@ -23022,9 +23022,9 @@ rack_process_option(struct tcpcb *tp, struct tcp_rack *rack, int sopt_name,
}
}
if (rack->r_mbuf_queue || rack->rc_always_pace || rack->r_use_cmp_ack)
- inp->inp_flags2 |= INP_SUPPORTS_MBUFQ;
+ tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
else
- inp->inp_flags2 &= ~INP_SUPPORTS_MBUFQ;
+ tp->t_flags2 &= ~TF2_SUPPORTS_MBUFQ;
/* A rate may be set irate or other, if so set seg size */
rack_update_seg(rack);
break;
diff --git a/sys/netinet/tcp_stacks/rack_bbr_common.c b/sys/netinet/tcp_stacks/rack_bbr_common.c
index 91bf32159004..625825fc7ed6 100644
--- a/sys/netinet/tcp_stacks/rack_bbr_common.c
+++ b/sys/netinet/tcp_stacks/rack_bbr_common.c
@@ -445,8 +445,8 @@ skip_vnet:
* been compressed. We assert the inp has
* the flag set to enable this!
*/
- KASSERT((inp->inp_flags2 & INP_MBUF_ACKCMP),
- ("tp:%p inp:%p no INP_MBUF_ACKCMP flags?", tp, inp));
+ KASSERT((tp->t_flags2 & TF2_MBUF_ACKCMP),
+ ("tp:%p no TF2_MBUF_ACKCMP flags?", tp));
tlen = 0;
drop_hdrlen = 0;
th = NULL;
diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c
index 40dd9b7f3aa9..a8a896b7ebe6 100644
--- a/sys/netinet/tcp_subr.c
+++ b/sys/netinet/tcp_subr.c
@@ -1073,7 +1073,7 @@ tcp_default_fb_init(struct tcpcb *tp, void **ptr)
/* Make sure we get no interesting mbuf queuing behavior */
/* All mbuf queue/ack compress flags should be off */
- tcp_lro_features_off(tptoinpcb(tp));
+ tcp_lro_features_off(tp);
/* Cancel the GP measurement in progress */
tp->t_flags &= ~TF_GPUTINPROG;
@@ -2270,7 +2270,7 @@ tcp_newtcpcb(struct inpcb *inp)
V_tcp_mssdflt;
/* All mbuf queue/ack compress flags should be off */
- tcp_lro_features_off(tptoinpcb(tp));
+ tcp_lro_features_off(tp);
callout_init_rw(&tp->t_callout, &inp->inp_lock, CALLOUT_RETURNUNLOCKED);
for (int i = 0; i < TT_N; i++)
@@ -4051,14 +4051,14 @@ tcp_default_switch_failed(struct tcpcb *tp)
/*
* If a switch fails we only need to
* care about two things:
- * a) The inp_flags2
+ * a) The t_flags2
* and
* b) The timer granularity.
* Timeouts, at least for now, don't use the
* old callout system in the other stacks so
* those are hopefully safe.
*/
- tcp_lro_features_off(tptoinpcb(tp));
+ tcp_lro_features_off(tp);
tcp_change_time_units(tp, TCP_TMR_GRANULARITY_TICKS);
}
@@ -4236,15 +4236,15 @@ tcp_handle_orphaned_packets(struct tcpcb *tp)
/*
* Called when a stack switch is occuring from the fini()
* of the old stack. We assue the init() as already been
- * run of the new stack and it has set the inp_flags2 to
+ * run of the new stack and it has set the t_flags2 to
* what it supports. This function will then deal with any
* differences i.e. cleanup packets that maybe queued that
* the newstack does not support.
*/
- if (tptoinpcb(tp)->inp_flags2 & INP_MBUF_L_ACKS)
+ if (tp->t_flags2 & TF2_MBUF_L_ACKS)
return;
- if ((tptoinpcb(tp)->inp_flags2 & INP_SUPPORTS_MBUFQ) == 0 &&
+ if ((tp->t_flags2 & TF2_SUPPORTS_MBUFQ) == 0 &&
!STAILQ_EMPTY(&tp->t_inqueue)) {
/*
* It is unsafe to process the packets since a
diff --git a/sys/netinet/tcp_syncache.c b/sys/netinet/tcp_syncache.c
index 5f4bae92974e..54a347c2669e 100644
--- a/sys/netinet/tcp_syncache.c
+++ b/sys/netinet/tcp_syncache.c
@@ -1724,7 +1724,7 @@ skip_alloc:
if (ltflags & TF_NOOPT)
sc->sc_flags |= SCF_NOOPT;
/* ECN Handshake */
- if (V_tcp_do_ecn && (inp->inp_flags2 & INP_CANNOT_DO_ECN) == 0)
+ if (V_tcp_do_ecn && (tp->t_flags2 & TF2_CANNOT_DO_ECN) == 0)
sc->sc_flags |= tcp_ecn_syncache_add(tcp_get_flags(th), iptos);
if (V_tcp_syncookies)
diff --git a/sys/netinet/tcp_var.h b/sys/netinet/tcp_var.h
index a3016a143b93..a8bd6f8732cc 100644
--- a/sys/netinet/tcp_var.h
+++ b/sys/netinet/tcp_var.h
@@ -677,16 +677,6 @@ tcp_output(struct tcpcb *tp)
return (rv);
}
-static inline void
-tcp_lro_features_off(struct inpcb *inp)
-{
- inp->inp_flags2 &= ~(INP_SUPPORTS_MBUFQ|
- INP_MBUF_QUEUE_READY|
- INP_DONT_SACK_QUEUE|
- INP_MBUF_ACKCMP|
- INP_MBUF_L_ACKS);
-}
-
/*
* tcp_output_unlock()
* Always returns unlocked, handles drop request from advanced stacks.
@@ -853,6 +843,12 @@ tcp_packets_this_ack(struct tcpcb *tp, tcp_seq ack)
#define TF2_ECN_USE_ECT1 0x00000800 /* Use ECT(1) marking on session */
#define TF2_TCP_ACCOUNTING 0x00001000 /* Do TCP accounting */
#define TF2_HPTS_CALLS 0x00002000 /* tcp_output() called via HPTS */
+#define TF2_MBUF_L_ACKS 0x00004000 /* large mbufs for ack compression */
+#define TF2_MBUF_ACKCMP 0x00008000 /* mbuf ack compression ok */
+#define TF2_SUPPORTS_MBUFQ 0x00010000 /* Supports the mbuf queue method */
+#define TF2_MBUF_QUEUE_READY 0x00020000 /* Inputs can be queued */
+#define TF2_DONT_SACK_QUEUE 0x00040000 /* Don't wake on sack */
+#define TF2_CANNOT_DO_ECN 0x00080000 /* The stack does not do ECN */
/*
* Structure to hold TCP options that are only used during segment
@@ -1543,6 +1539,15 @@ tcp_http_alloc_req_full(struct tcpcb *tp, struct http_req *req, uint64_t ts, int
int tcp_do_ack_accounting(struct tcpcb *tp, struct tcphdr *th, struct tcpopt *to, uint32_t tiwin, int mss);
#endif
+static inline void
+tcp_lro_features_off(struct tcpcb *tp)
+{
+ tp->t_flags2 &= ~(TF2_SUPPORTS_MBUFQ|
+ TF2_MBUF_QUEUE_READY|
+ TF2_DONT_SACK_QUEUE|
+ TF2_MBUF_ACKCMP|
+ TF2_MBUF_L_ACKS);
+}
static inline void
tcp_fields_to_host(struct tcphdr *th)