git: 7a17566770ff - main - tcp: Create fewer per-VNET zones
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Wed, 16 Sep 2026 12:25:37 UTC
The branch main has been updated by markj:
URL: https://cgit.FreeBSD.org/src/commit/?id=7a17566770ffc840813607bb2616365eba7bfca1
commit 7a17566770ffc840813607bb2616365eba7bfca1
Author: Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2026-09-16 12:11:17 +0000
Commit: Mark Johnston <markj@FreeBSD.org>
CommitDate: 2026-09-16 12:11:17 +0000
tcp: Create fewer per-VNET zones
We have a problem in UMA where creating new zones requires a traversal
of all existing zones (in order to find a unique name for the vm.uma
sysctl subtree). This means that per-VNET UMA zones can be expensive to
create if one creates many VNET jails in a row. This itself is a
problem, but I don't see a quick solution.
Ideally we would avoid creating per-VNET zones in the first place
(except when the desire to impose per-VNET limits necessitates this),
and it turns out that the TCP fastopen and SACK code create several of
these for no apparent reason.
So: make fastopen and SACK zones global. Move some fastopen structure
definitions into tcp_fastopen.c, as they don't need to be public.
Reported by: bapt ("creating many VNET jails in a row is slow")
Reviewed by: tuexen, glebius
MFC after: 3 weeks
Differential Revision: https://reviews.freebsd.org/D59708
---
sys/netinet/tcp_fastopen.c | 78 ++++++++++++++++++++++++++++++----------------
sys/netinet/tcp_fastopen.h | 45 +++++---------------------
sys/netinet/tcp_sack.c | 7 ++---
sys/netinet/tcp_subr.c | 21 +++++++------
4 files changed, 74 insertions(+), 77 deletions(-)
diff --git a/sys/netinet/tcp_fastopen.c b/sys/netinet/tcp_fastopen.c
index 4557a2c08cd2..11c551fc5bb1 100644
--- a/sys/netinet/tcp_fastopen.c
+++ b/sys/netinet/tcp_fastopen.c
@@ -241,6 +241,40 @@ struct tcp_fastopen_callout {
struct vnet *v;
};
+union tcp_fastopen_ip_addr {
+ struct in_addr v4;
+ struct in6_addr v6;
+};
+
+struct tcp_fastopen_ccache_entry {
+ TAILQ_ENTRY(tcp_fastopen_ccache_entry) cce_link;
+ union tcp_fastopen_ip_addr cce_client_ip; /* network byte order */
+ union tcp_fastopen_ip_addr cce_server_ip; /* network byte order */
+ uint16_t server_port; /* network byte order */
+ uint16_t server_mss; /* host byte order */
+ uint8_t af;
+ uint8_t cookie_len;
+ uint8_t cookie[TCP_FASTOPEN_MAX_COOKIE_LEN];
+ sbintime_t disable_time; /* non-zero value means path is disabled */
+};
+
+struct tcp_fastopen_ccache;
+
+struct tcp_fastopen_ccache_bucket {
+ struct mtx ccb_mtx;
+ TAILQ_HEAD(bucket_entries, tcp_fastopen_ccache_entry) ccb_entries;
+ int ccb_num_entries;
+ struct tcp_fastopen_ccache *ccb_ccache;
+};
+
+struct tcp_fastopen_ccache {
+ struct tcp_fastopen_ccache_bucket *base;
+ unsigned int bucket_limit;
+ unsigned int buckets;
+ unsigned int mask;
+ uint32_t secret;
+};
+
static struct tcp_fastopen_ccache_entry *tcp_fastopen_ccache_lookup(
struct in_conninfo *, struct tcp_fastopen_ccache_bucket **);
static struct tcp_fastopen_ccache_entry *tcp_fastopen_ccache_create(
@@ -366,8 +400,8 @@ VNET_DEFINE_STATIC(struct tcp_fastopen_keylist, tcp_fastopen_keys);
VNET_DEFINE_STATIC(struct tcp_fastopen_callout, tcp_fastopen_autokey_ctx);
#define V_tcp_fastopen_autokey_ctx VNET(tcp_fastopen_autokey_ctx)
-VNET_DEFINE_STATIC(uma_zone_t, counter_zone);
-#define V_counter_zone VNET(counter_zone)
+static uma_zone_t counter_zone;
+static uma_zone_t ccache_zone;
static MALLOC_DEFINE(M_TCP_FASTOPEN_CCACHE, "tfo_ccache", "TFO client cookie cache buckets");
@@ -380,11 +414,19 @@ VNET_DEFINE_STATIC(struct tcp_fastopen_ccache, tcp_fastopen_ccache);
void
tcp_fastopen_init(void)
+{
+ counter_zone = uma_zcreate("tfo", sizeof(unsigned int),
+ NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
+ ccache_zone = uma_zcreate("tfo_ccache_entries",
+ sizeof(struct tcp_fastopen_ccache_entry), NULL, NULL, NULL, NULL,
+ UMA_ALIGN_CACHE, 0);
+}
+
+void
+tcp_fastopen_vnet_init(void)
{
unsigned int i;
- V_counter_zone = uma_zcreate("tfo", sizeof(unsigned int),
- NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
rm_init(&V_tcp_fastopen_keylock, "tfo_keylock");
callout_init_rm(&V_tcp_fastopen_autokey_ctx.c,
&V_tcp_fastopen_keylock, 0);
@@ -426,23 +468,10 @@ tcp_fastopen_init(void)
}
V_tcp_fastopen_ccache.base[i].ccb_ccache = &V_tcp_fastopen_ccache;
}
-
- /*
- * Note that while the total number of entries in the cookie cache
- * is limited by the table management logic to
- * V_tcp_fastopen_ccache.buckets *
- * V_tcp_fastopen_ccache.bucket_limit, the total number of items in
- * this zone can exceed that amount by the number of CPUs in the
- * system times the maximum number of unallocated items that can be
- * present in each UMA per-CPU cache for this zone.
- */
- V_tcp_fastopen_ccache.zone = uma_zcreate("tfo_ccache_entries",
- sizeof(struct tcp_fastopen_ccache_entry), NULL, NULL, NULL, NULL,
- UMA_ALIGN_CACHE, 0);
}
void
-tcp_fastopen_destroy(void)
+tcp_fastopen_vnet_destroy(void)
{
struct tcp_fastopen_ccache_bucket *ccb;
unsigned int i;
@@ -453,21 +482,18 @@ tcp_fastopen_destroy(void)
mtx_destroy(&ccb->ccb_mtx);
}
- KASSERT(uma_zone_get_cur(V_tcp_fastopen_ccache.zone) == 0,
- ("%s: TFO ccache zone allocation count not 0", __func__));
- uma_zdestroy(V_tcp_fastopen_ccache.zone);
free(V_tcp_fastopen_ccache.base, M_TCP_FASTOPEN_CCACHE);
callout_drain(&V_tcp_fastopen_autokey_ctx.c);
rm_destroy(&V_tcp_fastopen_keylock);
- uma_zdestroy(V_counter_zone);
}
unsigned int *
tcp_fastopen_alloc_counter(void)
{
unsigned int *counter;
- counter = uma_zalloc(V_counter_zone, M_NOWAIT);
+
+ counter = uma_zalloc(counter_zone, M_NOWAIT);
if (counter)
*counter = 1;
return (counter);
@@ -477,7 +503,7 @@ void
tcp_fastopen_decrement_counter(unsigned int *counter)
{
if (*counter == 1)
- uma_zfree(V_counter_zone, counter);
+ uma_zfree(counter_zone, counter);
else
atomic_subtract_int(counter, 1);
}
@@ -1060,7 +1086,7 @@ tcp_fastopen_ccache_create(struct tcp_fastopen_ccache_bucket *ccb,
cce = NULL;
if (ccb->ccb_num_entries < V_tcp_fastopen_ccache.bucket_limit)
- cce = uma_zalloc(V_tcp_fastopen_ccache.zone, M_NOWAIT);
+ cce = uma_zalloc(ccache_zone, M_NOWAIT);
if (cce == NULL) {
/*
@@ -1138,7 +1164,7 @@ tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry *cce,
TAILQ_REMOVE(&ccb->ccb_entries, cce, cce_link);
ccb->ccb_num_entries--;
- uma_zfree(V_tcp_fastopen_ccache.zone, cce);
+ uma_zfree(ccache_zone, cce);
}
static int
diff --git a/sys/netinet/tcp_fastopen.h b/sys/netinet/tcp_fastopen.h
index f70de4bb8947..94a2e13d81e4 100644
--- a/sys/netinet/tcp_fastopen.h
+++ b/sys/netinet/tcp_fastopen.h
@@ -44,45 +44,11 @@ VNET_DECLARE(unsigned int, tcp_fastopen_server_enable);
#define V_tcp_fastopen_server_enable 0
#endif /* TCP_RFC7413 */
-union tcp_fastopen_ip_addr {
- struct in_addr v4;
- struct in6_addr v6;
-};
-
-struct tcp_fastopen_ccache_entry {
- TAILQ_ENTRY(tcp_fastopen_ccache_entry) cce_link;
- union tcp_fastopen_ip_addr cce_client_ip; /* network byte order */
- union tcp_fastopen_ip_addr cce_server_ip; /* network byte order */
- uint16_t server_port; /* network byte order */
- uint16_t server_mss; /* host byte order */
- uint8_t af;
- uint8_t cookie_len;
- uint8_t cookie[TCP_FASTOPEN_MAX_COOKIE_LEN];
- sbintime_t disable_time; /* non-zero value means path is disabled */
-};
-
-struct tcp_fastopen_ccache;
-
-struct tcp_fastopen_ccache_bucket {
- struct mtx ccb_mtx;
- TAILQ_HEAD(bucket_entries, tcp_fastopen_ccache_entry) ccb_entries;
- int ccb_num_entries;
- struct tcp_fastopen_ccache *ccb_ccache;
-};
-
-struct tcp_fastopen_ccache {
- uma_zone_t zone;
- struct tcp_fastopen_ccache_bucket *base;
- unsigned int bucket_limit;
- unsigned int buckets;
- unsigned int mask;
- uint32_t secret;
-};
-
struct tcpcb;
#ifdef TCP_RFC7413
void tcp_fastopen_init(void);
-void tcp_fastopen_destroy(void);
+void tcp_fastopen_vnet_init(void);
+void tcp_fastopen_vnet_destroy(void);
unsigned int *tcp_fastopen_alloc_counter(void);
void tcp_fastopen_decrement_counter(unsigned int *);
int tcp_fastopen_check_cookie(struct in_conninfo *, uint8_t *, unsigned int,
@@ -98,7 +64,12 @@ tcp_fastopen_init(void)
}
static __inline void
-tcp_fastopen_destroy(void)
+tcp_fastopen_vnet_init(void)
+{
+}
+
+static __inline void
+tcp_fastopen_vnet_destroy(void)
{
}
diff --git a/sys/netinet/tcp_sack.c b/sys/netinet/tcp_sack.c
index f31e88eda41a..d3bb02ca447f 100644
--- a/sys/netinet/tcp_sack.c
+++ b/sys/netinet/tcp_sack.c
@@ -115,8 +115,7 @@
#include <machine/in_cksum.h>
-VNET_DECLARE(struct uma_zone *, sack_hole_zone);
-#define V_sack_hole_zone VNET(sack_hole_zone)
+extern uma_zone_t tcp_sack_hole_zone;
SYSCTL_NODE(_net_inet_tcp, OID_AUTO, sack, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"TCP SACK");
@@ -466,7 +465,7 @@ tcp_sackhole_alloc(struct tcpcb *tp, tcp_seq start, tcp_seq end)
return NULL;
}
- hole = (struct sackhole *)uma_zalloc(V_sack_hole_zone, M_NOWAIT);
+ hole = uma_zalloc(tcp_sack_hole_zone, M_NOWAIT);
if (hole == NULL)
return NULL;
@@ -487,7 +486,7 @@ static void
tcp_sackhole_free(struct tcpcb *tp, struct sackhole *hole)
{
- uma_zfree(V_sack_hole_zone, hole);
+ uma_zfree(tcp_sack_hole_zone, hole);
tp->snd_numholes--;
atomic_subtract_int(&V_tcp_sack_globalholes, 1);
diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c
index 2320d6901fac..f29a5c5110f6 100644
--- a/sys/netinet/tcp_subr.c
+++ b/sys/netinet/tcp_subr.c
@@ -292,8 +292,8 @@ static int tcp_soreceive_stream;
SYSCTL_INT(_net_inet_tcp, OID_AUTO, soreceive_stream, CTLFLAG_RDTUN,
&tcp_soreceive_stream, 0, "Using soreceive_stream for TCP sockets");
-VNET_DEFINE(uma_zone_t, sack_hole_zone);
-#define V_sack_hole_zone VNET(sack_hole_zone)
+uma_zone_t tcp_sack_hole_zone;
+
VNET_DEFINE(uint32_t, tcp_map_entries_limit) = 0; /* unlimited */
static int
sysctl_net_inet_tcp_map_limit_check(SYSCTL_HANDLER_ARGS)
@@ -1460,10 +1460,8 @@ tcp_vnet_init(void *arg __unused)
tcp_hc_init();
TUNABLE_INT_FETCH("net.inet.tcp.sack.enable", &V_tcp_do_sack);
- V_sack_hole_zone = uma_zcreate("sackhole", sizeof(struct sackhole),
- NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
- tcp_fastopen_init();
+ tcp_fastopen_vnet_init();
COUNTER_ARRAY_ALLOC(V_tcps_states, TCP_NSTATES, M_WAITOK);
VNET_PCPUSTAT_ALLOC(tcpstat, M_WAITOK);
@@ -1583,12 +1581,16 @@ tcp_init(void *arg __unused)
#ifdef INET6
IP6PROTO_REGISTER(IPPROTO_TCP, tcp6_input, tcp6_ctlinput);
#endif
+
+ tcp_fastopen_init();
+ tcp_sack_hole_zone = uma_zcreate("sackhole", sizeof(struct sackhole),
+ NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
}
SYSINIT(tcp_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD, tcp_init, NULL);
#ifdef VIMAGE
static void
-tcp_destroy(void *unused __unused)
+tcp_vnet_destroy(void *unused __unused)
{
#ifdef TCP_HHOOK
int error;
@@ -1597,14 +1599,12 @@ tcp_destroy(void *unused __unused)
tcp_hc_destroy();
syncache_destroy();
in_pcbinfo_destroy(&V_tcbinfo);
- /* tcp_discardcb() clears the sack_holes up. */
- uma_zdestroy(V_sack_hole_zone);
/*
* Cannot free the zone until all tcpcbs are released as we attach
* the allocations to them.
*/
- tcp_fastopen_destroy();
+ tcp_fastopen_vnet_destroy();
COUNTER_ARRAY_FREE(V_tcps_states, TCP_NSTATES);
VNET_PCPUSTAT_FREE(tcpstat);
@@ -1624,7 +1624,8 @@ tcp_destroy(void *unused __unused)
}
#endif
}
-VNET_SYSUNINIT(tcp, SI_SUB_PROTO_DOMAIN, SI_ORDER_FOURTH, tcp_destroy, NULL);
+VNET_SYSUNINIT(tcp, SI_SUB_PROTO_DOMAIN, SI_ORDER_FOURTH, tcp_vnet_destroy,
+ NULL);
#endif
/*