git: d1e14cba4a79 - main - netipsec: Set the outer IPv6 flow label from the SA

From: R. Christian McDonald <rcm_at_FreeBSD.org>
Date: Thu, 19 Nov 2026 02:59:38 UTC
The branch main has been updated by rcm:

URL: https://cgit.FreeBSD.org/src/commit/?id=d1e14cba4a79c0b4fe7cff3c0afcc0ffb50f54b3

commit d1e14cba4a79c0b4fe7cff3c0afcc0ffb50f54b3
Author:     R. Christian McDonald <rcm@FreeBSD.org>
AuthorDate: 2026-11-19 02:58:21 +0000
Commit:     R. Christian McDonald <rcm@FreeBSD.org>
CommitDate: 2026-11-19 02:59:29 +0000

    netipsec: Set the outer IPv6 flow label from the SA
    
    Tunnel mode encapsulation leaves the flow label of the outer IPv6 header
    zero. ESP and AH have no transport ports (UDP-encapsulated ESP has them, but
    the same pair for every SA), so devices that hash IPv6 headers to spread
    traffic, link aggregates and ECMP routers alike, see no difference between
    the security associations of two gateways and put all of their tunnels on
    one path. lagg(4)'s own header hash does the same: for IPv6 its layer 4
    input is the flow label.
    
    Set the label to the low 20 bits of the SA's flow ID, the hash of its SPI
    that the previous change introduced. Each SA then carries one label for its
    lifetime; SPIs are unique within the SADB, so distinct SAs carry distinct
    labels unless the 20-bit hashes collide. A hasher that includes the flow
    label then spreads SAs over its paths while keeping the packets of each SA
    on one, which avoids reordering their sequence numbers across paths. A rekey
    starts a new SA and a new label. The label is set whether or not
    net.inet.ipsec.sa_flowid is enabled: that sysctl governs the local transmit
    queue selection, this label what the network sees. In a bundle whose tunnel
    mode SA is followed by transport mode SAs the outer header stays in place;
    relabel it from the outermost SA when the packet leaves, as that SA's
    sequence numbers are the first the path choice reorders and the previous
    change already took the flow ID from it. One transport SA shared by several
    tunnel SAs then stays on one path.
    
    RFC 6437 leaves the definition of a flow to the source and recommends
    uniformly distributed labels, a hash of the packet's fields being one
    stateless way to get them. RFC 6438 describes labelling at a tunnel entry
    point from the inner flow, which Linux IPsec and gre(4) with UDP
    encapsulation do, so that one tunnel spreads over several paths. ESP carries
    sequence numbers, so that spreading reorders them and a receiver with a
    small anti-replay window drops the late packets; keying the label on the SA
    is the choice that fits. RFC 6437 asks that a hash-derived label be hard to
    predict; this one is a function of the SPI in the same packet, so keying the
    hash would hide nothing, and the label is meant to stay the same for the
    life of the SA.
    
    AH treats the flow label as mutable and zeroes it before computing and
    verifying the ICV, so the label does not affect AH. Transport mode and SAs
    offloaded to a NIC are not affected. ip6_output() leaves the label of
    packets without socket options alone.
    
    Document the label in ipsec(4).
    
    Reviewed by:            kib, ae
    MFC after:              2 weeks
    Sponsored by:           Rubicon Communications, LLC ("Netgate")
    Differential Revision:  https://reviews.freebsd.org/D61196
---
 share/man/man4/ipsec.4                 |  11 +-
 sys/netipsec/ipsec_output.c            |  43 ++++-
 tests/sys/netipsec/tunnel/Makefile     |   1 +
 tests/sys/netipsec/tunnel/flowlabel.sh | 281 +++++++++++++++++++++++++++++++++
 4 files changed, 330 insertions(+), 6 deletions(-)

diff --git a/share/man/man4/ipsec.4 b/share/man/man4/ipsec.4
index dbc0787fe1c8..4cb976fbf532 100644
--- a/share/man/man4/ipsec.4
+++ b/share/man/man4/ipsec.4
@@ -27,7 +27,7 @@
 .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 .\" SUCH DAMAGE.
 .\"
-.Dd November 4, 2026
+.Dd November 16, 2026
 .Dt IPSEC 4
 .Os
 .Sh NAME
@@ -74,6 +74,15 @@ Transport mode is used to protect peer-to-peer communication between end nodes.
 Tunnel mode encapsulates IP packets within other IP packets
 and is designed for security gateways such as VPN endpoints.
 .Pp
+Tunnel mode encapsulation into IPv6, unless offloaded to a network interface,
+sets the flow label of the outer header to the low 20 bits of the flow ID of
+the security association, a hash of its SPI, whether or not the
+.Va net.inet.ipsec.sa_flowid
+variable described below is set.
+Routers and link aggregates that hash the flow label thus keep the packets
+of one SA on one path and may spread distinct SAs over several.
+In a bundle of SAs the label is that of the outermost SA.
+.Pp
 System configuration requires the
 .Xr crypto 4
 subsystem.
diff --git a/sys/netipsec/ipsec_output.c b/sys/netipsec/ipsec_output.c
index 50a34b83e5d0..0a35a6edf0d9 100644
--- a/sys/netipsec/ipsec_output.c
+++ b/sys/netipsec/ipsec_output.c
@@ -106,7 +106,7 @@
 		IPCOMPSTAT_INC(ipcomps_##name);	\
 } while (0)
 
-static int ipsec_encap(struct mbuf **mp, struct secasindex *saidx);
+static int ipsec_encap(struct mbuf **mp, struct secasvar *sav);
 static size_t ipsec_get_pmtu(struct secasvar *sav);
 
 /*
@@ -289,7 +289,7 @@ ipsec4_perform_request(struct ifnet *ifp, struct mbuf *m, struct ip *ip1,
 		ip->ip_len = htons(m->m_pkthdr.len);
 		ip->ip_sum = 0;
 		ip->ip_sum = in_cksum(m, ip->ip_hl << 2);
-		error = ipsec_encap(&m, &sav->sah->saidx);
+		error = ipsec_encap(&m, sav);
 		if (error != 0) {
 			DPRINTF(("%s: encapsulation for SPI 0x%08x failed "
 			    "with error %d\n", __func__, ntohl(sav->spi),
@@ -712,7 +712,7 @@ ipsec6_perform_request(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp,
 			error = ENXIO;   /*XXX*/
 			goto bad;
 		}
-		error = ipsec_encap(&m, &sav->sah->saidx);
+		error = ipsec_encap(&m, sav);
 		if (error != 0) {
 			DPRINTF(("%s: encapsulation for SPI 0x%08x failed "
 			    "with error %d\n", __func__, ntohl(sav->spi),
@@ -928,6 +928,10 @@ ipsec_process_done(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 	struct m_tag *mtag;
 #ifdef INET
 	struct ip *ip;
+#endif
+#ifdef INET6
+	struct ip6_hdr *ip6;
+	u_int i;
 #endif
 	int error;
 
@@ -1018,6 +1022,28 @@ ipsec_process_done(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 			goto bad;
 		}
 	}
+#ifdef INET6
+	/*
+	 * Transport mode requests that follow a tunnel mode request in a
+	 * bundle leave the outer IPv6 header in place.  Relabel it from the
+	 * outermost SA: it is the one whose sequence numbers reorder first
+	 * when the path differs between packets, and the inner SAs ride
+	 * along.  The policy's requests decide the mode, not the SA's own,
+	 * which may be "any".
+	 */
+	if (idx > 1 && saidx->dst.sa.sa_family == AF_INET6 &&
+	    sp->req[idx - 1]->saidx.mode == IPSEC_MODE_TRANSPORT) {
+		for (i = 0; i < idx - 1; i++) {
+			if (sp->req[i]->saidx.mode == IPSEC_MODE_TUNNEL)
+				break;
+		}
+		if (i < idx - 1) {
+			ip6 = mtod(m, struct ip6_hdr *);
+			ip6->ip6_flow = (ip6->ip6_flow & ~IPV6_FLOWLABEL_MASK) |
+			    (htonl(sav->flowid) & IPV6_FLOWLABEL_MASK);
+		}
+	}
+#endif
 
 	key_freesp(&sp), sp = NULL;	/* Release reference to SP */
 #if defined(INET) || defined(INET6)
@@ -1161,8 +1187,9 @@ ipsec_get_pmtu(struct secasvar *sav)
 }
 
 static int
-ipsec_encap(struct mbuf **mp, struct secasindex *saidx)
+ipsec_encap(struct mbuf **mp, struct secasvar *sav)
 {
+	struct secasindex *saidx;
 #ifdef INET6
 	struct ip6_hdr *ip6;
 #endif
@@ -1172,6 +1199,7 @@ ipsec_encap(struct mbuf **mp, struct secasindex *saidx)
 #endif
 	uint8_t itos, proto;
 
+	saidx = &sav->sah->saidx;
 	ip = mtod(*mp, struct ip *);
 	switch (ip->ip_v) {
 #ifdef INET
@@ -1233,7 +1261,12 @@ ipsec_encap(struct mbuf **mp, struct secasindex *saidx)
 		if (*mp == NULL)
 			return (ENOBUFS);
 		ip6 = mtod(*mp, struct ip6_hdr *);
-		ip6->ip6_flow = 0;
+		/*
+		 * Derive the outer flow label from the SA, so that routers
+		 * and link aggregates hashing on it keep the packets of one
+		 * SA on one path, which limits sequence number reordering.
+		 */
+		ip6->ip6_flow = htonl(sav->flowid) & IPV6_FLOWLABEL_MASK;
 		ip6->ip6_vfc = IPV6_VERSION;
 		ip6->ip6_hlim = V_ip6_defhlim;
 		ip6->ip6_nxt = proto;
diff --git a/tests/sys/netipsec/tunnel/Makefile b/tests/sys/netipsec/tunnel/Makefile
index ef172547462a..de1b56c4cc0b 100644
--- a/tests/sys/netipsec/tunnel/Makefile
+++ b/tests/sys/netipsec/tunnel/Makefile
@@ -13,6 +13,7 @@ ATF_TESTS_SH+=	empty \
 		aesni_aes_gcm_256 \
 		chacha20_poly1305 \
 		esp_iv \
+		flowlabel \
 		sa_flowid
 
 TEST_METADATA+= execenv="jail" \
diff --git a/tests/sys/netipsec/tunnel/flowlabel.sh b/tests/sys/netipsec/tunnel/flowlabel.sh
new file mode 100755
index 000000000000..7afd1c8cc21e
--- /dev/null
+++ b/tests/sys/netipsec/tunnel/flowlabel.sh
@@ -0,0 +1,281 @@
+
+. $(atf_get_srcdir)/utils.subr
+
+# The outer IPv6 header of a tunnel mode SA carries the low 20 bits of the
+# SA's flow ID, jenkins_hash32() of the SPI in host byte order with seed 0,
+# as its flow label.  Expected labels for the SPIs used below:
+#   0x1000 -> 0x625cd   0x1001 -> 0xead43   0x1002 -> 0x828ed
+#   0x1003 -> 0xb1c69   0x1275 -> 0 (the smallest SPI above 255 that hashes
+#   to a zero label)
+GW_A=2001:db8:23::2
+GW_B=2001:db8:34::2
+KEY="12345678901234567890"
+AKEY="12345678901234567890123456789012"
+
+# fl_setkey JAIL DIR SRC_LAN DST_LAN PROTO SPI_OUT SPI_IN
+fl_setkey()
+{
+	jname=$1; dir=$2; src=$3; dst=$4; proto=$5; spi_out=$6; spi_in=$7
+	if [ ${proto} = esp ]; then
+		alg="-E aes-gcm-16 \"${KEY}\""
+	else
+		alg="-A hmac-sha2-256 \"${AKEY}\""
+	fi
+	[ ${dir} = out ] && rdir=in || rdir=out
+	(
+		printf "flush;\nspdflush;\n"
+		printf "spdadd ${src} ${dst} any -P ${dir} ipsec ${proto}/tunnel/${GW_A}-${GW_B}/require;\n"
+		printf "spdadd ${dst} ${src} any -P ${rdir} ipsec ${proto}/tunnel/${GW_B}-${GW_A}/require;\n"
+		printf "add ${GW_A} ${GW_B} ${proto} ${spi_out} ${alg};\n"
+		printf "add ${GW_B} ${GW_A} ${proto} ${spi_in} ${alg};\n"
+	) > ${TMPDIR}/ipsec.${jname}.conf
+	atf_check -s exit:0 -o ignore jexec ${jname} setkey -f ${TMPDIR}/ipsec.${jname}.conf
+}
+
+# fl_run COUNT TARGET...: ping each target from hostA while capturing COUNT
+# packets that match fl_filter on ipsecA's public interface, by default ESP
+# or AH in both directions.  A test that expects no replies sets
+# fl_ping_status=ignore.
+fl_ping_status=exit:0
+fl_filter="ip6 and (esp or ah)"
+fl_run()
+{
+	count=$1; shift
+	rm -f ${TMPDIR}/fl.pcap ${TMPDIR}/tcpdump.err
+	jexec ipsecA tcpdump -i ${epair_PUB_A}a -n -c ${count} \
+	    -w ${TMPDIR}/fl.pcap "${fl_filter}" \
+	    > /dev/null 2> ${TMPDIR}/tcpdump.err &
+	tdpid=$!
+	# tcpdump reports the interface once the capture is open.
+	i=0
+	while ! grep -q "listening on" ${TMPDIR}/tcpdump.err 2>/dev/null; do
+		if ! kill -0 ${tdpid} 2>/dev/null || [ ${i} -ge 50 ]; then
+			cat ${TMPDIR}/tcpdump.err
+			atf_fail "tcpdump did not start capturing"
+		fi
+		sleep 0.1
+		i=$((i + 1))
+	done
+	for target in "$@"; do
+		atf_check -s ${fl_ping_status} -o ignore \
+		    jexec hostA ping -c 5 -i 0.2 ${target}
+	done
+	timeout 5 sh -c "while kill -0 ${tdpid} 2>/dev/null; do sleep 0.2; done"
+	kill ${tdpid} 2>/dev/null
+	wait ${tdpid} 2>/dev/null
+}
+
+# fl_check SPI LABEL: every captured packet of SPI carries LABEL, a hex
+# string without 0x; "0" means no flow label at all.
+fl_check()
+{
+	spi=$(printf "%08x" $1); label=$2
+	pk=$(tcpdump -n -v -r ${TMPDIR}/fl.pcap 2>/dev/null | grep "spi=0x${spi}")
+	n=$(echo "${pk}" | grep -c "spi=0x${spi}")
+	if [ ${n} -lt 5 ]; then
+		cat ${TMPDIR}/tcpdump.err
+		atf_fail "only ${n} packets with SPI 0x${spi} captured"
+	fi
+	if [ ${label} = 0 ]; then
+		bad=$(echo "${pk}" | grep -c "flowlabel")
+	else
+		bad=$(echo "${pk}" | grep -vc "flowlabel 0x${label},")
+	fi
+	if [ ${bad} -ne 0 ]; then
+		echo "${pk}"
+		atf_fail "${bad} of ${n} packets with SPI 0x${spi} lack flow label ${label}"
+	fi
+}
+
+fl_lab()
+{
+	ist_init
+	ist_labsetup
+	[ $1 = 4 ] && ist_v4_setup
+	ist_v6_setup
+}
+
+atf_test_case "v4in6" "cleanup"
+v4in6_head()
+{
+	atf_set descr 'Outer IPv6 flow label of an IPv4-in-IPv6 ESP tunnel is the SA hash'
+	atf_set require.user root
+}
+v4in6_body()
+{
+	fl_lab 4
+	fl_setkey ipsecA out 192.0.2.0/24 203.0.113.0/24 esp 0x1000 0x1001
+	fl_setkey ipsecB in 192.0.2.0/24 203.0.113.0/24 esp 0x1000 0x1001
+	fl_run 10 203.0.113.1
+	fl_check 0x1000 625cd
+	fl_check 0x1001 ead43
+}
+v4in6_cleanup()
+{
+	ist_cleanup
+}
+
+atf_test_case "v6in6" "cleanup"
+v6in6_head()
+{
+	atf_set descr 'Outer IPv6 flow label of an IPv6-in-IPv6 ESP tunnel is the SA hash'
+	atf_set require.user root
+}
+v6in6_body()
+{
+	fl_lab 6
+	fl_setkey ipsecA out 2001:db8:1::/64 2001:db8:45::/64 esp 0x1000 0x1001
+	fl_setkey ipsecB in 2001:db8:1::/64 2001:db8:45::/64 esp 0x1000 0x1001
+	fl_run 10 2001:db8:45::1
+	fl_check 0x1000 625cd
+	fl_check 0x1001 ead43
+}
+v6in6_cleanup()
+{
+	ist_cleanup
+}
+
+atf_test_case "ah" "cleanup"
+ah_head()
+{
+	atf_set descr 'An AH tunnel authenticates with the flow label set'
+	atf_set require.user root
+}
+ah_body()
+{
+	# IPv4 inner: an IPv6-in-IPv6 AH tunnel returns no traffic on an
+	# unpatched kernel either, so it cannot serve here.
+	fl_lab 4
+	fl_setkey ipsecA out 192.0.2.0/24 203.0.113.0/24 ah 0x1002 0x1003
+	fl_setkey ipsecB in 192.0.2.0/24 203.0.113.0/24 ah 0x1002 0x1003
+	fl_run 10 203.0.113.1
+	fl_check 0x1002 828ed
+	fl_check 0x1003 b1c69
+}
+ah_cleanup()
+{
+	ist_cleanup
+}
+
+atf_test_case "zero" "cleanup"
+zero_head()
+{
+	atf_set descr 'An SPI whose hash has a zero low 20 bits yields no flow label'
+	atf_set require.user root
+}
+zero_body()
+{
+	fl_lab 6
+	fl_setkey ipsecA out 2001:db8:1::/64 2001:db8:45::/64 esp 0x1275 0x1001
+	fl_setkey ipsecB in 2001:db8:1::/64 2001:db8:45::/64 esp 0x1275 0x1001
+	fl_run 10 2001:db8:45::1
+	fl_check 0x1275 0
+	fl_check 0x1001 ead43
+}
+zero_cleanup()
+{
+	ist_cleanup
+}
+
+atf_test_case "sa_flowid_off" "cleanup"
+sa_flowid_off_head()
+{
+	atf_set descr 'The flow label does not depend on net.inet.ipsec.sa_flowid'
+	atf_set require.user root
+}
+sa_flowid_off_body()
+{
+	fl_lab 6
+	atf_check -s exit:0 -o ignore jexec ipsecA sysctl net.inet.ipsec.sa_flowid=0
+	fl_setkey ipsecA out 2001:db8:1::/64 2001:db8:45::/64 esp 0x1000 0x1001
+	fl_setkey ipsecB in 2001:db8:1::/64 2001:db8:45::/64 esp 0x1000 0x1001
+	fl_run 10 2001:db8:45::1
+	fl_check 0x1000 625cd
+}
+sa_flowid_off_cleanup()
+{
+	ist_cleanup
+}
+
+# fl_bundle [AH_MODE]: two ESP tunnel SAs, one per policy, feed one AH
+# transport SA, added with setkey's -m AH_MODE (none: setkey's default
+# mode "any").  The outer header must carry the label of the AH SA on every
+# packet, not that of the tunnel SA which built it.  AH leaves the ESP
+# header readable, so the capture shows which tunnel SA each packet took.
+# Only the outbound packets carry AH, so only they are captured: when this
+# was written, ipsec6_common_input_cb() handed the ESP header beneath a
+# transport mode AH to the raw IPv6 input, so ipsecB did not decapsulate
+# the packets and did not reply.
+fl_bundle()
+{
+	ahmode=${1:+-m $1}
+	fl_lab 4
+	(
+		printf "flush;\nspdflush;\n"
+		printf "spdadd 192.0.2.0/24 203.0.113.0/25 any -P out ipsec esp/tunnel/${GW_A}-${GW_B}/unique:1 ah/transport//require;\n"
+		printf "spdadd 192.0.2.0/24 203.0.113.128/25 any -P out ipsec esp/tunnel/${GW_A}-${GW_B}/unique:2 ah/transport//require;\n"
+		printf "spdadd 203.0.113.0/24 192.0.2.0/24 any -P in ipsec esp/tunnel/${GW_B}-${GW_A}/require;\n"
+		printf "add ${GW_A} ${GW_B} esp 0x1000 -m tunnel -u 1 -E aes-gcm-16 \"${KEY}\";\n"
+		printf "add ${GW_A} ${GW_B} esp 0x1001 -m tunnel -u 2 -E aes-gcm-16 \"${KEY}\";\n"
+		printf "add ${GW_A} ${GW_B} ah 0x1002 ${ahmode} -A hmac-sha2-256 \"${AKEY}\";\n"
+		printf "add ${GW_B} ${GW_A} esp 0x1003 -m tunnel -E aes-gcm-16 \"${KEY}\";\n"
+	) > ${TMPDIR}/ipsec.ipsecA.conf
+	atf_check -s exit:0 -o ignore jexec ipsecA setkey -f ${TMPDIR}/ipsec.ipsecA.conf
+	(
+		printf "flush;\nspdflush;\n"
+		printf "spdadd 192.0.2.0/24 203.0.113.0/24 any -P in ipsec esp/tunnel/${GW_A}-${GW_B}/require;\n"
+		printf "spdadd 203.0.113.0/24 192.0.2.0/24 any -P out ipsec esp/tunnel/${GW_B}-${GW_A}/require;\n"
+		printf "add ${GW_A} ${GW_B} esp 0x1000 -m tunnel -E aes-gcm-16 \"${KEY}\";\n"
+		printf "add ${GW_A} ${GW_B} esp 0x1001 -m tunnel -E aes-gcm-16 \"${KEY}\";\n"
+		printf "add ${GW_A} ${GW_B} ah 0x1002 ${ahmode} -A hmac-sha2-256 \"${AKEY}\";\n"
+		printf "add ${GW_B} ${GW_A} esp 0x1003 -m tunnel -E aes-gcm-16 \"${KEY}\";\n"
+	) > ${TMPDIR}/ipsec.ipsecB.conf
+	atf_check -s exit:0 -o ignore jexec ipsecB setkey -f ${TMPDIR}/ipsec.ipsecB.conf
+	fl_ping_status=ignore
+	fl_filter="ip6 and ah"
+	fl_run 10 203.0.113.1 203.0.113.129
+	fl_check 0x1002 828ed
+	fl_check 0x1000 828ed
+	fl_check 0x1001 828ed
+}
+
+atf_test_case "bundle" "cleanup"
+bundle_head()
+{
+	atf_set descr 'A bundle of tunnel and transport mode SAs carries the label of the outermost SA'
+	atf_set require.user root
+}
+bundle_body()
+{
+	fl_bundle transport
+}
+bundle_cleanup()
+{
+	ist_cleanup
+}
+
+atf_test_case "bundle_any" "cleanup"
+bundle_any_head()
+{
+	atf_set descr 'The bundle label follows the policy when the transport SA has mode any'
+	atf_set require.user root
+}
+bundle_any_body()
+{
+	fl_bundle
+}
+bundle_any_cleanup()
+{
+	ist_cleanup
+}
+
+atf_init_test_cases()
+{
+	atf_add_test_case "v4in6"
+	atf_add_test_case "v6in6"
+	atf_add_test_case "ah"
+	atf_add_test_case "zero"
+	atf_add_test_case "sa_flowid_off"
+	atf_add_test_case "bundle"
+	atf_add_test_case "bundle_any"
+}