git: 744bfb213144 - main - Import the WireGuard driver from zx2c4.com.

From: John Baldwin <jhb_at_FreeBSD.org>
Date: Fri, 28 Oct 2022 20:55:57 UTC
The branch main has been updated by jhb:

URL: https://cgit.FreeBSD.org/src/commit/?id=744bfb213144c63cbaf38d91a1c4f7aebb9b9fbc

commit 744bfb213144c63cbaf38d91a1c4f7aebb9b9fbc
Author:     John Baldwin <jhb@FreeBSD.org>
AuthorDate: 2022-10-28 20:36:12 +0000
Commit:     John Baldwin <jhb@FreeBSD.org>
CommitDate: 2022-10-28 20:36:12 +0000

    Import the WireGuard driver from zx2c4.com.
    
    This commit brings back the driver from FreeBSD commit
    f187d6dfbf633665ba6740fe22742aec60ce02a2 plus subsequent fixes from
    upstream.
    
    Relative to upstream this commit includes a few other small fixes such
    as additional INET and INET6 #ifdef's, #include cleanups, and updates
    for recent API changes in main.
    
    Reviewed by:    pauamma, gbe, kevans, emaste
    Obtained from:  git@git.zx2c4.com:wireguard-freebsd @ 3cc22b2
    Sponsored by:   The FreeBSD Foundation
    Differential Revision:  https://reviews.freebsd.org/D36909
---
 etc/mtree/BSD.include.dist |    2 +
 include/Makefile           |    9 +-
 share/man/man4/Makefile    |    2 +
 share/man/man4/wg.4        |  213 +++
 sys/conf/NOTES             |    3 +
 sys/conf/files             |   12 +-
 sys/dev/wg/compat.h        |  118 ++
 sys/dev/wg/crypto.h        |  182 +++
 sys/dev/wg/if_wg.c         | 3055 ++++++++++++++++++++++++++++++++++++++++++++
 sys/dev/wg/if_wg.h         |   37 +
 sys/dev/wg/support.h       |   21 +
 sys/dev/wg/version.h       |    1 +
 sys/dev/wg/wg_cookie.c     |  500 ++++++++
 sys/dev/wg/wg_cookie.h     |   72 ++
 sys/dev/wg/wg_crypto.c     | 1830 ++++++++++++++++++++++++++
 sys/dev/wg/wg_noise.c      | 1410 ++++++++++++++++++++
 sys/dev/wg/wg_noise.h      |  131 ++
 sys/kern/kern_jail.c       |    1 +
 sys/modules/Makefile       |    4 +
 sys/modules/if_wg/Makefile |   10 +
 sys/net/if_types.h         |    1 +
 sys/netinet6/nd6.c         |    4 +-
 sys/sys/priv.h             |    1 +
 23 files changed, 7613 insertions(+), 6 deletions(-)

diff --git a/etc/mtree/BSD.include.dist b/etc/mtree/BSD.include.dist
index 192508bbf6f1..9a1fe1cd60a7 100644
--- a/etc/mtree/BSD.include.dist
+++ b/etc/mtree/BSD.include.dist
@@ -136,6 +136,8 @@
         ..
         vkbd
         ..
+        wg
+        ..
         wi
         ..
     ..
diff --git a/include/Makefile b/include/Makefile
index 80d2d9da8b06..988b0a56baa7 100644
--- a/include/Makefile
+++ b/include/Makefile
@@ -49,7 +49,7 @@ LSUBDIRS=	dev/acpica dev/agp dev/ciss dev/filemon dev/firewire \
 	dev/hwpmc dev/hyperv \
 	dev/ic dev/iicbus dev/io dev/mfi dev/mmc dev/nvme \
 	dev/ofw dev/pbio dev/pci ${_dev_powermac_nvram} dev/ppbus dev/pwm \
-	dev/smbus dev/speaker dev/tcp_log dev/veriexec dev/vkbd \
+	dev/smbus dev/speaker dev/tcp_log dev/veriexec dev/vkbd dev/wg \
 	fs/devfs fs/fdescfs fs/msdosfs fs/nfs fs/nullfs \
 	fs/procfs fs/smbfs fs/udf fs/unionfs \
 	geom/cache geom/concat geom/eli geom/gate geom/journal geom/label \
@@ -225,6 +225,10 @@ NVPAIRDIR=	${INCLUDEDIR}/sys
 MLX5=		mlx5io.h
 MLX5DIR=	${INCLUDEDIR}/dev/mlx5
 
+.PATH: ${SRCTOP}/sys/dev/wg
+WG=		if_wg.h
+WGDIR=		${INCLUDEDIR}/dev/wg
+
 INCSGROUPS=	INCS \
 		ACPICA \
 		AGP \
@@ -244,7 +248,8 @@ INCSGROUPS=	INCS \
 		RPC \
 		SECAUDIT \
 		TEKEN \
-		VERIEXEC
+		VERIEXEC \
+		WG
 
 .if ${MK_IPFILTER} != "no"
 INCSGROUPS+=	IPFILTER
diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile
index 4650d9d3ede8..413ac035003d 100644
--- a/share/man/man4/Makefile
+++ b/share/man/man4/Makefile
@@ -584,6 +584,7 @@ MAN=	aac.4 \
 	vtnet.4 \
 	watchdog.4 \
 	${_wbwd.4} \
+	wg.4 \
 	witness.4 \
 	wlan.4 \
 	wlan_acl.4 \
@@ -761,6 +762,7 @@ MLINKS+=vr.4 if_vr.4
 MLINKS+=vte.4 if_vte.4
 MLINKS+=vtnet.4 if_vtnet.4
 MLINKS+=watchdog.4 SW_WATCHDOG.4
+MLINKS+=wg.4 if_wg.4
 MLINKS+=${_wpi.4} ${_if_wpi.4}
 MLINKS+=xl.4 if_xl.4
 
diff --git a/share/man/man4/wg.4 b/share/man/man4/wg.4
new file mode 100644
index 000000000000..f2ae425002d7
--- /dev/null
+++ b/share/man/man4/wg.4
@@ -0,0 +1,213 @@
+.\" Copyright (c) 2020 Gordon Bergling <gbe@FreeBSD.org>
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd October 28, 2022
+.Dt WG 4
+.Os
+.Sh NAME
+.Nm wg
+.Nd "WireGuard - pseudo-device"
+.Sh SYNOPSIS
+To load the driver as a module at boot time, place the following line in
+.Xr loader.conf 5 :
+.Bd -literal -offset indent
+if_wg_load="YES"
+.Ed
+.Sh DESCRIPTION
+The
+.Nm
+driver provides Virtual Private Network (VPN) interfaces for the secure
+exchange of layer 3 traffic with other WireGuard peers using the WireGuard
+protocol.
+.Pp
+A
+.Nm
+interface recognises one or more peers, establishes a secure tunnel with
+each on demand, and tracks each peer's UDP endpoint for exchanging encrypted
+traffic with.
+.Pp
+The interfaces can be created at runtime using the
+.Ic ifconfig Cm wg Ns Ar N Cm create
+command.
+The interface itself can be configured with
+.Xr wg 8 .
+.Pp
+The following glossary provides a brief overview of WireGuard
+terminology:
+.Bl -tag -width indent -offset 3n
+.It Peer
+Peers exchange IPv4 or IPv6 traffic over secure tunnels.
+Each
+.Nm
+interface may be configured to recognise one or more peers.
+.It Key
+Each peer uses its private key and corresponding public key to
+identify itself to others.
+A peer configures a
+.Nm
+interface with its own private key and with the public keys of its peers.
+.It Pre-shared key
+In addition to the public keys, each peer pair may be configured with a
+unique pre-shared symmetric key.
+This is used in their handshake to guard against future compromise of the
+peers' encrypted tunnel if a quantum-computational attack on their
+Diffie-Hellman exchange becomes feasible.
+It is optional, but recommended.
+.It Allowed IPs
+A single
+.Nm
+interface may maintain concurrent tunnels connecting diverse networks.
+The interface therefore implements rudimentary routing and reverse-path
+filtering functions for its tunneled traffic.
+These functions reference a set of allowed IP ranges configured against
+each peer.
+.Pp
+The interface will route outbound tunneled traffic to the peer configured
+with the most specific matching allowed IP address range, or drop it
+if no such match exists.
+.Pp
+The interface will accept tunneled traffic only from the peer
+configured with the most specific matching allowed IP address range
+for the incoming traffic, or drop it if no such match exists.
+That is, tunneled traffic routed to a given peer cannot return through
+another peer of the same
+.Nm
+interface.
+This ensures that peers cannot spoof another's traffic.
+.It Handshake
+Two peers handshake to mutually authenticate each other and to
+establish a shared series of secret ephemeral encryption keys.
+Any peer may initiate a handshake.
+Handshakes occur only when there is traffic to send, and recur every
+two minutes during transfers.
+.It Connectionless
+Due to the handshake behavior, there is no connected or disconnected
+state.
+.El
+.Ss Keys
+Private keys for WireGuard can be generated from any sufficiently
+secure random source.
+The Curve25519 keys and the pre-shared keys are both 32 bytes
+long and are commonly encoded in base64 for ease of use.
+.Pp
+Keys can be generated with
+.Xr wg 8
+as follows:
+.Pp
+.Dl $ wg genkey
+.Pp
+Although a valid Curve25519 key must have 5 bits set to
+specific values, this is done by the interface and so it
+will accept any random 32-byte base64 string.
+.Sh EXAMPLES
+Create a
+.Nm
+interface and set random private key.
+.Bd -literal -offset indent
+# ifconfig wg0 create
+# wg genkey | wg set wg0 listen-port 54321 private-key /dev/stdin
+.Ed
+.Pp
+Retrieve the associated public key from a
+.Nm
+interface.
+.Bd -literal -offset indent
+$ wg show wg0 public-key
+.Ed
+.Pp
+Connect to a specific endpoint using its public-key and set the allowed IP address
+.Bd -literal -offset indent
+# wg set wg0 peer '7lWtsDdqaGB3EY9WNxRN3hVaHMtu1zXw71+bOjNOVUw=' endpoint 10.0.1.100:54321 allowed-ips 192.168.2.100/32
+.Ed
+.Pp
+Remove a peer
+.Bd -literal -offset indent
+# wg set wg0 peer '7lWtsDdqaGB3EY9WNxRN3hVaHMtu1zXw71+bOjNOVUw=' remove
+.Ed
+.Sh DIAGNOSTICS
+The
+.Nm
+interface supports runtime debugging, which can be enabled with:
+.Pp
+.D1 Ic ifconfig Cm wg Ns Ar N Cm debug
+.Pp
+Some common error messages include:
+.Bl -diag
+.It "Handshake for peer X did not complete after 5 seconds, retrying"
+Peer X did not reply to our initiation packet, for example because:
+.Bl -bullet
+.It
+The peer does not have the local interface configured as a peer.
+Peers must be able to mutually authenticate each other.
+.It
+The peer endpoint IP address is incorrectly configured.
+.It
+There are firewall rules preventing communication between hosts.
+.El
+.It "Invalid handshake initiation"
+The incoming handshake packet could not be processed.
+This is likely due to the local interface not containing
+the correct public key for the peer.
+.It "Invalid initiation MAC"
+The incoming handshake initiation packet had an invalid MAC.
+This is likely because the initiation sender has the wrong public key
+for the handshake receiver.
+.It "Packet has unallowed src IP from peer X"
+After decryption, an incoming data packet has a source IP address that
+is not assigned to the allowed IPs of Peer X.
+.El
+.Sh SEE ALSO
+.Xr inet 4 ,
+.Xr ip 4 ,
+.Xr netintro 4 ,
+.Xr ipf 5 ,
+.Xr pf.conf 5 ,
+.Xr ifconfig 8 ,
+.Xr ipfw 8 ,
+.Xr wg 8
+.Rs
+.%T WireGuard whitepaper
+.%U https://www.wireguard.com/papers/wireguard.pdf
+.Re
+.Sh HISTORY
+The
+.Nm
+device driver first appeared in
+.Fx 14.0 .
+.Sh AUTHORS
+The
+.Nm
+device driver written by
+.An Jason A. Donenfeld Aq Mt Jason@zx2c4.com ,
+.An Matt Dunwoodie Aq Mt ncon@nconroy.net ,
+and
+.An Kyle Evans Aq Mt kevans@FreeBSD.org .
+.Pp
+This manual page was written by
+.An Gordon Bergling Aq Mt gbe@FreeBSD.org
+and is based on the
+.Ox
+manual page written by
+.An David Gwynne Aq Mt dlg@openbsd.org .
diff --git a/sys/conf/NOTES b/sys/conf/NOTES
index 434c739c8b21..8a9c726b792c 100644
--- a/sys/conf/NOTES
+++ b/sys/conf/NOTES
@@ -961,6 +961,9 @@ device		enc
 # Link aggregation interface.
 device		lagg
 
+# WireGuard interface.
+device		wg
+
 #
 # Internet family options:
 #
diff --git a/sys/conf/files b/sys/conf/files
index f4f7cf6208e1..e47f6577e39c 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -750,8 +750,8 @@ crypto/sha2/sha256c.c		optional crypto | ekcd | geom_bde | \
 crypto/sha2/sha512c.c		optional crypto | geom_bde | zfs
 crypto/skein/skein.c		optional crypto | zfs
 crypto/skein/skein_block.c	optional crypto | zfs
-crypto/siphash/siphash.c	optional inet | inet6
-crypto/siphash/siphash_test.c	optional inet | inet6
+crypto/siphash/siphash.c	optional inet | inet6 | wg
+crypto/siphash/siphash_test.c	optional inet | inet6 | wg
 ddb/db_access.c			optional ddb
 ddb/db_break.c			optional ddb
 ddb/db_capture.c		optional ddb
@@ -3480,6 +3480,14 @@ dev/vt/vt_font.c		optional vt
 dev/vt/vt_sysmouse.c		optional vt
 dev/vte/if_vte.c		optional vte pci
 dev/watchdog/watchdog.c		standard
+dev/wg/if_wg.c			optional wg				\
+	compile-with "${NORMAL_C} -include $S/dev/wg/compat.h"
+dev/wg/wg_cookie.c		optional wg				\
+	compile-with "${NORMAL_C} -include $S/dev/wg/compat.h"
+dev/wg/wg_crypto.c		optional wg				\
+	compile-with "${NORMAL_C} -include $S/dev/wg/compat.h"
+dev/wg/wg_noise.c		optional wg				\
+	compile-with "${NORMAL_C} -include $S/dev/wg/compat.h"
 dev/wpi/if_wpi.c		optional wpi pci
 wpifw.c			optional wpifw					\
 	compile-with	"${AWK} -f $S/tools/fw_stub.awk wpi.fw:wpifw:153229 -mwpi -c${.TARGET}" \
diff --git a/sys/dev/wg/compat.h b/sys/dev/wg/compat.h
new file mode 100644
index 000000000000..101a771579d9
--- /dev/null
+++ b/sys/dev/wg/compat.h
@@ -0,0 +1,118 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ * Copyright (c) 2022 The FreeBSD Foundation
+ *
+ * compat.h contains code that is backported from FreeBSD's main branch.
+ * It is different from support.h, which is for code that is not _yet_ upstream.
+ */
+
+#include <sys/param.h>
+
+#if (__FreeBSD_version < 1400036 && __FreeBSD_version >= 1400000) || __FreeBSD_version < 1300519
+#define COMPAT_NEED_CHACHA20POLY1305_MBUF
+#endif
+
+#if __FreeBSD_version < 1400048
+#define COMPAT_NEED_CHACHA20POLY1305
+#endif
+
+#if __FreeBSD_version < 1400049
+#define COMPAT_NEED_CURVE25519
+#endif
+
+#if __FreeBSD_version < 0x7fffffff /* TODO: update this when implemented */
+#define COMPAT_NEED_BLAKE2S
+#endif
+
+#if __FreeBSD_version < 1400059
+#include <sys/sockbuf.h>
+#define sbcreatecontrol(a, b, c, d, e) sbcreatecontrol(a, b, c, d)
+#endif
+
+#if __FreeBSD_version < 1300507
+#include <sys/smp.h>
+#include <sys/gtaskqueue.h>
+
+struct taskqgroup_cpu {
+	LIST_HEAD(, grouptask)  tgc_tasks;
+	struct gtaskqueue       *tgc_taskq;
+	int     tgc_cnt;
+	int     tgc_cpu;
+};
+
+struct taskqgroup {
+	struct taskqgroup_cpu tqg_queue[MAXCPU];
+	/* Other members trimmed from compat. */
+};
+
+static inline void taskqgroup_drain_all(struct taskqgroup *tqg)
+{
+	struct gtaskqueue *q;
+
+	for (int i = 0; i < mp_ncpus; i++) {
+		q = tqg->tqg_queue[i].tgc_taskq;
+		if (q == NULL)
+			continue;
+		gtaskqueue_drain_all(q);
+	}
+}
+#endif
+
+#if __FreeBSD_version < 1300000
+#define VIMAGE
+
+#include <sys/types.h>
+#include <sys/limits.h>
+#include <sys/endian.h>
+#include <sys/socket.h>
+#include <sys/libkern.h>
+#include <sys/malloc.h>
+#include <sys/proc.h>
+#include <sys/lock.h>
+#include <sys/socketvar.h>
+#include <sys/protosw.h>
+#include <net/vnet.h>
+#include <net/if.h>
+#include <net/if_var.h>
+#include <vm/uma.h>
+
+#define taskqgroup_attach(a, b, c, d, e, f) taskqgroup_attach((a), (b), (c), -1, (f))
+#define taskqgroup_attach_cpu(a, b, c, d, e, f, g) taskqgroup_attach_cpu((a), (b), (c), (d), -1, (g))
+
+#undef NET_EPOCH_ENTER
+#define NET_EPOCH_ENTER(et) NET_EPOCH_ENTER_ET(et)
+#undef NET_EPOCH_EXIT
+#define NET_EPOCH_EXIT(et) NET_EPOCH_EXIT_ET(et)
+#define NET_EPOCH_CALL(f, c) epoch_call(net_epoch_preempt, (c), (f))
+#define NET_EPOCH_ASSERT() MPASS(in_epoch(net_epoch_preempt))
+
+#undef atomic_load_ptr
+#define atomic_load_ptr(p) (*(volatile __typeof(*p) *)(p))
+
+#endif
+
+#if __FreeBSD_version < 1202000
+static inline uint32_t arc4random_uniform(uint32_t bound)
+{
+	uint32_t ret, max_mod_bound;
+
+	if (bound < 2)
+		return 0;
+
+	max_mod_bound = (1 + ~bound) % bound;
+
+	do {
+		ret = arc4random();
+	} while (ret < max_mod_bound);
+
+	return ret % bound;
+}
+
+typedef void callout_func_t(void *);
+
+#ifndef CSUM_SND_TAG
+#define CSUM_SND_TAG 0x80000000
+#endif
+
+#endif
diff --git a/sys/dev/wg/crypto.h b/sys/dev/wg/crypto.h
new file mode 100644
index 000000000000..2115039321b1
--- /dev/null
+++ b/sys/dev/wg/crypto.h
@@ -0,0 +1,182 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ * Copyright (c) 2022 The FreeBSD Foundation
+ */
+
+#ifndef _WG_CRYPTO
+#define _WG_CRYPTO
+
+#include <sys/param.h>
+
+struct mbuf;
+
+int crypto_init(void);
+void crypto_deinit(void);
+
+enum chacha20poly1305_lengths {
+	XCHACHA20POLY1305_NONCE_SIZE = 24,
+	CHACHA20POLY1305_KEY_SIZE = 32,
+	CHACHA20POLY1305_AUTHTAG_SIZE = 16
+};
+
+#ifdef COMPAT_NEED_CHACHA20POLY1305
+void
+chacha20poly1305_encrypt(uint8_t *dst, const uint8_t *src, const size_t src_len,
+			 const uint8_t *ad, const size_t ad_len,
+			 const uint64_t nonce,
+			 const uint8_t key[CHACHA20POLY1305_KEY_SIZE]);
+
+bool
+chacha20poly1305_decrypt(uint8_t *dst, const uint8_t *src, const size_t src_len,
+			 const uint8_t *ad, const size_t ad_len,
+			 const uint64_t nonce,
+			 const uint8_t key[CHACHA20POLY1305_KEY_SIZE]);
+
+void
+xchacha20poly1305_encrypt(uint8_t *dst, const uint8_t *src,
+			  const size_t src_len, const uint8_t *ad,
+			  const size_t ad_len,
+			  const uint8_t nonce[XCHACHA20POLY1305_NONCE_SIZE],
+			  const uint8_t key[CHACHA20POLY1305_KEY_SIZE]);
+
+bool
+xchacha20poly1305_decrypt(uint8_t *dst, const uint8_t *src,
+			  const size_t src_len,  const uint8_t *ad,
+			  const size_t ad_len,
+			  const uint8_t nonce[XCHACHA20POLY1305_NONCE_SIZE],
+			  const uint8_t key[CHACHA20POLY1305_KEY_SIZE]);
+#else
+#include <sys/endian.h>
+#include <crypto/chacha20_poly1305.h>
+
+static inline void
+chacha20poly1305_encrypt(uint8_t *dst, const uint8_t *src, const size_t src_len,
+			 const uint8_t *ad, const size_t ad_len,
+			 const uint64_t nonce,
+			 const uint8_t key[CHACHA20POLY1305_KEY_SIZE])
+{
+	uint8_t nonce_bytes[8];
+
+	le64enc(nonce_bytes, nonce);
+	chacha20_poly1305_encrypt(dst, src, src_len, ad, ad_len,
+				  nonce_bytes, sizeof(nonce_bytes), key);
+}
+
+static inline bool
+chacha20poly1305_decrypt(uint8_t *dst, const uint8_t *src, const size_t src_len,
+			 const uint8_t *ad, const size_t ad_len,
+			 const uint64_t nonce,
+			 const uint8_t key[CHACHA20POLY1305_KEY_SIZE])
+{
+	uint8_t nonce_bytes[8];
+
+	le64enc(nonce_bytes, nonce);
+	return (chacha20_poly1305_decrypt(dst, src, src_len, ad, ad_len,
+					  nonce_bytes, sizeof(nonce_bytes), key));
+}
+
+static inline void
+xchacha20poly1305_encrypt(uint8_t *dst, const uint8_t *src,
+			  const size_t src_len, const uint8_t *ad,
+			  const size_t ad_len,
+			  const uint8_t nonce[XCHACHA20POLY1305_NONCE_SIZE],
+			  const uint8_t key[CHACHA20POLY1305_KEY_SIZE])
+{
+	xchacha20_poly1305_encrypt(dst, src, src_len, ad, ad_len, nonce, key);
+}
+
+static inline bool
+xchacha20poly1305_decrypt(uint8_t *dst, const uint8_t *src,
+			  const size_t src_len,  const uint8_t *ad,
+			  const size_t ad_len,
+			  const uint8_t nonce[XCHACHA20POLY1305_NONCE_SIZE],
+			  const uint8_t key[CHACHA20POLY1305_KEY_SIZE])
+{
+	return (xchacha20_poly1305_decrypt(dst, src, src_len, ad, ad_len, nonce, key));
+}
+#endif
+
+int
+chacha20poly1305_encrypt_mbuf(struct mbuf *, const uint64_t nonce,
+			      const uint8_t key[CHACHA20POLY1305_KEY_SIZE]);
+
+int
+chacha20poly1305_decrypt_mbuf(struct mbuf *, const uint64_t nonce,
+			      const uint8_t key[CHACHA20POLY1305_KEY_SIZE]);
+
+
+enum blake2s_lengths {
+	BLAKE2S_BLOCK_SIZE = 64,
+	BLAKE2S_HASH_SIZE = 32,
+	BLAKE2S_KEY_SIZE = 32
+};
+
+#ifdef COMPAT_NEED_BLAKE2S
+struct blake2s_state {
+	uint32_t h[8];
+	uint32_t t[2];
+	uint32_t f[2];
+	uint8_t buf[BLAKE2S_BLOCK_SIZE];
+	unsigned int buflen;
+	unsigned int outlen;
+};
+
+void blake2s_init(struct blake2s_state *state, const size_t outlen);
+
+void blake2s_init_key(struct blake2s_state *state, const size_t outlen,
+		      const uint8_t *key, const size_t keylen);
+
+void blake2s_update(struct blake2s_state *state, const uint8_t *in, size_t inlen);
+
+void blake2s_final(struct blake2s_state *state, uint8_t *out);
+
+static inline void blake2s(uint8_t *out, const uint8_t *in, const uint8_t *key,
+			   const size_t outlen, const size_t inlen, const size_t keylen)
+{
+	struct blake2s_state state;
+
+	if (keylen)
+		blake2s_init_key(&state, outlen, key, keylen);
+	else
+		blake2s_init(&state, outlen);
+
+	blake2s_update(&state, in, inlen);
+	blake2s_final(&state, out);
+}
+#endif
+
+#ifdef COMPAT_NEED_CURVE25519
+enum curve25519_lengths {
+        CURVE25519_KEY_SIZE = 32
+};
+
+bool curve25519(uint8_t mypublic[static CURVE25519_KEY_SIZE],
+		const uint8_t secret[static CURVE25519_KEY_SIZE],
+		const uint8_t basepoint[static CURVE25519_KEY_SIZE]);
+
+static inline bool
+curve25519_generate_public(uint8_t pub[static CURVE25519_KEY_SIZE],
+			   const uint8_t secret[static CURVE25519_KEY_SIZE])
+{
+	static const uint8_t basepoint[CURVE25519_KEY_SIZE] = { 9 };
+
+	return curve25519(pub, secret, basepoint);
+}
+
+static inline void curve25519_clamp_secret(uint8_t secret[static CURVE25519_KEY_SIZE])
+{
+        secret[0] &= 248;
+        secret[31] = (secret[31] & 127) | 64;
+}
+
+static inline void curve25519_generate_secret(uint8_t secret[CURVE25519_KEY_SIZE])
+{
+	arc4random_buf(secret, CURVE25519_KEY_SIZE);
+	curve25519_clamp_secret(secret);
+}
+#else
+#include <crypto/curve25519.h>
+#endif
+
+#endif
diff --git a/sys/dev/wg/if_wg.c b/sys/dev/wg/if_wg.c
new file mode 100644
index 000000000000..59979c087db2
--- /dev/null
+++ b/sys/dev/wg/if_wg.c
@@ -0,0 +1,3055 @@
+/* SPDX-License-Identifier: ISC
+ *
+ * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ * Copyright (C) 2019-2021 Matt Dunwoodie <ncon@noconroy.net>
+ * Copyright (c) 2019-2020 Rubicon Communications, LLC (Netgate)
+ * Copyright (c) 2021 Kyle Evans <kevans@FreeBSD.org>
+ * Copyright (c) 2022 The FreeBSD Foundation
+ */
+
+#include "opt_inet.h"
+#include "opt_inet6.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/counter.h>
+#include <sys/gtaskqueue.h>
+#include <sys/jail.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/mbuf.h>
+#include <sys/module.h>
+#include <sys/nv.h>
+#include <sys/priv.h>
+#include <sys/protosw.h>
+#include <sys/rmlock.h>
+#include <sys/rwlock.h>
+#include <sys/smp.h>
+#include <sys/socket.h>
+#include <sys/socketvar.h>
+#include <sys/sockio.h>
+#include <sys/sysctl.h>
+#include <sys/sx.h>
+#include <machine/_inttypes.h>
+#include <net/bpf.h>
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <net/if_clone.h>
+#include <net/if_types.h>
+#include <net/if_var.h>
+#include <net/netisr.h>
+#include <net/radix.h>
+#include <netinet/in.h>
+#include <netinet6/in6_var.h>
+#include <netinet/ip.h>
+#include <netinet/ip6.h>
+#include <netinet/ip_icmp.h>
+#include <netinet/icmp6.h>
+#include <netinet/udp_var.h>
+#include <netinet6/nd6.h>
+
+#include "support.h"
+#include "wg_noise.h"
+#include "wg_cookie.h"
+#include "version.h"
+#include "if_wg.h"
+
+#define DEFAULT_MTU		(ETHERMTU - 80)
+#define MAX_MTU			(IF_MAXMTU - 80)
+
+#define MAX_STAGED_PKT		128
+#define MAX_QUEUED_PKT		1024
+#define MAX_QUEUED_PKT_MASK	(MAX_QUEUED_PKT - 1)
+
+#define MAX_QUEUED_HANDSHAKES	4096
+
+#define REKEY_TIMEOUT_JITTER	334 /* 1/3 sec, round for arc4random_uniform */
+#define MAX_TIMER_HANDSHAKES	(90 / REKEY_TIMEOUT)
+#define NEW_HANDSHAKE_TIMEOUT	(REKEY_TIMEOUT + KEEPALIVE_TIMEOUT)
+#define UNDERLOAD_TIMEOUT	1
+
+#define DPRINTF(sc, ...) if (sc->sc_ifp->if_flags & IFF_DEBUG) if_printf(sc->sc_ifp, ##__VA_ARGS__)
+
+/* First byte indicating packet type on the wire */
+#define WG_PKT_INITIATION htole32(1)
+#define WG_PKT_RESPONSE htole32(2)
+#define WG_PKT_COOKIE htole32(3)
+#define WG_PKT_DATA htole32(4)
+
+#define WG_PKT_PADDING		16
+#define WG_KEY_SIZE		32
+
+struct wg_pkt_initiation {
+	uint32_t		t;
+	uint32_t		s_idx;
+	uint8_t			ue[NOISE_PUBLIC_KEY_LEN];
+	uint8_t			es[NOISE_PUBLIC_KEY_LEN + NOISE_AUTHTAG_LEN];
+	uint8_t			ets[NOISE_TIMESTAMP_LEN + NOISE_AUTHTAG_LEN];
+	struct cookie_macs	m;
+};
+
+struct wg_pkt_response {
+	uint32_t		t;
+	uint32_t		s_idx;
+	uint32_t		r_idx;
+	uint8_t			ue[NOISE_PUBLIC_KEY_LEN];
+	uint8_t			en[0 + NOISE_AUTHTAG_LEN];
+	struct cookie_macs	m;
+};
+
+struct wg_pkt_cookie {
+	uint32_t		t;
+	uint32_t		r_idx;
+	uint8_t			nonce[COOKIE_NONCE_SIZE];
+	uint8_t			ec[COOKIE_ENCRYPTED_SIZE];
+};
+
+struct wg_pkt_data {
+	uint32_t		t;
+	uint32_t		r_idx;
+	uint64_t		nonce;
+	uint8_t			buf[];
+};
+
+struct wg_endpoint {
+	union {
+		struct sockaddr		r_sa;
+		struct sockaddr_in	r_sin;
+#ifdef INET6
+		struct sockaddr_in6	r_sin6;
+#endif
+	} e_remote;
+	union {
+		struct in_addr		l_in;
+#ifdef INET6
+		struct in6_pktinfo	l_pktinfo6;
+#define l_in6 l_pktinfo6.ipi6_addr
+#endif
+	} e_local;
+};
+
+struct aip_addr {
+	uint8_t		length;
+	union {
+		uint8_t		bytes[16];
+		uint32_t	ip;
+		uint32_t	ip6[4];
+		struct in_addr	in;
+		struct in6_addr	in6;
+	};
+};
+
+struct wg_aip {
+	struct radix_node	 a_nodes[2];
+	LIST_ENTRY(wg_aip)	 a_entry;
+	struct aip_addr		 a_addr;
+	struct aip_addr		 a_mask;
+	struct wg_peer		*a_peer;
+	sa_family_t		 a_af;
+};
+
+struct wg_packet {
+	STAILQ_ENTRY(wg_packet)	 p_serial;
+	STAILQ_ENTRY(wg_packet)	 p_parallel;
+	struct wg_endpoint	 p_endpoint;
+	struct noise_keypair	*p_keypair;
+	uint64_t		 p_nonce;
+	struct mbuf		*p_mbuf;
+	int			 p_mtu;
+	sa_family_t		 p_af;
+	enum wg_ring_state {
+		WG_PACKET_UNCRYPTED,
+		WG_PACKET_CRYPTED,
+		WG_PACKET_DEAD,
+	}			 p_state;
+};
+
+STAILQ_HEAD(wg_packet_list, wg_packet);
+
+struct wg_queue {
+	struct mtx		 q_mtx;
+	struct wg_packet_list	 q_queue;
+	size_t			 q_len;
+};
+
+struct wg_peer {
+	TAILQ_ENTRY(wg_peer)		 p_entry;
+	uint64_t			 p_id;
+	struct wg_softc			*p_sc;
+
+	struct noise_remote		*p_remote;
+	struct cookie_maker		 p_cookie;
+
+	struct rwlock			 p_endpoint_lock;
+	struct wg_endpoint		 p_endpoint;
+
+	struct wg_queue	 		 p_stage_queue;
+	struct wg_queue	 		 p_encrypt_serial;
+	struct wg_queue	 		 p_decrypt_serial;
+
+	bool				 p_enabled;
+	bool				 p_need_another_keepalive;
+	uint16_t			 p_persistent_keepalive_interval;
+	struct callout			 p_new_handshake;
+	struct callout			 p_send_keepalive;
+	struct callout			 p_retry_handshake;
+	struct callout			 p_zero_key_material;
+	struct callout			 p_persistent_keepalive;
+
+	struct mtx			 p_handshake_mtx;
+	struct timespec			 p_handshake_complete;	/* nanotime */
+	int				 p_handshake_retries;
+
+	struct grouptask		 p_send;
+	struct grouptask		 p_recv;
+
+	counter_u64_t			 p_tx_bytes;
+	counter_u64_t			 p_rx_bytes;
+
+	LIST_HEAD(, wg_aip)		 p_aips;
+	size_t				 p_aips_num;
+};
+
+struct wg_socket {
+	struct socket	*so_so4;
+	struct socket	*so_so6;
+	uint32_t	 so_user_cookie;
+	int		 so_fibnum;
+	in_port_t	 so_port;
+};
+
+struct wg_softc {
+	LIST_ENTRY(wg_softc)	 sc_entry;
+	struct ifnet		*sc_ifp;
+	int			 sc_flags;
+
+	struct ucred		*sc_ucred;
+	struct wg_socket	 sc_socket;
+
+	TAILQ_HEAD(,wg_peer)	 sc_peers;
+	size_t			 sc_peers_num;
+
+	struct noise_local	*sc_local;
+	struct cookie_checker	 sc_cookie;
+
+	struct radix_node_head	*sc_aip4;
+	struct radix_node_head	*sc_aip6;
+
+	struct grouptask	 sc_handshake;
+	struct wg_queue		 sc_handshake_queue;
+
+	struct grouptask	*sc_encrypt;
+	struct grouptask	*sc_decrypt;
+	struct wg_queue		 sc_encrypt_parallel;
+	struct wg_queue		 sc_decrypt_parallel;
+	u_int			 sc_encrypt_last_cpu;
+	u_int			 sc_decrypt_last_cpu;
+
+	struct sx		 sc_lock;
+};
+
+#define	WGF_DYING	0x0001
+
+#define MAX_LOOPS	8
+#define MTAG_WGLOOP	0x77676c70 /* wglp */
+#ifndef ENOKEY
+#define	ENOKEY	ENOTCAPABLE
+#endif
+
+#define	GROUPTASK_DRAIN(gtask)			\
+	gtaskqueue_drain((gtask)->gt_taskqueue, &(gtask)->gt_task)
+
+#define BPF_MTAP2_AF(ifp, m, af) do { \
+		uint32_t __bpf_tap_af = (af); \
+		BPF_MTAP2(ifp, &__bpf_tap_af, sizeof(__bpf_tap_af), m); \
+	} while (0)
+
+static int clone_count;
+static uma_zone_t wg_packet_zone;
+static volatile unsigned long peer_counter = 0;
+static const char wgname[] = "wg";
+static unsigned wg_osd_jail_slot;
+
+static struct sx wg_sx;
+SX_SYSINIT(wg_sx, &wg_sx, "wg_sx");
+
+static LIST_HEAD(, wg_softc) wg_list = LIST_HEAD_INITIALIZER(wg_list);
+
+static TASKQGROUP_DEFINE(wg_tqg, mp_ncpus, 1);
+
+MALLOC_DEFINE(M_WG, "WG", "wireguard");
+
+VNET_DEFINE_STATIC(struct if_clone *, wg_cloner);
+
+#define	V_wg_cloner	VNET(wg_cloner)
+#define	WG_CAPS		IFCAP_LINKSTATE
+
+struct wg_timespec64 {
+	uint64_t	tv_sec;
+	uint64_t	tv_nsec;
*** 6905 LINES SKIPPED ***