git: 1243360b1a02 - stable/13 - ipsec: replace SECASVAR mtx by rmlock

From: Kristof Provost <kp_at_FreeBSD.org>
Date: Tue, 09 Aug 2022 13:47:18 UTC
The branch stable/13 has been updated by kp:

URL: https://cgit.FreeBSD.org/src/commit/?id=1243360b1a0230558e0deebf9bdba6ab48e6fff0

commit 1243360b1a0230558e0deebf9bdba6ab48e6fff0
Author:     Kristof Provost <kp@FreeBSD.org>
AuthorDate: 2022-06-23 20:35:29 +0000
Commit:     Kristof Provost <kp@FreeBSD.org>
CommitDate: 2022-08-09 13:46:57 +0000

    ipsec: replace SECASVAR  mtx by rmlock
    
    This mutex is a significant point of contention in the ipsec code, and
    can be relatively trivially replaced by a read-mostly lock.
    It does require a separate lock for the replay protection, which we do
    here by adding a separate mutex.
    
    This improves throughput (without replay protection) by 10-15%.
    
    MFC after:      3 weeks
    Sponsored by:   Orange Business Services
    Differential Revision:  https://reviews.freebsd.org/D35763
    
    (cherry picked from commit 0361f165f2193a098cfbfeef3a58ec2f1eaac0a1)
---
 sys/netipsec/ipsec.c        | 64 ++++++++++++++++++++++++++++++++++++---------
 sys/netipsec/key.c          | 46 ++++++++++++++++++--------------
 sys/netipsec/key_debug.c    |  4 +++
 sys/netipsec/keydb.h        | 18 ++++++++++---
 sys/netipsec/xform_ah.c     | 29 +++++++++++++-------
 sys/netipsec/xform_esp.c    | 24 +++++++++++------
 sys/netipsec/xform_ipcomp.c | 16 +++++++-----
 7 files changed, 142 insertions(+), 59 deletions(-)

diff --git a/sys/netipsec/ipsec.c b/sys/netipsec/ipsec.c
index cd24750607ea..5bb8d2412321 100644
--- a/sys/netipsec/ipsec.c
+++ b/sys/netipsec/ipsec.c
@@ -1192,6 +1192,8 @@ check_window(const struct secreplay *replay, uint64_t seq)
 {
 	int index, bit_location;
 
+	SECREPLAY_ASSERT(replay);
+
 	bit_location = seq & IPSEC_BITMAP_LOC_MASK;
 	index = (seq >> IPSEC_REDUNDANT_BIT_SHIFTS)
 		& IPSEC_BITMAP_INDEX_MASK(replay->bitmap_size);
@@ -1206,6 +1208,8 @@ advance_window(const struct secreplay *replay, uint64_t seq)
 	int i;
 	uint64_t index, index_cur, diff;
 
+	SECREPLAY_ASSERT(replay);
+
 	index_cur = replay->last >> IPSEC_REDUNDANT_BIT_SHIFTS;
 	index = seq >> IPSEC_REDUNDANT_BIT_SHIFTS;
 	diff = index - index_cur;
@@ -1226,6 +1230,8 @@ set_window(const struct secreplay *replay, uint64_t seq)
 {
 	int index, bit_location;
 
+	SECREPLAY_ASSERT(replay);
+
 	bit_location = seq & IPSEC_BITMAP_LOC_MASK;
 	index = (seq >> IPSEC_REDUNDANT_BIT_SHIFTS)
 		& IPSEC_BITMAP_INDEX_MASK(replay->bitmap_size);
@@ -1259,12 +1265,17 @@ ipsec_chkreplay(uint32_t seq, uint32_t *seqhigh, struct secasvar *sav)
 	replay = sav->replay;
 
 	/* No need to check replay if disabled. */
-	if (replay->wsize == 0)
+	if (replay->wsize == 0) {
 		return (1);
+	}
+
+	SECREPLAY_LOCK(replay);
 
 	/* Zero sequence number is not allowed. */
-	if (seq == 0 && replay->last == 0)
+	if (seq == 0 && replay->last == 0) {
+		SECREPLAY_UNLOCK(replay);
 		return (0);
+	}
 
 	window = replay->wsize << 3;		/* Size of window */
 	tl = (uint32_t)replay->last;		/* Top of window, lower part */
@@ -1282,10 +1293,13 @@ ipsec_chkreplay(uint32_t seq, uint32_t *seqhigh, struct secasvar *sav)
 		*seqhigh = th;
 		if (seq <= tl) {
 			/* Sequence number inside window - check against replay */
-			if (check_window(replay, seq))
+			if (check_window(replay, seq)) {
+				SECREPLAY_UNLOCK(replay);
 				return (0);
+			}
 		}
 
+		SECREPLAY_UNLOCK(replay);
 		/* Sequence number above top of window or not found in bitmap */
 		return (1);
 	}
@@ -1303,6 +1317,7 @@ ipsec_chkreplay(uint32_t seq, uint32_t *seqhigh, struct secasvar *sav)
 				ESPSTAT_INC(esps_wrap);
 			else if (sav->sah->saidx.proto == IPPROTO_AH)
 				AHSTAT_INC(ahs_wrap);
+			SECREPLAY_UNLOCK(replay);
 			return (0);
 		}
 
@@ -1322,8 +1337,11 @@ ipsec_chkreplay(uint32_t seq, uint32_t *seqhigh, struct secasvar *sav)
 			return (0);
 		*seqhigh = th - 1;
 		seqh = th - 1;
-		if (check_window(replay, seq))
+		if (check_window(replay, seq)) {
+			SECREPLAY_UNLOCK(replay);
 			return (0);
+		}
+		SECREPLAY_UNLOCK(replay);
 		return (1);
 	}
 
@@ -1344,6 +1362,7 @@ ipsec_chkreplay(uint32_t seq, uint32_t *seqhigh, struct secasvar *sav)
 				ESPSTAT_INC(esps_wrap);
 			else if (sav->sah->saidx.proto == IPPROTO_AH)
 				AHSTAT_INC(ahs_wrap);
+			SECREPLAY_UNLOCK(replay);
 			return (0);
 		}
 
@@ -1352,6 +1371,7 @@ ipsec_chkreplay(uint32_t seq, uint32_t *seqhigh, struct secasvar *sav)
 		    ipsec_sa2str(sav, buf, sizeof(buf))));
 	}
 
+	SECREPLAY_UNLOCK(replay);
 	return (1);
 }
 
@@ -1377,9 +1397,13 @@ ipsec_updatereplay(uint32_t seq, struct secasvar *sav)
 	if (replay->wsize == 0)
 		return (0);
 
+	SECREPLAY_LOCK(replay);
+
 	/* Zero sequence number is not allowed. */
-	if (seq == 0 && replay->last == 0)
+	if (seq == 0 && replay->last == 0) {
+		SECREPLAY_UNLOCK(replay);
 		return (1);
+	}
 
 	window = replay->wsize << 3;		/* Size of window */
 	tl = (uint32_t)replay->last;		/* Top of window, lower part */
@@ -1397,8 +1421,10 @@ ipsec_updatereplay(uint32_t seq, struct secasvar *sav)
 		seqh = th;
 		if (seq <= tl) {
 			/* Sequence number inside window - check against replay */
-			if (check_window(replay, seq))
+			if (check_window(replay, seq)) {
+				SECREPLAY_UNLOCK(replay);
 				return (1);
+			}
 			set_window(replay, seq);
 		} else {
 			advance_window(replay, ((uint64_t)seqh << 32) | seq);
@@ -1408,11 +1434,14 @@ ipsec_updatereplay(uint32_t seq, struct secasvar *sav)
 
 		/* Sequence number above top of window or not found in bitmap */
 		replay->count++;
+		SECREPLAY_UNLOCK(replay);
 		return (0);
 	}
 
-	if (!(sav->flags & SADB_X_SAFLAGS_ESN))
+	if (!(sav->flags & SADB_X_SAFLAGS_ESN)) {
+		SECREPLAY_UNLOCK(replay);
 		return (1);
+	}
 
 	/*
 	 * Seq is within [bl, 0xffffffff] and bl is within
@@ -1421,13 +1450,18 @@ ipsec_updatereplay(uint32_t seq, struct secasvar *sav)
 	 * subspace.
 	 */
 	if (tl < window - 1 && seq >= bl) {
-		if (th == 0)
+		if (th == 0) {
+			SECREPLAY_UNLOCK(replay);
 			return (1);
-		if (check_window(replay, seq))
+		}
+		if (check_window(replay, seq)) {
+			SECREPLAY_UNLOCK(replay);
 			return (1);
+		}
 
 		set_window(replay, seq);
 		replay->count++;
+		SECREPLAY_UNLOCK(replay);
 		return (0);
 	}
 
@@ -1438,13 +1472,17 @@ ipsec_updatereplay(uint32_t seq, struct secasvar *sav)
 	seqh = th + 1;
 
 	/* Don't let high part wrap. */
-	if (seqh == 0)
+	if (seqh == 0) {
+		SECREPLAY_UNLOCK(replay);
 		return (1);
+	}
 
 	advance_window(replay, ((uint64_t)seqh << 32) | seq);
 	set_window(replay, seq);
 	replay->last = ((uint64_t)seqh << 32) | seq;
 	replay->count++;
+
+	SECREPLAY_UNLOCK(replay);
 	return (0);
 }
 int
@@ -1480,17 +1518,17 @@ ipsec_updateid(struct secasvar *sav, crypto_session_t *new,
 	    printf("%s: SA(%p) moves cryptoid %p -> %p\n",
 		__func__, sav, *old, *new));
 	KEYDBG(IPSEC_DATA, kdebug_secasv(sav));
-	SECASVAR_LOCK(sav);
+	SECASVAR_WLOCK(sav);
 	if (sav->tdb_cryptoid != *old) {
 		/* cryptoid was already updated */
 		tmp = *new;
 		*new = sav->tdb_cryptoid;
 		*old = tmp;
-		SECASVAR_UNLOCK(sav);
+		SECASVAR_WUNLOCK(sav);
 		return (1);
 	}
 	sav->tdb_cryptoid = *new;
-	SECASVAR_UNLOCK(sav);
+	SECASVAR_WUNLOCK(sav);
 	return (0);
 }
 
diff --git a/sys/netipsec/key.c b/sys/netipsec/key.c
index 4bb46cd9a5c1..869d6b850aa0 100644
--- a/sys/netipsec/key.c
+++ b/sys/netipsec/key.c
@@ -2932,13 +2932,13 @@ key_newsav(const struct sadb_msghdr *mhp, struct secasindex *saidx,
 		*errp = ENOBUFS;
 		goto done;
 	}
-	sav->lock = malloc(sizeof(struct mtx), M_IPSEC_MISC,
+	sav->lock = malloc(sizeof(struct rmlock), M_IPSEC_MISC,
 	    M_NOWAIT | M_ZERO);
 	if (sav->lock == NULL) {
 		*errp = ENOBUFS;
 		goto done;
 	}
-	mtx_init(sav->lock, "ipsec association", NULL, MTX_DEF);
+	rm_init(sav->lock, "ipsec association");
 	sav->lft_c = uma_zalloc_pcpu(V_key_lft_zone, M_NOWAIT);
 	if (sav->lft_c == NULL) {
 		*errp = ENOBUFS;
@@ -3027,7 +3027,7 @@ done:
 	if (*errp != 0) {
 		if (sav != NULL) {
 			if (sav->lock != NULL) {
-				mtx_destroy(sav->lock);
+				rm_destroy(sav->lock);
 				free(sav->lock, M_IPSEC_MISC);
 			}
 			if (sav->lft_c != NULL)
@@ -3073,6 +3073,7 @@ key_cleansav(struct secasvar *sav)
 		sav->key_enc = NULL;
 	}
 	if (sav->replay != NULL) {
+		mtx_destroy(&sav->replay->lock);
 		if (sav->replay->bitmap != NULL)
 			free(sav->replay->bitmap, M_IPSEC_MISC);
 		free(sav->replay, M_IPSEC_MISC);
@@ -3107,7 +3108,7 @@ key_delsav(struct secasvar *sav)
 	 */
 	key_cleansav(sav);
 	if ((sav->flags & SADB_X_EXT_F_CLONED) == 0) {
-		mtx_destroy(sav->lock);
+		rm_destroy(sav->lock);
 		free(sav->lock, M_IPSEC_MISC);
 		uma_zfree_pcpu(V_key_lft_zone, sav->lft_c);
 	}
@@ -3238,7 +3239,7 @@ reset:
 		 * key_update() holds reference to this SA,
 		 * so it won't be deleted in meanwhile.
 		 */
-		SECASVAR_LOCK(sav);
+		SECASVAR_WLOCK(sav);
 		tmp = sav->lft_h;
 		sav->lft_h = lft_h;
 		lft_h = tmp;
@@ -3246,7 +3247,7 @@ reset:
 		tmp = sav->lft_s;
 		sav->lft_s = lft_s;
 		lft_s = tmp;
-		SECASVAR_UNLOCK(sav);
+		SECASVAR_WUNLOCK(sav);
 		if (lft_h != NULL)
 			free(lft_h, M_IPSEC_MISC);
 		if (lft_s != NULL)
@@ -3335,6 +3336,7 @@ key_setsaval(struct secasvar *sav, const struct sadb_msghdr *mhp)
 			error = ENOBUFS;
 			goto fail;
 		}
+		mtx_init(&sav->replay->lock, "ipsec replay", NULL, MTX_DEF);
 
 		if (replay != 0) {
 			/* number of 32b blocks to be allocated */
@@ -3552,6 +3554,8 @@ key_setdumpsa(struct secasvar *sav, uint8_t type, uint8_t satype,
 	};
 	uint32_t replay_count;
 
+	SECASVAR_RLOCK_TRACKER;
+
 	m = key_setsadbmsg(type, 0, satype, seq, pid, sav->refcnt);
 	if (m == NULL)
 		goto fail;
@@ -3566,16 +3570,16 @@ key_setdumpsa(struct secasvar *sav, uint8_t type, uint8_t satype,
 				goto fail;
 			break;
 
-		case SADB_X_EXT_SA2:
-			SECASVAR_LOCK(sav);
+		case SADB_X_EXT_SA2: {
+			SECASVAR_RLOCK(sav);
 			replay_count = sav->replay ? sav->replay->count : 0;
-			SECASVAR_UNLOCK(sav);
+			SECASVAR_RUNLOCK(sav);
 			m = key_setsadbxsa2(sav->sah->saidx.mode, replay_count,
 					sav->sah->saidx.reqid);
 			if (!m)
 				goto fail;
 			break;
-
+		}
 		case SADB_X_EXT_SA_REPLAY:
 			if (sav->replay == NULL ||
 			    sav->replay->wsize <= UINT8_MAX)
@@ -4477,6 +4481,8 @@ key_flush_sad(time_t now)
 	struct secashead *sah, *nextsah;
 	struct secasvar *sav, *nextsav;
 
+	SECASVAR_RLOCK_TRACKER;
+
 	LIST_INIT(&drainq);
 	LIST_INIT(&hexpireq);
 	LIST_INIT(&sexpireq);
@@ -4502,13 +4508,13 @@ key_flush_sad(time_t now)
 			/* lifetimes aren't specified */
 			if (sav->lft_h == NULL)
 				continue;
-			SECASVAR_LOCK(sav);
+			SECASVAR_RLOCK(sav);
 			/*
 			 * Check again with lock held, because it may
 			 * be updated by SADB_UPDATE.
 			 */
 			if (sav->lft_h == NULL) {
-				SECASVAR_UNLOCK(sav);
+				SECASVAR_RUNLOCK(sav);
 				continue;
 			}
 			/*
@@ -4525,7 +4531,7 @@ key_flush_sad(time_t now)
 			    now - sav->firstused > sav->lft_h->usetime) ||
 			    (sav->lft_h->bytes != 0 && counter_u64_fetch(
 			        sav->lft_c_bytes) > sav->lft_h->bytes)) {
-				SECASVAR_UNLOCK(sav);
+				SECASVAR_RUNLOCK(sav);
 				SAV_ADDREF(sav);
 				LIST_INSERT_HEAD(&hexpireq, sav, drainq);
 				continue;
@@ -4542,12 +4548,12 @@ key_flush_sad(time_t now)
 			    (sav->replay != NULL) && (
 			    (sav->replay->count > UINT32_80PCT) ||
 			    (sav->replay->last > UINT32_80PCT))))) {
-				SECASVAR_UNLOCK(sav);
+				SECASVAR_RUNLOCK(sav);
 				SAV_ADDREF(sav);
 				LIST_INSERT_HEAD(&sexpireq, sav, drainq);
 				continue;
 			}
-			SECASVAR_UNLOCK(sav);
+			SECASVAR_RUNLOCK(sav);
 		}
 	}
 	SAHTREE_RUNLOCK();
@@ -5245,11 +5251,11 @@ key_updateaddresses(struct socket *so, struct mbuf *m,
 	 * isnew == 0 -> we use the same @sah, that was used by @sav,
 	 *	and we use its reference for @newsav.
 	 */
-	SECASVAR_LOCK(sav);
+	SECASVAR_WLOCK(sav);
 	/* XXX: replace cntr with pointer? */
 	newsav->cntr = sav->cntr;
 	sav->flags |= SADB_X_EXT_F_CLONED;
-	SECASVAR_UNLOCK(sav);
+	SECASVAR_WUNLOCK(sav);
 
 	SAHTREE_WUNLOCK();
 
@@ -7265,6 +7271,8 @@ key_expire(struct secasvar *sav, int hard)
 	int error, len;
 	uint8_t satype;
 
+	SECASVAR_RLOCK_TRACKER;
+
 	IPSEC_ASSERT (sav != NULL, ("null sav"));
 	IPSEC_ASSERT (sav->sah != NULL, ("null sa header"));
 
@@ -7291,9 +7299,9 @@ key_expire(struct secasvar *sav, int hard)
 	m_cat(result, m);
 
 	/* create SA extension */
-	SECASVAR_LOCK(sav);
+	SECASVAR_RLOCK(sav);
 	replay_count = sav->replay ? sav->replay->count : 0;
-	SECASVAR_UNLOCK(sav);
+	SECASVAR_RUNLOCK(sav);
 
 	m = key_setsadbxsa2(sav->sah->saidx.mode, replay_count,
 			sav->sah->saidx.reqid);
diff --git a/sys/netipsec/key_debug.c b/sys/netipsec/key_debug.c
index 04946dc5c165..3705879397ff 100644
--- a/sys/netipsec/key_debug.c
+++ b/sys/netipsec/key_debug.c
@@ -808,12 +808,15 @@ kdebug_secreplay(struct secreplay *rpl)
 {
 	int len, l;
 
+	SECREPLAY_LOCK(rpl);
+
 	IPSEC_ASSERT(rpl != NULL, ("null rpl"));
 	printf(" secreplay{ count=%lu bitmap_size=%u wsize=%u last=%lu",
 	    rpl->count, rpl->bitmap_size, rpl->wsize, rpl->last);
 
 	if (rpl->bitmap == NULL) {
 		printf("  }\n");
+		SECREPLAY_UNLOCK(rpl);
 		return;
 	}
 
@@ -823,6 +826,7 @@ kdebug_secreplay(struct secreplay *rpl)
 			printf("%u", (((rpl->bitmap)[len] >> l) & 1) ? 1 : 0);
 	}
 	printf("    }\n");
+	SECREPLAY_UNLOCK(rpl);
 }
 #endif /* IPSEC_DEBUG */
 
diff --git a/sys/netipsec/keydb.h b/sys/netipsec/keydb.h
index 69c1cb29db34..a2da0da613e2 100644
--- a/sys/netipsec/keydb.h
+++ b/sys/netipsec/keydb.h
@@ -39,6 +39,7 @@
 #include <sys/counter.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
+#include <sys/rmlock.h>
 
 #include <netipsec/key_var.h>
 #include <opencrypto/_cryptodev.h>
@@ -157,7 +158,7 @@ struct secasvar {
 	struct seckey *key_enc;	        /* Key for Encryption */
 	struct secreplay *replay;	/* replay prevention */
 	struct secnatt *natt;		/* NAT-T config */
-	struct mtx *lock;		/* update/access lock */
+	struct rmlock *lock;		/* update/access lock */
 
 	const struct xformsw *tdb_xform;	/* transform */
 	const struct enc_xform *tdb_encalgxform;/* encoding algorithm */
@@ -187,9 +188,13 @@ struct secasvar {
 	volatile u_int refcnt;		/* reference count */
 };
 
-#define	SECASVAR_LOCK(_sav)		mtx_lock((_sav)->lock)
-#define	SECASVAR_UNLOCK(_sav)		mtx_unlock((_sav)->lock)
-#define	SECASVAR_LOCK_ASSERT(_sav)	mtx_assert((_sav)->lock, MA_OWNED)
+#define	SECASVAR_RLOCK_TRACKER		struct rm_priotracker _secas_tracker
+#define	SECASVAR_RLOCK(_sav)		rm_rlock((_sav)->lock, &_secas_tracker)
+#define	SECASVAR_RUNLOCK(_sav)		rm_runlock((_sav)->lock, &_secas_tracker)
+#define	SECASVAR_WLOCK(_sav)		rm_wlock((_sav)->lock)
+#define	SECASVAR_WUNLOCK(_sav)		rm_wunlock((_sav)->lock)
+#define	SECASVAR_LOCK_ASSERT(_sav)	rm_assert((_sav)->lock, RA_LOCKED)
+#define	SECASVAR_LOCK_WASSERT(_sav)	rm_assert((_sav)->lock, RA_WLOCKED)
 #define	SAV_ISGCM(_sav)							\
 			((_sav)->alg_enc == SADB_X_EALG_AESGCM8 ||	\
 			(_sav)->alg_enc == SADB_X_EALG_AESGCM12 ||	\
@@ -204,6 +209,7 @@ struct secasvar {
  *  (c) read only except during creation / free
  */
 struct secreplay {
+	struct mtx lock;
 	u_int64_t count;	/* (m) */
 	u_int wsize;		/* (c) window size, i.g. 4 bytes */
 	u_int64_t last;		/* (m) used by receiver */
@@ -212,6 +218,10 @@ struct secreplay {
 	int overflow;		/* (m) overflow flag */
 };
 
+#define SECREPLAY_LOCK(_r)	mtx_lock(&(_r)->lock)
+#define SECREPLAY_UNLOCK(_r)	mtx_unlock(&(_r)->lock)
+#define SECREPLAY_ASSERT(_r)	mtx_assert(&(_r)->lock, MA_OWNED)
+
 /* socket table due to send PF_KEY messages. */
 struct secreg {
 	LIST_ENTRY(secreg) chain;
diff --git a/sys/netipsec/xform_ah.c b/sys/netipsec/xform_ah.c
index 55dfe872092c..e9d30b26d8e7 100644
--- a/sys/netipsec/xform_ah.c
+++ b/sys/netipsec/xform_ah.c
@@ -543,6 +543,8 @@ ah_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 	int hl, rplen, authsize, ahsize, error;
 	uint32_t seqh;
 
+	SECASVAR_RLOCK_TRACKER;
+
 	IPSEC_ASSERT(sav != NULL, ("null SA"));
 	IPSEC_ASSERT(sav->key_auth != NULL, ("null authentication key"));
 	IPSEC_ASSERT(sav->tdb_authalgxform != NULL,
@@ -563,10 +565,10 @@ ah_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 	ah = (struct newah *)(mtod(m, caddr_t) + skip);
 
 	/* Check replay window, if applicable. */
-	SECASVAR_LOCK(sav);
+	SECASVAR_RLOCK(sav);
 	if (sav->replay != NULL && sav->replay->wsize != 0 &&
 	    ipsec_chkreplay(ntohl(ah->ah_seq), &seqh, sav) == 0) {
-		SECASVAR_UNLOCK(sav);
+		SECASVAR_RUNLOCK(sav);
 		AHSTAT_INC(ahs_replay);
 		DPRINTF(("%s: packet replay failure: %s\n", __func__,
 		    ipsec_sa2str(sav, buf, sizeof(buf))));
@@ -574,7 +576,7 @@ ah_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 		goto bad;
 	}
 	cryptoid = sav->tdb_cryptoid;
-	SECASVAR_UNLOCK(sav);
+	SECASVAR_RUNLOCK(sav);
 
 	/* Verify AH header length. */
 	hl = sizeof(struct ah) + (ah->ah_len * sizeof (u_int32_t));
@@ -698,6 +700,8 @@ ah_input_cb(struct cryptop *crp)
 	int authsize, rplen, ahsize, error, skip, protoff;
 	uint8_t nxt;
 
+	SECASVAR_RLOCK_TRACKER;
+
 	m = crp->crp_buf.cb_mbuf;
 	xd = crp->crp_opaque;
 	CURVNET_SET(xd->vnet);
@@ -778,14 +782,14 @@ ah_input_cb(struct cryptop *crp)
 
 		m_copydata(m, skip + offsetof(struct newah, ah_seq),
 			   sizeof (seq), (caddr_t) &seq);
-		SECASVAR_LOCK(sav);
+		SECASVAR_RLOCK(sav);
 		if (ipsec_updatereplay(ntohl(seq), sav)) {
-			SECASVAR_UNLOCK(sav);
+			SECASVAR_RUNLOCK(sav);
 			AHSTAT_INC(ahs_replay);
 			error = EACCES;
 			goto bad;
 		}
-		SECASVAR_UNLOCK(sav);
+		SECASVAR_RUNLOCK(sav);
 	}
 
 	/*
@@ -849,6 +853,8 @@ ah_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 	uint8_t prot;
 	uint32_t seqh;
 
+	SECASVAR_RLOCK_TRACKER;
+
 	IPSEC_ASSERT(sav != NULL, ("null SA"));
 	ahx = sav->tdb_authalgxform;
 	IPSEC_ASSERT(ahx != NULL, ("null authentication xform"));
@@ -938,13 +944,15 @@ ah_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 	    ipseczeroes);
 
 	/* Insert packet replay counter, as requested.  */
-	SECASVAR_LOCK(sav);
+	SECASVAR_RLOCK(sav);
 	if (sav->replay) {
+		SECREPLAY_LOCK(sav->replay);
 		if ((sav->replay->count == ~0 ||
 		    (!(sav->flags & SADB_X_SAFLAGS_ESN) &&
 		    ((uint32_t)sav->replay->count) == ~0)) &&
 		    (sav->flags & SADB_X_EXT_CYCSEQ) == 0) {
-			SECASVAR_UNLOCK(sav);
+			SECREPLAY_UNLOCK(sav->replay);
+			SECASVAR_RUNLOCK(sav);
 			DPRINTF(("%s: replay counter wrapped for SA %s/%08lx\n",
 			    __func__, ipsec_address(&sav->sah->saidx.dst, buf,
 			    sizeof(buf)), (u_long) ntohl(sav->spi)));
@@ -958,9 +966,10 @@ ah_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 #endif
 			sav->replay->count++;
 		ah->ah_seq = htonl((uint32_t)sav->replay->count);
+		SECREPLAY_UNLOCK(sav->replay);
 	}
 	cryptoid = sav->tdb_cryptoid;
-	SECASVAR_UNLOCK(sav);
+	SECASVAR_RUNLOCK(sav);
 
 	/* Get crypto descriptors. */
 	crp = crypto_getreq(cryptoid, M_NOWAIT);
@@ -1046,8 +1055,10 @@ ah_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 	crp->crp_opaque = xd;
 
 	if (sav->flags & SADB_X_SAFLAGS_ESN && sav->replay != NULL) {
+		SECREPLAY_LOCK(sav->replay);
 		seqh = htonl((uint32_t)(sav->replay->count >> IPSEC_SEQH_SHIFT));
 		memcpy(crp->crp_esn, &seqh, sizeof(seqh));
+		SECREPLAY_UNLOCK(sav->replay);
 	}
 
 	/* These are passed as-is to the callback. */
diff --git a/sys/netipsec/xform_esp.c b/sys/netipsec/xform_esp.c
index 01072cb4e2d6..222d1b14f1d1 100644
--- a/sys/netipsec/xform_esp.c
+++ b/sys/netipsec/xform_esp.c
@@ -274,6 +274,8 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 	uint32_t seqh;
 	const struct crypto_session_params *csp;
 
+	SECASVAR_RLOCK_TRACKER;
+
 	IPSEC_ASSERT(sav != NULL, ("null SA"));
 	IPSEC_ASSERT(sav->tdb_encalgxform != NULL, ("null encoding xform"));
 
@@ -329,10 +331,10 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 	/*
 	 * Check sequence number.
 	 */
-	SECASVAR_LOCK(sav);
+	SECASVAR_RLOCK(sav);
 	if (esph != NULL && sav->replay != NULL && sav->replay->wsize != 0) {
 		if (ipsec_chkreplay(ntohl(esp->esp_seq), &seqh, sav) == 0) {
-			SECASVAR_UNLOCK(sav);
+			SECASVAR_RUNLOCK(sav);
 			DPRINTF(("%s: packet replay check for %s\n", __func__,
 			    ipsec_sa2str(sav, buf, sizeof(buf))));
 			ESPSTAT_INC(esps_replay);
@@ -342,7 +344,7 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 		seqh = htonl(seqh);
 	}
 	cryptoid = sav->tdb_cryptoid;
-	SECASVAR_UNLOCK(sav);
+	SECASVAR_RUNLOCK(sav);
 
 	/* Update the counters */
 	ESPSTAT_ADD(esps_ibytes, m->m_pkthdr.len - (skip + hlen + alen));
@@ -493,6 +495,8 @@ esp_input_cb(struct cryptop *crp)
 	crypto_session_t cryptoid;
 	int hlen, skip, protoff, error, alen;
 
+	SECASVAR_RLOCK_TRACKER;
+
 	m = crp->crp_buf.cb_mbuf;
 	xd = crp->crp_opaque;
 	CURVNET_SET(xd->vnet);
@@ -569,16 +573,16 @@ esp_input_cb(struct cryptop *crp)
 
 		m_copydata(m, skip + offsetof(struct newesp, esp_seq),
 			   sizeof (seq), (caddr_t) &seq);
-		SECASVAR_LOCK(sav);
+		SECASVAR_RLOCK(sav);
 		if (ipsec_updatereplay(ntohl(seq), sav)) {
-			SECASVAR_UNLOCK(sav);
+			SECASVAR_RUNLOCK(sav);
 			DPRINTF(("%s: packet replay check for %s\n", __func__,
 			    ipsec_sa2str(sav, buf, sizeof(buf))));
 			ESPSTAT_INC(esps_replay);
 			error = EACCES;
 			goto bad;
 		}
-		SECASVAR_UNLOCK(sav);
+		SECASVAR_RUNLOCK(sav);
 	}
 
 	/* Determine the ESP header length */
@@ -693,6 +697,8 @@ esp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 	uint32_t seqh;
 	const struct crypto_session_params *csp;
 
+	SECASVAR_RLOCK_TRACKER;
+
 	IPSEC_ASSERT(sav != NULL, ("null SA"));
 	esph = sav->tdb_authalgxform;
 	espx = sav->tdb_encalgxform;
@@ -785,10 +791,11 @@ esp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 	/* Initialize ESP header. */
 	bcopy((caddr_t) &sav->spi, mtod(mo, caddr_t) + roff,
 	    sizeof(uint32_t));
-	SECASVAR_LOCK(sav);
+	SECASVAR_RLOCK(sav);
 	if (sav->replay) {
 		uint32_t replay;
 
+		SECREPLAY_LOCK(sav->replay);
 #ifdef REGRESSION
 		/* Emulate replay attack when ipsec_replay is TRUE. */
 		if (!V_ipsec_replay)
@@ -800,11 +807,12 @@ esp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 		    sizeof(uint32_t), sizeof(uint32_t));
 
 		seqh = htonl((uint32_t)(sav->replay->count >> IPSEC_SEQH_SHIFT));
+		SECREPLAY_UNLOCK(sav->replay);
 	}
 	cryptoid = sav->tdb_cryptoid;
 	if (SAV_ISCTRORGCM(sav))
 		cntr = sav->cntr++;
-	SECASVAR_UNLOCK(sav);
+	SECASVAR_RUNLOCK(sav);
 
 	/*
 	 * Add padding -- better to do it ourselves than use the crypto engine,
diff --git a/sys/netipsec/xform_ipcomp.c b/sys/netipsec/xform_ipcomp.c
index 760fd8dd2aa8..cb91db86e129 100644
--- a/sys/netipsec/xform_ipcomp.c
+++ b/sys/netipsec/xform_ipcomp.c
@@ -205,6 +205,8 @@ ipcomp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 	caddr_t addr;
 	int error, hlen = IPCOMP_HLENGTH;
 
+	SECASVAR_RLOCK_TRACKER;
+
 	/*
 	 * Check that the next header of the IPComp is not IPComp again, before
 	 * doing any real work.  Given it is not possible to do double
@@ -226,9 +228,9 @@ ipcomp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 		goto bad;
 	}
 
-	SECASVAR_LOCK(sav);
+	SECASVAR_RLOCK(sav);
 	cryptoid = sav->tdb_cryptoid;
-	SECASVAR_UNLOCK(sav);
+	SECASVAR_RUNLOCK(sav);
 
 	/* Get crypto descriptors */
 	crp = crypto_getreq(cryptoid, M_NOWAIT);
@@ -264,9 +266,9 @@ ipcomp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 	xd->vnet = curvnet;
 	xd->cryptoid = cryptoid;
 
-	SECASVAR_LOCK(sav);
+	SECASVAR_RLOCK(sav);
 	crp->crp_session = xd->cryptoid = sav->tdb_cryptoid;
-	SECASVAR_UNLOCK(sav);
+	SECASVAR_RUNLOCK(sav);
 
 	return crypto_dispatch(crp);
 bad:
@@ -405,6 +407,8 @@ ipcomp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 	crypto_session_t cryptoid;
 	int error, ralen, maxpacketsize;
 
+	SECASVAR_RLOCK_TRACKER;
+
 	IPSEC_ASSERT(sav != NULL, ("null SA"));
 	ipcompx = sav->tdb_compalgxform;
 	IPSEC_ASSERT(ipcompx != NULL, ("null compression xform"));
@@ -470,9 +474,9 @@ ipcomp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 	}
 
 	/* Ok now, we can pass to the crypto processing. */
-	SECASVAR_LOCK(sav);
+	SECASVAR_RLOCK(sav);
 	cryptoid = sav->tdb_cryptoid;
-	SECASVAR_UNLOCK(sav);
+	SECASVAR_RUNLOCK(sav);
 
 	/* Get crypto descriptors */
 	crp = crypto_getreq(cryptoid, M_NOWAIT);