git: 1243360b1a02 - stable/13 - ipsec: replace SECASVAR mtx by rmlock
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Tue, 09 Aug 2022 13:47:18 UTC
The branch stable/13 has been updated by kp:
URL: https://cgit.FreeBSD.org/src/commit/?id=1243360b1a0230558e0deebf9bdba6ab48e6fff0
commit 1243360b1a0230558e0deebf9bdba6ab48e6fff0
Author: Kristof Provost <kp@FreeBSD.org>
AuthorDate: 2022-06-23 20:35:29 +0000
Commit: Kristof Provost <kp@FreeBSD.org>
CommitDate: 2022-08-09 13:46:57 +0000
ipsec: replace SECASVAR mtx by rmlock
This mutex is a significant point of contention in the ipsec code, and
can be relatively trivially replaced by a read-mostly lock.
It does require a separate lock for the replay protection, which we do
here by adding a separate mutex.
This improves throughput (without replay protection) by 10-15%.
MFC after: 3 weeks
Sponsored by: Orange Business Services
Differential Revision: https://reviews.freebsd.org/D35763
(cherry picked from commit 0361f165f2193a098cfbfeef3a58ec2f1eaac0a1)
---
sys/netipsec/ipsec.c | 64 ++++++++++++++++++++++++++++++++++++---------
sys/netipsec/key.c | 46 ++++++++++++++++++--------------
sys/netipsec/key_debug.c | 4 +++
sys/netipsec/keydb.h | 18 ++++++++++---
sys/netipsec/xform_ah.c | 29 +++++++++++++-------
sys/netipsec/xform_esp.c | 24 +++++++++++------
sys/netipsec/xform_ipcomp.c | 16 +++++++-----
7 files changed, 142 insertions(+), 59 deletions(-)
diff --git a/sys/netipsec/ipsec.c b/sys/netipsec/ipsec.c
index cd24750607ea..5bb8d2412321 100644
--- a/sys/netipsec/ipsec.c
+++ b/sys/netipsec/ipsec.c
@@ -1192,6 +1192,8 @@ check_window(const struct secreplay *replay, uint64_t seq)
{
int index, bit_location;
+ SECREPLAY_ASSERT(replay);
+
bit_location = seq & IPSEC_BITMAP_LOC_MASK;
index = (seq >> IPSEC_REDUNDANT_BIT_SHIFTS)
& IPSEC_BITMAP_INDEX_MASK(replay->bitmap_size);
@@ -1206,6 +1208,8 @@ advance_window(const struct secreplay *replay, uint64_t seq)
int i;
uint64_t index, index_cur, diff;
+ SECREPLAY_ASSERT(replay);
+
index_cur = replay->last >> IPSEC_REDUNDANT_BIT_SHIFTS;
index = seq >> IPSEC_REDUNDANT_BIT_SHIFTS;
diff = index - index_cur;
@@ -1226,6 +1230,8 @@ set_window(const struct secreplay *replay, uint64_t seq)
{
int index, bit_location;
+ SECREPLAY_ASSERT(replay);
+
bit_location = seq & IPSEC_BITMAP_LOC_MASK;
index = (seq >> IPSEC_REDUNDANT_BIT_SHIFTS)
& IPSEC_BITMAP_INDEX_MASK(replay->bitmap_size);
@@ -1259,12 +1265,17 @@ ipsec_chkreplay(uint32_t seq, uint32_t *seqhigh, struct secasvar *sav)
replay = sav->replay;
/* No need to check replay if disabled. */
- if (replay->wsize == 0)
+ if (replay->wsize == 0) {
return (1);
+ }
+
+ SECREPLAY_LOCK(replay);
/* Zero sequence number is not allowed. */
- if (seq == 0 && replay->last == 0)
+ if (seq == 0 && replay->last == 0) {
+ SECREPLAY_UNLOCK(replay);
return (0);
+ }
window = replay->wsize << 3; /* Size of window */
tl = (uint32_t)replay->last; /* Top of window, lower part */
@@ -1282,10 +1293,13 @@ ipsec_chkreplay(uint32_t seq, uint32_t *seqhigh, struct secasvar *sav)
*seqhigh = th;
if (seq <= tl) {
/* Sequence number inside window - check against replay */
- if (check_window(replay, seq))
+ if (check_window(replay, seq)) {
+ SECREPLAY_UNLOCK(replay);
return (0);
+ }
}
+ SECREPLAY_UNLOCK(replay);
/* Sequence number above top of window or not found in bitmap */
return (1);
}
@@ -1303,6 +1317,7 @@ ipsec_chkreplay(uint32_t seq, uint32_t *seqhigh, struct secasvar *sav)
ESPSTAT_INC(esps_wrap);
else if (sav->sah->saidx.proto == IPPROTO_AH)
AHSTAT_INC(ahs_wrap);
+ SECREPLAY_UNLOCK(replay);
return (0);
}
@@ -1322,8 +1337,11 @@ ipsec_chkreplay(uint32_t seq, uint32_t *seqhigh, struct secasvar *sav)
return (0);
*seqhigh = th - 1;
seqh = th - 1;
- if (check_window(replay, seq))
+ if (check_window(replay, seq)) {
+ SECREPLAY_UNLOCK(replay);
return (0);
+ }
+ SECREPLAY_UNLOCK(replay);
return (1);
}
@@ -1344,6 +1362,7 @@ ipsec_chkreplay(uint32_t seq, uint32_t *seqhigh, struct secasvar *sav)
ESPSTAT_INC(esps_wrap);
else if (sav->sah->saidx.proto == IPPROTO_AH)
AHSTAT_INC(ahs_wrap);
+ SECREPLAY_UNLOCK(replay);
return (0);
}
@@ -1352,6 +1371,7 @@ ipsec_chkreplay(uint32_t seq, uint32_t *seqhigh, struct secasvar *sav)
ipsec_sa2str(sav, buf, sizeof(buf))));
}
+ SECREPLAY_UNLOCK(replay);
return (1);
}
@@ -1377,9 +1397,13 @@ ipsec_updatereplay(uint32_t seq, struct secasvar *sav)
if (replay->wsize == 0)
return (0);
+ SECREPLAY_LOCK(replay);
+
/* Zero sequence number is not allowed. */
- if (seq == 0 && replay->last == 0)
+ if (seq == 0 && replay->last == 0) {
+ SECREPLAY_UNLOCK(replay);
return (1);
+ }
window = replay->wsize << 3; /* Size of window */
tl = (uint32_t)replay->last; /* Top of window, lower part */
@@ -1397,8 +1421,10 @@ ipsec_updatereplay(uint32_t seq, struct secasvar *sav)
seqh = th;
if (seq <= tl) {
/* Sequence number inside window - check against replay */
- if (check_window(replay, seq))
+ if (check_window(replay, seq)) {
+ SECREPLAY_UNLOCK(replay);
return (1);
+ }
set_window(replay, seq);
} else {
advance_window(replay, ((uint64_t)seqh << 32) | seq);
@@ -1408,11 +1434,14 @@ ipsec_updatereplay(uint32_t seq, struct secasvar *sav)
/* Sequence number above top of window or not found in bitmap */
replay->count++;
+ SECREPLAY_UNLOCK(replay);
return (0);
}
- if (!(sav->flags & SADB_X_SAFLAGS_ESN))
+ if (!(sav->flags & SADB_X_SAFLAGS_ESN)) {
+ SECREPLAY_UNLOCK(replay);
return (1);
+ }
/*
* Seq is within [bl, 0xffffffff] and bl is within
@@ -1421,13 +1450,18 @@ ipsec_updatereplay(uint32_t seq, struct secasvar *sav)
* subspace.
*/
if (tl < window - 1 && seq >= bl) {
- if (th == 0)
+ if (th == 0) {
+ SECREPLAY_UNLOCK(replay);
return (1);
- if (check_window(replay, seq))
+ }
+ if (check_window(replay, seq)) {
+ SECREPLAY_UNLOCK(replay);
return (1);
+ }
set_window(replay, seq);
replay->count++;
+ SECREPLAY_UNLOCK(replay);
return (0);
}
@@ -1438,13 +1472,17 @@ ipsec_updatereplay(uint32_t seq, struct secasvar *sav)
seqh = th + 1;
/* Don't let high part wrap. */
- if (seqh == 0)
+ if (seqh == 0) {
+ SECREPLAY_UNLOCK(replay);
return (1);
+ }
advance_window(replay, ((uint64_t)seqh << 32) | seq);
set_window(replay, seq);
replay->last = ((uint64_t)seqh << 32) | seq;
replay->count++;
+
+ SECREPLAY_UNLOCK(replay);
return (0);
}
int
@@ -1480,17 +1518,17 @@ ipsec_updateid(struct secasvar *sav, crypto_session_t *new,
printf("%s: SA(%p) moves cryptoid %p -> %p\n",
__func__, sav, *old, *new));
KEYDBG(IPSEC_DATA, kdebug_secasv(sav));
- SECASVAR_LOCK(sav);
+ SECASVAR_WLOCK(sav);
if (sav->tdb_cryptoid != *old) {
/* cryptoid was already updated */
tmp = *new;
*new = sav->tdb_cryptoid;
*old = tmp;
- SECASVAR_UNLOCK(sav);
+ SECASVAR_WUNLOCK(sav);
return (1);
}
sav->tdb_cryptoid = *new;
- SECASVAR_UNLOCK(sav);
+ SECASVAR_WUNLOCK(sav);
return (0);
}
diff --git a/sys/netipsec/key.c b/sys/netipsec/key.c
index 4bb46cd9a5c1..869d6b850aa0 100644
--- a/sys/netipsec/key.c
+++ b/sys/netipsec/key.c
@@ -2932,13 +2932,13 @@ key_newsav(const struct sadb_msghdr *mhp, struct secasindex *saidx,
*errp = ENOBUFS;
goto done;
}
- sav->lock = malloc(sizeof(struct mtx), M_IPSEC_MISC,
+ sav->lock = malloc(sizeof(struct rmlock), M_IPSEC_MISC,
M_NOWAIT | M_ZERO);
if (sav->lock == NULL) {
*errp = ENOBUFS;
goto done;
}
- mtx_init(sav->lock, "ipsec association", NULL, MTX_DEF);
+ rm_init(sav->lock, "ipsec association");
sav->lft_c = uma_zalloc_pcpu(V_key_lft_zone, M_NOWAIT);
if (sav->lft_c == NULL) {
*errp = ENOBUFS;
@@ -3027,7 +3027,7 @@ done:
if (*errp != 0) {
if (sav != NULL) {
if (sav->lock != NULL) {
- mtx_destroy(sav->lock);
+ rm_destroy(sav->lock);
free(sav->lock, M_IPSEC_MISC);
}
if (sav->lft_c != NULL)
@@ -3073,6 +3073,7 @@ key_cleansav(struct secasvar *sav)
sav->key_enc = NULL;
}
if (sav->replay != NULL) {
+ mtx_destroy(&sav->replay->lock);
if (sav->replay->bitmap != NULL)
free(sav->replay->bitmap, M_IPSEC_MISC);
free(sav->replay, M_IPSEC_MISC);
@@ -3107,7 +3108,7 @@ key_delsav(struct secasvar *sav)
*/
key_cleansav(sav);
if ((sav->flags & SADB_X_EXT_F_CLONED) == 0) {
- mtx_destroy(sav->lock);
+ rm_destroy(sav->lock);
free(sav->lock, M_IPSEC_MISC);
uma_zfree_pcpu(V_key_lft_zone, sav->lft_c);
}
@@ -3238,7 +3239,7 @@ reset:
* key_update() holds reference to this SA,
* so it won't be deleted in meanwhile.
*/
- SECASVAR_LOCK(sav);
+ SECASVAR_WLOCK(sav);
tmp = sav->lft_h;
sav->lft_h = lft_h;
lft_h = tmp;
@@ -3246,7 +3247,7 @@ reset:
tmp = sav->lft_s;
sav->lft_s = lft_s;
lft_s = tmp;
- SECASVAR_UNLOCK(sav);
+ SECASVAR_WUNLOCK(sav);
if (lft_h != NULL)
free(lft_h, M_IPSEC_MISC);
if (lft_s != NULL)
@@ -3335,6 +3336,7 @@ key_setsaval(struct secasvar *sav, const struct sadb_msghdr *mhp)
error = ENOBUFS;
goto fail;
}
+ mtx_init(&sav->replay->lock, "ipsec replay", NULL, MTX_DEF);
if (replay != 0) {
/* number of 32b blocks to be allocated */
@@ -3552,6 +3554,8 @@ key_setdumpsa(struct secasvar *sav, uint8_t type, uint8_t satype,
};
uint32_t replay_count;
+ SECASVAR_RLOCK_TRACKER;
+
m = key_setsadbmsg(type, 0, satype, seq, pid, sav->refcnt);
if (m == NULL)
goto fail;
@@ -3566,16 +3570,16 @@ key_setdumpsa(struct secasvar *sav, uint8_t type, uint8_t satype,
goto fail;
break;
- case SADB_X_EXT_SA2:
- SECASVAR_LOCK(sav);
+ case SADB_X_EXT_SA2: {
+ SECASVAR_RLOCK(sav);
replay_count = sav->replay ? sav->replay->count : 0;
- SECASVAR_UNLOCK(sav);
+ SECASVAR_RUNLOCK(sav);
m = key_setsadbxsa2(sav->sah->saidx.mode, replay_count,
sav->sah->saidx.reqid);
if (!m)
goto fail;
break;
-
+ }
case SADB_X_EXT_SA_REPLAY:
if (sav->replay == NULL ||
sav->replay->wsize <= UINT8_MAX)
@@ -4477,6 +4481,8 @@ key_flush_sad(time_t now)
struct secashead *sah, *nextsah;
struct secasvar *sav, *nextsav;
+ SECASVAR_RLOCK_TRACKER;
+
LIST_INIT(&drainq);
LIST_INIT(&hexpireq);
LIST_INIT(&sexpireq);
@@ -4502,13 +4508,13 @@ key_flush_sad(time_t now)
/* lifetimes aren't specified */
if (sav->lft_h == NULL)
continue;
- SECASVAR_LOCK(sav);
+ SECASVAR_RLOCK(sav);
/*
* Check again with lock held, because it may
* be updated by SADB_UPDATE.
*/
if (sav->lft_h == NULL) {
- SECASVAR_UNLOCK(sav);
+ SECASVAR_RUNLOCK(sav);
continue;
}
/*
@@ -4525,7 +4531,7 @@ key_flush_sad(time_t now)
now - sav->firstused > sav->lft_h->usetime) ||
(sav->lft_h->bytes != 0 && counter_u64_fetch(
sav->lft_c_bytes) > sav->lft_h->bytes)) {
- SECASVAR_UNLOCK(sav);
+ SECASVAR_RUNLOCK(sav);
SAV_ADDREF(sav);
LIST_INSERT_HEAD(&hexpireq, sav, drainq);
continue;
@@ -4542,12 +4548,12 @@ key_flush_sad(time_t now)
(sav->replay != NULL) && (
(sav->replay->count > UINT32_80PCT) ||
(sav->replay->last > UINT32_80PCT))))) {
- SECASVAR_UNLOCK(sav);
+ SECASVAR_RUNLOCK(sav);
SAV_ADDREF(sav);
LIST_INSERT_HEAD(&sexpireq, sav, drainq);
continue;
}
- SECASVAR_UNLOCK(sav);
+ SECASVAR_RUNLOCK(sav);
}
}
SAHTREE_RUNLOCK();
@@ -5245,11 +5251,11 @@ key_updateaddresses(struct socket *so, struct mbuf *m,
* isnew == 0 -> we use the same @sah, that was used by @sav,
* and we use its reference for @newsav.
*/
- SECASVAR_LOCK(sav);
+ SECASVAR_WLOCK(sav);
/* XXX: replace cntr with pointer? */
newsav->cntr = sav->cntr;
sav->flags |= SADB_X_EXT_F_CLONED;
- SECASVAR_UNLOCK(sav);
+ SECASVAR_WUNLOCK(sav);
SAHTREE_WUNLOCK();
@@ -7265,6 +7271,8 @@ key_expire(struct secasvar *sav, int hard)
int error, len;
uint8_t satype;
+ SECASVAR_RLOCK_TRACKER;
+
IPSEC_ASSERT (sav != NULL, ("null sav"));
IPSEC_ASSERT (sav->sah != NULL, ("null sa header"));
@@ -7291,9 +7299,9 @@ key_expire(struct secasvar *sav, int hard)
m_cat(result, m);
/* create SA extension */
- SECASVAR_LOCK(sav);
+ SECASVAR_RLOCK(sav);
replay_count = sav->replay ? sav->replay->count : 0;
- SECASVAR_UNLOCK(sav);
+ SECASVAR_RUNLOCK(sav);
m = key_setsadbxsa2(sav->sah->saidx.mode, replay_count,
sav->sah->saidx.reqid);
diff --git a/sys/netipsec/key_debug.c b/sys/netipsec/key_debug.c
index 04946dc5c165..3705879397ff 100644
--- a/sys/netipsec/key_debug.c
+++ b/sys/netipsec/key_debug.c
@@ -808,12 +808,15 @@ kdebug_secreplay(struct secreplay *rpl)
{
int len, l;
+ SECREPLAY_LOCK(rpl);
+
IPSEC_ASSERT(rpl != NULL, ("null rpl"));
printf(" secreplay{ count=%lu bitmap_size=%u wsize=%u last=%lu",
rpl->count, rpl->bitmap_size, rpl->wsize, rpl->last);
if (rpl->bitmap == NULL) {
printf(" }\n");
+ SECREPLAY_UNLOCK(rpl);
return;
}
@@ -823,6 +826,7 @@ kdebug_secreplay(struct secreplay *rpl)
printf("%u", (((rpl->bitmap)[len] >> l) & 1) ? 1 : 0);
}
printf(" }\n");
+ SECREPLAY_UNLOCK(rpl);
}
#endif /* IPSEC_DEBUG */
diff --git a/sys/netipsec/keydb.h b/sys/netipsec/keydb.h
index 69c1cb29db34..a2da0da613e2 100644
--- a/sys/netipsec/keydb.h
+++ b/sys/netipsec/keydb.h
@@ -39,6 +39,7 @@
#include <sys/counter.h>
#include <sys/lock.h>
#include <sys/mutex.h>
+#include <sys/rmlock.h>
#include <netipsec/key_var.h>
#include <opencrypto/_cryptodev.h>
@@ -157,7 +158,7 @@ struct secasvar {
struct seckey *key_enc; /* Key for Encryption */
struct secreplay *replay; /* replay prevention */
struct secnatt *natt; /* NAT-T config */
- struct mtx *lock; /* update/access lock */
+ struct rmlock *lock; /* update/access lock */
const struct xformsw *tdb_xform; /* transform */
const struct enc_xform *tdb_encalgxform;/* encoding algorithm */
@@ -187,9 +188,13 @@ struct secasvar {
volatile u_int refcnt; /* reference count */
};
-#define SECASVAR_LOCK(_sav) mtx_lock((_sav)->lock)
-#define SECASVAR_UNLOCK(_sav) mtx_unlock((_sav)->lock)
-#define SECASVAR_LOCK_ASSERT(_sav) mtx_assert((_sav)->lock, MA_OWNED)
+#define SECASVAR_RLOCK_TRACKER struct rm_priotracker _secas_tracker
+#define SECASVAR_RLOCK(_sav) rm_rlock((_sav)->lock, &_secas_tracker)
+#define SECASVAR_RUNLOCK(_sav) rm_runlock((_sav)->lock, &_secas_tracker)
+#define SECASVAR_WLOCK(_sav) rm_wlock((_sav)->lock)
+#define SECASVAR_WUNLOCK(_sav) rm_wunlock((_sav)->lock)
+#define SECASVAR_LOCK_ASSERT(_sav) rm_assert((_sav)->lock, RA_LOCKED)
+#define SECASVAR_LOCK_WASSERT(_sav) rm_assert((_sav)->lock, RA_WLOCKED)
#define SAV_ISGCM(_sav) \
((_sav)->alg_enc == SADB_X_EALG_AESGCM8 || \
(_sav)->alg_enc == SADB_X_EALG_AESGCM12 || \
@@ -204,6 +209,7 @@ struct secasvar {
* (c) read only except during creation / free
*/
struct secreplay {
+ struct mtx lock;
u_int64_t count; /* (m) */
u_int wsize; /* (c) window size, i.g. 4 bytes */
u_int64_t last; /* (m) used by receiver */
@@ -212,6 +218,10 @@ struct secreplay {
int overflow; /* (m) overflow flag */
};
+#define SECREPLAY_LOCK(_r) mtx_lock(&(_r)->lock)
+#define SECREPLAY_UNLOCK(_r) mtx_unlock(&(_r)->lock)
+#define SECREPLAY_ASSERT(_r) mtx_assert(&(_r)->lock, MA_OWNED)
+
/* socket table due to send PF_KEY messages. */
struct secreg {
LIST_ENTRY(secreg) chain;
diff --git a/sys/netipsec/xform_ah.c b/sys/netipsec/xform_ah.c
index 55dfe872092c..e9d30b26d8e7 100644
--- a/sys/netipsec/xform_ah.c
+++ b/sys/netipsec/xform_ah.c
@@ -543,6 +543,8 @@ ah_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
int hl, rplen, authsize, ahsize, error;
uint32_t seqh;
+ SECASVAR_RLOCK_TRACKER;
+
IPSEC_ASSERT(sav != NULL, ("null SA"));
IPSEC_ASSERT(sav->key_auth != NULL, ("null authentication key"));
IPSEC_ASSERT(sav->tdb_authalgxform != NULL,
@@ -563,10 +565,10 @@ ah_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
ah = (struct newah *)(mtod(m, caddr_t) + skip);
/* Check replay window, if applicable. */
- SECASVAR_LOCK(sav);
+ SECASVAR_RLOCK(sav);
if (sav->replay != NULL && sav->replay->wsize != 0 &&
ipsec_chkreplay(ntohl(ah->ah_seq), &seqh, sav) == 0) {
- SECASVAR_UNLOCK(sav);
+ SECASVAR_RUNLOCK(sav);
AHSTAT_INC(ahs_replay);
DPRINTF(("%s: packet replay failure: %s\n", __func__,
ipsec_sa2str(sav, buf, sizeof(buf))));
@@ -574,7 +576,7 @@ ah_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
goto bad;
}
cryptoid = sav->tdb_cryptoid;
- SECASVAR_UNLOCK(sav);
+ SECASVAR_RUNLOCK(sav);
/* Verify AH header length. */
hl = sizeof(struct ah) + (ah->ah_len * sizeof (u_int32_t));
@@ -698,6 +700,8 @@ ah_input_cb(struct cryptop *crp)
int authsize, rplen, ahsize, error, skip, protoff;
uint8_t nxt;
+ SECASVAR_RLOCK_TRACKER;
+
m = crp->crp_buf.cb_mbuf;
xd = crp->crp_opaque;
CURVNET_SET(xd->vnet);
@@ -778,14 +782,14 @@ ah_input_cb(struct cryptop *crp)
m_copydata(m, skip + offsetof(struct newah, ah_seq),
sizeof (seq), (caddr_t) &seq);
- SECASVAR_LOCK(sav);
+ SECASVAR_RLOCK(sav);
if (ipsec_updatereplay(ntohl(seq), sav)) {
- SECASVAR_UNLOCK(sav);
+ SECASVAR_RUNLOCK(sav);
AHSTAT_INC(ahs_replay);
error = EACCES;
goto bad;
}
- SECASVAR_UNLOCK(sav);
+ SECASVAR_RUNLOCK(sav);
}
/*
@@ -849,6 +853,8 @@ ah_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
uint8_t prot;
uint32_t seqh;
+ SECASVAR_RLOCK_TRACKER;
+
IPSEC_ASSERT(sav != NULL, ("null SA"));
ahx = sav->tdb_authalgxform;
IPSEC_ASSERT(ahx != NULL, ("null authentication xform"));
@@ -938,13 +944,15 @@ ah_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
ipseczeroes);
/* Insert packet replay counter, as requested. */
- SECASVAR_LOCK(sav);
+ SECASVAR_RLOCK(sav);
if (sav->replay) {
+ SECREPLAY_LOCK(sav->replay);
if ((sav->replay->count == ~0 ||
(!(sav->flags & SADB_X_SAFLAGS_ESN) &&
((uint32_t)sav->replay->count) == ~0)) &&
(sav->flags & SADB_X_EXT_CYCSEQ) == 0) {
- SECASVAR_UNLOCK(sav);
+ SECREPLAY_UNLOCK(sav->replay);
+ SECASVAR_RUNLOCK(sav);
DPRINTF(("%s: replay counter wrapped for SA %s/%08lx\n",
__func__, ipsec_address(&sav->sah->saidx.dst, buf,
sizeof(buf)), (u_long) ntohl(sav->spi)));
@@ -958,9 +966,10 @@ ah_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
#endif
sav->replay->count++;
ah->ah_seq = htonl((uint32_t)sav->replay->count);
+ SECREPLAY_UNLOCK(sav->replay);
}
cryptoid = sav->tdb_cryptoid;
- SECASVAR_UNLOCK(sav);
+ SECASVAR_RUNLOCK(sav);
/* Get crypto descriptors. */
crp = crypto_getreq(cryptoid, M_NOWAIT);
@@ -1046,8 +1055,10 @@ ah_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
crp->crp_opaque = xd;
if (sav->flags & SADB_X_SAFLAGS_ESN && sav->replay != NULL) {
+ SECREPLAY_LOCK(sav->replay);
seqh = htonl((uint32_t)(sav->replay->count >> IPSEC_SEQH_SHIFT));
memcpy(crp->crp_esn, &seqh, sizeof(seqh));
+ SECREPLAY_UNLOCK(sav->replay);
}
/* These are passed as-is to the callback. */
diff --git a/sys/netipsec/xform_esp.c b/sys/netipsec/xform_esp.c
index 01072cb4e2d6..222d1b14f1d1 100644
--- a/sys/netipsec/xform_esp.c
+++ b/sys/netipsec/xform_esp.c
@@ -274,6 +274,8 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
uint32_t seqh;
const struct crypto_session_params *csp;
+ SECASVAR_RLOCK_TRACKER;
+
IPSEC_ASSERT(sav != NULL, ("null SA"));
IPSEC_ASSERT(sav->tdb_encalgxform != NULL, ("null encoding xform"));
@@ -329,10 +331,10 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
/*
* Check sequence number.
*/
- SECASVAR_LOCK(sav);
+ SECASVAR_RLOCK(sav);
if (esph != NULL && sav->replay != NULL && sav->replay->wsize != 0) {
if (ipsec_chkreplay(ntohl(esp->esp_seq), &seqh, sav) == 0) {
- SECASVAR_UNLOCK(sav);
+ SECASVAR_RUNLOCK(sav);
DPRINTF(("%s: packet replay check for %s\n", __func__,
ipsec_sa2str(sav, buf, sizeof(buf))));
ESPSTAT_INC(esps_replay);
@@ -342,7 +344,7 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
seqh = htonl(seqh);
}
cryptoid = sav->tdb_cryptoid;
- SECASVAR_UNLOCK(sav);
+ SECASVAR_RUNLOCK(sav);
/* Update the counters */
ESPSTAT_ADD(esps_ibytes, m->m_pkthdr.len - (skip + hlen + alen));
@@ -493,6 +495,8 @@ esp_input_cb(struct cryptop *crp)
crypto_session_t cryptoid;
int hlen, skip, protoff, error, alen;
+ SECASVAR_RLOCK_TRACKER;
+
m = crp->crp_buf.cb_mbuf;
xd = crp->crp_opaque;
CURVNET_SET(xd->vnet);
@@ -569,16 +573,16 @@ esp_input_cb(struct cryptop *crp)
m_copydata(m, skip + offsetof(struct newesp, esp_seq),
sizeof (seq), (caddr_t) &seq);
- SECASVAR_LOCK(sav);
+ SECASVAR_RLOCK(sav);
if (ipsec_updatereplay(ntohl(seq), sav)) {
- SECASVAR_UNLOCK(sav);
+ SECASVAR_RUNLOCK(sav);
DPRINTF(("%s: packet replay check for %s\n", __func__,
ipsec_sa2str(sav, buf, sizeof(buf))));
ESPSTAT_INC(esps_replay);
error = EACCES;
goto bad;
}
- SECASVAR_UNLOCK(sav);
+ SECASVAR_RUNLOCK(sav);
}
/* Determine the ESP header length */
@@ -693,6 +697,8 @@ esp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
uint32_t seqh;
const struct crypto_session_params *csp;
+ SECASVAR_RLOCK_TRACKER;
+
IPSEC_ASSERT(sav != NULL, ("null SA"));
esph = sav->tdb_authalgxform;
espx = sav->tdb_encalgxform;
@@ -785,10 +791,11 @@ esp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
/* Initialize ESP header. */
bcopy((caddr_t) &sav->spi, mtod(mo, caddr_t) + roff,
sizeof(uint32_t));
- SECASVAR_LOCK(sav);
+ SECASVAR_RLOCK(sav);
if (sav->replay) {
uint32_t replay;
+ SECREPLAY_LOCK(sav->replay);
#ifdef REGRESSION
/* Emulate replay attack when ipsec_replay is TRUE. */
if (!V_ipsec_replay)
@@ -800,11 +807,12 @@ esp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
sizeof(uint32_t), sizeof(uint32_t));
seqh = htonl((uint32_t)(sav->replay->count >> IPSEC_SEQH_SHIFT));
+ SECREPLAY_UNLOCK(sav->replay);
}
cryptoid = sav->tdb_cryptoid;
if (SAV_ISCTRORGCM(sav))
cntr = sav->cntr++;
- SECASVAR_UNLOCK(sav);
+ SECASVAR_RUNLOCK(sav);
/*
* Add padding -- better to do it ourselves than use the crypto engine,
diff --git a/sys/netipsec/xform_ipcomp.c b/sys/netipsec/xform_ipcomp.c
index 760fd8dd2aa8..cb91db86e129 100644
--- a/sys/netipsec/xform_ipcomp.c
+++ b/sys/netipsec/xform_ipcomp.c
@@ -205,6 +205,8 @@ ipcomp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
caddr_t addr;
int error, hlen = IPCOMP_HLENGTH;
+ SECASVAR_RLOCK_TRACKER;
+
/*
* Check that the next header of the IPComp is not IPComp again, before
* doing any real work. Given it is not possible to do double
@@ -226,9 +228,9 @@ ipcomp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
goto bad;
}
- SECASVAR_LOCK(sav);
+ SECASVAR_RLOCK(sav);
cryptoid = sav->tdb_cryptoid;
- SECASVAR_UNLOCK(sav);
+ SECASVAR_RUNLOCK(sav);
/* Get crypto descriptors */
crp = crypto_getreq(cryptoid, M_NOWAIT);
@@ -264,9 +266,9 @@ ipcomp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
xd->vnet = curvnet;
xd->cryptoid = cryptoid;
- SECASVAR_LOCK(sav);
+ SECASVAR_RLOCK(sav);
crp->crp_session = xd->cryptoid = sav->tdb_cryptoid;
- SECASVAR_UNLOCK(sav);
+ SECASVAR_RUNLOCK(sav);
return crypto_dispatch(crp);
bad:
@@ -405,6 +407,8 @@ ipcomp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
crypto_session_t cryptoid;
int error, ralen, maxpacketsize;
+ SECASVAR_RLOCK_TRACKER;
+
IPSEC_ASSERT(sav != NULL, ("null SA"));
ipcompx = sav->tdb_compalgxform;
IPSEC_ASSERT(ipcompx != NULL, ("null compression xform"));
@@ -470,9 +474,9 @@ ipcomp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
}
/* Ok now, we can pass to the crypto processing. */
- SECASVAR_LOCK(sav);
+ SECASVAR_RLOCK(sav);
cryptoid = sav->tdb_cryptoid;
- SECASVAR_UNLOCK(sav);
+ SECASVAR_RUNLOCK(sav);
/* Get crypto descriptors */
crp = crypto_getreq(cryptoid, M_NOWAIT);