svn commit: r360573 - in head/sys: kern sys
Gleb Smirnoff
glebius at FreeBSD.org
Sat May 2 23:38:15 UTC 2020
Author: glebius
Date: Sat May 2 23:38:13 2020
New Revision: 360573
URL: https://svnweb.freebsd.org/changeset/base/360573
Log:
Step 2.1: Build TLS workqueue from mbufs, not struct mbuf_ext_pgs.
Reviewed by: gallatin
Differential Revision: https://reviews.freebsd.org/D24598
Modified:
head/sys/kern/kern_mbuf.c
head/sys/kern/uipc_ktls.c
head/sys/sys/ktls.h
head/sys/sys/mbuf.h
Modified: head/sys/kern/kern_mbuf.c
==============================================================================
--- head/sys/kern/kern_mbuf.c Sat May 2 22:56:22 2020 (r360572)
+++ head/sys/kern/kern_mbuf.c Sat May 2 23:38:13 2020 (r360573)
@@ -1246,7 +1246,6 @@ mb_free_ext(struct mbuf *m)
break;
case EXT_PGS: {
#ifdef KERN_TLS
- struct mbuf_ext_pgs *pgs;
struct ktls_session *tls;
#endif
@@ -1254,11 +1253,10 @@ mb_free_ext(struct mbuf *m)
("%s: ext_free not set", __func__));
mref->m_ext.ext_free(mref);
#ifdef KERN_TLS
- pgs = &mref->m_ext_pgs;
- tls = pgs->tls;
+ tls = mref->m_ext_pgs.tls;
if (tls != NULL &&
!refcount_release_if_not_last(&tls->refcount))
- ktls_enqueue_to_free(pgs);
+ ktls_enqueue_to_free(mref);
else
#endif
uma_zfree(zone_mbuf, mref);
Modified: head/sys/kern/uipc_ktls.c
==============================================================================
--- head/sys/kern/uipc_ktls.c Sat May 2 22:56:22 2020 (r360572)
+++ head/sys/kern/uipc_ktls.c Sat May 2 23:38:13 2020 (r360573)
@@ -79,7 +79,7 @@ __FBSDID("$FreeBSD$");
struct ktls_wq {
struct mtx mtx;
- STAILQ_HEAD(, mbuf_ext_pgs) head;
+ STAILQ_HEAD(, mbuf) head;
bool running;
} __aligned(CACHE_LINE_SIZE);
@@ -1430,16 +1430,19 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls,
}
void
-ktls_enqueue_to_free(struct mbuf_ext_pgs *pgs)
+ktls_enqueue_to_free(struct mbuf *m)
{
+ struct mbuf_ext_pgs *pgs;
struct ktls_wq *wq;
bool running;
+ pgs = &m->m_ext_pgs;
+
/* Mark it for freeing. */
pgs->flags |= EPG_FLAG_2FREE;
wq = &ktls_wq[pgs->tls->wq_index];
mtx_lock(&wq->mtx);
- STAILQ_INSERT_TAIL(&wq->head, pgs, stailq);
+ STAILQ_INSERT_TAIL(&wq->head, m, m_ext_pgs.stailq);
running = wq->running;
mtx_unlock(&wq->mtx);
if (!running)
@@ -1472,7 +1475,7 @@ ktls_enqueue(struct mbuf *m, struct socket *so, int pa
wq = &ktls_wq[pgs->tls->wq_index];
mtx_lock(&wq->mtx);
- STAILQ_INSERT_TAIL(&wq->head, pgs, stailq);
+ STAILQ_INSERT_TAIL(&wq->head, m, m_ext_pgs.stailq);
running = wq->running;
mtx_unlock(&wq->mtx);
if (!running)
@@ -1481,11 +1484,12 @@ ktls_enqueue(struct mbuf *m, struct socket *so, int pa
}
static __noinline void
-ktls_encrypt(struct mbuf_ext_pgs *pgs)
+ktls_encrypt(struct mbuf *top)
{
struct ktls_session *tls;
struct socket *so;
- struct mbuf *m, *top;
+ struct mbuf *m;
+ struct mbuf_ext_pgs *pgs;
vm_paddr_t parray[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
struct iovec src_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
struct iovec dst_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
@@ -1493,15 +1497,14 @@ ktls_encrypt(struct mbuf_ext_pgs *pgs)
int error, i, len, npages, off, total_pages;
bool is_anon;
- so = pgs->so;
- tls = pgs->tls;
- top = __containerof(pgs, struct mbuf, m_ext_pgs);
- KASSERT(tls != NULL, ("tls = NULL, top = %p, pgs = %p\n", top, pgs));
- KASSERT(so != NULL, ("so = NULL, top = %p, pgs = %p\n", top, pgs));
+ so = top->m_ext_pgs.so;
+ tls = top->m_ext_pgs.tls;
+ KASSERT(tls != NULL, ("tls = NULL, top = %p\n", top));
+ KASSERT(so != NULL, ("so = NULL, top = %p\n", top));
#ifdef INVARIANTS
- pgs->so = NULL;
+ top->m_ext_pgs.so = NULL;
#endif
- total_pages = pgs->enc_cnt;
+ total_pages = top->m_ext_pgs.enc_cnt;
npages = 0;
/*
@@ -1631,10 +1634,8 @@ static void
ktls_work_thread(void *ctx)
{
struct ktls_wq *wq = ctx;
- struct mbuf_ext_pgs *p, *n;
- struct ktls_session *tls;
- struct mbuf *m;
- STAILQ_HEAD(, mbuf_ext_pgs) local_head;
+ struct mbuf *m, *n;
+ STAILQ_HEAD(, mbuf) local_head;
#if defined(__aarch64__) || defined(__amd64__) || defined(__i386__)
fpu_kern_thread(0);
@@ -1651,14 +1652,12 @@ ktls_work_thread(void *ctx)
STAILQ_CONCAT(&local_head, &wq->head);
mtx_unlock(&wq->mtx);
- STAILQ_FOREACH_SAFE(p, &local_head, stailq, n) {
- if (p->flags & EPG_FLAG_2FREE) {
- tls = p->tls;
- ktls_free(tls);
- m = __containerof(p, struct mbuf, m_ext_pgs);
+ STAILQ_FOREACH_SAFE(m, &local_head, m_ext_pgs.stailq, n) {
+ if (m->m_ext_pgs.flags & EPG_FLAG_2FREE) {
+ ktls_free(m->m_ext_pgs.tls);
uma_zfree(zone_mbuf, m);
} else {
- ktls_encrypt(p);
+ ktls_encrypt(m);
counter_u64_add(ktls_cnt_on, -1);
}
}
Modified: head/sys/sys/ktls.h
==============================================================================
--- head/sys/sys/ktls.h Sat May 2 22:56:22 2020 (r360572)
+++ head/sys/sys/ktls.h Sat May 2 23:38:13 2020 (r360573)
@@ -169,7 +169,6 @@ struct iovec;
struct ktls_session;
struct m_snd_tag;
struct mbuf;
-struct mbuf_ext_pgs;
struct sockbuf;
struct socket;
@@ -212,7 +211,7 @@ void ktls_frame(struct mbuf *m, struct ktls_session *t
uint8_t record_type);
void ktls_seq(struct sockbuf *sb, struct mbuf *m);
void ktls_enqueue(struct mbuf *m, struct socket *so, int page_count);
-void ktls_enqueue_to_free(struct mbuf_ext_pgs *pgs);
+void ktls_enqueue_to_free(struct mbuf *m);
int ktls_get_rx_mode(struct socket *so);
int ktls_set_tx_mode(struct socket *so, int mode);
int ktls_get_tx_mode(struct socket *so);
Modified: head/sys/sys/mbuf.h
==============================================================================
--- head/sys/sys/mbuf.h Sat May 2 22:56:22 2020 (r360572)
+++ head/sys/sys/mbuf.h Sat May 2 23:38:13 2020 (r360573)
@@ -372,7 +372,7 @@ struct mbuf {
struct ktls_session *tls;
struct socket *so;
uint64_t seqno;
- STAILQ_ENTRY(mbuf_ext_pgs) stailq;
+ STAILQ_ENTRY(mbuf) stailq;
} m_ext_pgs;
};
union {
More information about the svn-src-head
mailing list