svn commit: r360578 - head/sys/kern
Gleb Smirnoff
glebius at FreeBSD.org
Sun May 3 00:08:10 UTC 2020
Author: glebius
Date: Sun May 3 00:08:05 2020
New Revision: 360578
URL: https://svnweb.freebsd.org/changeset/base/360578
Log:
Step 2.5: Stop using 'struct mbuf_ext_pgs' in the kernel itself.
Reviewed by: gallatin
Differential Revision: https://reviews.freebsd.org/D24598
Modified:
head/sys/kern/kern_mbuf.c
head/sys/kern/kern_sendfile.c
head/sys/kern/uipc_ktls.c
head/sys/kern/uipc_mbuf.c
head/sys/kern/uipc_sockbuf.c
Modified: head/sys/kern/kern_mbuf.c
==============================================================================
--- head/sys/kern/kern_mbuf.c Sun May 3 00:03:39 2020 (r360577)
+++ head/sys/kern/kern_mbuf.c Sun May 3 00:08:05 2020 (r360578)
@@ -934,7 +934,6 @@ mb_unmapped_free_mext(struct mbuf *m)
static struct mbuf *
_mb_unmapped_to_ext(struct mbuf *m)
{
- struct mbuf_ext_pgs *ext_pgs;
struct mbuf *m_new, *top, *prev, *mref;
struct sf_buf *sf;
vm_page_t pg;
@@ -943,9 +942,8 @@ _mb_unmapped_to_ext(struct mbuf *m)
u_int ref_inc = 0;
MBUF_EXT_PGS_ASSERT(m);
- ext_pgs = &m->m_ext_pgs;
len = m->m_len;
- KASSERT(ext_pgs->tls == NULL, ("%s: can't convert TLS mbuf %p",
+ KASSERT(m->m_ext_pgs.tls == NULL, ("%s: can't convert TLS mbuf %p",
__func__, m));
/* See if this is the mbuf that holds the embedded refcount. */
@@ -963,11 +961,11 @@ _mb_unmapped_to_ext(struct mbuf *m)
off = mtod(m, vm_offset_t);
top = NULL;
- if (ext_pgs->hdr_len != 0) {
- if (off >= ext_pgs->hdr_len) {
- off -= ext_pgs->hdr_len;
+ if (m->m_ext_pgs.hdr_len != 0) {
+ if (off >= m->m_ext_pgs.hdr_len) {
+ off -= m->m_ext_pgs.hdr_len;
} else {
- seglen = ext_pgs->hdr_len - off;
+ seglen = m->m_ext_pgs.hdr_len - off;
segoff = off;
seglen = min(seglen, len);
off = 0;
@@ -981,8 +979,8 @@ _mb_unmapped_to_ext(struct mbuf *m)
seglen);
}
}
- pgoff = ext_pgs->first_pg_off;
- for (i = 0; i < ext_pgs->npgs && len > 0; i++) {
+ pgoff = m->m_ext_pgs.first_pg_off;
+ for (i = 0; i < m->m_ext_pgs.npgs && len > 0; i++) {
pglen = m_epg_pagelen(m, i, pgoff);
if (off >= pglen) {
off -= pglen;
@@ -1018,9 +1016,9 @@ _mb_unmapped_to_ext(struct mbuf *m)
pgoff = 0;
};
if (len != 0) {
- KASSERT((off + len) <= ext_pgs->trail_len,
+ KASSERT((off + len) <= m->m_ext_pgs.trail_len,
("off + len > trail (%d + %d > %d)", off, len,
- ext_pgs->trail_len));
+ m->m_ext_pgs.trail_len));
m_new = m_get(M_NOWAIT, MT_DATA);
if (m_new == NULL)
goto fail;
@@ -1119,22 +1117,20 @@ struct mbuf *
mb_alloc_ext_pgs(int how, m_ext_free_t ext_free)
{
struct mbuf *m;
- struct mbuf_ext_pgs *ext_pgs;
m = m_get(how, MT_DATA);
if (m == NULL)
return (NULL);
- ext_pgs = &m->m_ext_pgs;
- ext_pgs->npgs = 0;
- ext_pgs->nrdy = 0;
- ext_pgs->first_pg_off = 0;
- ext_pgs->last_pg_len = 0;
- ext_pgs->flags = 0;
- ext_pgs->hdr_len = 0;
- ext_pgs->trail_len = 0;
- ext_pgs->tls = NULL;
- ext_pgs->so = NULL;
+ m->m_ext_pgs.npgs = 0;
+ m->m_ext_pgs.nrdy = 0;
+ m->m_ext_pgs.first_pg_off = 0;
+ m->m_ext_pgs.last_pg_len = 0;
+ m->m_ext_pgs.flags = 0;
+ m->m_ext_pgs.hdr_len = 0;
+ m->m_ext_pgs.trail_len = 0;
+ m->m_ext_pgs.tls = NULL;
+ m->m_ext_pgs.so = NULL;
m->m_data = NULL;
m->m_flags |= (M_EXT | M_RDONLY | M_NOMAP);
m->m_ext.ext_type = EXT_PGS;
Modified: head/sys/kern/kern_sendfile.c
==============================================================================
--- head/sys/kern/kern_sendfile.c Sun May 3 00:03:39 2020 (r360577)
+++ head/sys/kern/kern_sendfile.c Sun May 3 00:08:05 2020 (r360578)
@@ -188,7 +188,6 @@ sendfile_free_mext(struct mbuf *m)
static void
sendfile_free_mext_pg(struct mbuf *m)
{
- struct mbuf_ext_pgs *ext_pgs;
vm_page_t pg;
int flags, i;
bool cache_last;
@@ -197,11 +196,10 @@ sendfile_free_mext_pg(struct mbuf *m)
("%s: m %p !M_EXT or !EXT_PGS", __func__, m));
cache_last = m->m_ext.ext_flags & EXT_FLAG_CACHE_LAST;
- ext_pgs = &m->m_ext_pgs;
flags = (m->m_ext.ext_flags & EXT_FLAG_NOCACHE) != 0 ? VPR_TRYFREE : 0;
- for (i = 0; i < ext_pgs->npgs; i++) {
- if (cache_last && i == ext_pgs->npgs - 1)
+ for (i = 0; i < m->m_ext_pgs.npgs; i++) {
+ if (cache_last && i == m->m_ext_pgs.npgs - 1)
flags = 0;
pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
vm_page_release(pg, flags);
@@ -692,7 +690,6 @@ vn_sendfile(struct file *fp, int sockfd, struct uio *h
#ifdef KERN_TLS
struct ktls_session *tls;
#endif
- struct mbuf_ext_pgs *ext_pgs;
struct mbuf *m, *mh, *mhtail;
struct sf_buf *sf;
struct shmfd *shmfd;
@@ -1029,7 +1026,6 @@ retry_space:
sfs->count++;
mtx_unlock(&sfs->mtx);
}
- ext_pgs = &m0->m_ext_pgs;
ext_pgs_idx = 0;
/* Append to mbuf chain. */
@@ -1038,18 +1034,18 @@ retry_space:
else
m = m0;
mtail = m0;
- ext_pgs->first_pg_off =
+ m0->m_ext_pgs.first_pg_off =
vmoff(i, off) & PAGE_MASK;
}
if (nios) {
mtail->m_flags |= M_NOTREADY;
- ext_pgs->nrdy++;
+ m0->m_ext_pgs.nrdy++;
}
m0->m_epg_pa[ext_pgs_idx] = VM_PAGE_TO_PHYS(pga);
- ext_pgs->npgs++;
+ m0->m_ext_pgs.npgs++;
xfs = xfsize(i, npages, off, space);
- ext_pgs->last_pg_len = xfs;
+ m0->m_ext_pgs.last_pg_len = xfs;
MBUF_EXT_PGS_ASSERT_SANITY(m0);
mtail->m_len += xfs;
mtail->m_ext.ext_size += PAGE_SIZE;
Modified: head/sys/kern/uipc_ktls.c
==============================================================================
--- head/sys/kern/uipc_ktls.c Sun May 3 00:03:39 2020 (r360577)
+++ head/sys/kern/uipc_ktls.c Sun May 3 00:08:05 2020 (r360578)
@@ -1287,14 +1287,12 @@ ktls_destroy(struct ktls_session *tls)
void
ktls_seq(struct sockbuf *sb, struct mbuf *m)
{
- struct mbuf_ext_pgs *pgs;
for (; m != NULL; m = m->m_next) {
KASSERT((m->m_flags & M_NOMAP) != 0,
("ktls_seq: mapped mbuf %p", m));
- pgs = &m->m_ext_pgs;
- pgs->seqno = sb->sb_tls_seqno;
+ m->m_ext_pgs.seqno = sb->sb_tls_seqno;
sb->sb_tls_seqno++;
}
}
@@ -1318,7 +1316,6 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls,
{
struct tls_record_layer *tlshdr;
struct mbuf *m;
- struct mbuf_ext_pgs *pgs;
uint64_t *noncep;
uint16_t tls_len;
int maxlen;
@@ -1341,13 +1338,12 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls,
("ktls_frame: mapped mbuf %p (top = %p)\n", m, top));
tls_len = m->m_len;
- pgs = &m->m_ext_pgs;
/* Save a reference to the session. */
- pgs->tls = ktls_hold(tls);
+ m->m_ext_pgs.tls = ktls_hold(tls);
- pgs->hdr_len = tls->params.tls_hlen;
- pgs->trail_len = tls->params.tls_tlen;
+ m->m_ext_pgs.hdr_len = tls->params.tls_hlen;
+ m->m_ext_pgs.trail_len = tls->params.tls_tlen;
if (tls->params.cipher_algorithm == CRYPTO_AES_CBC) {
int bs, delta;
@@ -1369,9 +1365,9 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls,
*/
bs = tls->params.tls_bs;
delta = (tls_len + tls->params.tls_tlen) & (bs - 1);
- pgs->trail_len -= delta;
+ m->m_ext_pgs.trail_len -= delta;
}
- m->m_len += pgs->hdr_len + pgs->trail_len;
+ m->m_len += m->m_ext_pgs.hdr_len + m->m_ext_pgs.trail_len;
/* Populate the TLS header. */
tlshdr = (void *)m->m_epg_hdr;
@@ -1386,7 +1382,7 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls,
tlshdr->tls_vminor = TLS_MINOR_VER_TWO;
tlshdr->tls_type = TLS_RLTYPE_APP;
/* save the real record type for later */
- pgs->record_type = record_type;
+ m->m_ext_pgs.record_type = record_type;
m->m_epg_trail[0] = record_type;
} else {
tlshdr->tls_vminor = tls->params.tls_vminor;
@@ -1423,8 +1419,8 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls,
*/
if (tls->mode == TCP_TLS_MODE_SW) {
m->m_flags |= M_NOTREADY;
- pgs->nrdy = pgs->npgs;
- *enq_cnt += pgs->npgs;
+ m->m_ext_pgs.nrdy = m->m_ext_pgs.npgs;
+ *enq_cnt += m->m_ext_pgs.npgs;
}
}
}
@@ -1432,15 +1428,12 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls,
void
ktls_enqueue_to_free(struct mbuf *m)
{
- struct mbuf_ext_pgs *pgs;
struct ktls_wq *wq;
bool running;
- pgs = &m->m_ext_pgs;
-
/* Mark it for freeing. */
- pgs->flags |= EPG_FLAG_2FREE;
- wq = &ktls_wq[pgs->tls->wq_index];
+ m->m_ext_pgs.flags |= EPG_FLAG_2FREE;
+ wq = &ktls_wq[m->m_ext_pgs.tls->wq_index];
mtx_lock(&wq->mtx);
STAILQ_INSERT_TAIL(&wq->head, m, m_ext_pgs.stailq);
running = wq->running;
@@ -1452,7 +1445,6 @@ ktls_enqueue_to_free(struct mbuf *m)
void
ktls_enqueue(struct mbuf *m, struct socket *so, int page_count)
{
- struct mbuf_ext_pgs *pgs;
struct ktls_wq *wq;
bool running;
@@ -1461,19 +1453,17 @@ ktls_enqueue(struct mbuf *m, struct socket *so, int pa
("ktls_enqueue: %p not unready & nomap mbuf\n", m));
KASSERT(page_count != 0, ("enqueueing TLS mbuf with zero page count"));
- pgs = &m->m_ext_pgs;
+ KASSERT(m->m_ext_pgs.tls->mode == TCP_TLS_MODE_SW, ("!SW TLS mbuf"));
- KASSERT(pgs->tls->mode == TCP_TLS_MODE_SW, ("!SW TLS mbuf"));
+ m->m_ext_pgs.enc_cnt = page_count;
- pgs->enc_cnt = page_count;
-
/*
* Save a pointer to the socket. The caller is responsible
* for taking an additional reference via soref().
*/
- pgs->so = so;
+ m->m_ext_pgs.so = so;
- wq = &ktls_wq[pgs->tls->wq_index];
+ wq = &ktls_wq[m->m_ext_pgs.tls->wq_index];
mtx_lock(&wq->mtx);
STAILQ_INSERT_TAIL(&wq->head, m, m_ext_pgs.stailq);
running = wq->running;
@@ -1489,7 +1479,6 @@ ktls_encrypt(struct mbuf *top)
struct ktls_session *tls;
struct socket *so;
struct mbuf *m;
- struct mbuf_ext_pgs *pgs;
vm_paddr_t parray[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
struct iovec src_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
struct iovec dst_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
@@ -1526,15 +1515,13 @@ ktls_encrypt(struct mbuf *top)
*/
error = 0;
for (m = top; npages != total_pages; m = m->m_next) {
- pgs = &m->m_ext_pgs;
-
- KASSERT(pgs->tls == tls,
+ KASSERT(m->m_ext_pgs.tls == tls,
("different TLS sessions in a single mbuf chain: %p vs %p",
- tls, pgs->tls));
+ tls, m->m_ext_pgs.tls));
KASSERT((m->m_flags & (M_NOMAP | M_NOTREADY)) ==
(M_NOMAP | M_NOTREADY),
("%p not unready & nomap mbuf (top = %p)\n", m, top));
- KASSERT(npages + pgs->npgs <= total_pages,
+ KASSERT(npages + m->m_ext_pgs.npgs <= total_pages,
("page count mismatch: top %p, total_pages %d, m %p", top,
total_pages, m));
@@ -1546,10 +1533,10 @@ ktls_encrypt(struct mbuf *top)
* (from sendfile), anonymous wired pages are
* allocated and assigned to the destination iovec.
*/
- is_anon = (pgs->flags & EPG_FLAG_ANON) != 0;
+ is_anon = (m->m_ext_pgs.flags & EPG_FLAG_ANON) != 0;
- off = pgs->first_pg_off;
- for (i = 0; i < pgs->npgs; i++, off = 0) {
+ off = m->m_ext_pgs.first_pg_off;
+ for (i = 0; i < m->m_ext_pgs.npgs; i++, off = 0) {
len = m_epg_pagelen(m, i, off);
src_iov[i].iov_len = len;
src_iov[i].iov_base =
@@ -1578,8 +1565,8 @@ retry_page:
error = (*tls->sw_encrypt)(tls,
(const struct tls_record_layer *)m->m_epg_hdr,
- m->m_epg_trail, src_iov, dst_iov, i, pgs->seqno,
- pgs->record_type);
+ m->m_epg_trail, src_iov, dst_iov, i, m->m_ext_pgs.seqno,
+ m->m_ext_pgs.record_type);
if (error) {
counter_u64_add(ktls_offload_failed_crypto, 1);
break;
@@ -1595,14 +1582,14 @@ retry_page:
m->m_ext.ext_free(m);
/* Replace them with the new pages. */
- for (i = 0; i < pgs->npgs; i++)
+ for (i = 0; i < m->m_ext_pgs.npgs; i++)
m->m_epg_pa[i] = parray[i];
/* Use the basic free routine. */
m->m_ext.ext_free = mb_free_mext_pgs;
/* Pages are now writable. */
- pgs->flags |= EPG_FLAG_ANON;
+ m->m_ext_pgs.flags |= EPG_FLAG_ANON;
}
/*
@@ -1612,7 +1599,7 @@ retry_page:
* yet-to-be-encrypted records having an associated
* session.
*/
- pgs->tls = NULL;
+ m->m_ext_pgs.tls = NULL;
ktls_free(tls);
}
Modified: head/sys/kern/uipc_mbuf.c
==============================================================================
--- head/sys/kern/uipc_mbuf.c Sun May 3 00:03:39 2020 (r360577)
+++ head/sys/kern/uipc_mbuf.c Sun May 3 00:08:05 2020 (r360578)
@@ -1427,7 +1427,6 @@ nospace:
static int
frags_per_mbuf(struct mbuf *m)
{
- struct mbuf_ext_pgs *ext_pgs;
int frags;
if ((m->m_flags & M_NOMAP) == 0)
@@ -1440,12 +1439,11 @@ frags_per_mbuf(struct mbuf *m)
* XXX: This overestimates the number of fragments by assuming
* all the backing physical pages are disjoint.
*/
- ext_pgs = &m->m_ext_pgs;
frags = 0;
- if (ext_pgs->hdr_len != 0)
+ if (m->m_ext_pgs.hdr_len != 0)
frags++;
- frags += ext_pgs->npgs;
- if (ext_pgs->trail_len != 0)
+ frags += m->m_ext_pgs.npgs;
+ if (m->m_ext_pgs.trail_len != 0)
frags++;
return (frags);
@@ -1628,12 +1626,10 @@ nospace:
void
mb_free_mext_pgs(struct mbuf *m)
{
- struct mbuf_ext_pgs *ext_pgs;
vm_page_t pg;
MBUF_EXT_PGS_ASSERT(m);
- ext_pgs = &m->m_ext_pgs;
- for (int i = 0; i < ext_pgs->npgs; i++) {
+ for (int i = 0; i < m->m_ext_pgs.npgs; i++) {
pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
vm_page_unwire_noq(pg);
vm_page_free(pg);
@@ -1644,7 +1640,6 @@ static struct mbuf *
m_uiotombuf_nomap(struct uio *uio, int how, int len, int maxseg, int flags)
{
struct mbuf *m, *mb, *prev;
- struct mbuf_ext_pgs *pgs;
vm_page_t pg_array[MBUF_PEXT_MAX_PGS];
int error, length, i, needed;
ssize_t total;
@@ -1677,8 +1672,7 @@ m_uiotombuf_nomap(struct uio *uio, int how, int len, i
else
prev->m_next = mb;
prev = mb;
- pgs = &mb->m_ext_pgs;
- pgs->flags = EPG_FLAG_ANON;
+ mb->m_ext_pgs.flags = EPG_FLAG_ANON;
needed = length = MIN(maxseg, total);
for (i = 0; needed > 0; i++, needed -= PAGE_SIZE) {
retry_page:
@@ -1693,16 +1687,16 @@ retry_page:
}
pg_array[i]->flags &= ~PG_ZERO;
mb->m_epg_pa[i] = VM_PAGE_TO_PHYS(pg_array[i]);
- pgs->npgs++;
+ mb->m_ext_pgs.npgs++;
}
- pgs->last_pg_len = length - PAGE_SIZE * (pgs->npgs - 1);
+ mb->m_ext_pgs.last_pg_len = length - PAGE_SIZE * (mb->m_ext_pgs.npgs - 1);
MBUF_EXT_PGS_ASSERT_SANITY(mb);
total -= length;
error = uiomove_fromphys(pg_array, 0, length, uio);
if (error != 0)
goto failed;
mb->m_len = length;
- mb->m_ext.ext_size += PAGE_SIZE * pgs->npgs;
+ mb->m_ext.ext_size += PAGE_SIZE * mb->m_ext_pgs.npgs;
if (flags & M_PKTHDR)
m->m_pkthdr.len += length;
}
@@ -1778,23 +1772,21 @@ m_uiotombuf(struct uio *uio, int how, int len, int ali
int
m_unmappedtouio(const struct mbuf *m, int m_off, struct uio *uio, int len)
{
- struct mbuf_ext_pgs *ext_pgs;
vm_page_t pg;
int error, i, off, pglen, pgoff, seglen, segoff;
MBUF_EXT_PGS_ASSERT(m);
- ext_pgs = __DECONST(void *, &m->m_ext_pgs);
error = 0;
/* Skip over any data removed from the front. */
off = mtod(m, vm_offset_t);
off += m_off;
- if (ext_pgs->hdr_len != 0) {
- if (off >= ext_pgs->hdr_len) {
- off -= ext_pgs->hdr_len;
+ if (m->m_ext_pgs.hdr_len != 0) {
+ if (off >= m->m_ext_pgs.hdr_len) {
+ off -= m->m_ext_pgs.hdr_len;
} else {
- seglen = ext_pgs->hdr_len - off;
+ seglen = m->m_ext_pgs.hdr_len - off;
segoff = off;
seglen = min(seglen, len);
off = 0;
@@ -1803,8 +1795,8 @@ m_unmappedtouio(const struct mbuf *m, int m_off, struc
&m->m_epg_hdr[segoff]), seglen, uio);
}
}
- pgoff = ext_pgs->first_pg_off;
- for (i = 0; i < ext_pgs->npgs && error == 0 && len > 0; i++) {
+ pgoff = m->m_ext_pgs.first_pg_off;
+ for (i = 0; i < m->m_ext_pgs.npgs && error == 0 && len > 0; i++) {
pglen = m_epg_pagelen(m, i, pgoff);
if (off >= pglen) {
off -= pglen;
@@ -1821,9 +1813,9 @@ m_unmappedtouio(const struct mbuf *m, int m_off, struc
pgoff = 0;
};
if (len != 0 && error == 0) {
- KASSERT((off + len) <= ext_pgs->trail_len,
+ KASSERT((off + len) <= m->m_ext_pgs.trail_len,
("off + len > trail (%d + %d > %d, m_off = %d)", off, len,
- ext_pgs->trail_len, m_off));
+ m->m_ext_pgs.trail_len, m_off));
error = uiomove(__DECONST(void *, &m->m_epg_trail[off]),
len, uio);
}
Modified: head/sys/kern/uipc_sockbuf.c
==============================================================================
--- head/sys/kern/uipc_sockbuf.c Sun May 3 00:03:39 2020 (r360577)
+++ head/sys/kern/uipc_sockbuf.c Sun May 3 00:08:05 2020 (r360578)
@@ -128,21 +128,18 @@ sbready_compress(struct sockbuf *sb, struct mbuf *m0,
(n->m_flags & M_NOMAP) &&
!mbuf_has_tls_session(m) &&
!mbuf_has_tls_session(n)) {
- struct mbuf_ext_pgs *mpgs, *npgs;
int hdr_len, trail_len;
- mpgs = &m->m_ext_pgs;
- npgs = &n->m_ext_pgs;
- hdr_len = npgs->hdr_len;
- trail_len = mpgs->trail_len;
+ hdr_len = n->m_ext_pgs.hdr_len;
+ trail_len = m->m_ext_pgs.trail_len;
if (trail_len != 0 && hdr_len != 0 &&
trail_len + hdr_len <= MBUF_PEXT_TRAIL_LEN) {
/* copy n's header to m's trailer */
memcpy(&m->m_epg_trail[trail_len],
n->m_epg_hdr, hdr_len);
- mpgs->trail_len += hdr_len;
+ m->m_ext_pgs.trail_len += hdr_len;
m->m_len += hdr_len;
- npgs->hdr_len = 0;
+ n->m_ext_pgs.hdr_len = 0;
n->m_len -= hdr_len;
}
}
More information about the svn-src-head
mailing list