svn commit: r352816 - in head/sys: kern sys

[Andrew Gallatin]

Author: gallatin
Date: Fri Sep 27 20:08:19 2019
New Revision: 352816
URL: https://svnweb.freebsd.org/changeset/base/352816

Log:
  kTLS: Fix a bug where we would not encrypt anon data inplace.
  
  Software Kernel TLS needs to allocate a new destination crypto
  buffer when encrypting data from the page cache, so as to avoid
  overwriting shared clear-text file data with encrypted data
  specific to a single socket. When the data is anonymous, eg, not
  tied to a file, then we can encrypt in place and avoid allocating
  a new page. This fixes a bug where the existing code always
  assumes the data is private, and never encrypts in place. This
  results in unneeded page allocations and potentially more memory
  bandwidth consumption when doing socket writes.
  
  When the code was written at Netflix, ktls_encrypt() looked at
  private sendfile flags to determine if the pages being encrypted
  where part of the page cache (coming from sendfile) or
  anonymous (coming from sosend). This was broken internally at
  Netflix when the sendfile flags were made private, and the
  M_WRITABLE() check was added. Unfortunately, M_WRITABLE() will
  always be false for M_NOMAP mbufs, since one cannot just mtod()
  them.
  
  This change introduces a new flags field to the mbuf_ext_pgs
  struct by stealing a byte from the tls hdr. Note that the current
  header is still 2 bytes larger than the largest header we
  support: AES-CBC with explicit IV. We set MBUF_PEXT_FLAG_ANON
  when creating an unmapped mbuf in m_uiotombuf_nomap() (which is
  the path that socket writes take), and we check for that flag in
  ktls_encrypt() when looking for anon pages.
  
  Reviewed by:	jhb
  Sponsored by:	Netflix
  Differential Revision:	https://reviews.freebsd.org/D21796

Modified:
  head/sys/kern/kern_mbuf.c
  head/sys/kern/uipc_ktls.c
  head/sys/kern/uipc_mbuf.c
  head/sys/sys/mbuf.h

Modified: head/sys/kern/kern_mbuf.c
==============================================================================
--- head/sys/kern/kern_mbuf.c	Fri Sep 27 19:26:52 2019	(r352815)
+++ head/sys/kern/kern_mbuf.c	Fri Sep 27 20:08:19 2019	(r352816)
@@ -1171,6 +1171,7 @@ mb_alloc_ext_pgs(int how, bool pkthdr, m_ext_free_t ex
 	ext_pgs->nrdy = 0;
 	ext_pgs->first_pg_off = 0;
 	ext_pgs->last_pg_len = 0;
+	ext_pgs->flags = 0;
 	ext_pgs->hdr_len = 0;
 	ext_pgs->trail_len = 0;
 	ext_pgs->tls = NULL;

Modified: head/sys/kern/uipc_ktls.c
==============================================================================
--- head/sys/kern/uipc_ktls.c	Fri Sep 27 19:26:52 2019	(r352815)
+++ head/sys/kern/uipc_ktls.c	Fri Sep 27 20:08:19 2019	(r352816)
@@ -1363,7 +1363,7 @@ ktls_encrypt(struct mbuf_ext_pgs *pgs)
 		 * (from sendfile), anonymous wired pages are
 		 * allocated and assigned to the destination iovec.
 		 */
-		is_anon = M_WRITABLE(m);
+		is_anon = (pgs->flags & MBUF_PEXT_FLAG_ANON) != 0;
 
 		off = pgs->first_pg_off;
 		for (i = 0; i < pgs->npgs; i++, off = 0) {
@@ -1416,6 +1416,9 @@ retry_page:
 
 			/* Use the basic free routine. */
 			m->m_ext.ext_free = mb_free_mext_pgs;
+
+			/* Pages are now writable. */
+			pgs->flags |= MBUF_PEXT_FLAG_ANON;
 		}
 
 		/*

Modified: head/sys/kern/uipc_mbuf.c
==============================================================================
--- head/sys/kern/uipc_mbuf.c	Fri Sep 27 19:26:52 2019	(r352815)
+++ head/sys/kern/uipc_mbuf.c	Fri Sep 27 20:08:19 2019	(r352816)
@@ -1664,6 +1664,7 @@ m_uiotombuf_nomap(struct uio *uio, int how, int len, i
 			prev->m_next = mb;
 		prev = mb;
 		pgs = mb->m_ext.ext_pgs;
+		pgs->flags = MBUF_PEXT_FLAG_ANON;
 		needed = length = MIN(maxseg, total);
 		for (i = 0; needed > 0; i++, needed -= PAGE_SIZE) {
 retry_page:

Modified: head/sys/sys/mbuf.h
==============================================================================
--- head/sys/sys/mbuf.h	Fri Sep 27 19:26:52 2019	(r352815)
+++ head/sys/sys/mbuf.h	Fri Sep 27 20:08:19 2019	(r352816)
@@ -312,7 +312,7 @@ struct socket;
  * - 21 (AES-CBC with explicit IV)
  * - 13 (AES-GCM with 8 byte explicit IV)
  */
-#define	MBUF_PEXT_HDR_LEN	24
+#define	MBUF_PEXT_HDR_LEN	23
 
 /*
  * TLS records for TLS 1.0-1.2 can have the following maximum trailer
@@ -333,6 +333,8 @@ struct socket;
 #define	MBUF_PEXT_MAX_BYTES						\
     (MBUF_PEXT_MAX_PGS * PAGE_SIZE + MBUF_PEXT_HDR_LEN + MBUF_PEXT_TRAIL_LEN)
 
+#define MBUF_PEXT_FLAG_ANON	1	/* Data can be encrypted in place. */
+
 /*
  * This struct is 256 bytes in size and is arranged so that the most
  * common case (accessing the first 4 pages of a 16KB TLS record) will
@@ -347,6 +349,7 @@ struct mbuf_ext_pgs {
 	uint16_t	last_pg_len;		/* Length of last page */
 	vm_paddr_t	pa[MBUF_PEXT_MAX_PGS];	/* phys addrs of pages */
 	char		hdr[MBUF_PEXT_HDR_LEN];	/* TLS header */
+	uint8_t		flags;			/* Flags */
 	struct ktls_session *tls;		/* TLS session */
 #if defined(__i386__) || \
     (defined(__powerpc__) && !defined(__powerpc64__) && defined(BOOKE))