svn commit: r364800 - head/sys/geom/eli
Alan Somers
asomers at FreeBSD.org
Wed Aug 26 02:44:36 UTC 2020
Author: asomers
Date: Wed Aug 26 02:44:35 2020
New Revision: 364800
URL: https://svnweb.freebsd.org/changeset/base/364800
Log:
geli: use unmapped I/O
Use unmapped I/O for geli. Unlike most geom providers, geli needs to
manipulate data on every read or write. Previously it would always map bios.
On my 16-core, dual socket server using geli atop md(4) devices, with 512B
sectors, this change increases geli IOPs by about 3x.
Note that geli still can't use unmapped I/O when data integrity verification
is enabled (but it could, with a little more work). And it can't use
unmapped I/O in combination with ZFS, because ZFS uses mapped bios.
Reviewed by: markj, kib, jhb, mjg, mat, bcr (manpages)
MFC after: 1 week
Sponsored by: Axcient
Differential Revision: https://reviews.freebsd.org/D25671
Modified:
head/sys/geom/eli/g_eli.c
head/sys/geom/eli/g_eli_privacy.c
Modified: head/sys/geom/eli/g_eli.c
==============================================================================
--- head/sys/geom/eli/g_eli.c Wed Aug 26 02:37:42 2020 (r364799)
+++ head/sys/geom/eli/g_eli.c Wed Aug 26 02:44:35 2020 (r364800)
@@ -49,6 +49,8 @@ __FBSDID("$FreeBSD$");
#include <sys/uio.h>
#include <sys/vnode.h>
+#include <machine/vmparam.h>
+
#include <vm/uma.h>
#include <geom/geom.h>
@@ -972,6 +974,15 @@ g_eli_create(struct gctl_req *req, struct g_class *mp,
*/
pp = g_new_providerf(gp, "%s%s", bpp->name, G_ELI_SUFFIX);
pp->flags |= G_PF_DIRECT_SEND | G_PF_DIRECT_RECEIVE;
+ if (CRYPTO_HAS_VMPAGE) {
+ /*
+ * On DMAP architectures we can use unmapped I/O. But don't
+ * use it with data integrity verification. That code hasn't
+ * been written yet.
+ */
+ if ((sc->sc_flags & G_ELI_FLAG_AUTH) == 0)
+ pp->flags |= G_PF_ACCEPT_UNMAPPED;
+ }
pp->mediasize = sc->sc_mediasize;
pp->sectorsize = sc->sc_sectorsize;
LIST_FOREACH(gap, &bpp->aliases, ga_next)
Modified: head/sys/geom/eli/g_eli_privacy.c
==============================================================================
--- head/sys/geom/eli/g_eli_privacy.c Wed Aug 26 02:37:42 2020 (r364799)
+++ head/sys/geom/eli/g_eli_privacy.c Wed Aug 26 02:44:35 2020 (r364800)
@@ -63,6 +63,28 @@ __FBSDID("$FreeBSD$");
MALLOC_DECLARE(M_ELI);
/*
+ * Copy data from a (potentially unmapped) bio to a kernelspace buffer.
+ *
+ * The buffer must have at least as much room as bp->bio_length.
+ */
+static void
+g_eli_bio_copyin(struct bio *bp, void *kaddr)
+{
+ struct uio uio;
+ struct iovec iov[1];
+
+ iov[0].iov_base = kaddr;
+ iov[0].iov_len = bp->bio_length;
+ uio.uio_iov = iov;
+ uio.uio_iovcnt = 1;
+ uio.uio_offset = 0;
+ uio.uio_resid = bp->bio_length;
+ uio.uio_segflg = UIO_SYSSPACE;
+ uio.uio_rw = UIO_READ;
+ uiomove_fromphys(bp->bio_ma, bp->bio_ma_offset, bp->bio_length, &uio);
+}
+
+/*
* The function is called after we read and decrypt data.
*
* g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> G_ELI_CRYPTO_READ_DONE -> g_io_deliver
@@ -98,8 +120,7 @@ g_eli_crypto_read_done(struct cryptop *crp)
*/
if (bp->bio_inbed < bp->bio_children)
return (0);
- free(bp->bio_driver2, M_ELI);
- bp->bio_driver2 = NULL;
+
if (bp->bio_error != 0) {
G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).",
bp->bio_error);
@@ -167,6 +188,11 @@ g_eli_crypto_write_done(struct cryptop *crp)
return (0);
}
cbp->bio_data = bp->bio_driver2;
+ /*
+ * Clear BIO_UNMAPPED, which was inherited from where we cloned the bio
+ * in g_eli_start, because we manually set bio_data
+ */
+ cbp->bio_flags &= ~BIO_UNMAPPED;
cbp->bio_done = g_eli_write_done;
cp = LIST_FIRST(&gp->consumer);
cbp->bio_to = cp->provider;
@@ -236,10 +262,12 @@ g_eli_crypto_run(struct g_eli_worker *wr, struct bio *
{
struct g_eli_softc *sc;
struct cryptop *crp;
+ vm_page_t *pages;
u_int i, nsec, secsize;
off_t dstoff;
- u_char *data;
+ u_char *data = NULL;
int error;
+ int pages_offset;
G_ELI_LOGREQ(3, bp, "%s", __func__);
@@ -258,16 +286,37 @@ g_eli_crypto_run(struct g_eli_worker *wr, struct bio *
if (bp->bio_cmd == BIO_WRITE) {
data = malloc(bp->bio_length, M_ELI, M_WAITOK);
bp->bio_driver2 = data;
- bcopy(bp->bio_data, data, bp->bio_length);
- } else
- data = bp->bio_data;
+ /*
+ * This copy could be eliminated by using crypto's output
+ * buffer, instead of using a single overwriting buffer.
+ */
+ if ((bp->bio_flags & BIO_UNMAPPED) != 0)
+ g_eli_bio_copyin(bp, data);
+ else
+ bcopy(bp->bio_data, data, bp->bio_length);
+ } else {
+ if ((bp->bio_flags & BIO_UNMAPPED) != 0) {
+ pages = bp->bio_ma;
+ pages_offset = bp->bio_ma_offset;
+ } else {
+ data = bp->bio_data;
+ }
+ }
for (i = 0, dstoff = bp->bio_offset; i < nsec; i++, dstoff += secsize) {
crp = crypto_getreq(wr->w_sid, M_WAITOK);
- crypto_use_buf(crp, data, secsize);
+ if (data) {
+ crypto_use_buf(crp, data, secsize);
+ data += secsize;
+ } else {
+ MPASS(pages != NULL);
+ crypto_use_vmpage(crp, pages, secsize, pages_offset);
+ pages_offset += secsize;
+ pages += pages_offset >> PAGE_SHIFT;
+ pages_offset &= PAGE_MASK;
+ }
crp->crp_opaque = (void *)bp;
- data += secsize;
if (bp->bio_cmd == BIO_WRITE) {
crp->crp_op = CRYPTO_OP_ENCRYPT;
crp->crp_callback = g_eli_crypto_write_done;
More information about the svn-src-head
mailing list