git: aacaeeee8ecd - main - nvmf: Permit failing I/O requests while disconnected

From: John Baldwin <jhb_at_FreeBSD.org>
Date: Wed, 05 Jun 2024 20:04:03 UTC
The branch main has been updated by jhb:

URL: https://cgit.FreeBSD.org/src/commit/?id=aacaeeee8ecd7084a33b2d8e140aad37b3b0eddc

commit aacaeeee8ecd7084a33b2d8e140aad37b3b0eddc
Author:     John Baldwin <jhb@FreeBSD.org>
AuthorDate: 2024-06-05 19:54:15 +0000
Commit:     John Baldwin <jhb@FreeBSD.org>
CommitDate: 2024-06-05 19:59:07 +0000

    nvmf: Permit failing I/O requests while disconnected
    
    Add a kern.nvmf.fail_on_disconnection sysctl similar to the
    kern.iscsi.fail_on_disconnection sysctl.  This causes pending I/O
    requests to fail with an error if an association is disconnected
    instead of requeueing to be retried once the association is
    reconnected.  As with iSCSI, the default is to queue and retry
    operations.
    
    Reviewed by:    imp
    Sponsored by:   Chelsio Communications
    Differential Revision:  https://reviews.freebsd.org/D45308
---
 share/man/man4/nvmf.4        | 24 +++++++++++++++++++++++-
 sys/dev/nvmf/host/nvmf.c     |  5 +++++
 sys/dev/nvmf/host/nvmf_ns.c  | 25 ++++++++++++++++++++-----
 sys/dev/nvmf/host/nvmf_sim.c | 10 ++++++++--
 sys/dev/nvmf/host/nvmf_var.h |  3 +++
 5 files changed, 59 insertions(+), 8 deletions(-)

diff --git a/share/man/man4/nvmf.4 b/share/man/man4/nvmf.4
index 8afbb4d9daaf..298365acefa9 100644
--- a/share/man/man4/nvmf.4
+++ b/share/man/man4/nvmf.4
@@ -3,7 +3,7 @@
 .\"
 .\" Copyright (c) 2024 Chelsio Communications, Inc.
 .\"
-.Dd May 2, 2024
+.Dd June 5, 2024
 .Dt NVMF 4
 .Os
 .Sh NAME
@@ -65,6 +65,28 @@ disk driver.
 Associations require a supported transport such as
 .Xr nvmf_tcp 4
 for associations using TCP/IP.
+.Sh SYSCTL VARIABLES
+The following variables are available as both
+.Xr sysctl 8
+variables and
+.Xr loader 8
+tunables:
+.Bl -tag -width indent
+.It Va kern.nvmf.fail_on_disconnection
+Determines the behavior when an association's connection is interrupted.
+By default, input/output operations are suspended while a host is disconnected.
+This includes operations pending at the time the association's connection was
+interrupted as well as new requests submitted while the host is disconnected.
+Once a new association is established, suspended I/O requests are retried.
+When set to 1, input/output operations fail with
+.Er EIO
+while a host is disconnected and
+.Xr nda 4
+peripherals are destroyed after the first failed I/O request.
+Note that any destroyed
+.Xr nda 4
+peripherals will be recreated after a new association is established.
+.El
 .Sh SEE ALSO
 .Xr nda 4 ,
 .Xr nvme 4 ,
diff --git a/sys/dev/nvmf/host/nvmf.c b/sys/dev/nvmf/host/nvmf.c
index 9684170c1de9..c309836ed8a8 100644
--- a/sys/dev/nvmf/host/nvmf.c
+++ b/sys/dev/nvmf/host/nvmf.c
@@ -15,6 +15,7 @@
 #include <sys/module.h>
 #include <sys/mutex.h>
 #include <sys/sx.h>
+#include <sys/sysctl.h>
 #include <sys/taskqueue.h>
 #include <dev/nvme/nvme.h>
 #include <dev/nvmf/nvmf.h>
@@ -23,6 +24,10 @@
 
 static struct cdevsw nvmf_cdevsw;
 
+bool nvmf_fail_disconnect = false;
+SYSCTL_BOOL(_kern_nvmf, OID_AUTO, fail_on_disconnection, CTLFLAG_RWTUN,
+    &nvmf_fail_disconnect, 0, "Fail I/O requests on connection failure");
+
 MALLOC_DEFINE(M_NVMF, "nvmf", "NVMe over Fabrics host");
 
 static void	nvmf_disconnect_task(void *arg, int pending);
diff --git a/sys/dev/nvmf/host/nvmf_ns.c b/sys/dev/nvmf/host/nvmf_ns.c
index 0727ca960a57..8381cc4aec54 100644
--- a/sys/dev/nvmf/host/nvmf_ns.c
+++ b/sys/dev/nvmf/host/nvmf_ns.c
@@ -84,13 +84,22 @@ nvmf_ns_biodone(struct bio *bio)
 	ns = bio->bio_dev->si_drv1;
 
 	/* If a request is aborted, resubmit or queue it for resubmission. */
-	if (bio->bio_error == ECONNABORTED) {
+	if (bio->bio_error == ECONNABORTED && !nvmf_fail_disconnect) {
 		bio->bio_error = 0;
 		bio->bio_driver2 = 0;
 		mtx_lock(&ns->lock);
 		if (ns->disconnected) {
-			TAILQ_INSERT_TAIL(&ns->pending_bios, bio, bio_queue);
-			mtx_unlock(&ns->lock);
+			if (nvmf_fail_disconnect) {
+				mtx_unlock(&ns->lock);
+				bio->bio_error = ECONNABORTED;
+				bio->bio_flags |= BIO_ERROR;
+				bio->bio_resid = bio->bio_bcount;
+				biodone(bio);
+			} else {
+				TAILQ_INSERT_TAIL(&ns->pending_bios, bio,
+				    bio_queue);
+				mtx_unlock(&ns->lock);
+			}
 		} else {
 			mtx_unlock(&ns->lock);
 			nvmf_ns_strategy(bio);
@@ -163,6 +172,7 @@ nvmf_ns_submit_bio(struct nvmf_namespace *ns, struct bio *bio)
 	struct nvme_dsm_range *dsm_range;
 	struct memdesc mem;
 	uint64_t lba, lba_count;
+	int error;
 
 	dsm_range = NULL;
 	memset(&cmd, 0, sizeof(cmd));
@@ -201,10 +211,15 @@ nvmf_ns_submit_bio(struct nvmf_namespace *ns, struct bio *bio)
 
 	mtx_lock(&ns->lock);
 	if (ns->disconnected) {
-		TAILQ_INSERT_TAIL(&ns->pending_bios, bio, bio_queue);
+		if (nvmf_fail_disconnect) {
+			error = ECONNABORTED;
+		} else {
+			TAILQ_INSERT_TAIL(&ns->pending_bios, bio, bio_queue);
+			error = 0;
+		}
 		mtx_unlock(&ns->lock);
 		free(dsm_range, M_NVMF);
-		return (0);
+		return (error);
 	}
 
 	req = nvmf_allocate_request(nvmf_select_io_queue(ns->sc), &cmd,
diff --git a/sys/dev/nvmf/host/nvmf_sim.c b/sys/dev/nvmf/host/nvmf_sim.c
index 00dad07889d1..71bb71dd4063 100644
--- a/sys/dev/nvmf/host/nvmf_sim.c
+++ b/sys/dev/nvmf/host/nvmf_sim.c
@@ -40,7 +40,10 @@ nvmf_ccb_done(union ccb *ccb)
 		return;
 
 	if (nvmf_cqe_aborted(&ccb->nvmeio.cpl)) {
-		ccb->ccb_h.status = CAM_REQUEUE_REQ;
+		if (nvmf_fail_disconnect)
+			ccb->ccb_h.status = CAM_DEV_NOT_THERE;
+		else
+			ccb->ccb_h.status = CAM_REQUEUE_REQ;
 		xpt_done(ccb);
 	} else if (ccb->nvmeio.cpl.status != 0) {
 		ccb->ccb_h.status = CAM_NVME_STATUS_ERROR;
@@ -106,7 +109,10 @@ nvmf_sim_io(struct nvmf_softc *sc, union ccb *ccb)
 	mtx_lock(&sc->sim_mtx);
 	if (sc->sim_disconnected) {
 		mtx_unlock(&sc->sim_mtx);
-		nvmeio->ccb_h.status = CAM_REQUEUE_REQ;
+		if (nvmf_fail_disconnect)
+			nvmeio->ccb_h.status = CAM_DEV_NOT_THERE;
+		else
+			nvmeio->ccb_h.status = CAM_REQUEUE_REQ;
 		xpt_done(ccb);
 		return;
 	}
diff --git a/sys/dev/nvmf/host/nvmf_var.h b/sys/dev/nvmf/host/nvmf_var.h
index 2fa0216baab8..cf88d2f7b01e 100644
--- a/sys/dev/nvmf/host/nvmf_var.h
+++ b/sys/dev/nvmf/host/nvmf_var.h
@@ -140,6 +140,9 @@ extern driver_t nvme_nvmf_driver;
 MALLOC_DECLARE(M_NVMF);
 #endif
 
+/* If true, I/O requests will fail while the host is disconnected. */
+extern bool nvmf_fail_disconnect;
+
 /* nvmf.c */
 void	nvmf_complete(void *arg, const struct nvme_completion *cqe);
 void	nvmf_io_complete(void *arg, size_t xfered, int error);