git: 35372e305863 - stable/13 - nvme: Introduce longer timeouts for admin queue

The branch stable/13 has been updated by mav:

URL: https://cgit.FreeBSD.org/src/commit/?id=35372e305863b7c372d8f019d3f422e7d2ee3be9

commit 35372e305863b7c372d8f019d3f422e7d2ee3be9
Author:     Alexander Motin <mav@FreeBSD.org>
AuthorDate: 2023-11-06 16:05:48 +0000
Commit:     Alexander Motin <mav@FreeBSD.org>
CommitDate: 2023-11-14 12:57:28 +0000

    nvme: Introduce longer timeouts for admin queue
    
    KIOXIA CD8 SSDs routinely take ~25 seconds to delete non-empty
    namespace.  In some cases like hot-plug it takes longer, triggering
    timeout and controller resets after just 30 seconds. Linux for many
    years has separate 60 seconds timeout for admin queue.  This patch
    does the same.  And it is good to be consistent.
    
    Sponsored by:   iXsystems, Inc.
    Reviewed by:    imp
    MFC after:      1 week
    Differential Revision:  https://reviews.freebsd.org/D42454
    
    (cherry picked from commit 8d6c0743e36e3cff9279c40468711a82db98df23)
---
 sys/dev/nvme/nvme_ctrlr.c   |  6 ++++++
 sys/dev/nvme/nvme_private.h |  2 ++
 sys/dev/nvme/nvme_qpair.c   |  2 ++
 sys/dev/nvme/nvme_sysctl.c  | 15 ++++++++++-----
 4 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/sys/dev/nvme/nvme_ctrlr.c b/sys/dev/nvme/nvme_ctrlr.c
index 84aa402af4f8..bcbc8dc26ee1 100644
--- a/sys/dev/nvme/nvme_ctrlr.c
+++ b/sys/dev/nvme/nvme_ctrlr.c
@@ -1437,6 +1437,12 @@ nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev)
 	to = NVME_CAP_LO_TO(cap_lo) + 1;
 	ctrlr->ready_timeout_in_ms = to * 500;
 
+	timeout_period = NVME_ADMIN_TIMEOUT_PERIOD;
+	TUNABLE_INT_FETCH("hw.nvme.admin_timeout_period", &timeout_period);
+	timeout_period = min(timeout_period, NVME_MAX_TIMEOUT_PERIOD);
+	timeout_period = max(timeout_period, NVME_MIN_TIMEOUT_PERIOD);
+	ctrlr->admin_timeout_period = timeout_period;
+
 	timeout_period = NVME_DEFAULT_TIMEOUT_PERIOD;
 	TUNABLE_INT_FETCH("hw.nvme.timeout_period", &timeout_period);
 	timeout_period = min(timeout_period, NVME_MAX_TIMEOUT_PERIOD);
diff --git a/sys/dev/nvme/nvme_private.h b/sys/dev/nvme/nvme_private.h
index e34dc20c2819..e2e2f5efeacc 100644
--- a/sys/dev/nvme/nvme_private.h
+++ b/sys/dev/nvme/nvme_private.h
@@ -85,6 +85,7 @@ MALLOC_DECLARE(M_NVME);
 #define NVME_MAX_CONSUMERS	(2)
 #define NVME_MAX_ASYNC_EVENTS	(8)
 
+#define NVME_ADMIN_TIMEOUT_PERIOD	(60)    /* in seconds */
 #define NVME_DEFAULT_TIMEOUT_PERIOD	(30)    /* in seconds */
 #define NVME_MIN_TIMEOUT_PERIOD		(5)
 #define NVME_MAX_TIMEOUT_PERIOD		(120)
@@ -287,6 +288,7 @@ struct nvme_controller {
 	uint32_t		int_coal_threshold;
 
 	/** timeout period in seconds */
+	uint32_t		admin_timeout_period;
 	uint32_t		timeout_period;
 
 	/** doorbell stride */
diff --git a/sys/dev/nvme/nvme_qpair.c b/sys/dev/nvme/nvme_qpair.c
index e31e4249b20c..d810d51fadce 100644
--- a/sys/dev/nvme/nvme_qpair.c
+++ b/sys/dev/nvme/nvme_qpair.c
@@ -1062,6 +1062,8 @@ nvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr)
 	if (req->timeout) {
 		if (req->cb_fn == nvme_completion_poll_cb)
 			timeout = 1;
+		else if (qpair->id == 0)
+			timeout = ctrlr->admin_timeout_period;
 		else
 			timeout = ctrlr->timeout_period;
 		tr->deadline = getsbinuptime() + timeout * SBT_1S;
diff --git a/sys/dev/nvme/nvme_sysctl.c b/sys/dev/nvme/nvme_sysctl.c
index 091c2f572eaf..c9c57ac91af5 100644
--- a/sys/dev/nvme/nvme_sysctl.c
+++ b/sys/dev/nvme/nvme_sysctl.c
@@ -132,8 +132,8 @@ nvme_sysctl_int_coal_threshold(SYSCTL_HANDLER_ARGS)
 static int
 nvme_sysctl_timeout_period(SYSCTL_HANDLER_ARGS)
 {
-	struct nvme_controller *ctrlr = arg1;
-	uint32_t newval = ctrlr->timeout_period;
+	uint32_t *ptr = arg1;
+	uint32_t newval = *ptr;
 	int error = sysctl_handle_int(oidp, &newval, 0, req);
 
 	if (error || (req->newptr == NULL))
@@ -143,7 +143,7 @@ nvme_sysctl_timeout_period(SYSCTL_HANDLER_ARGS)
 	    newval < NVME_MIN_TIMEOUT_PERIOD) {
 		return (EINVAL);
 	} else {
-		ctrlr->timeout_period = newval;
+		*ptr = newval;
 	}
 
 	return (0);
@@ -333,10 +333,15 @@ nvme_sysctl_initialize_ctrlr(struct nvme_controller *ctrlr)
 	    nvme_sysctl_int_coal_threshold, "IU",
 	    "Interrupt coalescing threshold");
 
+	SYSCTL_ADD_PROC(ctrlr_ctx, ctrlr_list, OID_AUTO,
+	    "admin_timeout_period", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
+	    &ctrlr->admin_timeout_period, 0, nvme_sysctl_timeout_period, "IU",
+	    "Timeout period for Admin queue (in seconds)");
+
 	SYSCTL_ADD_PROC(ctrlr_ctx, ctrlr_list, OID_AUTO,
 	    "timeout_period", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
-	    ctrlr, 0, nvme_sysctl_timeout_period, "IU",
-	    "Timeout period (in seconds)");
+	    &ctrlr->timeout_period, 0, nvme_sysctl_timeout_period, "IU",
+	    "Timeout period for I/O queues (in seconds)");
 
 	SYSCTL_ADD_PROC(ctrlr_ctx, ctrlr_list, OID_AUTO,
 	    "num_cmds", CTLTYPE_S64 | CTLFLAG_RD | CTLFLAG_MPSAFE,