git: 73c921ef1d44 - main - nvme: Add ability to override ioq to put the request on

The branch main has been updated by imp:

URL: https://cgit.FreeBSD.org/src/commit/?id=73c921ef1d44f6e1590957a4beb48a967e3ca8c8

commit 73c921ef1d44f6e1590957a4beb48a967e3ca8c8
Author:     Warner Losh <imp@FreeBSD.org>
AuthorDate: 2026-01-15 13:31:53 +0000
Commit:     Warner Losh <imp@FreeBSD.org>
CommitDate: 2026-01-15 13:31:53 +0000

    nvme: Add ability to override ioq to put the request on
    
    Sometimes the client device needs to manage the IOQ the request goes
    to. Expand the interface we have for the request to allow it to be set
    for this special use case.
    
    Sponsored by:           Netflix
    Reviewed by:            jhb
    Differential Revision:  https://reviews.freebsd.org/D54714
---
 sys/dev/nvme/nvme_ctrlr.c   |  4 +++-
 sys/dev/nvme/nvme_private.h | 20 +++++++++++++++++++-
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/sys/dev/nvme/nvme_ctrlr.c b/sys/dev/nvme/nvme_ctrlr.c
index 26194ccbb8f8..b75033300061 100644
--- a/sys/dev/nvme/nvme_ctrlr.c
+++ b/sys/dev/nvme/nvme_ctrlr.c
@@ -1877,8 +1877,10 @@ nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr,
     struct nvme_request *req)
 {
 	struct nvme_qpair       *qpair;
+	int32_t			ioq;
 
-	qpair = &ctrlr->ioq[QP(ctrlr, curcpu)];
+	ioq = req->ioq == NVME_IOQ_DEFAULT ? QP(ctrlr, curcpu) : req->ioq;
+	qpair = &ctrlr->ioq[ioq];
 	nvme_qpair_submit_request(qpair, req);
 }
 
diff --git a/sys/dev/nvme/nvme_private.h b/sys/dev/nvme/nvme_private.h
index 8837275e2ed5..b520b9946f7b 100644
--- a/sys/dev/nvme/nvme_private.h
+++ b/sys/dev/nvme/nvme_private.h
@@ -112,7 +112,9 @@ struct nvme_request {
 	struct memdesc			payload;
 	nvme_cb_fn_t			cb_fn;
 	void				*cb_arg;
-	int32_t				retries;
+	int16_t				retries;
+	uint16_t			ioq;
+#define NVME_IOQ_DEFAULT		0xffff
 	bool				payload_valid;
 	bool				timeout;
 	bool				spare[2];		/* Future use */
@@ -491,6 +493,7 @@ _nvme_allocate_request(const int how, nvme_cb_fn_t cb_fn, void *cb_arg)
 
 	req = malloc(sizeof(*req), M_NVME, how | M_ZERO);
 	if (req != NULL) {
+		req->ioq = NVME_IOQ_DEFAULT;
 		req->cb_fn = cb_fn;
 		req->cb_arg = cb_arg;
 		req->timeout = true;
@@ -551,6 +554,21 @@ nvme_allocate_request_ccb(union ccb *ccb, const int how, nvme_cb_fn_t cb_fn,
 
 #define nvme_free_request(req)	free(req, M_NVME)
 
+static __inline void
+nvme_request_set_ioq(struct nvme_controller *ctrlr, struct nvme_request *req, unt16_t ioq)
+{
+	/*
+	 * Note: NVMe queues are numbered 1-65535. The ioq here is numbered
+	 * 0-65534 to avoid off-by-one bugs, with 65535 being reserved for
+	 * DEFAULT.
+	 */
+	KASSERT(ioq == NVME_IOQ_DEFAULT || ioq < ctrlr->num_io_queues,
+	    ("ioq %d out of range 0..%d", ioq, ctrlr->num_io_queues));
+	if (ioq < 0 || ioq >= ctrlr->num_io_queues)
+		ioq = NVME_IOQ_DEFAULT;
+	req->ioq = ioq;
+}
+
 void	nvme_notify_async(struct nvme_controller *ctrlr,
 	    const struct nvme_completion *async_cpl,
 	    uint32_t log_page_id, void *log_page_buffer,