git: 4d3b659f24a8 - main - nvmf: Track SQ flow control

The branch main has been updated by jhb:

URL: https://cgit.FreeBSD.org/src/commit/?id=4d3b659f24a88f76557019e6f3192760bdebfbdf

commit 4d3b659f24a88f76557019e6f3192760bdebfbdf
Author:     John Baldwin <jhb@FreeBSD.org>
AuthorDate: 2024-11-11 16:39:05 +0000
Commit:     John Baldwin <jhb@FreeBSD.org>
CommitDate: 2024-11-11 16:39:05 +0000

    nvmf: Track SQ flow control
    
    This isn't really needed since the host driver never submits more
    commands to a queue than it can hold, but I noticed that the
    recently-added SQ head and tail sysctl nodes were not updating.  This
    fixes that and also uses these values to assert that there we never
    submit a command while a queue pair is full.
    
    Sponsored by:   Chelsio Communications
---
 sys/dev/nvmf/host/nvmf_qpair.c | 33 ++++++++++++++++++++++++++++-----
 sys/dev/nvmf/nvmf_transport.c  |  8 ++++++++
 sys/dev/nvmf/nvmf_transport.h  |  1 +
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/sys/dev/nvmf/host/nvmf_qpair.c b/sys/dev/nvmf/host/nvmf_qpair.c
index 1aeb0535eacf..b03ecfa081d3 100644
--- a/sys/dev/nvmf/host/nvmf_qpair.c
+++ b/sys/dev/nvmf/host/nvmf_qpair.c
@@ -115,8 +115,23 @@ nvmf_dispatch_command(struct nvmf_host_qpair *qp, struct nvmf_host_command *cmd)
 	struct nvmf_softc *sc = qp->sc;
 	struct nvme_command *sqe;
 	struct nvmf_capsule *nc;
+	uint16_t new_sqtail;
 	int error;
 
+	mtx_assert(&qp->lock, MA_OWNED);
+
+	qp->submitted++;
+
+	/*
+	 * Update flow control tracking.  This is just a sanity check.
+	 * Since num_commands == qsize - 1, there can never be too
+	 * many commands in flight.
+	 */
+	new_sqtail = (qp->sqtail + 1) % (qp->num_commands + 1);
+	KASSERT(new_sqtail != qp->sqhd, ("%s: qp %p is full", __func__, qp));
+	qp->sqtail = new_sqtail;
+	mtx_unlock(&qp->lock);
+
 	nc = cmd->req->nc;
 	sqe = nvmf_capsule_sqe(nc);
 
@@ -180,11 +195,23 @@ nvmf_receive_capsule(void *arg, struct nvmf_capsule *nc)
 		return;
 	}
 
+	/* Update flow control tracking. */
+	mtx_lock(&qp->lock);
+	if (qp->sq_flow_control) {
+		if (nvmf_sqhd_valid(nc))
+			qp->sqhd = le16toh(cqe->sqhd);
+	} else {
+		/*
+		 * If SQ FC is disabled, just advance the head for
+		 * each response capsule received.
+		 */
+		qp->sqhd = (qp->sqhd + 1) % (qp->num_commands + 1);
+	}
+
 	/*
 	 * If the queue has been shutdown due to an error, silently
 	 * drop the response.
 	 */
-	mtx_lock(&qp->lock);
 	if (qp->qp == NULL) {
 		device_printf(sc->dev,
 		    "received completion for CID %u on shutdown %s\n", cid,
@@ -215,8 +242,6 @@ nvmf_receive_capsule(void *arg, struct nvmf_capsule *nc)
 	} else {
 		cmd->req = STAILQ_FIRST(&qp->pending_requests);
 		STAILQ_REMOVE_HEAD(&qp->pending_requests, link);
-		qp->submitted++;
-		mtx_unlock(&qp->lock);
 		nvmf_dispatch_command(qp, cmd);
 	}
 
@@ -420,7 +445,5 @@ nvmf_submit_request(struct nvmf_request *req)
 	    ("%s: CID already busy", __func__));
 	qp->active_commands[cmd->cid] = cmd;
 	cmd->req = req;
-	qp->submitted++;
-	mtx_unlock(&qp->lock);
 	nvmf_dispatch_command(qp, cmd);
 }
diff --git a/sys/dev/nvmf/nvmf_transport.c b/sys/dev/nvmf/nvmf_transport.c
index ea4aee8cc7ae..316d1571e61d 100644
--- a/sys/dev/nvmf/nvmf_transport.c
+++ b/sys/dev/nvmf/nvmf_transport.c
@@ -180,6 +180,14 @@ nvmf_capsule_cqe(struct nvmf_capsule *nc)
 	return (&nc->nc_cqe);
 }
 
+bool
+nvmf_sqhd_valid(struct nvmf_capsule *nc)
+{
+	KASSERT(nc->nc_qe_len == sizeof(struct nvme_completion),
+	    ("%s: capsule %p is not a response capsule", __func__, nc));
+	return (nc->nc_sqhd_valid);
+}
+
 uint8_t
 nvmf_validate_command_capsule(struct nvmf_capsule *nc)
 {
diff --git a/sys/dev/nvmf/nvmf_transport.h b/sys/dev/nvmf/nvmf_transport.h
index 549170b25940..bbd830eba576 100644
--- a/sys/dev/nvmf/nvmf_transport.h
+++ b/sys/dev/nvmf/nvmf_transport.h
@@ -78,6 +78,7 @@ int	nvmf_transmit_capsule(struct nvmf_capsule *nc);
 void	nvmf_abort_capsule_data(struct nvmf_capsule *nc, int error);
 void *nvmf_capsule_sqe(struct nvmf_capsule *nc);
 void *nvmf_capsule_cqe(struct nvmf_capsule *nc);
+bool	nvmf_sqhd_valid(struct nvmf_capsule *nc);
 
 /* Controller-specific APIs. */