git: 931dd5feb0a1 - main - nvmf: Add sysctl nodes for each queue pair
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Sat, 02 Nov 2024 13:54:53 UTC
The branch main has been updated by jhb:
URL: https://cgit.FreeBSD.org/src/commit/?id=931dd5feb0a13d6f24d3de7055d344732de0e353
commit 931dd5feb0a13d6f24d3de7055d344732de0e353
Author: John Baldwin <jhb@FreeBSD.org>
AuthorDate: 2024-11-02 13:54:13 +0000
Commit: John Baldwin <jhb@FreeBSD.org>
CommitDate: 2024-11-02 13:54:13 +0000
nvmf: Add sysctl nodes for each queue pair
These report the queue size, queue head, queue tail, and the number of
commands submitted.
Sponsored by: Chelsio Communications
---
sys/dev/nvmf/host/nvmf.c | 10 ++++++++--
sys/dev/nvmf/host/nvmf_qpair.c | 42 +++++++++++++++++++++++++++++++++++++++++-
sys/dev/nvmf/host/nvmf_var.h | 5 ++++-
3 files changed, 53 insertions(+), 4 deletions(-)
diff --git a/sys/dev/nvmf/host/nvmf.c b/sys/dev/nvmf/host/nvmf.c
index 12023ebf96f9..9d06b501d675 100644
--- a/sys/dev/nvmf/host/nvmf.c
+++ b/sys/dev/nvmf/host/nvmf.c
@@ -266,7 +266,7 @@ nvmf_establish_connection(struct nvmf_softc *sc, struct nvmf_ivars *ivars)
/* Setup the admin queue. */
sc->admin = nvmf_init_qp(sc, ivars->hh->trtype, &ivars->hh->admin,
- "admin queue");
+ "admin queue", 0);
if (sc->admin == NULL) {
device_printf(sc->dev, "Failed to setup admin queue\n");
return (ENXIO);
@@ -279,7 +279,7 @@ nvmf_establish_connection(struct nvmf_softc *sc, struct nvmf_ivars *ivars)
for (u_int i = 0; i < sc->num_io_queues; i++) {
snprintf(name, sizeof(name), "I/O queue %u", i);
sc->io[i] = nvmf_init_qp(sc, ivars->hh->trtype,
- &ivars->io_params[i], name);
+ &ivars->io_params[i], name, i);
if (sc->io[i] == NULL) {
device_printf(sc->dev, "Failed to setup I/O queue %u\n",
i + 1);
@@ -453,6 +453,7 @@ nvmf_attach(device_t dev)
struct make_dev_args mda;
struct nvmf_softc *sc = device_get_softc(dev);
struct nvmf_ivars *ivars = device_get_ivars(dev);
+ struct sysctl_oid *oid;
uint64_t val;
u_int i;
int error;
@@ -467,6 +468,11 @@ nvmf_attach(device_t dev)
sx_init(&sc->connection_lock, "nvmf connection");
TASK_INIT(&sc->disconnect_task, 0, nvmf_disconnect_task, sc);
+ oid = SYSCTL_ADD_NODE(device_get_sysctl_ctx(dev),
+ SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "ioq",
+ CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "I/O Queues");
+ sc->ioq_oid_list = SYSCTL_CHILDREN(oid);
+
/* Claim the cdata pointer from ivars. */
sc->cdata = ivars->cdata;
ivars->cdata = NULL;
diff --git a/sys/dev/nvmf/host/nvmf_qpair.c b/sys/dev/nvmf/host/nvmf_qpair.c
index 96cb5a8b0465..1aeb0535eacf 100644
--- a/sys/dev/nvmf/host/nvmf_qpair.c
+++ b/sys/dev/nvmf/host/nvmf_qpair.c
@@ -10,6 +10,7 @@
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
+#include <sys/sysctl.h>
#include <dev/nvme/nvme.h>
#include <dev/nvmf/nvmf.h>
#include <dev/nvmf/nvmf_transport.h>
@@ -31,6 +32,7 @@ struct nvmf_host_qpair {
u_int num_commands;
uint16_t sqhd;
uint16_t sqtail;
+ uint64_t submitted;
struct mtx lock;
@@ -41,6 +43,7 @@ struct nvmf_host_qpair {
struct nvmf_host_command **active_commands;
char name[16];
+ struct sysctl_ctx_list sysctl_ctx;
};
struct nvmf_request *
@@ -212,6 +215,7 @@ nvmf_receive_capsule(void *arg, struct nvmf_capsule *nc)
} else {
cmd->req = STAILQ_FIRST(&qp->pending_requests);
STAILQ_REMOVE_HEAD(&qp->pending_requests, link);
+ qp->submitted++;
mtx_unlock(&qp->lock);
nvmf_dispatch_command(qp, cmd);
}
@@ -221,9 +225,39 @@ nvmf_receive_capsule(void *arg, struct nvmf_capsule *nc)
nvmf_free_request(req);
}
+static void
+nvmf_sysctls_qp(struct nvmf_softc *sc, struct nvmf_host_qpair *qp,
+ bool admin, u_int qid)
+{
+ struct sysctl_ctx_list *ctx = &qp->sysctl_ctx;
+ struct sysctl_oid *oid;
+ struct sysctl_oid_list *list;
+ char name[8];
+
+ if (admin) {
+ oid = SYSCTL_ADD_NODE(ctx,
+ SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO,
+ "adminq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Admin Queue");
+ } else {
+ snprintf(name, sizeof(name), "%u", qid);
+ oid = SYSCTL_ADD_NODE(ctx, sc->ioq_oid_list, OID_AUTO, name,
+ CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "I/O Queue");
+ }
+ list = SYSCTL_CHILDREN(oid);
+
+ SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "num_entries", CTLFLAG_RD,
+ NULL, qp->num_commands + 1, "Number of entries in queue");
+ SYSCTL_ADD_U16(ctx, list, OID_AUTO, "sq_head", CTLFLAG_RD, &qp->sqhd,
+ 0, "Current head of submission queue (as observed by driver)");
+ SYSCTL_ADD_U16(ctx, list, OID_AUTO, "sq_tail", CTLFLAG_RD, &qp->sqtail,
+ 0, "Current tail of submission queue (as observed by driver)");
+ SYSCTL_ADD_U64(ctx, list, OID_AUTO, "num_cmds", CTLFLAG_RD,
+ &qp->submitted, 0, "Number of commands submitted");
+}
+
struct nvmf_host_qpair *
nvmf_init_qp(struct nvmf_softc *sc, enum nvmf_trtype trtype,
- struct nvmf_handoff_qpair_params *handoff, const char *name)
+ struct nvmf_handoff_qpair_params *handoff, const char *name, u_int qid)
{
struct nvmf_host_command *cmd, *ncmd;
struct nvmf_host_qpair *qp;
@@ -236,6 +270,7 @@ nvmf_init_qp(struct nvmf_softc *sc, enum nvmf_trtype trtype,
qp->sqtail = handoff->sqtail;
strlcpy(qp->name, name, sizeof(qp->name));
mtx_init(&qp->lock, "nvmf qp", NULL, MTX_DEF);
+ (void)sysctl_ctx_init(&qp->sysctl_ctx);
/*
* Allocate a spare command slot for each pending AER command
@@ -258,6 +293,7 @@ nvmf_init_qp(struct nvmf_softc *sc, enum nvmf_trtype trtype,
qp->qp = nvmf_allocate_qpair(trtype, false, handoff, nvmf_qp_error,
qp, nvmf_receive_capsule, qp);
if (qp->qp == NULL) {
+ (void)sysctl_ctx_free(&qp->sysctl_ctx);
TAILQ_FOREACH_SAFE(cmd, &qp->free_commands, link, ncmd) {
TAILQ_REMOVE(&qp->free_commands, cmd, link);
free(cmd, M_NVMF);
@@ -268,6 +304,8 @@ nvmf_init_qp(struct nvmf_softc *sc, enum nvmf_trtype trtype,
return (NULL);
}
+ nvmf_sysctls_qp(sc, qp, handoff->admin, qid);
+
return (qp);
}
@@ -339,6 +377,7 @@ nvmf_destroy_qp(struct nvmf_host_qpair *qp)
struct nvmf_host_command *cmd, *ncmd;
nvmf_shutdown_qp(qp);
+ (void)sysctl_ctx_free(&qp->sysctl_ctx);
TAILQ_FOREACH_SAFE(cmd, &qp->free_commands, link, ncmd) {
TAILQ_REMOVE(&qp->free_commands, cmd, link);
@@ -381,6 +420,7 @@ nvmf_submit_request(struct nvmf_request *req)
("%s: CID already busy", __func__));
qp->active_commands[cmd->cid] = cmd;
cmd->req = req;
+ qp->submitted++;
mtx_unlock(&qp->lock);
nvmf_dispatch_command(qp, cmd);
}
diff --git a/sys/dev/nvmf/host/nvmf_var.h b/sys/dev/nvmf/host/nvmf_var.h
index adf6d8bde3d9..0e52f36a80a5 100644
--- a/sys/dev/nvmf/host/nvmf_var.h
+++ b/sys/dev/nvmf/host/nvmf_var.h
@@ -22,6 +22,7 @@ struct nvmf_aer;
struct nvmf_capsule;
struct nvmf_host_qpair;
struct nvmf_namespace;
+struct sysctl_oid_list;
typedef void nvmf_request_complete_t(void *, const struct nvme_completion *);
@@ -85,6 +86,8 @@ struct nvmf_softc {
u_int num_aer;
struct nvmf_aer *aer;
+ struct sysctl_oid_list *ioq_oid_list;
+
eventhandler_tag shutdown_pre_sync_eh;
eventhandler_tag shutdown_post_sync_eh;
};
@@ -200,7 +203,7 @@ bool nvmf_update_ns(struct nvmf_namespace *ns,
/* nvmf_qpair.c */
struct nvmf_host_qpair *nvmf_init_qp(struct nvmf_softc *sc,
enum nvmf_trtype trtype, struct nvmf_handoff_qpair_params *handoff,
- const char *name);
+ const char *name, u_int qid);
void nvmf_shutdown_qp(struct nvmf_host_qpair *qp);
void nvmf_destroy_qp(struct nvmf_host_qpair *qp);
struct nvmf_request *nvmf_allocate_request(struct nvmf_host_qpair *qp,