svn commit: r351828 - head/sys/dev/nvme
Warner Losh
imp at FreeBSD.org
Wed Sep 4 20:08:37 UTC 2019
Author: imp
Date: Wed Sep 4 20:08:36 2019
New Revision: 351828
URL: https://svnweb.freebsd.org/changeset/base/351828
Log:
Support doorbell strides != 0.
The NVMe standard (1.4) states
>>> 8.6 Doorbell Stride for Software Emulation
>>> The doorbell stride,...is useful in software emulation of an NVM
>>> Express controller. ... For hardware implementations of the NVM
>>> Express interface, the expected doorbell stride value is 0h.
However, hardware in the wild exists with a doorbell stride of 1
(meaning 8 byte separation). This change supports that hardware, as
well as software emulators as envisioned in Section 8.6. Since this is
the fast path, care has been taken to make this computation
efficient. The bit of math to compute an offset for each is replaced
by a memory load from cache of a pre-computed value.
MFC After: 3 days
Reviewed by: scottl@
Differential Revision: https://reviews.freebsd.org/D21514
Modified:
head/sys/dev/nvme/nvme_ctrlr.c
head/sys/dev/nvme/nvme_private.h
head/sys/dev/nvme/nvme_qpair.c
Modified: head/sys/dev/nvme/nvme_ctrlr.c
==============================================================================
--- head/sys/dev/nvme/nvme_ctrlr.c Wed Sep 4 19:32:50 2019 (r351827)
+++ head/sys/dev/nvme/nvme_ctrlr.c Wed Sep 4 20:08:36 2019 (r351828)
@@ -90,19 +90,25 @@ nvme_ctrlr_construct_io_qpairs(struct nvme_controller
struct nvme_qpair *qpair;
uint32_t cap_lo;
uint16_t mqes;
- int i, error, num_entries, num_trackers;
+ int i, error, num_entries, num_trackers, max_entries;
- num_entries = NVME_IO_ENTRIES;
- TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries);
-
/*
- * NVMe spec sets a hard limit of 64K max entries, but
- * devices may specify a smaller limit, so we need to check
- * the MQES field in the capabilities register.
+ * NVMe spec sets a hard limit of 64K max entries, but devices may
+ * specify a smaller limit, so we need to check the MQES field in the
+ * capabilities register. We have to cap the number of entries to the
+ * current stride allows for in BAR 0/1, otherwise the remainder entries
+ * are inaccessable. MQES should reflect this, and this is just a
+ * fail-safe.
*/
+ max_entries =
+ (rman_get_size(ctrlr->resource) - nvme_mmio_offsetof(doorbell[0])) /
+ (1 << (ctrlr->dstrd + 1));
+ num_entries = NVME_IO_ENTRIES;
+ TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries);
cap_lo = nvme_mmio_read_4(ctrlr, cap_lo);
mqes = NVME_CAP_LO_MQES(cap_lo);
num_entries = min(num_entries, mqes + 1);
+ num_entries = min(num_entries, max_entries);
num_trackers = NVME_IO_TRACKERS;
TUNABLE_INT_FETCH("hw.nvme.io_trackers", &num_trackers);
@@ -110,9 +116,9 @@ nvme_ctrlr_construct_io_qpairs(struct nvme_controller
num_trackers = max(num_trackers, NVME_MIN_IO_TRACKERS);
num_trackers = min(num_trackers, NVME_MAX_IO_TRACKERS);
/*
- * No need to have more trackers than entries in the submit queue.
- * Note also that for a queue size of N, we can only have (N-1)
- * commands outstanding, hence the "-1" here.
+ * No need to have more trackers than entries in the submit queue. Note
+ * also that for a queue size of N, we can only have (N-1) commands
+ * outstanding, hence the "-1" here.
*/
num_trackers = min(num_trackers, (num_entries-1));
@@ -1120,7 +1126,6 @@ nvme_ctrlr_construct(struct nvme_controller *ctrlr, de
uint32_t cap_lo;
uint32_t cap_hi;
uint32_t to;
- uint8_t dstrd;
uint8_t mpsmin;
int status, timeout_period;
@@ -1128,14 +1133,8 @@ nvme_ctrlr_construct(struct nvme_controller *ctrlr, de
mtx_init(&ctrlr->lock, "nvme ctrlr lock", NULL, MTX_DEF);
- /*
- * Software emulators may set the doorbell stride to something
- * other than zero, but this driver is not set up to handle that.
- */
cap_hi = nvme_mmio_read_4(ctrlr, cap_hi);
- dstrd = NVME_CAP_HI_DSTRD(cap_hi);
- if (dstrd != 0)
- return (ENXIO);
+ ctrlr->dstrd = NVME_CAP_HI_DSTRD(cap_hi) + 2;
mpsmin = NVME_CAP_HI_MPSMIN(cap_hi);
ctrlr->min_page_size = 1 << (12 + mpsmin);
Modified: head/sys/dev/nvme/nvme_private.h
==============================================================================
--- head/sys/dev/nvme/nvme_private.h Wed Sep 4 19:32:50 2019 (r351827)
+++ head/sys/dev/nvme/nvme_private.h Wed Sep 4 20:08:36 2019 (r351828)
@@ -297,6 +297,9 @@ struct nvme_controller {
/** timeout period in seconds */
uint32_t timeout_period;
+ /** doorbell stride */
+ uint32_t dstrd;
+
struct nvme_qpair adminq;
struct nvme_qpair *ioq;
Modified: head/sys/dev/nvme/nvme_qpair.c
==============================================================================
--- head/sys/dev/nvme/nvme_qpair.c Wed Sep 4 19:32:50 2019 (r351827)
+++ head/sys/dev/nvme/nvme_qpair.c Wed Sep 4 20:08:36 2019 (r351828)
@@ -622,8 +622,8 @@ nvme_qpair_process_completions(struct nvme_qpair *qpai
qpair->phase = !qpair->phase; /* 3 */
}
- nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].cq_hdbl,
- qpair->cq_head);
+ bus_space_write_4(qpair->ctrlr->bus_tag, qpair->ctrlr->bus_handle,
+ qpair->cq_hdbl_off, qpair->cq_head);
}
return (done != 0);
}
@@ -731,8 +731,15 @@ nvme_qpair_construct(struct nvme_qpair *qpair, uint32_
qpair->cpl_bus_addr = queuemem_phys + cmdsz;
prpmem_phys = queuemem_phys + cmdsz + cplsz;
- qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[id].sq_tdbl);
- qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[id].cq_hdbl);
+ /*
+ * Calcuate the stride of the doorbell register. Many emulators set this
+ * value to correspond to a cache line. However, some hardware has set
+ * it to various small values.
+ */
+ qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[0]) +
+ (id << (ctrlr->dstrd + 1));
+ qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[0]) +
+ (id << (ctrlr->dstrd + 1)) + (1 << ctrlr->dstrd);
TAILQ_INIT(&qpair->free_tr);
TAILQ_INIT(&qpair->outstanding_tr);
@@ -950,9 +957,8 @@ nvme_qpair_submit_tracker(struct nvme_qpair *qpair, st
wmb();
#endif
- nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].sq_tdbl,
- qpair->sq_tail);
-
+ bus_space_write_4(qpair->ctrlr->bus_tag, qpair->ctrlr->bus_handle,
+ qpair->sq_tdbl_off, qpair->sq_tail);
qpair->num_cmds++;
}
More information about the svn-src-head
mailing list