svn commit: r352883 - stable/12/sys/dev/nvme

[Warner Losh]

Author: imp
Date: Mon Sep 30 01:59:27 2019
New Revision: 352883
URL: https://svnweb.freebsd.org/changeset/base/352883

Log:
  MFC r351828:
  
    Support doorbell strides != 0.

Modified:
  stable/12/sys/dev/nvme/nvme_ctrlr.c
  stable/12/sys/dev/nvme/nvme_private.h
  stable/12/sys/dev/nvme/nvme_qpair.c
Directory Properties:
  stable/12/   (props changed)

Modified: stable/12/sys/dev/nvme/nvme_ctrlr.c
==============================================================================
--- stable/12/sys/dev/nvme/nvme_ctrlr.c	Mon Sep 30 01:25:37 2019	(r352882)
+++ stable/12/sys/dev/nvme/nvme_ctrlr.c	Mon Sep 30 01:59:27 2019	(r352883)
@@ -90,19 +90,25 @@ nvme_ctrlr_construct_io_qpairs(struct nvme_controller 
 	struct nvme_qpair	*qpair;
 	uint32_t		cap_lo;
 	uint16_t		mqes;
-	int			i, error, num_entries, num_trackers;
+	int			i, error, num_entries, num_trackers, max_entries;
 
-	num_entries = NVME_IO_ENTRIES;
-	TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries);
-
 	/*
-	 * NVMe spec sets a hard limit of 64K max entries, but
-	 *  devices may specify a smaller limit, so we need to check
-	 *  the MQES field in the capabilities register.
+	 * NVMe spec sets a hard limit of 64K max entries, but devices may
+	 * specify a smaller limit, so we need to check the MQES field in the
+	 * capabilities register. We have to cap the number of entries to the
+	 * current stride allows for in BAR 0/1, otherwise the remainder entries
+	 * are inaccessable. MQES should reflect this, and this is just a
+	 * fail-safe.
 	 */
+	max_entries =
+	    (rman_get_size(ctrlr->resource) - nvme_mmio_offsetof(doorbell[0])) /
+	    (1 << (ctrlr->dstrd + 1));
+	num_entries = NVME_IO_ENTRIES;
+	TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries);
 	cap_lo = nvme_mmio_read_4(ctrlr, cap_lo);
 	mqes = NVME_CAP_LO_MQES(cap_lo);
 	num_entries = min(num_entries, mqes + 1);
+	num_entries = min(num_entries, max_entries);
 
 	num_trackers = NVME_IO_TRACKERS;
 	TUNABLE_INT_FETCH("hw.nvme.io_trackers", &num_trackers);
@@ -110,9 +116,9 @@ nvme_ctrlr_construct_io_qpairs(struct nvme_controller 
 	num_trackers = max(num_trackers, NVME_MIN_IO_TRACKERS);
 	num_trackers = min(num_trackers, NVME_MAX_IO_TRACKERS);
 	/*
-	 * No need to have more trackers than entries in the submit queue.
-	 *  Note also that for a queue size of N, we can only have (N-1)
-	 *  commands outstanding, hence the "-1" here.
+	 * No need to have more trackers than entries in the submit queue.  Note
+	 * also that for a queue size of N, we can only have (N-1) commands
+	 * outstanding, hence the "-1" here.
 	 */
 	num_trackers = min(num_trackers, (num_entries-1));
 
@@ -1120,7 +1126,6 @@ nvme_ctrlr_construct(struct nvme_controller *ctrlr, de
 	uint32_t	cap_lo;
 	uint32_t	cap_hi;
 	uint32_t	to;
-	uint8_t		dstrd;
 	uint8_t		mpsmin;
 	int		status, timeout_period;
 
@@ -1128,14 +1133,8 @@ nvme_ctrlr_construct(struct nvme_controller *ctrlr, de
 
 	mtx_init(&ctrlr->lock, "nvme ctrlr lock", NULL, MTX_DEF);
 
-	/*
-	 * Software emulators may set the doorbell stride to something
-	 *  other than zero, but this driver is not set up to handle that.
-	 */
 	cap_hi = nvme_mmio_read_4(ctrlr, cap_hi);
-	dstrd = NVME_CAP_HI_DSTRD(cap_hi);
-	if (dstrd != 0)
-		return (ENXIO);
+	ctrlr->dstrd = NVME_CAP_HI_DSTRD(cap_hi) + 2;
 
 	mpsmin = NVME_CAP_HI_MPSMIN(cap_hi);
 	ctrlr->min_page_size = 1 << (12 + mpsmin);

Modified: stable/12/sys/dev/nvme/nvme_private.h
==============================================================================
--- stable/12/sys/dev/nvme/nvme_private.h	Mon Sep 30 01:25:37 2019	(r352882)
+++ stable/12/sys/dev/nvme/nvme_private.h	Mon Sep 30 01:59:27 2019	(r352883)
@@ -297,6 +297,9 @@ struct nvme_controller {
 	/** timeout period in seconds */
 	uint32_t		timeout_period;
 
+	/** doorbell stride */
+	uint32_t		dstrd;
+
 	struct nvme_qpair	adminq;
 	struct nvme_qpair	*ioq;
 

Modified: stable/12/sys/dev/nvme/nvme_qpair.c
==============================================================================
--- stable/12/sys/dev/nvme/nvme_qpair.c	Mon Sep 30 01:25:37 2019	(r352882)
+++ stable/12/sys/dev/nvme/nvme_qpair.c	Mon Sep 30 01:59:27 2019	(r352883)
@@ -622,8 +622,8 @@ nvme_qpair_process_completions(struct nvme_qpair *qpai
 			qpair->phase = !qpair->phase;			/* 3 */
 		}
 
-		nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].cq_hdbl,
-		    qpair->cq_head);
+		bus_space_write_4(qpair->ctrlr->bus_tag, qpair->ctrlr->bus_handle,
+		    qpair->cq_hdbl_off, qpair->cq_head);
 	}
 	return (done != 0);
 }
@@ -731,8 +731,15 @@ nvme_qpair_construct(struct nvme_qpair *qpair, uint32_
 	qpair->cpl_bus_addr = queuemem_phys + cmdsz;
 	prpmem_phys = queuemem_phys + cmdsz + cplsz;
 
-	qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[id].sq_tdbl);
-	qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[id].cq_hdbl);
+	/*
+	 * Calcuate the stride of the doorbell register. Many emulators set this
+	 * value to correspond to a cache line. However, some hardware has set
+	 * it to various small values.
+	 */
+	qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[0]) +
+	    (id << (ctrlr->dstrd + 1));
+	qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[0]) +
+	    (id << (ctrlr->dstrd + 1)) + (1 << ctrlr->dstrd);
 
 	TAILQ_INIT(&qpair->free_tr);
 	TAILQ_INIT(&qpair->outstanding_tr);
@@ -950,9 +957,8 @@ nvme_qpair_submit_tracker(struct nvme_qpair *qpair, st
 	wmb();
 #endif
 
-	nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].sq_tdbl,
-	    qpair->sq_tail);
-
+	bus_space_write_4(qpair->ctrlr->bus_tag, qpair->ctrlr->bus_handle,
+	    qpair->sq_tdbl_off, qpair->sq_tail);
 	qpair->num_cmds++;
 }