svn commit: r328521 - head/sys/dev/nvme

[Warner Losh]

Author: imp
Date: Mon Jan 29 00:00:52 2018
New Revision: 328521
URL: https://svnweb.freebsd.org/changeset/base/328521

Log:
  Use atomic load and stores to ensure that the compiler doesn't
  optimize away these loops. Change boolean to int to match what atomic
  API supplies. Remove wmb() since the atomic_store_rel() on status.done
  ensure the prior writes to status. It also fixes the fact that there
  wasn't a rmb() before reading done. This should also be more efficient
  since wmb() is fairly heavy weight.
  
  Sponsored by: Netflix
  Reviewed by: kib@, jim harris
  Differential Revision: https://reviews.freebsd.org/D14053

Modified:
  head/sys/dev/nvme/nvme.c
  head/sys/dev/nvme/nvme_ctrlr.c
  head/sys/dev/nvme/nvme_private.h

Modified: head/sys/dev/nvme/nvme.c
==============================================================================
--- head/sys/dev/nvme/nvme.c	Sun Jan 28 23:58:22 2018	(r328520)
+++ head/sys/dev/nvme/nvme.c	Mon Jan 29 00:00:52 2018	(r328521)
@@ -469,6 +469,5 @@ nvme_completion_poll_cb(void *arg, const struct nvme_c
 	 *  the request passed or failed.
 	 */
 	memcpy(&status->cpl, cpl, sizeof(*cpl));
-	wmb();
-	status->done = TRUE;
+	atomic_store_rel_int(&status->done, 1);
 }

Modified: head/sys/dev/nvme/nvme_ctrlr.c
==============================================================================
--- head/sys/dev/nvme/nvme_ctrlr.c	Sun Jan 28 23:58:22 2018	(r328520)
+++ head/sys/dev/nvme/nvme_ctrlr.c	Mon Jan 29 00:00:52 2018	(r328521)
@@ -404,10 +404,10 @@ nvme_ctrlr_identify(struct nvme_controller *ctrlr)
 {
 	struct nvme_completion_poll_status	status;
 
-	status.done = FALSE;
+	status.done = 0;
 	nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata,
 	    nvme_completion_poll_cb, &status);
-	while (status.done == FALSE)
+	while (!atomic_load_acq_int(&status.done))
 		pause("nvme", 1);
 	if (nvme_completion_is_error(&status.cpl)) {
 		nvme_printf(ctrlr, "nvme_identify_controller failed!\n");
@@ -431,10 +431,10 @@ nvme_ctrlr_set_num_qpairs(struct nvme_controller *ctrl
 	struct nvme_completion_poll_status	status;
 	int					cq_allocated, sq_allocated;
 
-	status.done = FALSE;
+	status.done = 0;
 	nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->num_io_queues,
 	    nvme_completion_poll_cb, &status);
-	while (status.done == FALSE)
+	while (!atomic_load_acq_int(&status.done))
 		pause("nvme", 1);
 	if (nvme_completion_is_error(&status.cpl)) {
 		nvme_printf(ctrlr, "nvme_ctrlr_set_num_qpairs failed!\n");
@@ -470,20 +470,20 @@ nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr
 	for (i = 0; i < ctrlr->num_io_queues; i++) {
 		qpair = &ctrlr->ioq[i];
 
-		status.done = FALSE;
+		status.done = 0;
 		nvme_ctrlr_cmd_create_io_cq(ctrlr, qpair, qpair->vector,
 		    nvme_completion_poll_cb, &status);
-		while (status.done == FALSE)
+		while (!atomic_load_acq_int(&status.done))
 			pause("nvme", 1);
 		if (nvme_completion_is_error(&status.cpl)) {
 			nvme_printf(ctrlr, "nvme_create_io_cq failed!\n");
 			return (ENXIO);
 		}
 
-		status.done = FALSE;
+		status.done = 0;
 		nvme_ctrlr_cmd_create_io_sq(qpair->ctrlr, qpair,
 		    nvme_completion_poll_cb, &status);
-		while (status.done == FALSE)
+		while (!atomic_load_acq_int(&status.done))
 			pause("nvme", 1);
 		if (nvme_completion_is_error(&status.cpl)) {
 			nvme_printf(ctrlr, "nvme_create_io_sq failed!\n");
@@ -693,10 +693,10 @@ nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr
 	ctrlr->async_event_config.raw = 0xFF;
 	ctrlr->async_event_config.bits.reserved = 0;
 
-	status.done = FALSE;
+	status.done = 0;
 	nvme_ctrlr_cmd_get_feature(ctrlr, NVME_FEAT_TEMPERATURE_THRESHOLD,
 	    0, NULL, 0, nvme_completion_poll_cb, &status);
-	while (status.done == FALSE)
+	while (!atomic_load_acq_int(&status.done))
 		pause("nvme", 1);
 	if (nvme_completion_is_error(&status.cpl) ||
 	    (status.cpl.cdw0 & 0xFFFF) == 0xFFFF ||

Modified: head/sys/dev/nvme/nvme_private.h
==============================================================================
--- head/sys/dev/nvme/nvme_private.h	Sun Jan 28 23:58:22 2018	(r328520)
+++ head/sys/dev/nvme/nvme_private.h	Mon Jan 29 00:00:52 2018	(r328521)
@@ -128,7 +128,7 @@ extern int32_t		nvme_retry_count;
 struct nvme_completion_poll_status {
 
 	struct nvme_completion	cpl;
-	boolean_t		done;
+	int			done;
 };
 
 #define NVME_REQUEST_VADDR	1