svn commit: r330669 - stable/11/sys/dev/nvme

[Alexander Motin]

Author: mav
Date: Thu Mar  8 21:33:52 2018
New Revision: 330669
URL: https://svnweb.freebsd.org/changeset/base/330669

Log:
  MFC r328521 (by imp):
  Use atomic load and stores to ensure that the compiler doesn't
  optimize away these loops. Change boolean to int to match what atomic
  API supplies. Remove wmb() since the atomic_store_rel() on status.done
  ensure the prior writes to status. It also fixes the fact that there
  wasn't a rmb() before reading done. This should also be more efficient
  since wmb() is fairly heavy weight.

Modified:
  stable/11/sys/dev/nvme/nvme.c
  stable/11/sys/dev/nvme/nvme_ctrlr.c
  stable/11/sys/dev/nvme/nvme_private.h
Directory Properties:
  stable/11/   (props changed)

Modified: stable/11/sys/dev/nvme/nvme.c
==============================================================================
--- stable/11/sys/dev/nvme/nvme.c	Thu Mar  8 19:24:10 2018	(r330668)
+++ stable/11/sys/dev/nvme/nvme.c	Thu Mar  8 21:33:52 2018	(r330669)
@@ -467,6 +467,5 @@ nvme_completion_poll_cb(void *arg, const struct nvme_c
 	 *  the request passed or failed.
 	 */
 	memcpy(&status->cpl, cpl, sizeof(*cpl));
-	wmb();
-	status->done = TRUE;
+	atomic_store_rel_int(&status->done, 1);
 }

Modified: stable/11/sys/dev/nvme/nvme_ctrlr.c
==============================================================================
--- stable/11/sys/dev/nvme/nvme_ctrlr.c	Thu Mar  8 19:24:10 2018	(r330668)
+++ stable/11/sys/dev/nvme/nvme_ctrlr.c	Thu Mar  8 21:33:52 2018	(r330669)
@@ -402,10 +402,10 @@ nvme_ctrlr_identify(struct nvme_controller *ctrlr)
 {
 	struct nvme_completion_poll_status	status;
 
-	status.done = FALSE;
+	status.done = 0;
 	nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata,
 	    nvme_completion_poll_cb, &status);
-	while (status.done == FALSE)
+	while (!atomic_load_acq_int(&status.done))
 		pause("nvme", 1);
 	if (nvme_completion_is_error(&status.cpl)) {
 		nvme_printf(ctrlr, "nvme_identify_controller failed!\n");
@@ -429,10 +429,10 @@ nvme_ctrlr_set_num_qpairs(struct nvme_controller *ctrl
 	struct nvme_completion_poll_status	status;
 	int					cq_allocated, sq_allocated;
 
-	status.done = FALSE;
+	status.done = 0;
 	nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->num_io_queues,
 	    nvme_completion_poll_cb, &status);
-	while (status.done == FALSE)
+	while (!atomic_load_acq_int(&status.done))
 		pause("nvme", 1);
 	if (nvme_completion_is_error(&status.cpl)) {
 		nvme_printf(ctrlr, "nvme_ctrlr_set_num_qpairs failed!\n");
@@ -468,20 +468,20 @@ nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr
 	for (i = 0; i < ctrlr->num_io_queues; i++) {
 		qpair = &ctrlr->ioq[i];
 
-		status.done = FALSE;
+		status.done = 0;
 		nvme_ctrlr_cmd_create_io_cq(ctrlr, qpair, qpair->vector,
 		    nvme_completion_poll_cb, &status);
-		while (status.done == FALSE)
+		while (!atomic_load_acq_int(&status.done))
 			pause("nvme", 1);
 		if (nvme_completion_is_error(&status.cpl)) {
 			nvme_printf(ctrlr, "nvme_create_io_cq failed!\n");
 			return (ENXIO);
 		}
 
-		status.done = FALSE;
+		status.done = 0;
 		nvme_ctrlr_cmd_create_io_sq(qpair->ctrlr, qpair,
 		    nvme_completion_poll_cb, &status);
-		while (status.done == FALSE)
+		while (!atomic_load_acq_int(&status.done))
 			pause("nvme", 1);
 		if (nvme_completion_is_error(&status.cpl)) {
 			nvme_printf(ctrlr, "nvme_create_io_sq failed!\n");
@@ -691,10 +691,10 @@ nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr
 	ctrlr->async_event_config.raw = 0xFF;
 	ctrlr->async_event_config.bits.reserved = 0;
 
-	status.done = FALSE;
+	status.done = 0;
 	nvme_ctrlr_cmd_get_feature(ctrlr, NVME_FEAT_TEMPERATURE_THRESHOLD,
 	    0, NULL, 0, nvme_completion_poll_cb, &status);
-	while (status.done == FALSE)
+	while (!atomic_load_acq_int(&status.done))
 		pause("nvme", 1);
 	if (nvme_completion_is_error(&status.cpl) ||
 	    (status.cpl.cdw0 & 0xFFFF) == 0xFFFF ||

Modified: stable/11/sys/dev/nvme/nvme_private.h
==============================================================================
--- stable/11/sys/dev/nvme/nvme_private.h	Thu Mar  8 19:24:10 2018	(r330668)
+++ stable/11/sys/dev/nvme/nvme_private.h	Thu Mar  8 21:33:52 2018	(r330669)
@@ -126,7 +126,7 @@ extern int32_t		nvme_retry_count;
 struct nvme_completion_poll_status {
 
 	struct nvme_completion	cpl;
-	boolean_t		done;
+	int			done;
 };
 
 #define NVME_REQUEST_VADDR	1