svn commit: r361523 - head/sys/dev/ena

[Marcin Wojtas]

Author: mw
Date: Tue May 26 15:50:30 2020
New Revision: 361523
URL: https://svnweb.freebsd.org/changeset/base/361523

Log:
  Rework ENA Tx buffer ring size reconfiguration
  
  This method has been aligned with the way how the Rx queue size is being
  updated - so it's now done synchronously instead of resetting the
  device.
  
  Moreover, the input parameter is now being validated if it's a power of
  2. Without this, it can cause kernel panic.
  
  Submitted by:  Michal Krawczyk <mk at semihalf.com>
  Obtained from: Semihalf
  Sponsored by:  Amazon, Inc.

Modified:
  head/sys/dev/ena/ena.c
  head/sys/dev/ena/ena.h
  head/sys/dev/ena/ena_sysctl.c

Modified: head/sys/dev/ena/ena.c
==============================================================================
--- head/sys/dev/ena/ena.c	Tue May 26 15:48:27 2020	(r361522)
+++ head/sys/dev/ena/ena.c	Tue May 26 15:50:30 2020	(r361523)
@@ -1135,6 +1135,55 @@ ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t 
 }
 
 int
+ena_update_buf_ring_size(struct ena_adapter *adapter,
+    uint32_t new_buf_ring_size)
+{
+	uint32_t old_buf_ring_size;
+	int rc = 0;
+	bool dev_was_up;
+
+	ENA_LOCK_LOCK(adapter);
+
+	old_buf_ring_size = adapter->buf_ring_size;
+	adapter->buf_ring_size = new_buf_ring_size;
+
+	dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter);
+	ena_down(adapter);
+
+	/* Reconfigure buf ring for all Tx rings. */
+	ena_free_all_io_rings_resources(adapter);
+	ena_init_io_rings_advanced(adapter);
+	if (dev_was_up) {
+		/*
+		 * If ena_up() fails, it's not because of recent buf_ring size
+		 * changes. Because of that, we just want to revert old drbr
+		 * value and trigger the reset because something else had to
+		 * go wrong.
+		 */
+		rc = ena_up(adapter);
+		if (unlikely(rc != 0)) {
+			device_printf(adapter->pdev,
+			    "Failed to configure device after setting new drbr size: %u. Reverting old value: %u and triggering the reset\n",
+			    new_buf_ring_size, old_buf_ring_size);
+
+			/* Revert old size and trigger the reset */
+			adapter->buf_ring_size = old_buf_ring_size;
+			ena_free_all_io_rings_resources(adapter);
+			ena_init_io_rings_advanced(adapter);
+
+			ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET,
+			    adapter);
+			ena_trigger_reset(adapter, ENA_REGS_RESET_OS_TRIGGER);
+
+		}
+	}
+
+	ENA_LOCK_UNLOCK(adapter);
+
+	return (rc);
+}
+
+int
 ena_update_queue_size(struct ena_adapter *adapter, uint32_t new_tx_size,
     uint32_t new_rx_size)
 {

Modified: head/sys/dev/ena/ena.h
==============================================================================
--- head/sys/dev/ena/ena.h	Tue May 26 15:48:27 2020	(r361522)
+++ head/sys/dev/ena/ena.h	Tue May 26 15:50:30 2020	(r361523)
@@ -426,7 +426,7 @@ struct ena_adapter {
 
 	uint32_t tx_offload_cap;
 
-	uint16_t buf_ring_size;
+	uint32_t buf_ring_size;
 
 	/* RSS*/
 	uint8_t	rss_ind_tbl[ENA_RX_RSS_TABLE_SIZE];
@@ -497,6 +497,8 @@ void	ena_down(struct ena_adapter *adapter);
 int	ena_restore_device(struct ena_adapter *adapter);
 void	ena_destroy_device(struct ena_adapter *adapter, bool graceful);
 int	ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num);
+int	ena_update_buf_ring_size(struct ena_adapter *adapter,
+    uint32_t new_buf_ring_size);
 int	ena_update_queue_size(struct ena_adapter *adapter, uint32_t new_tx_size,
     uint32_t new_rx_size);
 

Modified: head/sys/dev/ena/ena_sysctl.c
==============================================================================
--- head/sys/dev/ena/ena_sysctl.c	Tue May 26 15:48:27 2020	(r361522)
+++ head/sys/dev/ena/ena_sysctl.c	Tue May 26 15:50:30 2020	(r361523)
@@ -307,8 +307,9 @@ ena_sysctl_add_tuneables(struct ena_adapter *adapter)
 
 	/* Tuneable number of buffers in the buf-ring (drbr) */
 	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "buf_ring_size",
-	    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, adapter, 0,
-	    ena_sysctl_buf_ring_size, "I", "Size of the bufring");
+	    CTLTYPE_U32 | CTLFLAG_RW | CTLFLAG_MPSAFE, adapter, 0,
+	    ena_sysctl_buf_ring_size, "I",
+	    "Size of the Tx buffer ring (drbr).");
 
 	/* Tuneable number of the Rx ring size */
 	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_queue_size",
@@ -322,31 +323,38 @@ static int
 ena_sysctl_buf_ring_size(SYSCTL_HANDLER_ARGS)
 {
 	struct ena_adapter *adapter = arg1;
-	int val;
+	uint32_t val;
 	int error;
 
 	val = 0;
-	error = sysctl_wire_old_buffer(req, sizeof(int));
+	error = sysctl_wire_old_buffer(req, sizeof(val));
 	if (error == 0) {
 		val = adapter->buf_ring_size;
 		error = sysctl_handle_int(oidp, &val, 0, req);
 	}
 	if (error != 0 || req->newptr == NULL)
 		return (error);
-	if (val < 0)
+
+	if (!powerof2(val) || val == 0) {
+		device_printf(adapter->pdev,
+		    "Requested new Tx buffer ring size (%u) is not a power of 2\n",
+		    val);
 		return (EINVAL);
+	}
 
-	device_printf(adapter->pdev,
-	    "Requested new buf ring size: %d. Old size: %d\n",
-	    val, adapter->buf_ring_size);
-
 	if (val != adapter->buf_ring_size) {
-		adapter->buf_ring_size = val;
-		adapter->reset_reason = ENA_REGS_RESET_OS_TRIGGER;
-		ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
+		device_printf(adapter->pdev,
+		    "Requested new Tx buffer ring size: %d. Old size: %d\n",
+		    val, adapter->buf_ring_size);
+
+		error = ena_update_buf_ring_size(adapter, val);
+	} else {
+		device_printf(adapter->pdev,
+		    "New Tx buffer ring size is the same as already used: %u\n",
+		    adapter->buf_ring_size);
 	}
 
-	return (0);
+	return (error);
 }
 
 static int