svn commit: r319200 - head/sys/dev/ena

[Zbigniew Bodek]

Author: zbb
Date: Tue May 30 11:58:51 2017
New Revision: 319200
URL: https://svnweb.freebsd.org/changeset/base/319200

Log:
  Move ENA's hw stats updating routine to separate task
  
  Initially, stats were being updated each time OS was requesting for
  the first statistic.
  To read statistics from hw, condvar was used. cv_timedwait cannot be
  called when unsleepable lock is held, and this happens when FreeBSD
  is requesting statistic.
  Seperate task is reading statistics from NIC each 1 second.
  
  Submitted by:   Michal Krawczyk <mk at semihalf.com>
  Obtained from:  Semihalf
  Sponsored by:   Amazon.com Inc.
  Differential revision: https://reviews.freebsd.org/D10926

Modified:
  head/sys/dev/ena/ena.c
  head/sys/dev/ena/ena.h

Modified: head/sys/dev/ena/ena.c
==============================================================================
--- head/sys/dev/ena/ena.c	Tue May 30 11:56:54 2017	(r319199)
+++ head/sys/dev/ena/ena.c	Tue May 30 11:58:51 2017	(r319200)
@@ -141,6 +141,7 @@ static void	ena_free_irqs(struct ena_adapter*);
 static void	ena_disable_msix(struct ena_adapter *);
 static void	ena_unmask_all_io_irqs(struct ena_adapter *);
 static int	ena_rss_configure(struct ena_adapter *);
+static void	ena_update_hw_stats(void *, int);
 static int	ena_up_complete(struct ena_adapter *);
 static int	ena_up(struct ena_adapter *);
 static void	ena_down(struct ena_adapter *);
@@ -2058,6 +2059,25 @@ static int ena_rss_configure(struct ena_adapter *adapt
 	return 0;
 }
 
+static void
+ena_update_hw_stats(void *arg, int pending)
+{
+	struct ena_adapter *adapter = arg;
+	int rc;
+
+	for (;;) {
+		if (!adapter->up)
+			return;
+
+		rc = ena_update_stats_counters(adapter);
+		if (rc)
+			ena_trace(ENA_WARNING,
+			    "Error updating stats counters, rc = %d", rc);
+
+		pause("ena update hw stats", hz);
+	}
+}
+
 static int
 ena_up_complete(struct ena_adapter *adapter)
 {
@@ -2141,6 +2161,8 @@ ena_up(struct ena_adapter *adapter)
 		callout_reset_sbt(&adapter->timer_service, SBT_1S, SBT_1S,
 		    ena_timer_service, (void *)adapter, 0);
 
+		taskqueue_enqueue(adapter->stats_tq, &adapter->stats_task);
+
 		adapter->up = true;
 	}
 
@@ -2193,24 +2215,8 @@ ena_get_counter(if_t ifp, ift_counter cnt)
 {
 	struct ena_adapter *adapter;
 	struct ena_hw_stats *stats;
-	int rc;
 
 	adapter = if_getsoftc(ifp);
-
-	/*
-	 * Update only when asking for first counter and interface is up.
-	 * Usually asks for all statistics in sequence.
-	 */
-	if (adapter->up) {
-		if (cnt == 0) {
-			rc = ena_update_stats_counters(adapter);
-			if (rc) {
-				ena_trace(ENA_WARNING,
-				    "Error updating stats counters, rc = %d",
-				    rc);
-			}
-		}
-	}
 	stats = &adapter->hw_stats;
 
 	switch (cnt) {
@@ -2501,6 +2507,10 @@ ena_down(struct ena_adapter *adapter)
 		if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE,
 		    IFF_DRV_RUNNING);
 
+		/* Drain task responsible for updating hw stats */
+		while (taskqueue_cancel(adapter->stats_tq, &adapter->stats_task, NULL))
+			taskqueue_drain(adapter->stats_tq, &adapter->stats_task);
+
 		ena_free_io_irq(adapter);
 
 		ena_destroy_all_io_queues(adapter);
@@ -3616,6 +3626,18 @@ ena_attach(device_t pdev)
 	taskqueue_start_threads(&adapter->reset_tq, 1, PI_NET,
 	    "%s rstq", device_get_nameunit(adapter->pdev));
 
+	/* Initialize task queue responsible for updating hw stats */
+	TASK_INIT(&adapter->stats_task, 0, ena_update_hw_stats, adapter);
+	adapter->stats_tq = taskqueue_create_fast("ena_stats_update",
+	    M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->stats_tq);
+	if (adapter->stats_tq == NULL) {
+		device_printf(adapter->pdev,
+		    "Unable to create taskqueue for updating hw stats\n");
+		goto err_stats_tq;
+	}
+	taskqueue_start_threads(&adapter->stats_tq, 1, PI_REALTIME,
+	    "%s stats tq", device_get_nameunit(adapter->pdev));
+
 	/* Initialize statistics */
 	ena_alloc_counters((counter_u64_t *)&adapter->dev_stats,
 	    sizeof(struct ena_stats_dev));
@@ -3628,6 +3650,8 @@ ena_attach(device_t pdev)
 	adapter->running = true;
 	return (0);
 
+err_stats_tq:
+	taskqueue_free(adapter->reset_tq);
 err_reset_tq:
 	ena_free_mgmnt_irq(adapter);
 	ena_disable_msix(adapter);
@@ -3682,6 +3706,8 @@ ena_detach(device_t pdev)
 	sx_xlock(&adapter->ioctl_sx);
 	ena_down(adapter);
 	sx_unlock(&adapter->ioctl_sx);
+
+	taskqueue_free(adapter->stats_tq);
 
 	if (adapter->ifp != NULL) {
 		ether_ifdetach(adapter->ifp);

Modified: head/sys/dev/ena/ena.h
==============================================================================
--- head/sys/dev/ena/ena.h	Tue May 30 11:56:54 2017	(r319199)
+++ head/sys/dev/ena/ena.h	Tue May 30 11:58:51 2017	(r319200)
@@ -403,6 +403,10 @@ struct ena_adapter {
 	uint32_t missing_tx_max_queues;
 	uint32_t missing_tx_threshold;
 
+	/* Task updating hw stats */
+	struct task stats_task;
+	struct taskqueue *stats_tq;
+
 	/* Statistics */
 	struct ena_stats_dev dev_stats;
 	struct ena_hw_stats hw_stats;