svn commit: r327528 - in head: share/man/man4 sys/dev/cxgbe

[Navdeep Parhar]

Author: np
Date: Wed Jan  3 19:24:57 2018
New Revision: 327528
URL: https://svnweb.freebsd.org/changeset/base/327528

Log:
  cxgbe(4): Add a knob to enable/disable PCIe relaxed ordering.  Disable it by
  default when running on Intel CPUs.
  
  This is a crude fix for the performance issues alluded to in these Linux commits:
  https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=87e09cdec4dae08acdb4aa49beb793c19d73e73e
  https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=a99b646afa8a02571ea298bedca6592d818229cd
  
  MFC after:	1 week
  Sponsored by:	Chelsio Communications

Modified:
  head/share/man/man4/cxgbe.4
  head/sys/dev/cxgbe/t4_main.c

Modified: head/share/man/man4/cxgbe.4
==============================================================================
--- head/share/man/man4/cxgbe.4	Wed Jan  3 19:24:21 2018	(r327527)
+++ head/share/man/man4/cxgbe.4	Wed Jan  3 19:24:57 2018	(r327528)
@@ -243,6 +243,13 @@ Permitted interrupt types.
 Bit 0 represents INTx (line interrupts), bit 1 MSI, and bit 2 MSI-X.
 The default is 7 (all allowed).
 The driver selects the best possible type out of the allowed types.
+.It Va hw.cxgbe.pcie_relaxed_ordering
+PCIe Relaxed Ordering.
+-1 indicates the driver should determine whether to enable or disable PCIe RO.
+0 disables PCIe RO.
+1 enables PCIe RO.
+2 indicates the driver should not modify the PCIe RO setting.
+The default is -1.
 .It Va hw.cxgbe.fw_install
 0 prohibits the driver from installing a firmware on the card.
 1 allows the driver to install a new firmware if internal driver

Modified: head/sys/dev/cxgbe/t4_main.c
==============================================================================
--- head/sys/dev/cxgbe/t4_main.c	Wed Jan  3 19:24:21 2018	(r327527)
+++ head/sys/dev/cxgbe/t4_main.c	Wed Jan  3 19:24:57 2018	(r327528)
@@ -63,6 +63,8 @@ __FBSDID("$FreeBSD$");
 #include <net/rss_config.h>
 #endif
 #if defined(__i386__) || defined(__amd64__)
+#include <machine/md_var.h>
+#include <machine/cputypes.h>
 #include <vm/vm.h>
 #include <vm/pmap.h>
 #endif
@@ -454,7 +456,17 @@ TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine)
 
 static int t4_num_vis = 1;
 TUNABLE_INT("hw.cxgbe.num_vis", &t4_num_vis);
+/*
+ * PCIe Relaxed Ordering.
+ * -1: driver should figure out a good value.
+ * 0: disable RO.
+ * 1: enable RO.
+ * 2: leave RO alone.
+ */
+static int pcie_relaxed_ordering = -1;
+TUNABLE_INT("hw.cxgbe.pcie_relaxed_ordering", &pcie_relaxed_ordering);
 
+
 /* Functions used by VIs to obtain unique MAC addresses for each VI. */
 static int vi_mac_funcs[] = {
 	FW_VI_FUNC_ETH,
@@ -856,10 +868,16 @@ t4_attach(device_t dev)
 
 		pci_set_max_read_req(dev, 4096);
 		v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
-		v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
-		pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
-
 		sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
+		if (pcie_relaxed_ordering == 0 &&
+		    (v | PCIEM_CTL_RELAXED_ORD_ENABLE) != 0) {
+			v &= ~PCIEM_CTL_RELAXED_ORD_ENABLE;
+			pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
+		} else if (pcie_relaxed_ordering == 1 &&
+		    (v & PCIEM_CTL_RELAXED_ORD_ENABLE) == 0) {
+			v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
+			pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
+		}
 	}
 
 	sc->sge_gts_reg = MYPF_REG(A_SGE_PF_GTS);
@@ -9961,6 +9979,14 @@ tweak_tunables(void)
 	if (t4_num_vis > nitems(vi_mac_funcs)) {
 		t4_num_vis = nitems(vi_mac_funcs);
 		printf("cxgbe: number of VIs limited to %d\n", t4_num_vis);
+	}
+
+	if (pcie_relaxed_ordering < 0 || pcie_relaxed_ordering > 2) {
+		pcie_relaxed_ordering = 1;
+#if defined(__i386__) || defined(__amd64__)
+		if (cpu_vendor_id == CPU_VENDOR_INTEL)
+			pcie_relaxed_ordering = 0;
+#endif
 	}
 }