svn commit: r347374 - stable/11/sys/amd64/vmm/io

[John Baldwin]

Author: jhb
Date: Wed May  8 23:24:47 2019
New Revision: 347374
URL: https://svnweb.freebsd.org/changeset/base/347374

Log:
  MFC 333174: Use PCI power-mgmt to reset a device if FLR fails.
  
  A large number of devices don't support PCIe FLR, in particular
  graphics adapters. Use PCI power management to perform the
  reset if FLR fails or isn't available, by cycling the device
  through the D3 state.
  
  This has been tested by a number of users with Nvidia and AMD GPUs.

Modified:
  stable/11/sys/amd64/vmm/io/ppt.c
Directory Properties:
  stable/11/   (props changed)

Modified: stable/11/sys/amd64/vmm/io/ppt.c
==============================================================================
--- stable/11/sys/amd64/vmm/io/ppt.c	Wed May  8 23:13:27 2019	(r347373)
+++ stable/11/sys/amd64/vmm/io/ppt.c	Wed May  8 23:24:47 2019	(r347374)
@@ -351,6 +351,30 @@ ppt_is_mmio(struct vm *vm, vm_paddr_t gpa)
 	return (FALSE);
 }
 
+static void
+ppt_pci_reset(device_t dev)
+{
+	int ps;
+
+	if (pcie_flr(dev,
+		     max(pcie_get_max_completion_timeout(dev) / 1000, 10),
+		     true))
+		return;
+
+	/*
+	 * If FLR fails, attempt a power-management reset by cycling
+	 * the device in/out of D3 state.
+	 * PCI spec says we can only go into D3 state from D0 state.
+	 * Transition from D[12] into D0 before going to D3 state.
+	 */
+	ps = pci_get_powerstate(dev);
+	if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
+		pci_set_powerstate(dev, PCI_POWERSTATE_D0);
+	if (pci_get_powerstate(dev) != PCI_POWERSTATE_D3)
+		pci_set_powerstate(dev, PCI_POWERSTATE_D3);
+	pci_set_powerstate(dev, ps);
+}
+
 int
 ppt_assign_device(struct vm *vm, int bus, int slot, int func)
 {
@@ -366,9 +390,7 @@ ppt_assign_device(struct vm *vm, int bus, int slot, in
 			return (EBUSY);
 
 		pci_save_state(ppt->dev);
-		pcie_flr(ppt->dev,
-		    max(pcie_get_max_completion_timeout(ppt->dev) / 1000, 10),
-		    true);
+		ppt_pci_reset(ppt->dev);
 		pci_restore_state(ppt->dev);
 		ppt->vm = vm;
 		iommu_add_device(vm_iommu_domain(vm), pci_get_rid(ppt->dev));
@@ -391,9 +413,7 @@ ppt_unassign_device(struct vm *vm, int bus, int slot, 
 			return (EBUSY);
 
 		pci_save_state(ppt->dev);
-		pcie_flr(ppt->dev,
-		    max(pcie_get_max_completion_timeout(ppt->dev) / 1000, 10),
-		    true);
+		ppt_pci_reset(ppt->dev);
 		pci_restore_state(ppt->dev);
 		ppt_unmap_mmio(vm, ppt);
 		ppt_teardown_msi(ppt);