mfi(4) patch to add MSI-X support, possibly address command timeouts

[John Baldwin]

I'd like some folks to test a patch to the mfi(4) driver that may help to 
address issues several folks have reported.  The patch does two things, first 
it adds some dummy reads of PCI registers when checking device status in the 
interrupt handler to "flush" the writes to ACK interrupts.  The Linux 
megaraid-sas driver uses this approach and some folks have tested a patch from 
Scott Long which had a somewhat similar effect.  Second, it enables the use of 
MSI-X interrupts for many newer devices.

The patch is available below and at www.freebsd.org/~jhb/patches/mfi.patch

Index: mfi_pci.c
===================================================================
--- mfi_pci.c	(revision 224613)
+++ mfi_pci.c	(working copy)
@@ -169,7 +169,7 @@
 	struct mfi_softc *sc;
 	struct mfi_ident *m;
 	uint32_t command;
-	int error;
+	int count, error;
 
 	sc = device_get_softc(dev);
 	bzero(sc, sizeof(*sc));
@@ -226,6 +226,29 @@
 		goto out;
 	}
 
+	/* Allocate IRQ resource. */
+	sc->mfi_irq_rid = 0;
+	switch (pci_get_device(sc->mfi_dev)) {
+	case 0x0060:	/* SAS1078R */
+	case 0x007c:	/* SAS1078DE */
+	case 0x0413:	/* Verde ZCR */
+		/* Do not use MSI-X for these systems. */
+		break;
+	default:
+		count = 1;
+		if (pci_alloc_msix(sc->mfi_dev, &count) == 0) {
+			device_printf(sc->mfi_dev, "Using MSI-X\n");
+			sc->mfi_irq_rid = 1;
+		}
+		break;
+	}
+	if ((sc->mfi_irq = bus_alloc_resource_any(sc->mfi_dev, SYS_RES_IRQ,
+	    &sc->mfi_irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
+		device_printf(sc->mfi_dev, "Cannot allocate interrupt\n");
+		error = EINVAL;
+		goto out;
+	}
+
 	error = mfi_attach(sc);
 out:
 	if (error) {
@@ -280,6 +303,8 @@
 		bus_release_resource(sc->mfi_dev, SYS_RES_MEMORY,
 		    sc->mfi_regs_rid, sc->mfi_regs_resource);
 	}
+	if (sc->mfi_irq_rid != 0)
+		pci_release_msi(sc->mfi_dev);
 
 	return;
 }
Index: mfi.c
===================================================================
--- mfi.c	(revision 224613)
+++ mfi.c	(working copy)
@@ -157,6 +157,9 @@
 mfi_enable_intr_xscale(struct mfi_softc *sc)
 {
 	MFI_WRITE4(sc, MFI_OMSK, 0x01);
+
+	/* Dummy read to force PCI flush. */
+	(void)MFI_READ4(sc, MFI_OMSK);
 }
 
 static void
@@ -168,6 +171,9 @@
 	} else if (sc->mfi_flags & MFI_FLAGS_GEN2) {
 		MFI_WRITE4(sc, MFI_OMSK, ~MFI_GEN2_EIM);
 	}
+
+	/* Dummy read to force PCI flush. */
+	(void)MFI_READ4(sc, MFI_OMSK);
 }
 
 static int32_t
@@ -192,6 +198,9 @@
 		return 1;
 
 	MFI_WRITE4(sc, MFI_OSTS, status);
+
+	/* Dummy read to force PCI flush. */
+	(void)MFI_READ4(sc, MFI_OSTS);
 	return 0;
 }
 
@@ -212,6 +221,9 @@
 	}
 
 	MFI_WRITE4(sc, MFI_ODCR0, status);
+
+	/* Dummy read to force PCI flush. */
+	(void)MFI_READ4(sc, MFI_OSTS);
 	return 0;
 }
 
@@ -484,15 +496,8 @@
 	mtx_unlock(&sc->mfi_io_lock);
 
 	/*
-	 * Set up the interrupt handler.  XXX This should happen in
-	 * mfi_pci.c
+	 * Set up the interrupt handler.
 	 */
-	sc->mfi_irq_rid = 0;
-	if ((sc->mfi_irq = bus_alloc_resource_any(sc->mfi_dev, SYS_RES_IRQ,
-	    &sc->mfi_irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
-		device_printf(sc->mfi_dev, "Cannot allocate interrupt\n");
-		return (EINVAL);
-	}
 	if (bus_setup_intr(sc->mfi_dev, sc->mfi_irq, INTR_MPSAFE|INTR_TYPE_BIO,
 	    NULL, mfi_intr, sc, &sc->mfi_intr)) {
 		device_printf(sc->mfi_dev, "Cannot set up interrupt\n");

-- 
John Baldwin