git: 7f46deccbed7 - main - x86/iommu: Reduce the number of queued invalidation interrupts

The branch main has been updated by alc:

URL: https://cgit.FreeBSD.org/src/commit/?id=7f46deccbed74436b62f8fd02655ff4ad89f1023

commit 7f46deccbed74436b62f8fd02655ff4ad89f1023
Author:     Alan Cox <alc@FreeBSD.org>
AuthorDate: 2022-07-31 19:28:30 +0000
Commit:     Alan Cox <alc@FreeBSD.org>
CommitDate: 2022-08-06 18:05:58 +0000

    x86/iommu: Reduce the number of queued invalidation interrupts
    
    Restructure dmar_qi_task() so as to reduce the number of invalidation
    completion interrupts.  Specifically, because processing completed
    invalidations in dmar_qi_task() can take quite some time, don't reenable
    completion interrupts until processing has completed a first time. Then,
    check a second time after reenabling completion interrupts, so that
    any invalidations that complete just before interrupts are reenabled
    do not linger until a future invalidation might raise an interrupt.
    (Recent changes have made checking for completed invalidations cheap; no
    locking is required.)
    
    Reviewed by:    kib
    MFC after:      1 week
    Differential Revision:  https://reviews.freebsd.org/D36054
---
 sys/x86/iommu/intel_qi.c | 45 +++++++++++++++++++++++++++++----------------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/sys/x86/iommu/intel_qi.c b/sys/x86/iommu/intel_qi.c
index baaf5b472a2c..8a8e656083e3 100644
--- a/sys/x86/iommu/intel_qi.c
+++ b/sys/x86/iommu/intel_qi.c
@@ -411,14 +411,34 @@ dmar_qi_intr(void *arg)
 	return (FILTER_HANDLED);
 }
 
+static void
+dmar_qi_drain_tlb_flush(struct dmar_unit *unit)
+{
+	struct iommu_map_entry *entry, *head;
+
+	for (head = unit->tlb_flush_head;; head = entry) {
+		entry = (struct iommu_map_entry *)
+		    atomic_load_acq_ptr((uintptr_t *)&head->tlb_flush_next);
+		if (entry == NULL ||
+		    !dmar_qi_seq_processed(unit, &entry->gseq))
+			break;
+		unit->tlb_flush_head = entry;
+		iommu_gas_free_entry(head);
+		if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0)
+			iommu_gas_free_region(entry);
+		else
+			iommu_gas_free_space(entry);
+	}
+}
+
 static void
 dmar_qi_task(void *arg, int pending __unused)
 {
 	struct dmar_unit *unit;
-	struct iommu_map_entry *entry, *head;
 	uint32_t ics;
 
 	unit = arg;
+	dmar_qi_drain_tlb_flush(unit);
 
 	/*
 	 * Request an interrupt on the completion of the next invalidation
@@ -428,23 +448,16 @@ dmar_qi_task(void *arg, int pending __unused)
 	if ((ics & DMAR_ICS_IWC) != 0) {
 		ics = DMAR_ICS_IWC;
 		dmar_write4(unit, DMAR_ICS_REG, ics);
-	}
 
-	for (;;) {
-		head = unit->tlb_flush_head;
-		entry = (struct iommu_map_entry *)
-		    atomic_load_acq_ptr((uintptr_t *)&head->tlb_flush_next);
-		if (entry == NULL)
-			break;
-		if (!dmar_qi_seq_processed(unit, &entry->gseq))
-			break;
-		unit->tlb_flush_head = entry;
-		iommu_gas_free_entry(head);
-		if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0)
-			iommu_gas_free_region(entry);
-		else
-			iommu_gas_free_space(entry);
+		/*
+		 * Drain a second time in case the DMAR processes an entry
+		 * after the first call and before clearing DMAR_ICS_IWC.
+		 * Otherwise, such entries will linger until a later entry
+		 * that requests an interrupt is processed.
+		 */
+		dmar_qi_drain_tlb_flush(unit);
 	}
+
 	if (unit->inv_seq_waiters > 0) {
 		/*
 		 * Acquire the DMAR lock so that wakeup() is called only after