svn commit: r320125 - head/sys/x86/iommu
Konstantin Belousov
kib at FreeBSD.org
Mon Jun 19 21:48:53 UTC 2017
Author: kib
Date: Mon Jun 19 21:48:52 2017
New Revision: 320125
URL: https://svnweb.freebsd.org/changeset/base/320125
Log:
Fix batched unload for DMAR busdma in qi mode.
Do not queue dmar_map_entries with zeroed gseq to
dmar_qi_invalidate_locked(). Zero gseq stops the processing in the qi
task. Do not assign possibly uninitialized on-stack gseq to map
entries when requeuing them on unit tlb_flush queue. Random garbage
in gsec is interpreted as too high invalidation sequence number and
again stop the processing in the task.
Make the sequence numbers generation completely contained in
dmar_qi_invalidate_locked() and dmar_qi_emit_wait_seq(). Upper code
directly passes boolean requesting emiting wait command instead of
trying to provide hint to avoid it by passing NULL gseq pointer.
Microoptimize the requeueing to tlb_flush queue by doing it for the
whole queue.
Diagnosed and tested by: Brett Gutstein <bgutstein at rice.edu>
Discussed with: alc
Sponsored by: The FreeBSD Foundation
MFC after: 1 week
Modified:
head/sys/x86/iommu/intel_ctx.c
head/sys/x86/iommu/intel_dmar.h
head/sys/x86/iommu/intel_qi.c
Modified: head/sys/x86/iommu/intel_ctx.c
==============================================================================
--- head/sys/x86/iommu/intel_ctx.c Mon Jun 19 21:09:50 2017 (r320124)
+++ head/sys/x86/iommu/intel_ctx.c Mon Jun 19 21:48:52 2017 (r320125)
@@ -703,7 +703,7 @@ dmar_domain_unload_entry(struct dmar_map_entry *entry,
if (unit->qi_enabled) {
DMAR_LOCK(unit);
dmar_qi_invalidate_locked(entry->domain, entry->start,
- entry->end - entry->start, &entry->gseq);
+ entry->end - entry->start, &entry->gseq, true);
if (!free)
entry->flags |= DMAR_MAP_ENTRY_QI_NF;
TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link);
@@ -715,16 +715,14 @@ dmar_domain_unload_entry(struct dmar_map_entry *entry,
}
}
-static struct dmar_qi_genseq *
-dmar_domain_unload_gseq(struct dmar_domain *domain,
- struct dmar_map_entry *entry, struct dmar_qi_genseq *gseq)
+static bool
+dmar_domain_unload_emit_wait(struct dmar_domain *domain,
+ struct dmar_map_entry *entry)
{
- if (TAILQ_NEXT(entry, dmamap_link) != NULL)
- return (NULL);
- if (domain->batch_no++ % dmar_batch_coalesce != 0)
- return (NULL);
- return (gseq);
+ if (TAILQ_NEXT(entry, dmamap_link) == NULL)
+ return (true);
+ return (domain->batch_no++ % dmar_batch_coalesce == 0);
}
void
@@ -733,7 +731,6 @@ dmar_domain_unload(struct dmar_domain *domain,
{
struct dmar_unit *unit;
struct dmar_map_entry *entry, *entry1;
- struct dmar_qi_genseq gseq;
int error;
unit = domain->dmar;
@@ -757,17 +754,11 @@ dmar_domain_unload(struct dmar_domain *domain,
KASSERT(unit->qi_enabled, ("loaded entry left"));
DMAR_LOCK(unit);
TAILQ_FOREACH(entry, entries, dmamap_link) {
- entry->gseq.gen = 0;
- entry->gseq.seq = 0;
dmar_qi_invalidate_locked(domain, entry->start, entry->end -
- entry->start, dmar_domain_unload_gseq(domain, entry,
- &gseq));
+ entry->start, &entry->gseq,
+ dmar_domain_unload_emit_wait(domain, entry));
}
- TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
- entry->gseq = gseq;
- TAILQ_REMOVE(entries, entry, dmamap_link);
- TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link);
- }
+ TAILQ_CONCAT(&unit->tlb_flush_entries, entries, dmamap_link);
DMAR_UNLOCK(unit);
}
Modified: head/sys/x86/iommu/intel_dmar.h
==============================================================================
--- head/sys/x86/iommu/intel_dmar.h Mon Jun 19 21:09:50 2017 (r320124)
+++ head/sys/x86/iommu/intel_dmar.h Mon Jun 19 21:48:52 2017 (r320125)
@@ -305,7 +305,7 @@ void dmar_disable_qi_intr(struct dmar_unit *unit);
int dmar_init_qi(struct dmar_unit *unit);
void dmar_fini_qi(struct dmar_unit *unit);
void dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t start,
- dmar_gaddr_t size, struct dmar_qi_genseq *pseq);
+ dmar_gaddr_t size, struct dmar_qi_genseq *psec, bool emit_wait);
void dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit);
void dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit);
void dmar_qi_invalidate_iec_glob(struct dmar_unit *unit);
Modified: head/sys/x86/iommu/intel_qi.c
==============================================================================
--- head/sys/x86/iommu/intel_qi.c Mon Jun 19 21:09:50 2017 (r320124)
+++ head/sys/x86/iommu/intel_qi.c Mon Jun 19 21:48:52 2017 (r320125)
@@ -171,7 +171,8 @@ dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32
}
static void
-dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq)
+dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq,
+ bool emit_wait)
{
struct dmar_qi_genseq gsec;
uint32_t seq;
@@ -192,7 +193,10 @@ dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct d
seq = unit->inv_waitd_seq++;
pseq->gen = unit->inv_waitd_gen;
pseq->seq = seq;
- dmar_qi_emit_wait_descr(unit, seq, true, true, false);
+ if (emit_wait) {
+ dmar_qi_ensure(unit, 1);
+ dmar_qi_emit_wait_descr(unit, seq, true, true, false);
+ }
}
static void
@@ -215,7 +219,7 @@ dmar_qi_wait_for_seq(struct dmar_unit *unit, const str
void
dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t base,
- dmar_gaddr_t size, struct dmar_qi_genseq *pseq)
+ dmar_gaddr_t size, struct dmar_qi_genseq *pseq, bool emit_wait)
{
struct dmar_unit *unit;
dmar_gaddr_t isize;
@@ -232,10 +236,7 @@ dmar_qi_invalidate_locked(struct dmar_domain *domain,
DMAR_IQ_DESCR_IOTLB_DID(domain->domain),
base | am);
}
- if (pseq != NULL) {
- dmar_qi_ensure(unit, 1);
- dmar_qi_emit_wait_seq(unit, pseq);
- }
+ dmar_qi_emit_wait_seq(unit, pseq, emit_wait);
dmar_qi_advance_tail(unit);
}
@@ -247,7 +248,7 @@ dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *u
DMAR_ASSERT_LOCKED(unit);
dmar_qi_ensure(unit, 2);
dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0);
- dmar_qi_emit_wait_seq(unit, &gseq);
+ dmar_qi_emit_wait_seq(unit, &gseq, true);
dmar_qi_advance_tail(unit);
dmar_qi_wait_for_seq(unit, &gseq, false);
}
@@ -261,7 +262,7 @@ dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit
dmar_qi_ensure(unit, 2);
dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB |
DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0);
- dmar_qi_emit_wait_seq(unit, &gseq);
+ dmar_qi_emit_wait_seq(unit, &gseq, true);
dmar_qi_advance_tail(unit);
dmar_qi_wait_for_seq(unit, &gseq, false);
}
@@ -274,7 +275,7 @@ dmar_qi_invalidate_iec_glob(struct dmar_unit *unit)
DMAR_ASSERT_LOCKED(unit);
dmar_qi_ensure(unit, 2);
dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0);
- dmar_qi_emit_wait_seq(unit, &gseq);
+ dmar_qi_emit_wait_seq(unit, &gseq, true);
dmar_qi_advance_tail(unit);
dmar_qi_wait_for_seq(unit, &gseq, false);
}
@@ -298,7 +299,7 @@ dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int s
DMAR_IQ_DESCR_IEC_IM(l), 0);
}
dmar_qi_ensure(unit, 1);
- dmar_qi_emit_wait_seq(unit, &gseq);
+ dmar_qi_emit_wait_seq(unit, &gseq, true);
dmar_qi_advance_tail(unit);
/*
@@ -344,8 +345,7 @@ dmar_qi_task(void *arg, int pending __unused)
entry = TAILQ_FIRST(&unit->tlb_flush_entries);
if (entry == NULL)
break;
- if ((entry->gseq.gen == 0 && entry->gseq.seq == 0) ||
- !dmar_qi_seq_processed(unit, &entry->gseq))
+ if (!dmar_qi_seq_processed(unit, &entry->gseq))
break;
TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link);
DMAR_UNLOCK(unit);
@@ -432,7 +432,7 @@ dmar_fini_qi(struct dmar_unit *unit)
DMAR_LOCK(unit);
/* quisce */
dmar_qi_ensure(unit, 1);
- dmar_qi_emit_wait_seq(unit, &gseq);
+ dmar_qi_emit_wait_seq(unit, &gseq, true);
dmar_qi_advance_tail(unit);
dmar_qi_wait_for_seq(unit, &gseq, false);
/* only after the quisce, disable queue */
More information about the svn-src-head
mailing list