svn commit: r351921 - head/sys/powerpc/aim

[Justin Hibbits]

Author: jhibbits
Date: Fri Sep  6 02:45:46 2019
New Revision: 351921
URL: https://svnweb.freebsd.org/changeset/base/351921

Log:
  powerpc64/pmap: Simplify the code path for moea64_pte_replace_native()
  
  Summary:
  MOEA64_PTE_REPLACE() is called often with the pmap lock held, and
  sometimes with the page pv lock held.  The less work done while holding
  a lock, the better.  Since we are intending to replace the same PTE
  (same hash index), we don't need to recalculate anything, just flat
  replace the PTE.  This cuts more than 200 instructions off the
  invalidating code path.  In addition, we don't need to replace a PTE
  that's not occupied by this PVO.
  
  Reviewed by:	luporl
  Differential Revision:	https://reviews.freebsd.org/D21515

Modified:
  head/sys/powerpc/aim/moea64_native.c

Modified: head/sys/powerpc/aim/moea64_native.c
==============================================================================
--- head/sys/powerpc/aim/moea64_native.c	Fri Sep  6 02:28:03 2019	(r351920)
+++ head/sys/powerpc/aim/moea64_native.c	Fri Sep  6 02:45:46 2019	(r351921)
@@ -358,6 +358,45 @@ moea64_pte_unset_native(mmu_t mmu, struct pvo_entry *p
 }
 
 static int64_t
+moea64_pte_replace_inval_native(mmu_t mmu, struct pvo_entry *pvo,
+    volatile struct lpte *pt)
+{
+	struct lpte properpt;
+	uint64_t ptelo;
+
+	moea64_pte_from_pvo(pvo, &properpt);
+
+	rw_rlock(&moea64_eviction_lock);
+	if ((be64toh(pt->pte_hi & LPTE_AVPN_MASK)) !=
+	    (properpt.pte_hi & LPTE_AVPN_MASK)) {
+		/* Evicted */
+		STAT_MOEA64(moea64_pte_overflow--);
+		rw_runlock(&moea64_eviction_lock);
+		return (-1);
+	}
+
+	/*
+	 * Replace the pte, briefly locking it to collect RC bits. No
+	 * atomics needed since this is protected against eviction by the lock.
+	 */
+	isync();
+	critical_enter();
+	pt->pte_hi = be64toh((pt->pte_hi & ~LPTE_VALID) | LPTE_LOCKED);
+	PTESYNC();
+	TLBIE(pvo->pvo_vpn);
+	ptelo = be64toh(pt->pte_lo);
+	EIEIO();
+	pt->pte_lo = htobe64(properpt.pte_lo);
+	EIEIO();
+	pt->pte_hi = htobe64(properpt.pte_hi); /* Release lock */
+	PTESYNC();
+	critical_exit();
+	rw_runlock(&moea64_eviction_lock);
+
+	return (ptelo & (LPTE_CHG | LPTE_REF));
+}
+
+static int64_t
 moea64_pte_replace_native(mmu_t mmu, struct pvo_entry *pvo, int flags)
 {
 	volatile struct lpte *pt = moea64_pteg_table + pvo->pvo_pte.slot;
@@ -379,8 +418,7 @@ moea64_pte_replace_native(mmu_t mmu, struct pvo_entry 
 		rw_runlock(&moea64_eviction_lock);
 	} else {
 		/* Otherwise, need reinsertion and deletion */
-		ptelo = moea64_pte_unset_native(mmu, pvo);
-		moea64_pte_insert_native(mmu, pvo);
+		ptelo = moea64_pte_replace_inval_native(mmu, pvo, pt);
 	}
 
 	return (ptelo);