svn commit: r355819 - in head/sys: arm/arm arm64/arm64 i386/i386 mips/mips powerpc/powerpc riscv/riscv

[Jeff Roberson]

Author: jeff
Date: Mon Dec 16 20:15:04 2019
New Revision: 355819
URL: https://svnweb.freebsd.org/changeset/base/355819

Log:
  Repeat the spinlock_enter/exit pattern from amd64 on other architectures to
  fix an assert violation introduced in r355784.  Without this spinlock_exit()
  may see owepreempt and switch before reducing the spinlock count.  amd64
  had been optimized to do a single critical enter/exit regardless of the
  number of spinlocks which avoided the problem and this optimization had
  not been applied elsewhere.
  
  Reported by:	emaste
  Suggested by:	rlibby
  Discussed with:	jhb, rlibby
  Tested by:	manu (arm64)

Modified:
  head/sys/arm/arm/machdep.c
  head/sys/arm64/arm64/machdep.c
  head/sys/i386/i386/machdep.c
  head/sys/mips/mips/machdep.c
  head/sys/powerpc/powerpc/machdep.c
  head/sys/riscv/riscv/machdep.c

Modified: head/sys/arm/arm/machdep.c
==============================================================================
--- head/sys/arm/arm/machdep.c	Mon Dec 16 20:07:04 2019	(r355818)
+++ head/sys/arm/arm/machdep.c	Mon Dec 16 20:15:04 2019	(r355819)
@@ -389,9 +389,9 @@ spinlock_enter(void)
 		cspr = disable_interrupts(PSR_I | PSR_F);
 		td->td_md.md_spinlock_count = 1;
 		td->td_md.md_saved_cspr = cspr;
+		critical_enter();
 	} else
 		td->td_md.md_spinlock_count++;
-	critical_enter();
 }
 
 void
@@ -401,11 +401,12 @@ spinlock_exit(void)
 	register_t cspr;
 
 	td = curthread;
-	critical_exit();
 	cspr = td->td_md.md_saved_cspr;
 	td->td_md.md_spinlock_count--;
-	if (td->td_md.md_spinlock_count == 0)
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
 		restore_interrupts(cspr);
+	}
 }
 
 /*

Modified: head/sys/arm64/arm64/machdep.c
==============================================================================
--- head/sys/arm64/arm64/machdep.c	Mon Dec 16 20:07:04 2019	(r355818)
+++ head/sys/arm64/arm64/machdep.c	Mon Dec 16 20:15:04 2019	(r355819)
@@ -635,9 +635,9 @@ spinlock_enter(void)
 		daif = intr_disable();
 		td->td_md.md_spinlock_count = 1;
 		td->td_md.md_saved_daif = daif;
+		critical_enter();
 	} else
 		td->td_md.md_spinlock_count++;
-	critical_enter();
 }
 
 void
@@ -647,11 +647,12 @@ spinlock_exit(void)
 	register_t daif;
 
 	td = curthread;
-	critical_exit();
 	daif = td->td_md.md_saved_daif;
 	td->td_md.md_spinlock_count--;
-	if (td->td_md.md_spinlock_count == 0)
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
 		intr_restore(daif);
+	}
 }
 
 #ifndef	_SYS_SYSPROTO_H_

Modified: head/sys/i386/i386/machdep.c
==============================================================================
--- head/sys/i386/i386/machdep.c	Mon Dec 16 20:07:04 2019	(r355818)
+++ head/sys/i386/i386/machdep.c	Mon Dec 16 20:15:04 2019	(r355819)
@@ -2679,9 +2679,9 @@ spinlock_enter(void)
 		flags = intr_disable();
 		td->td_md.md_spinlock_count = 1;
 		td->td_md.md_saved_flags = flags;
+		critical_enter();
 	} else
 		td->td_md.md_spinlock_count++;
-	critical_enter();
 }
 
 void
@@ -2691,11 +2691,12 @@ spinlock_exit(void)
 	register_t flags;
 
 	td = curthread;
-	critical_exit();
 	flags = td->td_md.md_saved_flags;
 	td->td_md.md_spinlock_count--;
-	if (td->td_md.md_spinlock_count == 0)
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
 		intr_restore(flags);
+	}
 }
 
 #if defined(I586_CPU) && !defined(NO_F00F_HACK)

Modified: head/sys/mips/mips/machdep.c
==============================================================================
--- head/sys/mips/mips/machdep.c	Mon Dec 16 20:07:04 2019	(r355818)
+++ head/sys/mips/mips/machdep.c	Mon Dec 16 20:15:04 2019	(r355819)
@@ -516,9 +516,9 @@ spinlock_enter(void)
 		intr = intr_disable();
 		td->td_md.md_spinlock_count = 1;
 		td->td_md.md_saved_intr = intr;
+		critical_enter();
 	} else
 		td->td_md.md_spinlock_count++;
-	critical_enter();
 }
 
 void
@@ -528,11 +528,12 @@ spinlock_exit(void)
 	register_t intr;
 
 	td = curthread;
-	critical_exit();
 	intr = td->td_md.md_saved_intr;
 	td->td_md.md_spinlock_count--;
-	if (td->td_md.md_spinlock_count == 0)
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
 		intr_restore(intr);
+	}
 }
 
 /*

Modified: head/sys/powerpc/powerpc/machdep.c
==============================================================================
--- head/sys/powerpc/powerpc/machdep.c	Mon Dec 16 20:07:04 2019	(r355818)
+++ head/sys/powerpc/powerpc/machdep.c	Mon Dec 16 20:15:04 2019	(r355819)
@@ -499,9 +499,9 @@ spinlock_enter(void)
 		msr = intr_disable();
 		td->td_md.md_spinlock_count = 1;
 		td->td_md.md_saved_msr = msr;
+		critical_enter();
 	} else
 		td->td_md.md_spinlock_count++;
-	critical_enter();
 }
 
 void
@@ -511,10 +511,10 @@ spinlock_exit(void)
 	register_t msr;
 
 	td = curthread;
-	critical_exit();
 	msr = td->td_md.md_saved_msr;
 	td->td_md.md_spinlock_count--;
 	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
 		intr_restore(msr);
 		nop_prio_medium();
 	}

Modified: head/sys/riscv/riscv/machdep.c
==============================================================================
--- head/sys/riscv/riscv/machdep.c	Mon Dec 16 20:07:04 2019	(r355818)
+++ head/sys/riscv/riscv/machdep.c	Mon Dec 16 20:15:04 2019	(r355819)
@@ -489,9 +489,9 @@ spinlock_enter(void)
 		reg = intr_disable();
 		td->td_md.md_spinlock_count = 1;
 		td->td_md.md_saved_sstatus_ie = reg;
+		critical_enter();
 	} else
 		td->td_md.md_spinlock_count++;
-	critical_enter();
 }
 
 void
@@ -501,11 +501,12 @@ spinlock_exit(void)
 	register_t sstatus_ie;
 
 	td = curthread;
-	critical_exit();
 	sstatus_ie = td->td_md.md_saved_sstatus_ie;
 	td->td_md.md_spinlock_count--;
-	if (td->td_md.md_spinlock_count == 0)
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
 		intr_restore(sstatus_ie);
+	}
 }
 
 #ifndef	_SYS_SYSPROTO_H_