svn commit: r357809 - head/sys/kern

[Mateusz Guzik]

Author: mjg
Date: Wed Feb 12 11:17:18 2020
New Revision: 357809
URL: https://svnweb.freebsd.org/changeset/base/357809

Log:
  rms: use smp_rendezvous_cpus_retry instead of a hand-rolled variant

Modified:
  head/sys/kern/kern_rmlock.c

Modified: head/sys/kern/kern_rmlock.c
==============================================================================
--- head/sys/kern/kern_rmlock.c	Wed Feb 12 11:16:55 2020	(r357808)
+++ head/sys/kern/kern_rmlock.c	Wed Feb 12 11:17:18 2020	(r357809)
@@ -991,17 +991,17 @@ rms_runlock(struct rmslock *rms)
 
 struct rmslock_ipi {
 	struct rmslock *rms;
-	cpuset_t signal;
+	struct smp_rendezvous_cpus_retry_arg srcra;
 };
 
 static void
-rms_wlock_IPI(void *arg)
+rms_action_func(void *arg)
 {
 	struct rmslock_ipi *rmsipi;
 	struct rmslock *rms;
 	int readers;
 
-	rmsipi = arg;
+	rmsipi = __containerof(arg, struct rmslock_ipi, srcra);
 	rms = rmsipi->rms;
 
 	if (*zpcpu_get(rms->readers_influx))
@@ -1009,65 +1009,40 @@ rms_wlock_IPI(void *arg)
 	readers = zpcpu_replace(rms->readers_pcpu, 0);
 	if (readers != 0)
 		atomic_add_int(&rms->readers, readers);
-	CPU_CLR_ATOMIC(curcpu, &rmsipi->signal);
+	smp_rendezvous_cpus_done(arg);
 }
 
 static void
+rms_wait_func(void *arg, int cpu)
+{
+	struct rmslock_ipi *rmsipi;
+	struct rmslock *rms;
+	int *in_op;
+
+	rmsipi = __containerof(arg, struct rmslock_ipi, srcra);
+	rms = rmsipi->rms;
+
+	in_op = zpcpu_get_cpu(rms->readers_influx, cpu);
+	while (atomic_load_int(in_op))
+		cpu_spinwait();
+}
+
+static void
 rms_wlock_switch(struct rmslock *rms)
 {
 	struct rmslock_ipi rmsipi;
-	int *in_op;
-	int cpu;
 
 	MPASS(rms->readers == 0);
 	MPASS(rms->writers == 1);
 
 	rmsipi.rms = rms;
 
-	/*
-	 * Publishes rms->writers. rlock and runlock will get this ordered
-	 * via IPI in the worst case.
-	 */
-	atomic_thread_fence_rel();
-
-	/*
-	 * Collect reader counts from all CPUs using an IPI. The handler can
-	 * find itself running while the interrupted CPU was doing either
-	 * rlock or runlock in which case it will fail.
-	 *
-	 * Successful attempts clear the cpu id in the bitmap.
-	 *
-	 * In case of failure we observe all failing CPUs not executing there to
-	 * determine when to make the next attempt. Note that threads having
-	 * the var set have preemption disabled.  Setting of readers_influx
-	 * only uses compiler barriers making these loads unreliable, which is
-	 * fine -- the IPI handler will always see the correct result.
-	 *
-	 * We retry until all counts are collected. Forward progress is
-	 * guaranteed by that fact that the total number of threads which can
-	 * be caught like this is finite and they all are going to block on
-	 * their own.
-	 */
-	CPU_COPY(&all_cpus, &rmsipi.signal);
-	for (;;) {
-		smp_rendezvous_cpus(
-		    rmsipi.signal,
-		    smp_no_rendezvous_barrier,
-		    rms_wlock_IPI,
-		    smp_no_rendezvous_barrier,
-		    &rmsipi);
-
-		if (CPU_EMPTY(&rmsipi.signal))
-			break;
-
-		CPU_FOREACH(cpu) {
-			if (!CPU_ISSET(cpu, &rmsipi.signal))
-				continue;
-			in_op = zpcpu_get_cpu(rms->readers_influx, cpu);
-			while (atomic_load_int(in_op))
-				cpu_spinwait();
-		}
-	}
+	smp_rendezvous_cpus_retry(all_cpus,
+	    smp_no_rendezvous_barrier,
+	    rms_action_func,
+	    smp_no_rendezvous_barrier,
+	    rms_wait_func,
+	    &rmsipi.srcra);
 }
 
 void