svn commit: r344229 - stable/12/sys/i386/include

[Konstantin Belousov]

Author: kib
Date: Sun Feb 17 09:57:43 2019
New Revision: 344229
URL: https://svnweb.freebsd.org/changeset/base/344229

Log:
  MFC r343723:
  i386: Do not ever store to other-CPU counter64 slot.

Modified:
  stable/12/sys/i386/include/counter.h
Directory Properties:
  stable/12/   (props changed)

Modified: stable/12/sys/i386/include/counter.h
==============================================================================
--- stable/12/sys/i386/include/counter.h	Sun Feb 17 09:56:11 2019	(r344228)
+++ stable/12/sys/i386/include/counter.h	Sun Feb 17 09:57:43 2019	(r344229)
@@ -72,7 +72,12 @@ counter_64_inc_8b(uint64_t *p, int64_t inc)
 }
 
 #ifdef IN_SUBR_COUNTER_C
-static inline uint64_t
+struct counter_u64_fetch_cx8_arg {
+	uint64_t res;
+	uint64_t *p;
+};
+
+static uint64_t
 counter_u64_read_one_8b(uint64_t *p)
 {
 	uint32_t res_lo, res_high;
@@ -87,9 +92,22 @@ counter_u64_read_one_8b(uint64_t *p)
 	return (res_lo + ((uint64_t)res_high << 32));
 }
 
+static void
+counter_u64_fetch_cx8_one(void *arg1)
+{
+	struct counter_u64_fetch_cx8_arg *arg;
+	uint64_t val;
+
+	arg = arg1;
+	val = counter_u64_read_one_8b((uint64_t *)((char *)arg->p +
+	    UMA_PCPU_ALLOC_SIZE * PCPU_GET(cpuid)));
+	atomic_add_64(&arg->res, val);
+}
+
 static inline uint64_t
 counter_u64_fetch_inline(uint64_t *p)
 {
+	struct counter_u64_fetch_cx8_arg arg;
 	uint64_t res;
 	int i;
 
@@ -108,9 +126,10 @@ counter_u64_fetch_inline(uint64_t *p)
 		}
 		critical_exit();
 	} else {
-		CPU_FOREACH(i)
-			res += counter_u64_read_one_8b((uint64_t *)((char *)p +
-			    UMA_PCPU_ALLOC_SIZE * i));
+		arg.p = p;
+		arg.res = 0;
+		smp_rendezvous(NULL, counter_u64_fetch_cx8_one, NULL, &arg);
+		res = arg.res;
 	}
 	return (res);
 }