git: b1d30a83af2b - stable/13 - uma: Micro-optimize memory trashing

The branch stable/13 has been updated by mav:

URL: https://cgit.FreeBSD.org/src/commit/?id=b1d30a83af2b7e2c58dd674490440d3a948ad75c

commit b1d30a83af2b7e2c58dd674490440d3a948ad75c
Author:     Alexander Motin <mav@FreeBSD.org>
AuthorDate: 2023-11-09 18:07:46 +0000
Commit:     Alexander Motin <mav@FreeBSD.org>
CommitDate: 2023-12-09 02:33:45 +0000

    uma: Micro-optimize memory trashing
    
    Use u_long for memory accesses instead of uint32_t.  On my tests on
    amd64 this by ~30% reduces time spent in those functions thanks to
    bigger 64bit accesses.  i386 still uses 32bit accesses.
    
    MFC after:      1 month
    
    (cherry picked from commit 7c566d6cfc7bfb913bad89d87386fa21dce8c2e6)
---
 sys/vm/uma_dbg.c | 62 ++++++++++++++++++++++++--------------------------------
 1 file changed, 26 insertions(+), 36 deletions(-)

diff --git a/sys/vm/uma_dbg.c b/sys/vm/uma_dbg.c
index 36567f3b3968..76dd2bfde2fe 100644
--- a/sys/vm/uma_dbg.c
+++ b/sys/vm/uma_dbg.c
@@ -53,7 +53,7 @@
 #include <vm/uma_dbg.h>
 #include <vm/memguard.h>
 
-static const uint32_t uma_junk = 0xdeadc0de;
+static const u_long uma_junk = (u_long)0xdeadc0dedeadc0de;
 
 /*
  * Checks an item to make sure it hasn't been overwritten since it was freed,
@@ -64,27 +64,20 @@ static const uint32_t uma_junk = 0xdeadc0de;
 int
 trash_ctor(void *mem, int size, void *arg, int flags)
 {
-	int cnt;
-	uint32_t *p;
+	u_long *p = mem, *e;
 
 #ifdef DEBUG_MEMGUARD
 	if (is_memguard_addr(mem))
 		return (0);
 #endif
 
-	cnt = size / sizeof(uma_junk);
-
-	for (p = mem; cnt > 0; cnt--, p++)
-		if (*p != uma_junk) {
-#ifdef INVARIANTS
-			panic("Memory modified after free %p(%d) val=%x @ %p\n",
-			    mem, size, *p, p);
-#else
-			printf("Memory modified after free %p(%d) val=%x @ %p\n",
-			    mem, size, *p, p);
-#endif
-			return (0);
-		}
+	e = p + size / sizeof(*p);
+	for (; p < e; p++) {
+		if (__predict_true(*p == uma_junk))
+			continue;
+		panic("Memory modified after free %p(%d) val=%lx @ %p\n",
+		    mem, size, *p, p);
+	}
 	return (0);
 }
 
@@ -97,17 +90,15 @@ trash_ctor(void *mem, int size, void *arg, int flags)
 void
 trash_dtor(void *mem, int size, void *arg)
 {
-	int cnt;
-	uint32_t *p;
+	u_long *p = mem, *e;
 
 #ifdef DEBUG_MEMGUARD
 	if (is_memguard_addr(mem))
 		return;
 #endif
 
-	cnt = size / sizeof(uma_junk);
-
-	for (p = mem; cnt > 0; cnt--, p++)
+	e = p + size / sizeof(*p);
+	for (; p < e; p++)
 		*p = uma_junk;
 }
 
@@ -140,8 +131,7 @@ int
 mtrash_ctor(void *mem, int size, void *arg, int flags)
 {
 	struct malloc_type **ksp;
-	uint32_t *p = mem;
-	int cnt;
+	u_long *p = mem, *e;
 
 #ifdef DEBUG_MEMGUARD
 	if (is_memguard_addr(mem))
@@ -151,15 +141,16 @@ mtrash_ctor(void *mem, int size, void *arg, int flags)
 	size -= sizeof(struct malloc_type *);
 	ksp = (struct malloc_type **)mem;
 	ksp += size / sizeof(struct malloc_type *);
-	cnt = size / sizeof(uma_junk);
-
-	for (p = mem; cnt > 0; cnt--, p++)
-		if (*p != uma_junk) {
-			printf("Memory modified after free %p(%d) val=%x @ %p\n",
-			    mem, size, *p, p);
-			panic("Most recently used by %s\n", (*ksp == NULL)?
-			    "none" : (*ksp)->ks_shortdesc);
-		}
+
+	e = p + size / sizeof(*p);
+	for (; p < e; p++) {
+		if (__predict_true(*p == uma_junk))
+			continue;
+		printf("Memory modified after free %p(%d) val=%lx @ %p\n",
+		    mem, size, *p, p);
+		panic("Most recently used by %s\n", (*ksp == NULL)?
+		    "none" : (*ksp)->ks_shortdesc);
+	}
 	return (0);
 }
 
@@ -172,8 +163,7 @@ mtrash_ctor(void *mem, int size, void *arg, int flags)
 void
 mtrash_dtor(void *mem, int size, void *arg)
 {
-	int cnt;
-	uint32_t *p;
+	u_long *p = mem, *e;
 
 #ifdef DEBUG_MEMGUARD
 	if (is_memguard_addr(mem))
@@ -181,9 +171,9 @@ mtrash_dtor(void *mem, int size, void *arg)
 #endif
 
 	size -= sizeof(struct malloc_type *);
-	cnt = size / sizeof(uma_junk);
 
-	for (p = mem; cnt > 0; cnt--, p++)
+	e = p + size / sizeof(*p);
+	for (; p < e; p++)
 		*p = uma_junk;
 }