git: b1d30a83af2b - stable/13 - uma: Micro-optimize memory trashing
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Sat, 09 Dec 2023 02:34:04 UTC
The branch stable/13 has been updated by mav: URL: https://cgit.FreeBSD.org/src/commit/?id=b1d30a83af2b7e2c58dd674490440d3a948ad75c commit b1d30a83af2b7e2c58dd674490440d3a948ad75c Author: Alexander Motin <mav@FreeBSD.org> AuthorDate: 2023-11-09 18:07:46 +0000 Commit: Alexander Motin <mav@FreeBSD.org> CommitDate: 2023-12-09 02:33:45 +0000 uma: Micro-optimize memory trashing Use u_long for memory accesses instead of uint32_t. On my tests on amd64 this by ~30% reduces time spent in those functions thanks to bigger 64bit accesses. i386 still uses 32bit accesses. MFC after: 1 month (cherry picked from commit 7c566d6cfc7bfb913bad89d87386fa21dce8c2e6) --- sys/vm/uma_dbg.c | 62 ++++++++++++++++++++++++-------------------------------- 1 file changed, 26 insertions(+), 36 deletions(-) diff --git a/sys/vm/uma_dbg.c b/sys/vm/uma_dbg.c index 36567f3b3968..76dd2bfde2fe 100644 --- a/sys/vm/uma_dbg.c +++ b/sys/vm/uma_dbg.c @@ -53,7 +53,7 @@ #include <vm/uma_dbg.h> #include <vm/memguard.h> -static const uint32_t uma_junk = 0xdeadc0de; +static const u_long uma_junk = (u_long)0xdeadc0dedeadc0de; /* * Checks an item to make sure it hasn't been overwritten since it was freed, @@ -64,27 +64,20 @@ static const uint32_t uma_junk = 0xdeadc0de; int trash_ctor(void *mem, int size, void *arg, int flags) { - int cnt; - uint32_t *p; + u_long *p = mem, *e; #ifdef DEBUG_MEMGUARD if (is_memguard_addr(mem)) return (0); #endif - cnt = size / sizeof(uma_junk); - - for (p = mem; cnt > 0; cnt--, p++) - if (*p != uma_junk) { -#ifdef INVARIANTS - panic("Memory modified after free %p(%d) val=%x @ %p\n", - mem, size, *p, p); -#else - printf("Memory modified after free %p(%d) val=%x @ %p\n", - mem, size, *p, p); -#endif - return (0); - } + e = p + size / sizeof(*p); + for (; p < e; p++) { + if (__predict_true(*p == uma_junk)) + continue; + panic("Memory modified after free %p(%d) val=%lx @ %p\n", + mem, size, *p, p); + } return (0); } @@ -97,17 +90,15 @@ trash_ctor(void *mem, int size, void *arg, int flags) void trash_dtor(void *mem, int size, void *arg) { - int cnt; - uint32_t *p; + u_long *p = mem, *e; #ifdef DEBUG_MEMGUARD if (is_memguard_addr(mem)) return; #endif - cnt = size / sizeof(uma_junk); - - for (p = mem; cnt > 0; cnt--, p++) + e = p + size / sizeof(*p); + for (; p < e; p++) *p = uma_junk; } @@ -140,8 +131,7 @@ int mtrash_ctor(void *mem, int size, void *arg, int flags) { struct malloc_type **ksp; - uint32_t *p = mem; - int cnt; + u_long *p = mem, *e; #ifdef DEBUG_MEMGUARD if (is_memguard_addr(mem)) @@ -151,15 +141,16 @@ mtrash_ctor(void *mem, int size, void *arg, int flags) size -= sizeof(struct malloc_type *); ksp = (struct malloc_type **)mem; ksp += size / sizeof(struct malloc_type *); - cnt = size / sizeof(uma_junk); - - for (p = mem; cnt > 0; cnt--, p++) - if (*p != uma_junk) { - printf("Memory modified after free %p(%d) val=%x @ %p\n", - mem, size, *p, p); - panic("Most recently used by %s\n", (*ksp == NULL)? - "none" : (*ksp)->ks_shortdesc); - } + + e = p + size / sizeof(*p); + for (; p < e; p++) { + if (__predict_true(*p == uma_junk)) + continue; + printf("Memory modified after free %p(%d) val=%lx @ %p\n", + mem, size, *p, p); + panic("Most recently used by %s\n", (*ksp == NULL)? + "none" : (*ksp)->ks_shortdesc); + } return (0); } @@ -172,8 +163,7 @@ mtrash_ctor(void *mem, int size, void *arg, int flags) void mtrash_dtor(void *mem, int size, void *arg) { - int cnt; - uint32_t *p; + u_long *p = mem, *e; #ifdef DEBUG_MEMGUARD if (is_memguard_addr(mem)) @@ -181,9 +171,9 @@ mtrash_dtor(void *mem, int size, void *arg) #endif size -= sizeof(struct malloc_type *); - cnt = size / sizeof(uma_junk); - for (p = mem; cnt > 0; cnt--, p++) + e = p + size / sizeof(*p); + for (; p < e; p++) *p = uma_junk; }