svn commit: r332896 - head/sys/kern
Mateusz Guzik
mjg at FreeBSD.org
Mon Apr 23 22:28:50 UTC 2018
Author: mjg
Date: Mon Apr 23 22:28:49 2018
New Revision: 332896
URL: https://svnweb.freebsd.org/changeset/base/332896
Log:
malloc: stop reading the subzone if MALLOC_DEBUG_MAXZONES == 1 (the default)
malloc was showing at the top of profile during while running microbenchmarks.
#define DTMALLOC_PROBE_MAX 2
struct malloc_type_internal {
uint32_t mti_probes[DTMALLOC_PROBE_MAX];
u_char mti_zone;
struct malloc_type_stats mti_stats[MAXCPU];
};
Reading mti_zone it wastes a cacheline to hold mti_probes + mti_zone
(which we know is 0) + part of malloc stats of the first cpu which on top
induces false-sharing.
In particular will-it-scale lock1_processes -t 128 -s 10:
before: average:45879692
after: average:51655596
Note the counters can be padded but the right fix is to move them to
counter(9), leaving the struct read-only after creation (modulo dtrace
probes).
Modified:
head/sys/kern/kern_malloc.c
Modified: head/sys/kern/kern_malloc.c
==============================================================================
--- head/sys/kern/kern_malloc.c Mon Apr 23 21:10:33 2018 (r332895)
+++ head/sys/kern/kern_malloc.c Mon Apr 23 22:28:49 2018 (r332896)
@@ -296,22 +296,49 @@ SYSCTL_UINT(_debug_malloc, OID_AUTO, zone_offset, CTLF
&zone_offset, 0, "Separate malloc types by examining the "
"Nth character in the malloc type short description.");
-static u_int
-mtp_get_subzone(const char *desc)
+static void
+mtp_set_subzone(struct malloc_type *mtp)
{
+ struct malloc_type_internal *mtip;
+ const char *desc;
size_t len;
u_int val;
+ mtip = mtp->ks_handle;
+ desc = mtp->ks_shortdesc;
if (desc == NULL || (len = strlen(desc)) == 0)
- return (0);
- val = desc[zone_offset % len];
- return (val % numzones);
+ val = 0;
+ else
+ val = desc[zone_offset % len];
+ mtip->mti_zone = (val % numzones);
}
+
+static inline u_int
+mtp_get_subzone(struct malloc_type *mtp)
+{
+ struct malloc_type_internal *mtip;
+
+ mtip = mtp->ks_handle;
+
+ KASSERT(mtip->mti_zone < numzones,
+ ("mti_zone %u out of range %d",
+ mtip->mti_zone, numzones));
+ return (mtip->mti_zone);
+}
#elif MALLOC_DEBUG_MAXZONES == 0
#error "MALLOC_DEBUG_MAXZONES must be positive."
#else
+static void
+mtp_set_subzone(struct malloc_type *mtp)
+{
+ struct malloc_type_internal *mtip;
+
+ mtip = mtp->ks_handle;
+ mtip->mti_zone = 0;
+}
+
static inline u_int
-mtp_get_subzone(const char *desc)
+mtp_get_subzone(struct malloc_type *mtp)
{
return (0);
@@ -521,7 +548,6 @@ void *
malloc(size_t size, struct malloc_type *mtp, int flags)
{
int indx;
- struct malloc_type_internal *mtip;
caddr_t va;
uma_zone_t zone;
#if defined(DEBUG_REDZONE)
@@ -534,14 +560,10 @@ malloc(size_t size, struct malloc_type *mtp, int flags
#endif
if (size <= kmem_zmax) {
- mtip = mtp->ks_handle;
if (size & KMEM_ZMASK)
size = (size & ~KMEM_ZMASK) + KMEM_ZBASE;
indx = kmemsize[size >> KMEM_ZSHIFT];
- KASSERT(mtip->mti_zone < numzones,
- ("mti_zone %u out of range %d",
- mtip->mti_zone, numzones));
- zone = kmemzones[indx].kz_zone[mtip->mti_zone];
+ zone = kmemzones[indx].kz_zone[mtp_get_subzone(mtp)];
#ifdef MALLOC_PROFILE
krequests[size >> KMEM_ZSHIFT]++;
#endif
@@ -571,7 +593,6 @@ malloc_domain(size_t size, struct malloc_type *mtp, in
int flags)
{
int indx;
- struct malloc_type_internal *mtip;
caddr_t va;
uma_zone_t zone;
#if defined(DEBUG_REDZONE)
@@ -583,14 +604,10 @@ malloc_domain(size_t size, struct malloc_type *mtp, in
return (va);
#endif
if (size <= kmem_zmax) {
- mtip = mtp->ks_handle;
if (size & KMEM_ZMASK)
size = (size & ~KMEM_ZMASK) + KMEM_ZBASE;
indx = kmemsize[size >> KMEM_ZSHIFT];
- KASSERT(mtip->mti_zone < numzones,
- ("mti_zone %u out of range %d",
- mtip->mti_zone, numzones));
- zone = kmemzones[indx].kz_zone[mtip->mti_zone];
+ zone = kmemzones[indx].kz_zone[mtp_get_subzone(mtp)];
#ifdef MALLOC_PROFILE
krequests[size >> KMEM_ZSHIFT]++;
#endif
@@ -973,7 +990,7 @@ malloc_init(void *data)
mtip = uma_zalloc(mt_zone, M_WAITOK | M_ZERO);
mtp->ks_handle = mtip;
- mtip->mti_zone = mtp_get_subzone(mtp->ks_shortdesc);
+ mtp_set_subzone(mtp);
mtx_lock(&malloc_mtx);
mtp->ks_next = kmemstatistics;
More information about the svn-src-all
mailing list