svn commit: r335579 - head/sys/vm
Jeff Roberson
jroberson at jroberson.net
Sat Jun 23 09:10:48 UTC 2018
On Sat, 23 Jun 2018, Jeff Roberson wrote:
> Author: jeff
> Date: Sat Jun 23 08:10:09 2018
> New Revision: 335579
> URL: https://svnweb.freebsd.org/changeset/base/335579
>
> Log:
> Sort uma_zone fields according to 64 byte cache line with adjacent line
> prefetch on 64bit architectures. Prior to this, two lines were needed
> for the fast path and each line may fetch an unused adjacent neighbor.
> - Move fields used by the fast path into a single line.
> - Move constants into the adjacent line which is mostly used for
> the spare bucket alloc 'medium path'.
> - Unpad the mtx which is only used by the fast path and place it in
I got a little overexcited about fast. The mtx is only used by the slow
path.
Jeff
> a line with rarely used data. This aligns the cachelines better and
> eliminates 128 bytes of wasted space.
>
> This gives a 45% improvement on a will-it-scale test on a 24 core machine.
>
> Reviewed by: mmacy
>
> Modified:
> head/sys/vm/uma_int.h
>
> Modified: head/sys/vm/uma_int.h
> ==============================================================================
> --- head/sys/vm/uma_int.h Sat Jun 23 07:14:08 2018 (r335578)
> +++ head/sys/vm/uma_int.h Sat Jun 23 08:10:09 2018 (r335579)
> @@ -222,9 +222,8 @@ typedef struct uma_domain * uma_domain_t;
> *
> */
> struct uma_keg {
> - struct mtx_padalign uk_lock; /* Lock for the keg */
> + struct mtx uk_lock; /* Lock for the keg */
> struct uma_hash uk_hash;
> -
> LIST_HEAD(,uma_zone) uk_zones; /* Keg's zones */
>
> uint32_t uk_cursor; /* Domain alloc cursor. */
> @@ -315,40 +314,48 @@ typedef struct uma_zone_domain * uma_zone_domain_t;
> *
> */
> struct uma_zone {
> - struct mtx_padalign uz_lock; /* Lock for the zone */
> - struct mtx_padalign *uz_lockptr;
> - const char *uz_name; /* Text name of the zone */
> -
> - LIST_ENTRY(uma_zone) uz_link; /* List of all zones in keg */
> + /* Offset 0, used in alloc/free fast/medium fast path and const. */
> + struct mtx *uz_lockptr;
> + const char *uz_name; /* Text name of the zone */
> struct uma_zone_domain *uz_domain; /* per-domain buckets */
> -
> - LIST_HEAD(,uma_klink) uz_kegs; /* List of kegs. */
> - struct uma_klink uz_klink; /* klink for first keg. */
> -
> - uma_slaballoc uz_slab; /* Allocate a slab from the backend. */
> + uint32_t uz_flags; /* Flags inherited from kegs */
> + uint32_t uz_size; /* Size inherited from kegs */
> uma_ctor uz_ctor; /* Constructor for each allocation */
> uma_dtor uz_dtor; /* Destructor */
> uma_init uz_init; /* Initializer for each item */
> uma_fini uz_fini; /* Finalizer for each item. */
> +
> + /* Offset 64, used in bucket replenish. */
> uma_import uz_import; /* Import new memory to cache. */
> uma_release uz_release; /* Release memory from cache. */
> void *uz_arg; /* Import/release argument. */
> -
> - uint32_t uz_flags; /* Flags inherited from kegs */
> - uint32_t uz_size; /* Size inherited from kegs */
> -
> - volatile u_long uz_allocs UMA_ALIGN; /* Total number of allocations */
> - volatile u_long uz_fails; /* Total number of alloc failures */
> - volatile u_long uz_frees; /* Total number of frees */
> - uint64_t uz_sleeps; /* Total number of alloc sleeps */
> + uma_slaballoc uz_slab; /* Allocate a slab from the backend. */
> uint16_t uz_count; /* Amount of items in full bucket */
> uint16_t uz_count_min; /* Minimal amount of items there */
> + /* 32bit pad on 64bit. */
> + LIST_ENTRY(uma_zone) uz_link; /* List of all zones in keg */
> + LIST_HEAD(,uma_klink) uz_kegs; /* List of kegs. */
>
> + /* Offset 128 Rare. */
> + /*
> + * The lock is placed here to avoid adjacent line prefetcher
> + * in fast paths and to take up space near infrequently accessed
> + * members to reduce alignment overhead.
> + */
> + struct mtx uz_lock; /* Lock for the zone */
> + struct uma_klink uz_klink; /* klink for first keg. */
> /* The next two fields are used to print a rate-limited warnings. */
> const char *uz_warning; /* Warning to print on failure */
> struct timeval uz_ratecheck; /* Warnings rate-limiting */
> -
> struct task uz_maxaction; /* Task to run when at limit */
> +
> + /* 16 bytes of pad. */
> +
> + /* Offset 256, atomic stats. */
> + volatile u_long uz_allocs UMA_ALIGN; /* Total number of allocations */
> + volatile u_long uz_fails; /* Total number of alloc failures */
> + volatile u_long uz_frees; /* Total number of frees */
> + uint64_t uz_sleeps; /* Total number of alloc sleeps */
>
> /*
> * This HAS to be the last item because we adjust the zone size
>
More information about the svn-src-head
mailing list