svn commit: r335579 - head/sys/vm

Jeff Roberson jroberson at jroberson.net
Sat Jun 23 09:10:48 UTC 2018


On Sat, 23 Jun 2018, Jeff Roberson wrote:

> Author: jeff
> Date: Sat Jun 23 08:10:09 2018
> New Revision: 335579
> URL: https://svnweb.freebsd.org/changeset/base/335579
>
> Log:
>  Sort uma_zone fields according to 64 byte cache line with adjacent line
>  prefetch on 64bit architectures.  Prior to this, two lines were needed
>  for the fast path and each line may fetch an unused adjacent neighbor.
>   - Move fields used by the fast path into a single line.
>   - Move constants into the adjacent line which is mostly used for
>     the spare bucket alloc 'medium path'.
>   - Unpad the mtx which is only used by the fast path and place it in

I got a little overexcited about fast.  The mtx is only used by the slow 
path.

Jeff

>     a line with rarely used data.  This aligns the cachelines better and
>     eliminates 128 bytes of wasted space.
>
>  This gives a 45% improvement on a will-it-scale test on a 24 core machine.
>
>  Reviewed by:	mmacy
>
> Modified:
>  head/sys/vm/uma_int.h
>
> Modified: head/sys/vm/uma_int.h
> ==============================================================================
> --- head/sys/vm/uma_int.h	Sat Jun 23 07:14:08 2018	(r335578)
> +++ head/sys/vm/uma_int.h	Sat Jun 23 08:10:09 2018	(r335579)
> @@ -222,9 +222,8 @@ typedef struct uma_domain * uma_domain_t;
>  *
>  */
> struct uma_keg {
> -	struct mtx_padalign	uk_lock;	/* Lock for the keg */
> +	struct mtx	uk_lock;	/* Lock for the keg */
> 	struct uma_hash	uk_hash;
> -
> 	LIST_HEAD(,uma_zone)	uk_zones;	/* Keg's zones */
>
> 	uint32_t	uk_cursor;	/* Domain alloc cursor. */
> @@ -315,40 +314,48 @@ typedef struct uma_zone_domain * uma_zone_domain_t;
>  *
>  */
> struct uma_zone {
> -	struct mtx_padalign	uz_lock;	/* Lock for the zone */
> -	struct mtx_padalign	*uz_lockptr;
> -	const char		*uz_name;	/* Text name of the zone */
> -
> -	LIST_ENTRY(uma_zone)	uz_link;	/* List of all zones in keg */
> +	/* Offset 0, used in alloc/free fast/medium fast path and const. */
> +	struct mtx	*uz_lockptr;
> +	const char	*uz_name;	/* Text name of the zone */
> 	struct uma_zone_domain	*uz_domain;	/* per-domain buckets */
> -
> -	LIST_HEAD(,uma_klink)	uz_kegs;	/* List of kegs. */
> -	struct uma_klink	uz_klink;	/* klink for first keg. */
> -
> -	uma_slaballoc	uz_slab;	/* Allocate a slab from the backend. */
> +	uint32_t	uz_flags;	/* Flags inherited from kegs */
> +	uint32_t	uz_size;	/* Size inherited from kegs */
> 	uma_ctor	uz_ctor;	/* Constructor for each allocation */
> 	uma_dtor	uz_dtor;	/* Destructor */
> 	uma_init	uz_init;	/* Initializer for each item */
> 	uma_fini	uz_fini;	/* Finalizer for each item. */
> +
> +	/* Offset 64, used in bucket replenish. */
> 	uma_import	uz_import;	/* Import new memory to cache. */
> 	uma_release	uz_release;	/* Release memory from cache. */
> 	void		*uz_arg;	/* Import/release argument. */
> -
> -	uint32_t	uz_flags;	/* Flags inherited from kegs */
> -	uint32_t	uz_size;	/* Size inherited from kegs */
> -
> -	volatile u_long	uz_allocs UMA_ALIGN; /* Total number of allocations */
> -	volatile u_long	uz_fails;	/* Total number of alloc failures */
> -	volatile u_long	uz_frees;	/* Total number of frees */
> -	uint64_t	uz_sleeps;	/* Total number of alloc sleeps */
> +	uma_slaballoc	uz_slab;	/* Allocate a slab from the backend. */
> 	uint16_t	uz_count;	/* Amount of items in full bucket */
> 	uint16_t	uz_count_min;	/* Minimal amount of items there */
> +	/* 32bit pad on 64bit. */
> +	LIST_ENTRY(uma_zone)	uz_link;	/* List of all zones in keg */
> +	LIST_HEAD(,uma_klink)	uz_kegs;	/* List of kegs. */
>
> +	/* Offset 128 Rare. */
> +	/*
> +	 * The lock is placed here to avoid adjacent line prefetcher
> +	 * in fast paths and to take up space near infrequently accessed
> +	 * members to reduce alignment overhead.
> +	 */
> +	struct mtx	uz_lock;	/* Lock for the zone */
> +	struct uma_klink	uz_klink;	/* klink for first keg. */
> 	/* The next two fields are used to print a rate-limited warnings. */
> 	const char	*uz_warning;	/* Warning to print on failure */
> 	struct timeval	uz_ratecheck;	/* Warnings rate-limiting */
> -
> 	struct task	uz_maxaction;	/* Task to run when at limit */
> +
> +	/* 16 bytes of pad. */
> +
> +	/* Offset 256, atomic stats. */
> +	volatile u_long	uz_allocs UMA_ALIGN; /* Total number of allocations */
> +	volatile u_long	uz_fails;	/* Total number of alloc failures */
> +	volatile u_long	uz_frees;	/* Total number of frees */
> +	uint64_t	uz_sleeps;	/* Total number of alloc sleeps */
>
> 	/*
> 	 * This HAS to be the last item because we adjust the zone size
>


More information about the svn-src-head mailing list