svn commit: r348764 - head/sys/vm

[Alexander Motin]

Author: mav
Date: Thu Jun  6 23:57:28 2019
New Revision: 348764
URL: https://svnweb.freebsd.org/changeset/base/348764

Log:
  Allow UMA hash tables to expand faster then 2x in 20 seconds.
  
  ZFS ABD allocates tons of 4KB chunks via UMA, requiring huge hash tables.
  With initial hash table size of only 32 elements it takes ~20 expansions
  or ~400 seconds to adapt to handling 220GB ZFS ARC.  During that time not
  only the hash table is highly inefficient, but also each of those expan-
  sions takes significant time with the lock held, blocking operation.
  
  On my test system with 256GB of RAM and ZFS pool of 28 HDDs this change
  reduces time needed to first time read 240GB from ~300-400s, during which
  system is quite busy and unresponsive, to only ~150s with light CPU load
  and just 5 sub-second CPU spikes to expand the hash table.
  
  MFC after:	2 weeks
  Sponsored by:	iXsystems, Inc.

Modified:
  head/sys/vm/uma_core.c

Modified: head/sys/vm/uma_core.c
==============================================================================
--- head/sys/vm/uma_core.c	Thu Jun  6 22:56:55 2019	(r348763)
+++ head/sys/vm/uma_core.c	Thu Jun  6 23:57:28 2019	(r348764)
@@ -261,7 +261,7 @@ static void keg_small_init(uma_keg_t keg);
 static void keg_large_init(uma_keg_t keg);
 static void zone_foreach(void (*zfunc)(uma_zone_t));
 static void zone_timeout(uma_zone_t zone);
-static int hash_alloc(struct uma_hash *);
+static int hash_alloc(struct uma_hash *, u_int);
 static int hash_expand(struct uma_hash *, struct uma_hash *);
 static void hash_free(struct uma_hash *hash);
 static void uma_timeout(void *);
@@ -572,6 +572,7 @@ static void
 zone_timeout(uma_zone_t zone)
 {
 	uma_keg_t keg = zone->uz_keg;
+	u_int slabs;
 
 	KEG_LOCK(keg);
 	/*
@@ -582,7 +583,8 @@ zone_timeout(uma_zone_t zone)
 	 * may be a little aggressive.  Should I allow for two collisions max?
 	 */
 	if (keg->uk_flags & UMA_ZONE_HASH &&
-	    keg->uk_pages / keg->uk_ppera >= keg->uk_hash.uh_hashsize) {
+	    (slabs = keg->uk_pages / keg->uk_ppera) >
+	     keg->uk_hash.uh_hashsize) {
 		struct uma_hash newhash;
 		struct uma_hash oldhash;
 		int ret;
@@ -593,9 +595,8 @@ zone_timeout(uma_zone_t zone)
 		 * I have to do everything in stages and check for
 		 * races.
 		 */
-		newhash = keg->uk_hash;
 		KEG_UNLOCK(keg);
-		ret = hash_alloc(&newhash);
+		ret = hash_alloc(&newhash, 1 << fls(slabs));
 		KEG_LOCK(keg);
 		if (ret) {
 			if (hash_expand(&keg->uk_hash, &newhash)) {
@@ -627,16 +628,13 @@ zone_timeout(uma_zone_t zone)
  *	1 on success and 0 on failure.
  */
 static int
-hash_alloc(struct uma_hash *hash)
+hash_alloc(struct uma_hash *hash, u_int size)
 {
-	u_int oldsize;
 	size_t alloc;
 
-	oldsize = hash->uh_hashsize;
-
-	/* We're just going to go to a power of two greater */
-	if (oldsize)  {
-		hash->uh_hashsize = oldsize * 2;
+	KASSERT(powerof2(size), ("hash size must be power of 2"));
+	if (size > UMA_HASH_SIZE_INIT)  {
+		hash->uh_hashsize = size;
 		alloc = sizeof(hash->uh_slab_hash[0]) * hash->uh_hashsize;
 		hash->uh_slab_hash = (struct slabhead *)malloc(alloc,
 		    M_UMAHASH, M_NOWAIT);
@@ -1711,7 +1709,7 @@ keg_ctor(void *mem, int size, void *udata, int flags)
 	}
 
 	if (keg->uk_flags & UMA_ZONE_HASH)
-		hash_alloc(&keg->uk_hash);
+		hash_alloc(&keg->uk_hash, 0);
 
 	CTR5(KTR_UMA, "keg_ctor %p zone %s(%p) out %d free %d\n",
 	    keg, zone->uz_name, zone,