svn commit: r313756 - head/sys/kern

[Mark Johnston]

Author: markj
Date: Wed Feb 15 01:50:58 2017
New Revision: 313756
URL: https://svnweb.freebsd.org/changeset/base/313756

Log:
  Apply MADV_FREE to exec_map entries only after a lowmem event.
  
  This effectively provides the same benefit as applying MADV_FREE inline
  upon every execve, since the page daemon invokes lowmem handlers prior to
  scanning the inactive queue. It also has less overhead; the cost of
  applying MADV_FREE is very noticeable on many-CPU systems since it includes
  that of a TLB shootdown of global PTEs. For instance, this change nearly
  halves the system CPU usage during a buildkernel on a 128-vCPU EC2
  instance (with some other patches applied).
  
  Benchmarked by:	cperciva (earlier version)
  Reviewed by:	kib
  MFC after:	2 weeks
  Differential Revision:	https://reviews.freebsd.org/D9586

Modified:
  head/sys/kern/kern_exec.c

Modified: head/sys/kern/kern_exec.c
==============================================================================
--- head/sys/kern/kern_exec.c	Wed Feb 15 01:35:26 2017	(r313755)
+++ head/sys/kern/kern_exec.c	Wed Feb 15 01:50:58 2017	(r313756)
@@ -1320,6 +1320,7 @@ err_exit:
 
 struct exec_args_kva {
 	vm_offset_t addr;
+	u_int gen;
 	SLIST_ENTRY(exec_args_kva) next;
 };
 
@@ -1327,6 +1328,7 @@ static DPCPU_DEFINE(struct exec_args_kva
 
 static SLIST_HEAD(, exec_args_kva) exec_args_kva_freelist;
 static struct mtx exec_args_kva_mtx;
+static u_int exec_args_gen;
 
 static void
 exec_prealloc_args_kva(void *arg __unused)
@@ -1339,6 +1341,7 @@ exec_prealloc_args_kva(void *arg __unuse
 	for (i = 0; i < exec_map_entries; i++) {
 		argkva = malloc(sizeof(*argkva), M_PARGS, M_WAITOK);
 		argkva->addr = kmap_alloc_wait(exec_map, exec_map_entry_size);
+		argkva->gen = exec_args_gen;
 		SLIST_INSERT_HEAD(&exec_args_kva_freelist, argkva, next);
 	}
 }
@@ -1364,15 +1367,16 @@ exec_alloc_args_kva(void **cookie)
 }
 
 static void
-exec_free_args_kva(void *cookie)
+exec_release_args_kva(struct exec_args_kva *argkva, u_int gen)
 {
-	struct exec_args_kva *argkva;
 	vm_offset_t base;
 
-	argkva = cookie;
 	base = argkva->addr;
-
-	vm_map_madvise(exec_map, base, base + exec_map_entry_size, MADV_FREE);
+	if (argkva->gen != gen) {
+		vm_map_madvise(exec_map, base, base + exec_map_entry_size,
+		    MADV_FREE);
+		argkva->gen = gen;
+	}
 	if (!atomic_cmpset_ptr((uintptr_t *)DPCPU_PTR(exec_args_kva),
 	    (uintptr_t)NULL, (uintptr_t)argkva)) {
 		mtx_lock(&exec_args_kva_mtx);
@@ -1382,6 +1386,46 @@ exec_free_args_kva(void *cookie)
 	}
 }
 
+static void
+exec_free_args_kva(void *cookie)
+{
+
+	exec_release_args_kva(cookie, exec_args_gen);
+}
+
+static void
+exec_args_kva_lowmem(void *arg __unused)
+{
+	SLIST_HEAD(, exec_args_kva) head;
+	struct exec_args_kva *argkva;
+	u_int gen;
+	int i;
+
+	gen = atomic_fetchadd_int(&exec_args_gen, 1) + 1;
+
+	/*
+	 * Force an madvise of each KVA range. Any currently allocated ranges
+	 * will have MADV_FREE applied once they are freed.
+	 */
+	SLIST_INIT(&head);
+	mtx_lock(&exec_args_kva_mtx);
+	SLIST_SWAP(&head, &exec_args_kva_freelist, exec_args_kva);
+	mtx_unlock(&exec_args_kva_mtx);
+	while ((argkva = SLIST_FIRST(&head)) != NULL) {
+		SLIST_REMOVE_HEAD(&head, next);
+		exec_release_args_kva(argkva, gen);
+	}
+
+	CPU_FOREACH(i) {
+		argkva = (void *)atomic_readandclear_ptr(
+		    (uintptr_t *)DPCPU_ID_PTR(i, exec_args_kva));
+		if (argkva != NULL)
+			exec_release_args_kva(argkva, gen);
+	}
+}
+EVENTHANDLER_DEFINE(vm_lowmem, exec_args_kva_lowmem, NULL,
+    EVENTHANDLER_PRI_ANY);
+
 /*
  * Allocate temporary demand-paged, zero-filled memory for the file name,
  * argument, and environment strings.