svn commit: r280327 - in head/sys: kern vm

[Alan Cox]

Author: alc
Date: Sat Mar 21 17:56:55 2015
New Revision: 280327
URL: https://svnweb.freebsd.org/changeset/base/280327

Log:
  Introduce vm_object_color() and use it in mmap(2) to set the color of
  named objects to zero before the virtual address is selected.  Previously,
  the color setting was delayed until after the virtual address was
  selected.  In rtld, this delay effectively prevented the mapping of a
  shared library's code section using superpages.  Now, for example, we see
  the first 1 MB of libc's code on armv6 mapped by a superpage after we've
  gotten through the initial cold misses that bring the first 1 MB of code
  into memory.  (With the page clustering that we perform on read faults,
  this happens quickly.)
  
  Differential Revision:	https://reviews.freebsd.org/D2013
  Reviewed by:	jhb, kib
  Tested by:	Svatopluk Kraus (armv6)
  MFC after:	6 weeks

Modified:
  head/sys/kern/kern_exec.c
  head/sys/vm/vm_fault.c
  head/sys/vm/vm_mmap.c
  head/sys/vm/vm_object.h
  head/sys/vm/vnode_pager.c

Modified: head/sys/kern/kern_exec.c
==============================================================================
--- head/sys/kern/kern_exec.c	Sat Mar 21 16:54:01 2015	(r280326)
+++ head/sys/kern/kern_exec.c	Sat Mar 21 17:56:55 2015	(r280327)
@@ -933,10 +933,7 @@ exec_map_first_page(imgp)
 		return (EACCES);
 	VM_OBJECT_WLOCK(object);
 #if VM_NRESERVLEVEL > 0
-	if ((object->flags & OBJ_COLORED) == 0) {
-		object->flags |= OBJ_COLORED;
-		object->pg_color = 0;
-	}
+	vm_object_color(object, 0);
 #endif
 	ma[0] = vm_page_grab(object, 0, VM_ALLOC_NORMAL);
 	if (ma[0]->valid != VM_PAGE_BITS_ALL) {

Modified: head/sys/vm/vm_fault.c
==============================================================================
--- head/sys/vm/vm_fault.c	Sat Mar 21 16:54:01 2015	(r280326)
+++ head/sys/vm/vm_fault.c	Sat Mar 21 17:56:55 2015	(r280327)
@@ -522,11 +522,8 @@ fast_failed:
 			fs.m = NULL;
 			if (!vm_page_count_severe() || P_KILLED(curproc)) {
 #if VM_NRESERVLEVEL > 0
-				if ((fs.object->flags & OBJ_COLORED) == 0) {
-					fs.object->flags |= OBJ_COLORED;
-					fs.object->pg_color = atop(vaddr) -
-					    fs.pindex;
-				}
+				vm_object_color(fs.object, atop(vaddr) -
+				    fs.pindex);
 #endif
 				alloc_req = P_KILLED(curproc) ?
 				    VM_ALLOC_SYSTEM : VM_ALLOC_NORMAL;

Modified: head/sys/vm/vm_mmap.c
==============================================================================
--- head/sys/vm/vm_mmap.c	Sat Mar 21 16:54:01 2015	(r280326)
+++ head/sys/vm/vm_mmap.c	Sat Mar 21 17:56:55 2015	(r280327)
@@ -45,6 +45,7 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_compat.h"
 #include "opt_hwpmc_hooks.h"
+#include "opt_vm.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -1387,17 +1388,22 @@ vm_mmap_vnode(struct thread *td, vm_size
 	objsize = round_page(va.va_size);
 	if (va.va_nlink == 0)
 		flags |= MAP_NOSYNC;
-	if (obj->type == OBJT_VNODE)
+	if (obj->type == OBJT_VNODE) {
 		obj = vm_pager_allocate(OBJT_VNODE, vp, objsize, prot, foff,
 		    cred);
-	else {
+		if (obj == NULL) {
+			error = ENOMEM;
+			goto done;
+		}
+	} else {
 		KASSERT(obj->type == OBJT_DEFAULT || obj->type == OBJT_SWAP,
 		    ("wrong object type"));
-		vm_object_reference(obj);
-	}
-	if (obj == NULL) {
-		error = ENOMEM;
-		goto done;
+		VM_OBJECT_WLOCK(obj);
+		vm_object_reference_locked(obj);
+#if VM_NRESERVLEVEL > 0
+		vm_object_color(obj, 0);
+#endif
+		VM_OBJECT_WUNLOCK(obj);
 	}
 	*objp = obj;
 	*flagsp = flags;

Modified: head/sys/vm/vm_object.h
==============================================================================
--- head/sys/vm/vm_object.h	Sat Mar 21 16:54:01 2015	(r280326)
+++ head/sys/vm/vm_object.h	Sat Mar 21 17:56:55 2015	(r280327)
@@ -257,6 +257,30 @@ vm_object_set_flag(vm_object_t object, u
 	object->flags |= bits;
 }
 
+/*
+ *	Conditionally set the object's color, which (1) enables the allocation
+ *	of physical memory reservations for anonymous objects and larger-than-
+ *	superpage-sized named objects and (2) determines the first page offset
+ *	within the object at which a reservation may be allocated.  In other
+ *	words, the color determines the alignment of the object with respect
+ *	to the largest superpage boundary.  When mapping named objects, like
+ *	files or POSIX shared memory objects, the color should be set to zero
+ *	before a virtual address is selected for the mapping.  In contrast,
+ *	for anonymous objects, the color may be set after the virtual address
+ *	is selected.
+ *
+ *	The object must be locked.
+ */
+static __inline void
+vm_object_color(vm_object_t object, u_short color)
+{
+
+	if ((object->flags & OBJ_COLORED) == 0) {
+		object->pg_color = color;
+		object->flags |= OBJ_COLORED;
+	}
+}
+
 void vm_object_clear_flag(vm_object_t object, u_short bits);
 void vm_object_pip_add(vm_object_t object, short i);
 void vm_object_pip_subtract(vm_object_t object, short i);

Modified: head/sys/vm/vnode_pager.c
==============================================================================
--- head/sys/vm/vnode_pager.c	Sat Mar 21 16:54:01 2015	(r280326)
+++ head/sys/vm/vnode_pager.c	Sat Mar 21 17:56:55 2015	(r280327)
@@ -53,6 +53,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_vm.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/proc.h>
@@ -242,6 +244,9 @@ retry:
 		VI_UNLOCK(vp);
 	} else {
 		object->ref_count++;
+#if VM_NRESERVLEVEL > 0
+		vm_object_color(object, 0);
+#endif
 		VM_OBJECT_WUNLOCK(object);
 	}
 	vref(vp);