svn commit: r246834 - user/attilio/vmc-playground/sys/vm

[Attilio Rao]

Author: attilio
Date: Fri Feb 15 14:48:06 2013
New Revision: 246834
URL: http://svnweb.freebsd.org/changeset/base/246834

Log:
  Improve dynamic branch prediction and i-cache utilization:
  - Use predict_false() to tag boot-time cache decisions
  - Compact boot-time cache allocation into a separate, non-inline,
    function that won't be called most of the times.
  
  Sponsored by:	EMC / Isilon storage division

Modified:
  user/attilio/vmc-playground/sys/vm/vm_radix.c

Modified: user/attilio/vmc-playground/sys/vm/vm_radix.c
==============================================================================
--- user/attilio/vmc-playground/sys/vm/vm_radix.c	Fri Feb 15 14:22:23 2013	(r246833)
+++ user/attilio/vmc-playground/sys/vm/vm_radix.c	Fri Feb 15 14:48:06 2013	(r246834)
@@ -128,6 +128,18 @@ vm_radix_node_zone_dtor(void *mem, int s
 }
 #endif
 
+static struct vm_radix_node *
+vm_radix_carve_bootcache(void)
+{
+	struct vm_radix_node *rnode;
+
+	if (boot_cache_cnt == VM_RADIX_BOOT_CACHE)
+		panic("%s: Increase VM_RADIX_BOOT_CACHE", __func__);
+	rnode = &boot_cache[boot_cache_cnt];
+	boot_cache_cnt++;
+	return (rnode);
+}
+
 /*
  * Allocate a radix node.  Pre-allocation ensures that the request will be
  * always successfully satisfied.
@@ -137,12 +149,9 @@ vm_radix_node_get(vm_pindex_t owner, uin
 {
 	struct vm_radix_node *rnode;
 
-	if (boot_cache_cnt <= VM_RADIX_BOOT_CACHE) {
-		if (boot_cache_cnt == VM_RADIX_BOOT_CACHE)
-			panic("%s: Increase VM_RADIX_BOOT_CACHE", __func__);
-		rnode = &boot_cache[boot_cache_cnt];
-		boot_cache_cnt++;
-	} else {
+	if (__predict_false(boot_cache_cnt <= VM_RADIX_BOOT_CACHE))
+		rnode = vm_radix_carve_bootcache();
+	else {
 		rnode = uma_zalloc(vm_radix_node_zone, M_NOWAIT | M_ZERO);
 
 		/*
@@ -173,7 +182,8 @@ static __inline void
 vm_radix_node_put(struct vm_radix_node *rnode)
 {
 
-	if (rnode > boot_cache && rnode <= &boot_cache[VM_RADIX_BOOT_CACHE])
+	if (__predict_false(rnode > boot_cache &&
+	    rnode <= &boot_cache[VM_RADIX_BOOT_CACHE]))
 		return;
 	uma_zfree(vm_radix_node_zone, rnode);
 }