svn commit: r255363 - head/sys/kern

[Alexander Motin]

Author: mav
Date: Sat Sep  7 15:16:30 2013
New Revision: 255363
URL: http://svnweb.freebsd.org/changeset/base/255363

Log:
  Micro-optimize cpu_search(), allowing compiler to use more efficient inline
  ffsl() implementation, when it is available, instead of homegrown iteration.
  
  On dual-E5645 amd64 system (2x6x2 cores) under heavy I/O load that reduces
  time spent inside cpu_search() from 19% to 13%, while IOPS increased by 5%.

Modified:
  head/sys/kern/sched_ule.c

Modified: head/sys/kern/sched_ule.c
==============================================================================
--- head/sys/kern/sched_ule.c	Sat Sep  7 14:15:13 2013	(r255362)
+++ head/sys/kern/sched_ule.c	Sat Sep  7 15:16:30 2013	(r255363)
@@ -667,10 +667,14 @@ cpu_search(const struct cpu_group *cg, s
 	}
 
 	/* Iterate through the child CPU groups and then remaining CPUs. */
-	for (i = cg->cg_children, cpu = mp_maxid; i >= 0; ) {
+	for (i = cg->cg_children, cpu = mp_maxid; ; ) {
 		if (i == 0) {
+#ifdef HAVE_INLINE_FFSL
+			cpu = CPU_FFS(&cpumask) - 1;
+#else
 			while (cpu >= 0 && !CPU_ISSET(cpu, &cpumask))
 				cpu--;
+#endif
 			if (cpu < 0)
 				break;
 			child = NULL;
@@ -695,6 +699,7 @@ cpu_search(const struct cpu_group *cg, s
 				break;
 			}
 		} else {			/* Handle child CPU. */
+			CPU_CLR(cpu, &cpumask);
 			tdq = TDQ_CPU(cpu);
 			load = tdq->tdq_load * 256;
 			rndptr = DPCPU_PTR(randomval);
@@ -742,8 +747,11 @@ cpu_search(const struct cpu_group *cg, s
 			i--;
 			if (i == 0 && CPU_EMPTY(&cpumask))
 				break;
-		} else
+		}
+#ifndef HAVE_INLINE_FFSL
+		else
 			cpu--;
+#endif
 	}
 	return (total);
 }