svn commit: r268211 - head/sys/kern

[Konstantin Belousov]

Author: kib
Date: Thu Jul  3 11:06:27 2014
New Revision: 268211
URL: http://svnweb.freebsd.org/changeset/base/268211

Log:
  Micro-manage clang to get the expected inlining for cpu_search().
  Mark cpu_search_lowest/cpu_search_highest/cpu_search_both as noinline,
  while cpu_search() gets always_inline.  With the attributes set,
  cpu_search() is inlined in wrappers, and if()s with constant
  conditionals are optimized.
  
  On some tests on many-core machine, the hwpmc reported samples for
  cpu_search*() are reduced from 25% total to 9%.
  
  Submitted by:	"Rang, Anton" <anton.rang at isilon.com>
  MFC after:	1 week

Modified:
  head/sys/kern/sched_ule.c

Modified: head/sys/kern/sched_ule.c
==============================================================================
--- head/sys/kern/sched_ule.c	Thu Jul  3 10:59:42 2014	(r268210)
+++ head/sys/kern/sched_ule.c	Thu Jul  3 11:06:27 2014	(r268211)
@@ -622,12 +622,14 @@ struct cpu_search {
 	for ((cpu) = 0; (cpu) <= mp_maxid; (cpu)++)		\
 		if (CPU_ISSET(cpu, &mask))
 
-static __inline int cpu_search(const struct cpu_group *cg, struct cpu_search *low,
-    struct cpu_search *high, const int match);
-int cpu_search_lowest(const struct cpu_group *cg, struct cpu_search *low);
-int cpu_search_highest(const struct cpu_group *cg, struct cpu_search *high);
-int cpu_search_both(const struct cpu_group *cg, struct cpu_search *low,
+static __always_inline int cpu_search(const struct cpu_group *cg,
+    struct cpu_search *low, struct cpu_search *high, const int match);
+int __noinline cpu_search_lowest(const struct cpu_group *cg,
+    struct cpu_search *low);
+int __noinline cpu_search_highest(const struct cpu_group *cg,
     struct cpu_search *high);
+int __noinline cpu_search_both(const struct cpu_group *cg,
+    struct cpu_search *low, struct cpu_search *high);
 
 /*
  * Search the tree of cpu_groups for the lowest or highest loaded cpu
@@ -640,7 +642,7 @@ int cpu_search_both(const struct cpu_gro
  * match argument.  It is reduced to the minimum set for each case.  It is
  * also recursive to the depth of the tree.
  */
-static __inline int
+static __always_inline int
 cpu_search(const struct cpu_group *cg, struct cpu_search *low,
     struct cpu_search *high, const int match)
 {