git: d704d7189dba - main - sched_ule: Skip the CPU search for threads bound to one CPU

From: Andrew Gallatin <gallatin_at_FreeBSD.org>
Date: Thu, 19 Nov 2026 14:55:09 UTC
The branch main has been updated by gallatin:

URL: https://cgit.FreeBSD.org/src/commit/?id=d704d7189dbaed8db143ac084b474a79420dd3b3

commit d704d7189dbaed8db143ac084b474a79420dd3b3
Author:     Nick Banks <nickbanks@netflix.com>
AuthorDate: 2026-11-19 14:27:38 +0000
Commit:     Andrew Gallatin <gallatin@FreeBSD.org>
CommitDate: 2026-11-19 14:54:45 +0000

    sched_ule: Skip the CPU search for threads bound to one CPU
    
    sched_pickcpu() returns early only for pinned threads (td_pinned != 0).
    A thread whose cpuset holds a single CPU, as set by
    cpuset_setaffinity(2) or pthread_setaffinity_np(3), is not pinned in
    that sense. Unless its last CPU is idle, it falls through to
    cpu_search_lowest() over the whole topology, even though the only
    possible answer is that one CPU.
    
    The search is costly in a common layout: one worker thread bound to each
    CPU, with a NIC interrupt thread on the same CPU waking its worker for
    every batch of received packets. The worker's CPU is busy running the
    interrupt thread at wakeup, so the idle-CPU shortcut never applies, and
    every wakeup walks the cpu_group tree.
    
    sched_pickcpu() now checks whether the thread's cpuset mask holds one
    CPU: CPU_FFS() finds the first CPU, and CPU_FFS_AT() looks for another
    after it. If there is none, sched_pickcpu() returns that CPU without
    the search. Add CPU_FFS_AT(), on top of __BIT_FFS_AT() and documented
    in cpuset(9), and a SCHED_STATS counter, pickcpu_onecpu.
    
    On a 64-thread AMD EPYC 7502P (8-CPU LLC groups) under a sustained UDP
    receive load with one bound worker per CPU, kernel CPU time in the
    receive path fell by 60%, from 1.72 to 0.69 profile samples per second.
    sched_pickcpu() went from 73% of the receive path to 4.8%, and
    cpu_search_lowest() dropped out of the profile.
    
    Reviewed by: gallatin, olce
    Sponsored by: Netflix
    Differential Revision: https://reviews.freebsd.org/D61512
---
 share/man/man9/Makefile |  1 +
 share/man/man9/cpuset.9 | 24 +++++++++++++++++++++++-
 sys/kern/sched_ule.c    | 15 +++++++++++++--
 sys/sys/cpuset.h        |  1 +
 4 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/share/man/man9/Makefile b/share/man/man9/Makefile
index fb46f9cee884..9454872d3e53 100644
--- a/share/man/man9/Makefile
+++ b/share/man/man9/Makefile
@@ -964,6 +964,7 @@ MLINKS+=cpuset.9 CPUSET_T_INITIALIZER.9 \
 	cpuset.9 CPU_EMPTY.9 \
 	cpuset.9 CPU_ISFULLSET.9 \
 	cpuset.9 CPU_FFS.9 \
+	cpuset.9 CPU_FFS_AT.9 \
 	cpuset.9 CPU_COUNT.9 \
 	cpuset.9 CPU_SUBSET.9 \
 	cpuset.9 CPU_OVERLAP.9 \
diff --git a/share/man/man9/cpuset.9 b/share/man/man9/cpuset.9
index f96226890d90..df8ba4a49d4f 100644
--- a/share/man/man9/cpuset.9
+++ b/share/man/man9/cpuset.9
@@ -22,7 +22,7 @@
 .\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 .\" POSSIBILITY OF SUCH DAMAGE.
 .\"
-.Dd August 7, 2025
+.Dd November 19, 2026
 .Dt CPUSET 9
 .Os
 .Sh NAME
@@ -39,6 +39,7 @@
 .Nm CPU_EMPTY ,
 .Nm CPU_ISFULLSET ,
 .Nm CPU_FFS ,
+.Nm CPU_FFS_AT ,
 .Nm CPU_COUNT ,
 .Nm CPU_SUBSET ,
 .Nm CPU_OVERLAP ,
@@ -79,6 +80,8 @@
 .Ft int
 .Fn CPU_FFS "cpuset_t *cpuset"
 .Ft int
+.Fn CPU_FFS_AT "cpuset_t *cpuset" "long start"
+.Ft int
 .Fn CPU_COUNT "cpuset_t *cpuset"
 .\"
 .Ft bool
@@ -248,6 +251,25 @@ index parameter to any other
 macro, you must subtract one from the result.
 .Pp
 The
+.Fn CPU_FFS_AT
+macro returns the 1-index of the first (lowest) CPU in
+.Fa cpuset
+at or after the 0-indexed
+.Fa start ,
+or zero if there is none.
+Passing a non-zero result of
+.Fn CPU_FFS
+or
+.Fn CPU_FFS_AT
+as
+.Fa start
+finds the next CPU in the set.
+A
+.Fa start
+of zero searches from the first CPU again,
+so a loop over the set must stop when the result is zero.
+.Pp
+The
 .Fn CPU_COUNT
 macro returns the total number of CPUs in
 .Fa cpuset .
diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c
index edd08f4ecb6f..1d59ec14d043 100644
--- a/sys/kern/sched_ule.c
+++ b/sys/kern/sched_ule.c
@@ -1423,6 +1423,7 @@ SCHED_STAT_DEFINE(pickcpu_affinity, "Picked cpu based on affinity");
 SCHED_STAT_DEFINE(pickcpu_lowest, "Selected lowest load");
 SCHED_STAT_DEFINE(pickcpu_local, "Migrated to current cpu");
 SCHED_STAT_DEFINE(pickcpu_migration, "Selection may have caused migration");
+SCHED_STAT_DEFINE(pickcpu_onecpu, "Cpuset allows a single cpu");
 
 static int
 sched_pickcpu(struct thread *td, int flags)
@@ -1431,7 +1432,7 @@ sched_pickcpu(struct thread *td, int flags)
 	struct td_sched *ts;
 	struct tdq *tdq;
 	cpuset_t *mask;
-	int cpu, pri, r, self, intr;
+	int cpu, cpu_ffs, pri, r, self, intr;
 
 	self = PCPU_GET(cpuid);
 	ts = td_get_sched(td);
@@ -1444,6 +1445,17 @@ sched_pickcpu(struct thread *td, int flags)
 	 */
 	if ((flags & SRQ_OURSELF) || !THREAD_CAN_MIGRATE(td))
 		return (ts->ts_cpu);
+	/*
+	 * A thread whose cpuset holds one cpu can only run there, so skip
+	 * the topology search.
+	 */
+	mask = &td->td_cpuset->cs_mask;
+	cpu_ffs = CPU_FFS(mask);
+	MPASS(cpu_ffs > 0);
+	if (cpu_ffs == CPU_SETSIZE || CPU_FFS_AT(mask, cpu_ffs) == 0) {
+		SCHED_STAT_INC(pickcpu_onecpu);
+		return (cpu_ffs - 1);
+	}
 	/*
 	 * Prefer to run interrupt threads on the processors that generate
 	 * the interrupt.
@@ -1509,7 +1521,6 @@ llc:
 	if (ccg == cpu_top)
 		ccg = NULL;
 	cpu = -1;
-	mask = &td->td_cpuset->cs_mask;
 	pri = td->td_priority;
 	r = TD_IS_RUNNING(td);
 	/*
diff --git a/sys/sys/cpuset.h b/sys/sys/cpuset.h
index 92c14b0de5eb..7421235b7351 100644
--- a/sys/sys/cpuset.h
+++ b/sys/sys/cpuset.h
@@ -69,6 +69,7 @@
 #define	CPU_OR_ATOMIC(d, s)		__BIT_OR_ATOMIC(CPU_SETSIZE, d, s)
 #define	CPU_COPY_STORE_REL(f, t)	__BIT_COPY_STORE_REL(CPU_SETSIZE, f, t)
 #define	CPU_FFS(p)			__BIT_FFS(CPU_SETSIZE, p)
+#define	CPU_FFS_AT(p, start)		__BIT_FFS_AT(CPU_SETSIZE, p, start)
 #define	CPU_FLS(p)			__BIT_FLS(CPU_SETSIZE, p)
 #define	CPU_FOREACH_ISSET(i, p)		__BIT_FOREACH_ISSET(CPU_SETSIZE, i, p)
 #define	CPU_FOREACH_ISCLR(i, p)		__BIT_FOREACH_ISCLR(CPU_SETSIZE, i, p)