git: d704d7189dba - main - sched_ule: Skip the CPU search for threads bound to one CPU
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Thu, 19 Nov 2026 14:55:09 UTC
The branch main has been updated by gallatin:
URL: https://cgit.FreeBSD.org/src/commit/?id=d704d7189dbaed8db143ac084b474a79420dd3b3
commit d704d7189dbaed8db143ac084b474a79420dd3b3
Author: Nick Banks <nickbanks@netflix.com>
AuthorDate: 2026-11-19 14:27:38 +0000
Commit: Andrew Gallatin <gallatin@FreeBSD.org>
CommitDate: 2026-11-19 14:54:45 +0000
sched_ule: Skip the CPU search for threads bound to one CPU
sched_pickcpu() returns early only for pinned threads (td_pinned != 0).
A thread whose cpuset holds a single CPU, as set by
cpuset_setaffinity(2) or pthread_setaffinity_np(3), is not pinned in
that sense. Unless its last CPU is idle, it falls through to
cpu_search_lowest() over the whole topology, even though the only
possible answer is that one CPU.
The search is costly in a common layout: one worker thread bound to each
CPU, with a NIC interrupt thread on the same CPU waking its worker for
every batch of received packets. The worker's CPU is busy running the
interrupt thread at wakeup, so the idle-CPU shortcut never applies, and
every wakeup walks the cpu_group tree.
sched_pickcpu() now checks whether the thread's cpuset mask holds one
CPU: CPU_FFS() finds the first CPU, and CPU_FFS_AT() looks for another
after it. If there is none, sched_pickcpu() returns that CPU without
the search. Add CPU_FFS_AT(), on top of __BIT_FFS_AT() and documented
in cpuset(9), and a SCHED_STATS counter, pickcpu_onecpu.
On a 64-thread AMD EPYC 7502P (8-CPU LLC groups) under a sustained UDP
receive load with one bound worker per CPU, kernel CPU time in the
receive path fell by 60%, from 1.72 to 0.69 profile samples per second.
sched_pickcpu() went from 73% of the receive path to 4.8%, and
cpu_search_lowest() dropped out of the profile.
Reviewed by: gallatin, olce
Sponsored by: Netflix
Differential Revision: https://reviews.freebsd.org/D61512
---
share/man/man9/Makefile | 1 +
share/man/man9/cpuset.9 | 24 +++++++++++++++++++++++-
sys/kern/sched_ule.c | 15 +++++++++++++--
sys/sys/cpuset.h | 1 +
4 files changed, 38 insertions(+), 3 deletions(-)
diff --git a/share/man/man9/Makefile b/share/man/man9/Makefile
index fb46f9cee884..9454872d3e53 100644
--- a/share/man/man9/Makefile
+++ b/share/man/man9/Makefile
@@ -964,6 +964,7 @@ MLINKS+=cpuset.9 CPUSET_T_INITIALIZER.9 \
cpuset.9 CPU_EMPTY.9 \
cpuset.9 CPU_ISFULLSET.9 \
cpuset.9 CPU_FFS.9 \
+ cpuset.9 CPU_FFS_AT.9 \
cpuset.9 CPU_COUNT.9 \
cpuset.9 CPU_SUBSET.9 \
cpuset.9 CPU_OVERLAP.9 \
diff --git a/share/man/man9/cpuset.9 b/share/man/man9/cpuset.9
index f96226890d90..df8ba4a49d4f 100644
--- a/share/man/man9/cpuset.9
+++ b/share/man/man9/cpuset.9
@@ -22,7 +22,7 @@
.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
.\" POSSIBILITY OF SUCH DAMAGE.
.\"
-.Dd August 7, 2025
+.Dd November 19, 2026
.Dt CPUSET 9
.Os
.Sh NAME
@@ -39,6 +39,7 @@
.Nm CPU_EMPTY ,
.Nm CPU_ISFULLSET ,
.Nm CPU_FFS ,
+.Nm CPU_FFS_AT ,
.Nm CPU_COUNT ,
.Nm CPU_SUBSET ,
.Nm CPU_OVERLAP ,
@@ -79,6 +80,8 @@
.Ft int
.Fn CPU_FFS "cpuset_t *cpuset"
.Ft int
+.Fn CPU_FFS_AT "cpuset_t *cpuset" "long start"
+.Ft int
.Fn CPU_COUNT "cpuset_t *cpuset"
.\"
.Ft bool
@@ -248,6 +251,25 @@ index parameter to any other
macro, you must subtract one from the result.
.Pp
The
+.Fn CPU_FFS_AT
+macro returns the 1-index of the first (lowest) CPU in
+.Fa cpuset
+at or after the 0-indexed
+.Fa start ,
+or zero if there is none.
+Passing a non-zero result of
+.Fn CPU_FFS
+or
+.Fn CPU_FFS_AT
+as
+.Fa start
+finds the next CPU in the set.
+A
+.Fa start
+of zero searches from the first CPU again,
+so a loop over the set must stop when the result is zero.
+.Pp
+The
.Fn CPU_COUNT
macro returns the total number of CPUs in
.Fa cpuset .
diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c
index edd08f4ecb6f..1d59ec14d043 100644
--- a/sys/kern/sched_ule.c
+++ b/sys/kern/sched_ule.c
@@ -1423,6 +1423,7 @@ SCHED_STAT_DEFINE(pickcpu_affinity, "Picked cpu based on affinity");
SCHED_STAT_DEFINE(pickcpu_lowest, "Selected lowest load");
SCHED_STAT_DEFINE(pickcpu_local, "Migrated to current cpu");
SCHED_STAT_DEFINE(pickcpu_migration, "Selection may have caused migration");
+SCHED_STAT_DEFINE(pickcpu_onecpu, "Cpuset allows a single cpu");
static int
sched_pickcpu(struct thread *td, int flags)
@@ -1431,7 +1432,7 @@ sched_pickcpu(struct thread *td, int flags)
struct td_sched *ts;
struct tdq *tdq;
cpuset_t *mask;
- int cpu, pri, r, self, intr;
+ int cpu, cpu_ffs, pri, r, self, intr;
self = PCPU_GET(cpuid);
ts = td_get_sched(td);
@@ -1444,6 +1445,17 @@ sched_pickcpu(struct thread *td, int flags)
*/
if ((flags & SRQ_OURSELF) || !THREAD_CAN_MIGRATE(td))
return (ts->ts_cpu);
+ /*
+ * A thread whose cpuset holds one cpu can only run there, so skip
+ * the topology search.
+ */
+ mask = &td->td_cpuset->cs_mask;
+ cpu_ffs = CPU_FFS(mask);
+ MPASS(cpu_ffs > 0);
+ if (cpu_ffs == CPU_SETSIZE || CPU_FFS_AT(mask, cpu_ffs) == 0) {
+ SCHED_STAT_INC(pickcpu_onecpu);
+ return (cpu_ffs - 1);
+ }
/*
* Prefer to run interrupt threads on the processors that generate
* the interrupt.
@@ -1509,7 +1521,6 @@ llc:
if (ccg == cpu_top)
ccg = NULL;
cpu = -1;
- mask = &td->td_cpuset->cs_mask;
pri = td->td_priority;
r = TD_IS_RUNNING(td);
/*
diff --git a/sys/sys/cpuset.h b/sys/sys/cpuset.h
index 92c14b0de5eb..7421235b7351 100644
--- a/sys/sys/cpuset.h
+++ b/sys/sys/cpuset.h
@@ -69,6 +69,7 @@
#define CPU_OR_ATOMIC(d, s) __BIT_OR_ATOMIC(CPU_SETSIZE, d, s)
#define CPU_COPY_STORE_REL(f, t) __BIT_COPY_STORE_REL(CPU_SETSIZE, f, t)
#define CPU_FFS(p) __BIT_FFS(CPU_SETSIZE, p)
+#define CPU_FFS_AT(p, start) __BIT_FFS_AT(CPU_SETSIZE, p, start)
#define CPU_FLS(p) __BIT_FLS(CPU_SETSIZE, p)
#define CPU_FOREACH_ISSET(i, p) __BIT_FOREACH_ISSET(CPU_SETSIZE, i, p)
#define CPU_FOREACH_ISCLR(i, p) __BIT_FOREACH_ISCLR(CPU_SETSIZE, i, p)