git: 42490d5cd29d - main - sched: New scheduler interface definition scheme
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Tue, 06 Oct 2026 13:39:26 UTC
The branch main has been updated by olce:
URL: https://cgit.FreeBSD.org/src/commit/?id=42490d5cd29d7a6acd3c28dacfbdc527c2d39028
commit 42490d5cd29d7a6acd3c28dacfbdc527c2d39028
Author: Olivier Certner <olce@FreeBSD.org>
AuthorDate: 2026-09-07 16:19:53 +0000
Commit: Olivier Certner <olce@FreeBSD.org>
CommitDate: 2026-10-06 13:27:41 +0000
sched: New scheduler interface definition scheme
Define the scheduler interface once and for all (in 'sys/sys/sched.h')
and remove all code duplication related to it (function signatures, slot
names, dispatch, scheduler instance declaration), making it easier to
modify the interface or to add new schedulers.
This is implemented by defining the interface as X macros, which are
passed a macro (and additional arguments for it) that is "called" with
a variable number of arguments describing a single function of the
interface. The convention used for a function's parameters is that each
parameter is represented with two macro arguments, the first one being
the type and the second one being the name. Helper macros allow to
process arguments described in this convention in order to generate
a list of arguments for function definitions (currently, up to
4 function parameters). The chosen convention eliminates the need for
any specific declaration of functions depending on their number of
arguments.
DECLARE_SCHEDULER() has been changed to automatically declare and fill
a 'struct sched_instance' structure. Its number of parameters has been
reduced to the minimum, and the symbol part used to produce distinct
structure names can now start with a digit (useful for 4BSD).
The new implementation has the following additional benefits:
1. The name of implementations for the interface functions is imposed
(same name as the interface function but with the scheduler symbol
that was passed to DECLARE_SCHEDULER() added just after the 'sched_'
prefix, e.g., for ULE, 'sched_ule_*()') and checked at compile-time.
2. Missing functions in an implementation are detected at compile-time
(and not at runtime with a NULL dereference).
3. Failures to fill correctly the 'struct sched_instance' object
associated to an implementation are eliminated.
Preserve comments/categories of interface functions by moving comments
that were in the old explicit interface declarations to the SCHED_ITF*()
macros, and while here, marginally improve some (and move the common
documentation for sched_initticks() from the implementations to the
interface). Several of them are not exact, but this will be fixed in
a separate commit.
Make the active scheduler instance object internal, as it is not used
outside of the shim implementation and one of the purposes of the
interface is also to hide the actual implementation. If the need
arises, it would be easy to add scheduler-specific routines as new
fields in 'struct sched_instance' and to extend the machinery to
automatically assign routines returning EOPNOTSUPP to these fields, and
then to provide a macro to register non-trivial implementation, instead
of making the current instance object visible again.
Additionally, in order to avoid collision with C and C++ keywords:
1. Fields in 'struct sched_instance' are now suffixed with '_impl', so
that, e.g., 'switch()' does not require special treatment anymore and
the corresponding implementations can be renamed back to
`sched_*_switch()`. This suffix avoids any collision with existing
C++ keywords as new functions are added to the interface. (See
SCHED_ITF_FIELD_NAME().)
2. Actual interface functions' C declarations do not list parameter
names anymore (sched_class() was causing a problem in C++ with its
'class' parameter).
allowing to remove the bracketing with '#ifndef __cplusplus'.
Reviewed by: mchoo, kib
Fixes: ce38acee8d0b ("Add kern/sched_shim.c") (+ some followups)
MFC after: 1 week
Sponsored by: The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D59988
---
sys/arm/arm/mp_machdep.c | 2 +-
sys/arm64/arm64/mp_machdep.c | 2 +-
sys/kern/init_main.c | 2 +-
sys/kern/sched_4bsd.c | 54 +----
sys/kern/sched_shim.c | 96 ++------
sys/kern/sched_ule.c | 62 +-----
sys/powerpc/aim/mp_cpudep.c | 2 +-
sys/powerpc/booke/mp_cpudep.c | 2 +-
sys/riscv/riscv/mp_machdep.c | 2 +-
sys/sys/sched.h | 507 ++++++++++++++++++++++++++++--------------
sys/x86/x86/mp_x86.c | 2 +-
11 files changed, 379 insertions(+), 354 deletions(-)
diff --git a/sys/arm/arm/mp_machdep.c b/sys/arm/arm/mp_machdep.c
index 9a4dc4e503e1..a745adef20bc 100644
--- a/sys/arm/arm/mp_machdep.c
+++ b/sys/arm/arm/mp_machdep.c
@@ -177,7 +177,7 @@ init_secondary(int cpu)
pc->pc_curthread = pc->pc_idlethread;
pc->pc_curpcb = pc->pc_idlethread->td_pcb;
set_curthread(pc->pc_idlethread);
- schedinit_ap();
+ sched_init_ap();
#ifdef VFP
vfp_init();
#endif
diff --git a/sys/arm64/arm64/mp_machdep.c b/sys/arm64/arm64/mp_machdep.c
index 30f4b30e64a7..2883e7dc74bf 100644
--- a/sys/arm64/arm64/mp_machdep.c
+++ b/sys/arm64/arm64/mp_machdep.c
@@ -280,7 +280,7 @@ init_secondary(uint64_t cpu)
/* Initialize curthread */
KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
pcpup->pc_curthread = pcpup->pc_idlethread;
- schedinit_ap();
+ sched_init_ap();
/* Initialize curpmap to match TTBR0's current setting. */
pmap0 = vmspace_pmap(&vmspace0);
diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index cfca5d9d1296..829583cb687c 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -493,7 +493,7 @@ proc0_init(void *dummy __unused)
* Initialise scheduler resources.
* Add scheduler specific parts to proc, thread as needed.
*/
- schedinit(); /* scheduler gets its house in order */
+ sched_init(); /* scheduler gets its house in order */
/*
* Create process 0.
diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c
index 5f0246323dea..431957c048eb 100644
--- a/sys/kern/sched_4bsd.c
+++ b/sys/kern/sched_4bsd.c
@@ -1010,7 +1010,7 @@ sched_4bsd_sleep(struct thread *td, int pri)
}
static void
-sched_4bsd_sswitch(struct thread *td, int flags)
+sched_4bsd_switch(struct thread *td, int flags)
{
struct thread *newtd;
struct mtx *tmtx;
@@ -1842,54 +1842,4 @@ sched_4bsd_find_l2_neighbor(int cpu)
return (-1);
}
-struct sched_instance sched_4bsd_instance = {
-#define SLOT(name) .name = sched_4bsd_##name
- SLOT(load),
- SLOT(rr_interval),
- SLOT(runnable),
- SLOT(exit),
- SLOT(fork),
- SLOT(fork_exit),
- SLOT(class),
- SLOT(nice),
- SLOT(ap_entry),
- SLOT(exit_thread),
- SLOT(estcpu),
- SLOT(fork_thread),
- SLOT(ithread_prio),
- SLOT(lend_prio),
- SLOT(lend_user_prio),
- SLOT(lend_user_prio_cond),
- SLOT(pctcpu),
- SLOT(prio),
- SLOT(sleep),
- SLOT(sswitch),
- SLOT(throw),
- SLOT(unlend_prio),
- SLOT(user_prio),
- SLOT(userret_slowpath),
- SLOT(add),
- SLOT(choose),
- SLOT(clock),
- SLOT(idletd),
- SLOT(preempt),
- SLOT(relinquish),
- SLOT(rem),
- SLOT(wakeup),
- SLOT(bind),
- SLOT(unbind),
- SLOT(is_bound),
- SLOT(affinity),
- SLOT(sizeof_proc),
- SLOT(sizeof_thread),
- SLOT(tdname),
- SLOT(clear_tdname),
- SLOT(find_l2_neighbor),
- SLOT(init),
- SLOT(init_ap),
- SLOT(setup),
- SLOT(initticks),
- SLOT(sysinit),
-#undef SLOT
-};
-DECLARE_SCHEDULER(fourbsd_sched_selector, "4BSD", &sched_4bsd_instance);
+DECLARE_SCHEDULER(4bsd, "4BSD");
diff --git a/sys/kern/sched_shim.c b/sys/kern/sched_shim.c
index 9935c4e40aa8..8fdd12e5f84d 100644
--- a/sys/kern/sched_shim.c
+++ b/sys/kern/sched_shim.c
@@ -18,76 +18,10 @@
#include <sys/sched.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
-#include <machine/ifunc.h>
-
-const struct sched_instance *active_sched;
-#define __DEFINE_SHIM(__m, __r, __n, __p, __a) \
- DEFINE_IFUNC(, __r, __n, __p) \
- { \
- return (active_sched->__m); \
- }
-#define DEFINE_SHIM0(__m, __r, __n) \
- __DEFINE_SHIM(__m, __r, __n, (void), ())
-#define DEFINE_SHIM1(__m, __r, __n, __t1, __a1) \
- __DEFINE_SHIM(__m, __r, __n, (__t1 __a1), (__a1))
-#define DEFINE_SHIM2(__m, __r, __n, __t1, __a1, __t2, __a2) \
- __DEFINE_SHIM(__m, __r, __n, (__t1 __a1, __t2 __a2), (__a1, __a2))
-
-DEFINE_SHIM0(load, int, sched_load)
-DEFINE_SHIM0(rr_interval, int, sched_rr_interval)
-DEFINE_SHIM0(runnable, bool, sched_runnable)
-DEFINE_SHIM2(exit, void, sched_exit, struct proc *, p,
- struct thread *, childtd)
-DEFINE_SHIM2(fork, void, sched_fork, struct thread *, td,
- struct thread *, childtd)
-DEFINE_SHIM1(fork_exit, void, sched_fork_exit, struct thread *, td)
-DEFINE_SHIM2(class, void, sched_class, struct thread *, td, int, class)
-DEFINE_SHIM2(nice, void, sched_nice, struct proc *, p, int, nice)
-DEFINE_SHIM0(ap_entry, void, sched_ap_entry)
-DEFINE_SHIM2(exit_thread, void, sched_exit_thread, struct thread *, td,
- struct thread *, child)
-DEFINE_SHIM1(estcpu, u_int, sched_estcpu, struct thread *, td)
-DEFINE_SHIM2(fork_thread, void, sched_fork_thread, struct thread *, td,
- struct thread *, child)
-DEFINE_SHIM2(ithread_prio, void, sched_ithread_prio, struct thread *, td,
- u_char, prio)
-DEFINE_SHIM2(lend_prio, void, sched_lend_prio, struct thread *, td,
- u_char, prio)
-DEFINE_SHIM2(lend_user_prio, void, sched_lend_user_prio, struct thread *, td,
- u_char, pri)
-DEFINE_SHIM2(lend_user_prio_cond, void, sched_lend_user_prio_cond,
- struct thread *, td, u_char, pri)
-DEFINE_SHIM1(pctcpu, fixpt_t, sched_pctcpu, struct thread *, td)
-DEFINE_SHIM2(prio, void, sched_prio, struct thread *, td, u_char, prio)
-DEFINE_SHIM2(sleep, void, sched_sleep, struct thread *, td, int, prio)
-DEFINE_SHIM2(sswitch, void, sched_switch, struct thread *, td, int, flags)
-DEFINE_SHIM1(throw, void, sched_throw, struct thread *, td)
-DEFINE_SHIM2(unlend_prio, void, sched_unlend_prio, struct thread *, td,
- u_char, prio)
-DEFINE_SHIM2(user_prio, void, sched_user_prio, struct thread *, td,
- u_char, prio)
-DEFINE_SHIM1(userret_slowpath, void, sched_userret_slowpath,
- struct thread *, td)
-DEFINE_SHIM2(add, void, sched_add, struct thread *, td, int, flags)
-DEFINE_SHIM0(choose, struct thread *, sched_choose)
-DEFINE_SHIM2(clock, void, sched_clock, struct thread *, td, int, cnt)
-DEFINE_SHIM1(idletd, void, sched_idletd, void *, dummy)
-DEFINE_SHIM1(preempt, void, sched_preempt, struct thread *, td)
-DEFINE_SHIM1(relinquish, void, sched_relinquish, struct thread *, td)
-DEFINE_SHIM1(rem, void, sched_rem, struct thread *, td)
-DEFINE_SHIM2(wakeup, void, sched_wakeup, struct thread *, td, int, srqflags)
-DEFINE_SHIM2(bind, void, sched_bind, struct thread *, td, int, cpu)
-DEFINE_SHIM1(unbind, void, sched_unbind, struct thread *, td)
-DEFINE_SHIM1(is_bound, int, sched_is_bound, struct thread *, td)
-DEFINE_SHIM1(affinity, void, sched_affinity, struct thread *, td)
-DEFINE_SHIM0(sizeof_proc, int, sched_sizeof_proc)
-DEFINE_SHIM0(sizeof_thread, int, sched_sizeof_thread)
-DEFINE_SHIM1(tdname, char *, sched_tdname, struct thread *, td)
-DEFINE_SHIM1(clear_tdname, void, sched_clear_tdname, struct thread *, td)
-DEFINE_SHIM1(find_l2_neighbor, int, sched_find_l2_neighbor, int, cpu)
-DEFINE_SHIM0(init_ap, void, schedinit_ap)
+#include <machine/ifunc.h>
+static const struct sched_instance *active_sched;
SCHED_STAT_DEFINE(ithread_demotions, "Interrupt thread priority demotions");
SCHED_STAT_DEFINE(ithread_preemptions,
@@ -117,6 +51,18 @@ int __read_mostly dtrace_vtime_active;
dtrace_vtime_switch_func_t dtrace_vtime_switch_func;
#endif
+/*
+ * Define dispatchers for all functions described by SCHED_ITF_DISPATCH().
+ */
+#define DEFINE_SHIM(args, ret_type, fn, ...) \
+ DEFINE_IFUNC(, ret_type, SCHED_ITF_FUNCTION_NAME(fn), \
+ (SCHED_ITF_ARGS_PROTO_NAME(__VA_ARGS__))) \
+ { \
+ return (active_sched->SCHED_ITF_FIELD_NAME(fn)); \
+ }
+SCHED_ITF_DISPATCH(DEFINE_SHIM, );
+#undef DEFINE_SHIM
+
static char sched_name[32] = "ULE";
SET_DECLARE(sched_instance_set, struct sched_selection);
@@ -155,27 +101,27 @@ sched_instance_select(void)
}
void
-schedinit(void)
+sched_init(void)
{
if (active_sched == NULL)
panic("Cannot find scheduler %s", sched_name);
- active_sched->init();
+ active_sched->SCHED_ITF_FIELD_NAME(init)();
}
struct cpu_group __read_mostly *cpu_top; /* CPU topology */
static void
-sched_setup(void *dummy)
+sched_setup(void)
{
cpu_top = smp_topo();
- active_sched->setup();
+ active_sched->SCHED_ITF_FIELD_NAME(setup)();
}
SYSINIT(sched_setup, SI_SUB_RUN_QUEUE, SI_ORDER_FIRST, sched_setup, NULL);
static void
-sched_initticks(void *dummy)
+sched_initticks(void)
{
- active_sched->initticks();
+ active_sched->SCHED_ITF_FIELD_NAME(initticks)();
}
SYSINIT(sched_initticks, SI_SUB_CLOCKS, SI_ORDER_THIRD, sched_initticks,
NULL);
@@ -183,7 +129,7 @@ SYSINIT(sched_initticks, SI_SUB_CLOCKS, SI_ORDER_THIRD, sched_initticks,
static void
sched_sysinit(void)
{
- active_sched->sysinit();
+ active_sched->SCHED_ITF_FIELD_NAME(sysinit)();
}
SYSINIT(sched_sysinit, SI_SUB_LAST, SI_ORDER_FIRST, sched_sysinit, NULL);
diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c
index ea5f63311387..453e3958faf9 100644
--- a/sys/kern/sched_ule.c
+++ b/sys/kern/sched_ule.c
@@ -1899,14 +1899,14 @@ sched_ule_init(void)
}
/*
- * schedinit_ap() is needed prior to calling sched_throw(NULL) to ensure that
+ * sched_init_ap() is needed prior to calling sched_throw(NULL) to ensure that
* the pcpu requirements are met for any calls in the period between curthread
* initialization and sched_throw(). One can safely add threads to the queue
* before sched_throw(), for instance, as long as the thread lock is setup
* correctly.
*
* TDQ_SELF() relies on the below sched pcpu setting; it may be used only
- * after schedinit_ap().
+ * after sched_init_ap().
*/
static void
sched_ule_init_ap(void)
@@ -2320,7 +2320,7 @@ thread_unblock_switch(struct thread *td, struct mtx *mtx)
* be assigned elsewhere via binding.
*/
static void
-sched_ule_sswitch(struct thread *td, int flags)
+sched_ule_switch(struct thread *td, int flags)
{
struct thread *newtd;
struct tdq *tdq;
@@ -3240,7 +3240,7 @@ sched_ule_ap_entry(void)
tdq = TDQ_SELF();
- /* This should have been setup in schedinit_ap(). */
+ /* This should have been setup in sched_init_ap(). */
THREAD_LOCKPTR_ASSERT(curthread, TDQ_LOCKPTR(tdq));
TDQ_LOCK(tdq);
@@ -3418,58 +3418,6 @@ sched_ule_find_l2_neighbor(int cpu)
}
#endif
-struct sched_instance sched_ule_instance = {
-#define SLOT(name) .name = sched_ule_##name
- SLOT(load),
- SLOT(rr_interval),
- SLOT(runnable),
- SLOT(exit),
- SLOT(fork),
- SLOT(fork_exit),
- SLOT(class),
- SLOT(nice),
- SLOT(ap_entry),
- SLOT(exit_thread),
- SLOT(estcpu),
- SLOT(fork_thread),
- SLOT(ithread_prio),
- SLOT(lend_prio),
- SLOT(lend_user_prio),
- SLOT(lend_user_prio_cond),
- SLOT(pctcpu),
- SLOT(prio),
- SLOT(sleep),
- SLOT(sswitch),
- SLOT(throw),
- SLOT(unlend_prio),
- SLOT(user_prio),
- SLOT(userret_slowpath),
- SLOT(add),
- SLOT(choose),
- SLOT(clock),
- SLOT(idletd),
- SLOT(preempt),
- SLOT(relinquish),
- SLOT(rem),
- SLOT(wakeup),
- SLOT(bind),
- SLOT(unbind),
- SLOT(is_bound),
- SLOT(affinity),
- SLOT(sizeof_proc),
- SLOT(sizeof_thread),
- SLOT(tdname),
- SLOT(clear_tdname),
- SLOT(find_l2_neighbor),
- SLOT(init),
- SLOT(init_ap),
- SLOT(setup),
- SLOT(initticks),
- SLOT(sysinit),
-#undef SLOT
-};
-DECLARE_SCHEDULER(ule_sched_selector, "ULE", &sched_ule_instance);
-
static int
sysctl_kern_quantum(SYSCTL_HANDLER_ARGS)
{
@@ -3551,3 +3499,5 @@ SYSCTL_INT(_kern_sched_ule, OID_AUTO, always_steal, CTLFLAG_RWTUN,
&always_steal, 0,
"Always run the stealer from the idle thread");
#endif
+
+DECLARE_SCHEDULER(ule, "ULE");
diff --git a/sys/powerpc/aim/mp_cpudep.c b/sys/powerpc/aim/mp_cpudep.c
index 98acfc1a5c37..6bfec392f166 100644
--- a/sys/powerpc/aim/mp_cpudep.c
+++ b/sys/powerpc/aim/mp_cpudep.c
@@ -132,7 +132,7 @@ cpudep_ap_bootstrap(void)
#endif
pcpup->pc_curpcb = pcpup->pc_curthread->td_pcb;
sp = pcpup->pc_curpcb->pcb_sp;
- schedinit_ap();
+ sched_init_ap();
return (sp);
}
diff --git a/sys/powerpc/booke/mp_cpudep.c b/sys/powerpc/booke/mp_cpudep.c
index a57c80102045..e38ce1bded99 100644
--- a/sys/powerpc/booke/mp_cpudep.c
+++ b/sys/powerpc/booke/mp_cpudep.c
@@ -83,7 +83,7 @@ cpudep_ap_bootstrap(void)
#endif
pcpup->pc_curpcb = pcpup->pc_curthread->td_pcb;
sp = pcpup->pc_curpcb->pcb_sp;
- schedinit_ap();
+ sched_init_ap();
/* XXX shouldn't the pcb_sp be checked/forced for alignment here?? */
diff --git a/sys/riscv/riscv/mp_machdep.c b/sys/riscv/riscv/mp_machdep.c
index 9c81d9ee1043..5402c6d388ed 100644
--- a/sys/riscv/riscv/mp_machdep.c
+++ b/sys/riscv/riscv/mp_machdep.c
@@ -180,7 +180,7 @@ init_secondary(uint64_t hart)
/* Initialize curthread */
KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
pcpup->pc_curthread = pcpup->pc_idlethread;
- schedinit_ap();
+ sched_init_ap();
/* Setup and enable interrupts */
intr_pic_init_secondary();
diff --git a/sys/sys/sched.h b/sys/sys/sched.h
index e4cf848cf641..9d8524ee6464 100644
--- a/sys/sys/sched.h
+++ b/sys/sys/sched.h
@@ -36,6 +36,11 @@
/*-
* Copyright (c) 2002-2008, Jeffrey Roberson <jeff@freebsd.org>
* All rights reserved.
+ * Copyright (c) 2026 The FreeBSD Foundation
+ *
+ * Portions of this software were developed by Olivier Certner
+ * <olce@FreeBSD.org> at Kumacom SARL under sponsorship from the FreeBSD
+ * Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -71,51 +76,316 @@
#include <sys/linker_set.h>
#include <sys/sdt.h>
-struct proc;
-struct thread;
+/*
+ * Scheduler interface.
+ *
+ * The scheduler interface is the list of functions that external users can
+ * call. It consists of two parts: Functions that must be provided directly by
+ * a scheduler implementation (called an "instance"), and functions with
+ * a common implementation (which may indirectly call instance functions).
+ *
+ * The former are declared in an abstract manner with some of the SCHED_ITF_*()
+ * macros (SCHED_ITF_INTERNAL() being a notable exception), described in detail
+ * below. A fUN() line declares a function, and the final interface function
+ * name is the symbol in the third argument prefixed by 'sched_'. E.g., such
+ * lines:
+ *
+ * FUN((__VA_ARGS__), int, load)
+ * FUN((__VA_ARGS__), void, switch, struct thread *, td, int, flags)
+ *
+ * lead to the C declarations (among others):
+ *
+ * int sched_load(void);
+ * void sched_switch(struct thread *, int);
+ *
+ * and each implementation (instance) must implement corresponding functions,
+ * e.g., for ULE:
+ *
+ * int sched_ule_load(void);
+ * void sched_ule_switch(struct thread *td, int flags);
+ *
+ * Such interface functions are grouped and commented in the SCHED_ITF*()
+ * macros listing them.
+ *
+ * Functions with a common implementation are declared under "Common functions"
+ * below as commented plain C declarations.
+ */
+
+/*
+ * Macros SCHED_ITF_*() define functions that must be provided by an instance.
+ * Please see their herald comments for details about their differences. They
+ * are X macros to avoid repetitions in function signatures, slot names,
+ * dispatch code, and scheduler instance declaration. They take as an argument
+ * the FUN() macro, which is expanded on each interface's function descriptor.
+ *
+ * The arguments passed to FUN() are:
+ * - The variable arguments to the X macro, as the first argument.
+ * - The function descriptor, consisting of:
+ * - Field name in 'struct sched_instance'.
+ * - Return type.
+ * - Function name; prefixed by 'sched_' for the publicly visible functions,
+ * and by 'sched_<instance_name>_' for instance's implementation.
+ * - Function arguments, each being represented as a pair of successive macro
+ * parameters, the first being the argument's type and the second its name.
+ * Note that FUN() is passed a variable number of arguments for flexibility.
+ *
+ * In SCHED_ITF_*() macros, please keep the function's parameters on a separate
+ * line for clarity, even if everything would fit on a single line.
+ */
+
+/* sched_add() arguments. */
+#define SRQ_BORING 0x0000 /* No special circumstances. */
+#define SRQ_YIELDING 0x0001 /* We are yielding (from mi_switch). */
+#define SRQ_OURSELF 0x0002 /* It is ourself (from mi_switch). */
+#define SRQ_INTR 0x0004 /* It is probably urgent. */
+#define SRQ_PREEMPTED 0x0008 /* has been preempted.. be kind */
+#define SRQ_BORROWING 0x0010 /* Priority updated due to prio_lend */
+#define SRQ_HOLD 0x0020 /* Return holding original td lock */
+#define SRQ_HOLDTD 0x0040 /* Return holding td lock */
/*
- * General scheduling info.
+ * Functions to be directly dispatched to the active scheduler instance.
*
- * sched_load:
- * Total runnable non-ithread threads in the system.
+ * These are implemented using ifuncs for zero-cost dispatch.
+ */
+#define SCHED_ITF_DISPATCH(FUN, ...) \
+ /* \
+ * General scheduling info. \
+ * \
+ * sched_load: \
+ * Total runnable non-ithread threads in the system. \
+ * \
+ * sched_runnable: \
+ * Runnable threads for this processor. \
+ */ \
+ FUN((__VA_ARGS__), int, load) \
+ FUN((__VA_ARGS__), int, rr_interval) \
+ FUN((__VA_ARGS__), bool, runnable) \
+ \
+ /* \
+ * Proc related scheduling hooks. \
+ */ \
+ FUN((__VA_ARGS__), void, exit, \
+ struct proc *, p, struct thread *, child) \
+ FUN((__VA_ARGS__), void, fork, \
+ struct thread *, td, struct thread *, child) \
+ FUN((__VA_ARGS__), void, fork_exit, \
+ struct thread *, td) \
+ FUN((__VA_ARGS__), void, class, \
+ struct thread *, td, int, class) \
+ FUN((__VA_ARGS__), void, nice, \
+ struct proc *, p, int, nice) \
+ \
+ /* \
+ * Threads are switched in and out, block on resources, \
+ * have temporary priorities inherited from their procs, \
+ * and use up cpu time. \
+ */ \
+ FUN((__VA_ARGS__), void, ap_entry) \
+ FUN((__VA_ARGS__), void, exit_thread, \
+ struct thread *, td, struct thread *, child) \
+ FUN((__VA_ARGS__), u_int, estcpu, \
+ struct thread *, td) \
+ FUN((__VA_ARGS__), void, fork_thread, \
+ struct thread *, td, struct thread *, child) \
+ FUN((__VA_ARGS__), void, ithread_prio, \
+ struct thread *, td, u_char, prio) \
+ FUN((__VA_ARGS__), void, lend_prio, \
+ struct thread *, td, u_char, prio) \
+ FUN((__VA_ARGS__), void, lend_user_prio, \
+ struct thread *, td, u_char, pri) \
+ FUN((__VA_ARGS__), void, lend_user_prio_cond, \
+ struct thread *, td, u_char, pri) \
+ FUN((__VA_ARGS__), fixpt_t, pctcpu, \
+ struct thread *, td) \
+ FUN((__VA_ARGS__), void, prio, \
+ struct thread *, td, u_char, prio) \
+ FUN((__VA_ARGS__), void, sleep, \
+ struct thread *, td, int, prio) \
+ FUN((__VA_ARGS__), void, switch, \
+ struct thread *, td, int, flags) \
+ FUN((__VA_ARGS__), void, throw, \
+ struct thread *, td) \
+ FUN((__VA_ARGS__), void, unlend_prio, \
+ struct thread *, td, u_char, prio) \
+ FUN((__VA_ARGS__), void, user_prio, \
+ struct thread *, td, u_char, prio) \
+ FUN((__VA_ARGS__), void, userret_slowpath, \
+ struct thread *, td) \
+ \
+ /* \
+ * Threads are moved on and off of run queues \
+ */ \
+ FUN((__VA_ARGS__), void, add, \
+ struct thread *, td, int, flags) \
+ FUN((__VA_ARGS__), struct thread *, choose) \
+ FUN((__VA_ARGS__), void, clock, \
+ struct thread *, td, int, cnt) \
+ FUN((__VA_ARGS__), void, idletd, \
+ void *, dummy) \
+ FUN((__VA_ARGS__), void, preempt, \
+ struct thread *, td) \
+ FUN((__VA_ARGS__), void, relinquish, \
+ struct thread *, td) \
+ FUN((__VA_ARGS__), void, rem, \
+ struct thread *, td) \
+ FUN((__VA_ARGS__), void, wakeup, \
+ struct thread *, td, int, srqflags) \
+ \
+ /* \
+ * Binding makes cpu affinity permanent while pinning is used \
+ * to temporarily hold a thread on a particular CPU. \
+ */ \
+ FUN((__VA_ARGS__), void, bind, \
+ struct thread *, td, int, cpu) \
+ FUN((__VA_ARGS__), void, unbind, \
+ struct thread *, td) \
+ FUN((__VA_ARGS__), int, is_bound, \
+ struct thread *, td) \
+ FUN((__VA_ARGS__), void, affinity, \
+ struct thread *, td) \
+ \
+ /* \
+ * These procedures tell the process data structure allocation \
+ * code how many bytes to actually allocate. \
+ */ \
+ FUN((__VA_ARGS__), int, sizeof_proc) \
+ FUN((__VA_ARGS__), int, sizeof_thread) \
+ \
+ /* \
+ * This routine provides a consistent thread name for use with \
+ * KTR graphing FUNctions. \
+ */ \
+ FUN((__VA_ARGS__), char *, tdname, \
+ struct thread *, td) \
+ FUN((__VA_ARGS__), void, clear_tdname, \
+ struct thread *, td) \
+ \
+ /* \
+ * Find an L2 neighbor of the given CPU or return -1 if none \
+ * found. This does not distinguish among multiple L2 \
+ * if the given CPU has more than one (it will always return \
+ * the same result in that case). \
+ */ \
+ FUN((__VA_ARGS__), int, find_l2_neighbor, \
+ int, cpu) \
+ \
+ /* \
+ * Fixup scheduler state for secondary APs \
+ */ \
+ FUN((__VA_ARGS__), void, init_ap)
+
+/*
+ * Instance functions to be specifically wrapped.
+ *
+ * An explicit common implementation must be provided for these, and it is
+ * expected to call the instance's implementation.
+ */
+#define SCHED_ITF_WRAP(FUN, ...) \
+ /* \
+ * Initialize scheduler state for proc0 and thread0. \
+ */ \
+ FUN((__VA_ARGS__), void, init)
+
+/*
+ * Instance functions used internally.
*
- * sched_runnable:
- * Runnable threads for this processor.
+ * Functions here are not made available for external callers.
*/
-int sched_load(void);
-int sched_rr_interval(void);
-bool sched_runnable(void);
+#define SCHED_ITF_INTERNAL(FUN, ...) \
+ /* \
+ * Setup run queues ('cpu_top' filled by the shim machinery). \
+ */ \
+ FUN((__VA_ARGS__), void, setup) \
+ \
+ /* \
+ * Determines time constants after stathz and hz are setup. \
+ */ \
+ FUN((__VA_ARGS__), void, initticks) \
+ \
+ /* \
+ * Final steps after all the rest has been initialized. \
+ */ \
+ FUN((__VA_ARGS__), void, sysinit)
+
+/* All parts at once, but common functions. */
+#define SCHED_ITF_NOCOMMON(FUN, ...) \
+ SCHED_ITF_DISPATCH(FUN, __VA_ARGS__) \
+ SCHED_ITF_WRAP(FUN, __VA_ARGS__) \
+ SCHED_ITF_INTERNAL(FUN, __VA_ARGS__)
-/*
- * Proc related scheduling hooks.
+/*
+ * Machinery to help define FUN() macros.
*/
-void sched_exit(struct proc *p, struct thread *childtd);
-void sched_fork(struct thread *td, struct thread *childtd);
-void sched_fork_exit(struct thread *td);
-void sched_class(struct thread *td, int class);
-void sched_nice(struct proc *p, int nice);
+#define _SCHED_ITF_ARG1(a0, ...) \
+ a0
+#define _SCHED_ITF_ARG9(a0, a1, a2, a3, a4, a5, a6, a7, a8, ...) \
+ a8
+#define _SCHED_ITF_NARGS(...) \
+ _SCHED_ITF_ARG9(__VA_ARGS__ __VA_OPT__(,) 4, ERR, 3, ERR, 2, \
+ ERR, 1, ERR, 0)
+#define _SCHED_ITF_NARGS_DISP(PREFIX, disp_args, full_args) \
+ __CONCAT(__CONCAT(PREFIX, _), _SCHED_ITF_NARGS disp_args) \
+ full_args
+#define _SCHED_ITF_ARGS_PAIR_ERR \
+ "Two macro arguments per interface function argument expected."
+#define _SCHED_ITF_ARGS_PROTO_NAME_0() \
+ void
+#define _SCHED_ITF_ARGS_PROTO_NAME_1(t1, a1) \
+ t1 a1
+#define _SCHED_ITF_ARGS_PROTO_NAME_2(t1, a1, t2, a2) \
+ t1 a1, t2 a2
+#define _SCHED_ITF_ARGS_PROTO_NAME_3(t1, a1, t2, a2, t3, a3) \
+ t1 a1, t2 a2, t3 a3
+#define _SCHED_ITF_ARGS_PROTO_NAME_4(t1, a1, t2, a2, t3, a3, t4, a4) \
+ t1 a1, t2 a2, t3 a3, t4 a4
+#define _SCHED_ITF_ARGS_PROTO_NAME_ERR(...) \
+ _SCHED_ITF_ARGS_PAIR_ERR
+#define SCHED_ITF_ARGS_PROTO_NAME(...) \
+ _SCHED_ITF_NARGS_DISP(_SCHED_ITF_ARGS_PROTO_NAME, \
+ (__VA_ARGS__), (__VA_ARGS__))
+#define _SCHED_ITF_ARGS_PROTO_NONAME_0() \
+ void
+#define _SCHED_ITF_ARGS_PROTO_NONAME_1(t1, a1) \
+ t1
+#define _SCHED_ITF_ARGS_PROTO_NONAME_2(t1, a1, t2, a2) \
+ t1, t2
+#define _SCHED_ITF_ARGS_PROTO_NONAME_3(t1, a1, t2, a2, t3, a3) \
+ t1, t2, t3
+#define _SCHED_ITF_ARGS_PROTO_NONAME_4(t1, a1, t2, a2, t3, a3, t4, a4) \
+ t1, t2, t3, t4
+#define _SCHED_ITF_ARGS_PROTO_NONAME_ERR(...) \
+ _SCHED_ITF_ARGS_PAIR_ERR
+#define SCHED_ITF_ARGS_PROTO_NONAME(...) \
+ _SCHED_ITF_NARGS_DISP(_SCHED_ITF_ARGS_PROTO_NONAME, \
+ (__VA_ARGS__), (__VA_ARGS__))
+
+#define SCHED_ITF_FIELD_NAME(fn) \
+ __CONCAT(fn, _impl)
+#define SCHED_ITF_FUNCTION_NAME(fn) \
+ __CONCAT(sched_, fn)
+#define _SCHED_ITF_IMPL_FUNCTION_NAME(sched_name, fn) \
+ SCHED_ITF_FUNCTION_NAME(__CONCAT(__CONCAT(sched_name, _), \
+ fn))
/*
- * Threads are switched in and out, block on resources, have temporary
- * priorities inherited from their procs, and use up cpu time.
+ * C declaration of first part of interface functions.
+ */
+struct proc;
+struct thread;
+
+/* Eliding argument names to avoid collisions with C++ keywords. */
+#define _SCHED_ITF_FUN(args, ret_type, fn, ...) \
+ ret_type \
+ SCHED_ITF_FUNCTION_NAME(fn) \
+ (SCHED_ITF_ARGS_PROTO_NONAME(__VA_ARGS__));
+SCHED_ITF_DISPATCH(_SCHED_ITF_FUN, );
+SCHED_ITF_WRAP(_SCHED_ITF_FUN, );
+#undef _SCHED_ITF_FUN
+
+/*
+ * Common functions.
*/
-void sched_ap_entry(void);
-void sched_exit_thread(struct thread *td, struct thread *child);
-u_int sched_estcpu(struct thread *td);
-void sched_fork_thread(struct thread *td, struct thread *child);
-void sched_ithread_prio(struct thread *td, u_char prio);
-void sched_lend_prio(struct thread *td, u_char prio);
-void sched_lend_user_prio(struct thread *td, u_char pri);
-void sched_lend_user_prio_cond(struct thread *td, u_char pri);
-fixpt_t sched_pctcpu(struct thread *td);
-void sched_prio(struct thread *td, u_char prio);
-void sched_sleep(struct thread *td, int prio);
-void sched_switch(struct thread *td, int flags);
-void sched_throw(struct thread *td);
-void sched_unlend_prio(struct thread *td, u_char prio);
-void sched_user_prio(struct thread *td, u_char prio);
-void sched_userret_slowpath(struct thread *td);
static inline void
sched_userret(struct thread *td)
@@ -136,43 +406,6 @@ sched_userret(struct thread *td)
sched_userret_slowpath(td);
}
-/*
- * Threads are moved on and off of run queues
- */
-void sched_add(struct thread *td, int flags);
-struct thread *sched_choose(void);
-void sched_clock(struct thread *td, int cnt);
-void sched_idletd(void *);
-void sched_preempt(struct thread *td);
-void sched_relinquish(struct thread *td);
-void sched_rem(struct thread *td);
-void sched_wakeup(struct thread *td, int srqflags);
-
-/*
- * Binding makes cpu affinity permanent while pinning is used to temporarily
- * hold a thread on a particular CPU.
- */
-void sched_bind(struct thread *td, int cpu);
-static __inline void sched_pin(void);
-void sched_unbind(struct thread *td);
-static __inline void sched_unpin(void);
-int sched_is_bound(struct thread *td);
-void sched_affinity(struct thread *td);
-
-/*
- * These procedures tell the process data structure allocation code how
- * many bytes to actually allocate.
- */
-int sched_sizeof_proc(void);
-int sched_sizeof_thread(void);
-
-/*
- * This routine provides a consistent thread name for use with KTR graphing
- * functions.
- */
-char *sched_tdname(struct thread *td);
-void sched_clear_tdname(struct thread *td);
-
static __inline void
sched_pin(void)
{
@@ -188,19 +421,9 @@ sched_unpin(void)
curthread->td_pinned--;
}
-void ast_scheduler(struct thread *td, int tda);
-
-/* sched_add arguments (formerly setrunqueue) */
-#define SRQ_BORING 0x0000 /* No special circumstances. */
-#define SRQ_YIELDING 0x0001 /* We are yielding (from mi_switch). */
-#define SRQ_OURSELF 0x0002 /* It is ourself (from mi_switch). */
-#define SRQ_INTR 0x0004 /* It is probably urgent. */
-#define SRQ_PREEMPTED 0x0008 /* has been preempted.. be kind */
-#define SRQ_BORROWING 0x0010 /* Priority updated due to prio_lend */
-#define SRQ_HOLD 0x0020 /* Return holding original td lock */
-#define SRQ_HOLDTD 0x0040 /* Return holding td lock */
-
-/* Scheduler stats. */
+/*
+ * Scheduler statistics.
+ */
#ifdef SCHED_STATS
DPCPU_DECLARE(long, sched_switch_stats[SWT_COUNT]);
@@ -237,6 +460,7 @@ SYSINIT(name, SI_SUB_LAST, SI_ORDER_MIDDLE, name ## _add_proc, NULL);
SCHED_STAT_DECLARE(ithread_demotions);
SCHED_STAT_DECLARE(ithread_preemptions);
+/* Dtrace. */
SDT_PROBE_DECLARE(sched, , , change__pri);
SDT_PROBE_DECLARE(sched, , , dequeue);
SDT_PROBE_DECLARE(sched, , , enqueue);
@@ -254,95 +478,50 @@ extern dtrace_vtime_switch_func_t dtrace_vtime_switch_func;
#endif
/*
- * Fixup scheduler state for proc0 and thread0
- */
-void schedinit(void);
-
-/*
- * Fixup scheduler state for secondary APs
- */
-void schedinit_ap(void);
-
-/*
- * Find an L2 neighbor of the given CPU or return -1 if none found. This
- * does not distinguish among multiple L2 neighbors if the given CPU has
- * more than one (it will always return the same result in that case).
- */
-int sched_find_l2_neighbor(int cpu);
-
-/*
- * The scheduler selection interface uses names that are reserved words in
- * C++, causing problems for downstream projects that use C++ in the
- * kernel.
+ * Scheduler instance structure.
*/
-#ifndef __cplusplus
-
+#define _SCHED_ITF_FUN(args, ret_type, fn, ...) \
+ ret_type (*SCHED_ITF_FIELD_NAME(fn)) \
+ (SCHED_ITF_ARGS_PROTO_NONAME(__VA_ARGS__));
struct sched_instance {
- int (*load)(void);
- int (*rr_interval)(void);
- bool (*runnable)(void);
- void (*exit)(struct proc *p, struct thread *childtd);
- void (*fork)(struct thread *td, struct thread *childtd);
- void (*fork_exit)(struct thread *td);
- void (*class)(struct thread *td, int class);
- void (*nice)(struct proc *p, int nice);
- void (*ap_entry)(void);
- void (*exit_thread)(struct thread *td, struct thread *child);
- u_int (*estcpu)(struct thread *td);
- void (*fork_thread)(struct thread *td, struct thread *child);
- void (*ithread_prio)(struct thread *td, u_char prio);
- void (*lend_prio)(struct thread *td, u_char prio);
- void (*lend_user_prio)(struct thread *td, u_char pri);
- void (*lend_user_prio_cond)(struct thread *td, u_char pri);
- fixpt_t (*pctcpu)(struct thread *td);
- void (*prio)(struct thread *td, u_char prio);
- void (*sleep)(struct thread *td, int prio);
- void (*sswitch)(struct thread *td, int flags);
- void (*throw)(struct thread *td);
- void (*unlend_prio)(struct thread *td, u_char prio);
- void (*user_prio)(struct thread *td, u_char prio);
- void (*userret_slowpath)(struct thread *td);
- void (*add)(struct thread *td, int flags);
*** 97 LINES SKIPPED ***