From bba2c3615bd6cfee7456d1130f2e6b01b3f4e9ba Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 22 Jun 2026 05:32:54 -1000 Subject: sched_ext: Move sources under kernel/sched/ext/ The sched_ext sources had grown to ten ext* files directly under kernel/sched/. Move them into a new kernel/sched/ext/ subdirectory and drop the now-redundant ext_ prefix. ext.c/h keep their names. kernel/sched/ext.{c,h} -> kernel/sched/ext/ext.{c,h} kernel/sched/ext_internal.h -> kernel/sched/ext/internal.h kernel/sched/ext_types.h -> kernel/sched/ext/types.h kernel/sched/ext_idle.{c,h} -> kernel/sched/ext/idle.{c,h} kernel/sched/ext_cid.{c,h} -> kernel/sched/ext/cid.{c,h} kernel/sched/ext_arena.{c,h} -> kernel/sched/ext/arena.{c,h} The include paths in build_policy.c and sched.h, the MAINTAINERS glob, and a few documentation and comment references are updated to match. No code or symbol changes. Suggested-by: Linus Torvalds Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/build_policy.c | 18 +- kernel/sched/ext.c | 10882 ------------------------------------------ kernel/sched/ext.h | 95 - kernel/sched/ext/arena.c | 131 + kernel/sched/ext/arena.h | 18 + kernel/sched/ext/cid.c | 707 +++ kernel/sched/ext/cid.h | 271 ++ kernel/sched/ext/ext.c | 10882 ++++++++++++++++++++++++++++++++++++++++++ kernel/sched/ext/ext.h | 95 + kernel/sched/ext/idle.c | 1511 ++++++ kernel/sched/ext/idle.h | 27 + kernel/sched/ext/internal.h | 1653 +++++++ kernel/sched/ext/types.h | 144 + kernel/sched/ext_arena.c | 131 - kernel/sched/ext_arena.h | 18 - kernel/sched/ext_cid.c | 707 --- kernel/sched/ext_cid.h | 271 -- kernel/sched/ext_idle.c | 1511 ------ kernel/sched/ext_idle.h | 27 - kernel/sched/ext_internal.h | 1653 ------- kernel/sched/ext_types.h | 144 - kernel/sched/sched.h | 2 +- 22 files changed, 15449 insertions(+), 15449 deletions(-) delete mode 100644 kernel/sched/ext.c delete mode 100644 kernel/sched/ext.h create mode 100644 kernel/sched/ext/arena.c create mode 100644 kernel/sched/ext/arena.h create mode 100644 kernel/sched/ext/cid.c create mode 100644 kernel/sched/ext/cid.h create mode 100644 kernel/sched/ext/ext.c create mode 100644 kernel/sched/ext/ext.h create mode 100644 kernel/sched/ext/idle.c create mode 100644 kernel/sched/ext/idle.h create mode 100644 kernel/sched/ext/internal.h create mode 100644 kernel/sched/ext/types.h delete mode 100644 kernel/sched/ext_arena.c delete mode 100644 kernel/sched/ext_arena.h delete mode 100644 kernel/sched/ext_cid.c delete mode 100644 kernel/sched/ext_cid.h delete mode 100644 kernel/sched/ext_idle.c delete mode 100644 kernel/sched/ext_idle.h delete mode 100644 kernel/sched/ext_internal.h delete mode 100644 kernel/sched/ext_types.h (limited to 'kernel') diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c index 067979a7b69e..d74b54f81992 100644 --- a/kernel/sched/build_policy.c +++ b/kernel/sched/build_policy.c @@ -61,15 +61,15 @@ # include # include # include -# include "ext_types.h" -# include "ext_internal.h" -# include "ext_cid.h" -# include "ext_arena.h" -# include "ext_idle.h" -# include "ext.c" -# include "ext_cid.c" -# include "ext_arena.c" -# include "ext_idle.c" +# include "ext/types.h" +# include "ext/internal.h" +# include "ext/cid.h" +# include "ext/arena.h" +# include "ext/idle.h" +# include "ext/ext.c" +# include "ext/cid.c" +# include "ext/arena.c" +# include "ext/idle.c" #endif #include "syscalls.c" diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c deleted file mode 100644 index 0db6fa2daea3..000000000000 --- a/kernel/sched/ext.c +++ /dev/null @@ -1,10882 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst - * - * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2022 Tejun Heo - * Copyright (c) 2022 David Vernet - */ - -static DEFINE_RAW_SPINLOCK(scx_sched_lock); - -/* - * NOTE: sched_ext is in the process of growing multiple scheduler support and - * scx_root usage is in a transitional state. Naked dereferences are safe if the - * caller is one of the tasks attached to SCX and explicit RCU dereference is - * necessary otherwise. Naked scx_root dereferences trigger sparse warnings but - * are used as temporary markers to indicate that the dereferences need to be - * updated to point to the associated scheduler instances rather than scx_root. - */ -struct scx_sched __rcu *scx_root; - -/* - * All scheds, writers must hold both scx_enable_mutex and scx_sched_lock. - * Readers can hold either or rcu_read_lock(). - */ -static LIST_HEAD(scx_sched_all); - -#ifdef CONFIG_EXT_SUB_SCHED -static const struct rhashtable_params scx_sched_hash_params = { - .key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id), - .key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id), - .head_offset = offsetof(struct scx_sched, hash_node), - .insecure_elasticity = true, /* inserted under scx_sched_lock */ -}; - -static struct rhashtable scx_sched_hash; -#endif - -/* see SCX_OPS_TID_TO_TASK */ -static const struct rhashtable_params scx_tid_hash_params = { - .key_len = sizeof_field(struct sched_ext_entity, tid), - .key_offset = offsetof(struct sched_ext_entity, tid), - .head_offset = offsetof(struct sched_ext_entity, tid_hash_node), - .insecure_elasticity = true, /* inserted/removed under scx_tasks_lock */ -}; -static struct rhashtable scx_tid_hash; - -/* - * During exit, a task may schedule after losing its PIDs. When disabling the - * BPF scheduler, we need to be able to iterate tasks in every state to - * guarantee system safety. Maintain a dedicated task list which contains every - * task between its fork and eventual free. - */ -static DEFINE_RAW_SPINLOCK(scx_tasks_lock); -static LIST_HEAD(scx_tasks); - -/* ops enable/disable */ -static DEFINE_MUTEX(scx_enable_mutex); -DEFINE_STATIC_KEY_FALSE(__scx_enabled); -DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); -static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED); -static DEFINE_RAW_SPINLOCK(scx_bypass_lock); -static bool scx_init_task_enabled; -static bool scx_switching_all; -DEFINE_STATIC_KEY_FALSE(__scx_switched_all); -static DEFINE_STATIC_KEY_FALSE(__scx_tid_to_task_enabled); - -/* - * True once SCX_OPS_TID_TO_TASK has been negotiated with the root scheduler - * and the tid->task table is live. Wraps the static key so callers don't - * take the address, and hints "likely enabled" for the common case where - * the feature is in use. - */ -static inline bool scx_tid_to_task_enabled(void) -{ - return static_branch_likely(&__scx_tid_to_task_enabled); -} - -static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0); -static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0); - -/* Global cursor for the per-CPU tid allocator. Starts at 1; tid 0 is reserved. */ -static atomic64_t scx_tid_cursor = ATOMIC64_INIT(1); - -#ifdef CONFIG_EXT_SUB_SCHED -/* - * The sub sched being enabled. Used by scx_disable_and_exit_task() to exit - * tasks for the sub-sched being enabled. Use a global variable instead of a - * per-task field as all enables are serialized. - */ -static struct scx_sched *scx_enabling_sub_sched; -#else -#define scx_enabling_sub_sched (struct scx_sched *)NULL -#endif /* CONFIG_EXT_SUB_SCHED */ - -/* - * A monotonically increasing sequence number that is incremented every time a - * scheduler is enabled. This can be used to check if any custom sched_ext - * scheduler has ever been used in the system. - */ -static atomic_long_t scx_enable_seq = ATOMIC_LONG_INIT(0); - -/* - * Watchdog interval. All scx_sched's share a single watchdog timer and the - * interval is half of the shortest sch->watchdog_timeout. - */ -static unsigned long scx_watchdog_interval; - -/* - * The last time the delayed work was run. This delayed work relies on - * ksoftirqd being able to run to service timer interrupts, so it's possible - * that this work itself could get wedged. To account for this, we check that - * it's not stalled in the timer tick, and trigger an error if it is. - */ -static unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; - -static struct delayed_work scx_watchdog_work; - -/* - * For %SCX_KICK_WAIT: Each CPU has a pointer to an array of kick_sync sequence - * numbers. The arrays are allocated with kvzalloc() as size can exceed percpu - * allocator limits on large machines. O(nr_cpu_ids^2) allocation, allocated - * lazily when enabling and freed when disabling to avoid waste when sched_ext - * isn't active. - */ -struct scx_kick_syncs { - struct rcu_head rcu; - unsigned long syncs[]; -}; - -static DEFINE_PER_CPU(struct scx_kick_syncs __rcu *, scx_kick_syncs); - -/* - * Per-CPU buffered allocator state for p->scx.tid. Each CPU pulls a chunk of - * SCX_TID_CHUNK ids from scx_tid_cursor and hands them out locally without - * further synchronization. See scx_alloc_tid(). - */ -struct scx_tid_alloc { - u64 next; - u64 end; -}; -static DEFINE_PER_CPU(struct scx_tid_alloc, scx_tid_alloc); - -/* - * Direct dispatch marker. - * - * Non-NULL values are used for direct dispatch from enqueue path. A valid - * pointer points to the task currently being enqueued. An ERR_PTR value is used - * to indicate that direct dispatch has already happened. - */ -static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); - -static const struct rhashtable_params dsq_hash_params = { - .key_len = sizeof_field(struct scx_dispatch_q, id), - .key_offset = offsetof(struct scx_dispatch_q, id), - .head_offset = offsetof(struct scx_dispatch_q, hash_node), -}; - -static LLIST_HEAD(dsqs_to_free); - -/* string formatting from BPF */ -struct scx_bstr_buf { - u64 data[MAX_BPRINTF_VARARGS]; - char line[SCX_EXIT_MSG_LEN]; -}; - -static DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock); -static struct scx_bstr_buf scx_exit_bstr_buf; - -/* ops debug dump */ -static DEFINE_RAW_SPINLOCK(scx_dump_lock); - -struct scx_dump_data { - s32 cpu; - bool first; - s32 cursor; - struct seq_buf *s; - const char *prefix; - struct scx_bstr_buf buf; -}; - -static struct scx_dump_data scx_dump_data = { - .cpu = -1, -}; - -/* /sys/kernel/sched_ext interface */ -static struct kset *scx_kset; - -/* - * Parameters that can be adjusted through /sys/module/sched_ext/parameters. - * There usually is no reason to modify these as normal scheduler operation - * shouldn't be affected by them. The knobs are primarily for debugging. - */ -static unsigned int scx_slice_bypass_us = SCX_SLICE_BYPASS / NSEC_PER_USEC; -static unsigned int scx_bypass_lb_intv_us = SCX_BYPASS_LB_DFL_INTV_US; - -static int set_slice_us(const char *val, const struct kernel_param *kp) -{ - return param_set_uint_minmax(val, kp, 100, 100 * USEC_PER_MSEC); -} - -static const struct kernel_param_ops slice_us_param_ops = { - .set = set_slice_us, - .get = param_get_uint, -}; - -static int set_bypass_lb_intv_us(const char *val, const struct kernel_param *kp) -{ - return param_set_uint_minmax(val, kp, 0, 10 * USEC_PER_SEC); -} - -static const struct kernel_param_ops bypass_lb_intv_us_param_ops = { - .set = set_bypass_lb_intv_us, - .get = param_get_uint, -}; - -#undef MODULE_PARAM_PREFIX -#define MODULE_PARAM_PREFIX "sched_ext." - -module_param_cb(slice_bypass_us, &slice_us_param_ops, &scx_slice_bypass_us, 0600); -MODULE_PARM_DESC(slice_bypass_us, "bypass slice in microseconds, applied on [un]load (100us to 100ms)"); -module_param_cb(bypass_lb_intv_us, &bypass_lb_intv_us_param_ops, &scx_bypass_lb_intv_us, 0600); -MODULE_PARM_DESC(bypass_lb_intv_us, "bypass load balance interval in microseconds (0 (disable) to 10s)"); - -#undef MODULE_PARAM_PREFIX - -#define CREATE_TRACE_POINTS -#include - -static void run_deferred(struct rq *rq); -static bool task_dead_and_done(struct task_struct *p); -static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); -static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind); - -__printf(5, 6) bool __scx_exit(struct scx_sched *sch, - enum scx_exit_kind kind, s64 exit_code, - s32 exit_cpu, const char *fmt, ...) -{ - va_list args; - bool ret; - - va_start(args, fmt); - ret = scx_vexit(sch, kind, exit_code, exit_cpu, fmt, args); - va_end(args); - - return ret; -} - -#define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op) - -static long jiffies_delta_msecs(unsigned long at, unsigned long now) -{ - if (time_after(at, now)) - return jiffies_to_msecs(at - now); - else - return -(long)jiffies_to_msecs(now - at); -} - -static bool u32_before(u32 a, u32 b) -{ - return (s32)(a - b) < 0; -} - -#ifdef CONFIG_EXT_SUB_SCHED -/** - * scx_parent - Find the parent sched - * @sch: sched to find the parent of - * - * Returns the parent scheduler or %NULL if @sch is root. - */ -static struct scx_sched *scx_parent(struct scx_sched *sch) -{ - if (sch->level) - return sch->ancestors[sch->level - 1]; - else - return NULL; -} - -/** - * scx_next_descendant_pre - find the next descendant for pre-order walk - * @pos: the current position (%NULL to initiate traversal) - * @root: sched whose descendants to walk - * - * To be used by scx_for_each_descendant_pre(). Find the next descendant to - * visit for pre-order traversal of @root's descendants. @root is included in - * the iteration and the first node to be visited. - */ -static struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, - struct scx_sched *root) -{ - struct scx_sched *next; - - lockdep_assert(lockdep_is_held(&scx_enable_mutex) || - lockdep_is_held(&scx_sched_lock)); - - /* if first iteration, visit @root */ - if (!pos) - return root; - - /* visit the first child if exists */ - next = list_first_entry_or_null(&pos->children, struct scx_sched, sibling); - if (next) - return next; - - /* no child, visit my or the closest ancestor's next sibling */ - while (pos != root) { - if (!list_is_last(&pos->sibling, &scx_parent(pos)->children)) - return list_next_entry(pos, sibling); - pos = scx_parent(pos); - } - - return NULL; -} - -static struct scx_sched *scx_find_sub_sched(u64 cgroup_id) -{ - return rhashtable_lookup(&scx_sched_hash, &cgroup_id, - scx_sched_hash_params); -} - -static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) -{ - rcu_assign_pointer(p->scx.sched, sch); -} -#else /* CONFIG_EXT_SUB_SCHED */ -static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; } -static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } -static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} -#endif /* CONFIG_EXT_SUB_SCHED */ - -/** - * scx_is_descendant - Test whether sched is a descendant - * @sch: sched to test - * @ancestor: ancestor sched to test against - * - * Test whether @sch is a descendant of @ancestor. - */ -static bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor) -{ - if (sch->level < ancestor->level) - return false; - return sch->ancestors[ancestor->level] == ancestor; -} - -/** - * scx_for_each_descendant_pre - pre-order walk of a sched's descendants - * @pos: iteration cursor - * @root: sched to walk the descendants of - * - * Walk @root's descendants. @root is included in the iteration and the first - * node to be visited. Must be called with either scx_enable_mutex or - * scx_sched_lock held. - */ -#define scx_for_each_descendant_pre(pos, root) \ - for ((pos) = scx_next_descendant_pre(NULL, (root)); (pos); \ - (pos) = scx_next_descendant_pre((pos), (root))) - -static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch, s32 cpu) -{ - return &sch->pnode[cpu_to_node(cpu)]->global_dsq; -} - -static struct scx_dispatch_q *find_user_dsq(struct scx_sched *sch, u64 dsq_id) -{ - return rhashtable_lookup(&sch->dsq_hash, &dsq_id, dsq_hash_params); -} - -static const struct sched_class *scx_setscheduler_class(struct task_struct *p) -{ - if (p->sched_class == &stop_sched_class) - return &stop_sched_class; - - return __setscheduler_class(p->policy, p->prio); -} - -static struct scx_dispatch_q *bypass_dsq(struct scx_sched *sch, s32 cpu) -{ - return &per_cpu_ptr(sch->pcpu, cpu)->bypass_dsq; -} - -static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu) -{ -#ifdef CONFIG_EXT_SUB_SCHED - /* - * If @sch is a sub-sched which is bypassing, its tasks should go into - * the bypass DSQs of the nearest ancestor which is not bypassing. The - * not-bypassing ancestor is responsible for scheduling all tasks from - * bypassing sub-trees. If all ancestors including root are bypassing, - * all tasks should go to the root's bypass DSQs. - * - * Whenever a sched starts bypassing, all runnable tasks in its subtree - * are re-enqueued after scx_bypassing() is turned on, guaranteeing that - * all tasks are transferred to the right DSQs. - */ - while (scx_parent(sch) && scx_bypassing(sch, cpu)) - sch = scx_parent(sch); -#endif /* CONFIG_EXT_SUB_SCHED */ - - return bypass_dsq(sch, cpu); -} - -/** - * bypass_dsp_enabled - Check if bypass dispatch path is enabled - * @sch: scheduler to check - * - * When a descendant scheduler enters bypass mode, bypassed tasks are scheduled - * by the nearest non-bypassing ancestor, or the root scheduler if all ancestors - * are bypassing. In the former case, the ancestor is not itself bypassing but - * its bypass DSQs will be populated with bypassed tasks from descendants. Thus, - * the ancestor's bypass dispatch path must be active even though its own - * bypass_depth remains zero. - * - * This function checks bypass_dsp_enable_depth which is managed separately from - * bypass_depth to enable this decoupling. See enable_bypass_dsp() and - * disable_bypass_dsp(). - */ -static bool bypass_dsp_enabled(struct scx_sched *sch) -{ - return unlikely(atomic_read(&sch->bypass_dsp_enable_depth)); -} - -/** - * rq_is_open - Is the rq available for immediate execution of an SCX task? - * @rq: rq to test - * @enq_flags: optional %SCX_ENQ_* of the task being enqueued - * - * Returns %true if @rq is currently open for executing an SCX task. After a - * %false return, @rq is guaranteed to invoke SCX dispatch path at least once - * before going to idle and not inserting a task into @rq's local DSQ after a - * %false return doesn't cause @rq to stall. - */ -static bool rq_is_open(struct rq *rq, u64 enq_flags) -{ - lockdep_assert_rq_held(rq); - - /* - * A higher-priority class task is either running or in the process of - * waking up on @rq. - */ - if (sched_class_above(rq->next_class, &ext_sched_class)) - return false; - - /* - * @rq is either in transition to or in idle and there is no - * higher-priority class task waking up on it. - */ - if (sched_class_above(&ext_sched_class, rq->next_class)) - return true; - - /* - * @rq is either picking, in transition to, or running an SCX task. - */ - - /* - * If we're in the dispatch path holding rq lock, $curr may or may not - * be ready depending on whether the on-going dispatch decides to extend - * $curr's slice. We say yes here and resolve it at the end of dispatch. - * See balance_one(). - */ - if (rq->scx.flags & SCX_RQ_IN_BALANCE) - return true; - - /* - * %SCX_ENQ_PREEMPT clears $curr's slice if on SCX and kicks dispatch, - * so allow it to avoid spuriously triggering reenq on a combined - * PREEMPT|IMMED insertion. - */ - if (enq_flags & SCX_ENQ_PREEMPT) - return true; - - /* - * @rq is either in transition to or running an SCX task and can't go - * idle without another SCX dispatch cycle. - */ - return false; -} - -/* - * Track the rq currently locked. - * - * This allows kfuncs to safely operate on rq from any scx ops callback, - * knowing which rq is already locked. - */ -DEFINE_PER_CPU(struct rq *, scx_locked_rq_state); - -static inline void update_locked_rq(struct rq *rq) -{ - /* - * Check whether @rq is actually locked. This can help expose bugs - * or incorrect assumptions about the context in which a kfunc or - * callback is executed. - */ - if (rq) - lockdep_assert_rq_held(rq); - __this_cpu_write(scx_locked_rq_state, rq); -} - -/* - * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not - * clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit. - */ -#define SCX_CALL_OP(sch, op, locked_rq, args...) \ -do { \ - struct rq *__prev_locked_rq; \ - \ - if (locked_rq) { \ - __prev_locked_rq = scx_locked_rq(); \ - update_locked_rq(locked_rq); \ - } \ - (sch)->ops.op(args); \ - if (locked_rq) \ - update_locked_rq(__prev_locked_rq); \ -} while (0) - -/* - * Flipped on enable per sch->is_cid_type. Declared in ext_internal.h so - * subsystem inlines can read it. - */ -DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type); - -/* - * scx_cpu_arg() wraps a cpu arg being handed to an SCX op. For cid-form - * schedulers it resolves to the matching cid; for cpu-form it passes @cpu - * through. scx_cpu_ret() is the inverse for a cpu/cid returned from an op - * (currently only ops.select_cpu); it validates the BPF-supplied cid and - * triggers scx_error() on @sch if invalid. - */ -static s32 scx_cpu_arg(s32 cpu) -{ - if (scx_is_cid_type()) - return __scx_cpu_to_cid(cpu); - return cpu; -} - -static s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid) -{ - if (cpu_or_cid < 0 || !scx_is_cid_type()) - return cpu_or_cid; - return scx_cid_to_cpu(sch, cpu_or_cid); -} - -#define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \ -({ \ - struct rq *__prev_locked_rq; \ - __typeof__((sch)->ops.op(args)) __ret; \ - \ - if (locked_rq) { \ - __prev_locked_rq = scx_locked_rq(); \ - update_locked_rq(locked_rq); \ - } \ - __ret = (sch)->ops.op(args); \ - if (locked_rq) \ - update_locked_rq(__prev_locked_rq); \ - __ret; \ -}) - -/* - * SCX_CALL_OP_TASK*() invokes an SCX op that takes one or two task arguments - * and records them in current->scx.kf_tasks[] for the duration of the call. A - * kfunc invoked from inside such an op can then use - * scx_kf_arg_task_ok() to verify that its task argument is one of - * those subject tasks. - * - * Every SCX_CALL_OP_TASK*() call site invokes its op with @p's rq lock held - - * either via the @locked_rq argument here, or (for ops.select_cpu()) via @p's - * pi_lock held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu. - * So if kf_tasks[] is set, @p's scheduler-protected fields are stable. - * - * kf_tasks[] can not stack, so task-based SCX ops must not nest. The - * WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants - * while a previous one is still in progress. - */ -#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \ -do { \ - WARN_ON_ONCE(current->scx.kf_tasks[0]); \ - current->scx.kf_tasks[0] = task; \ - SCX_CALL_OP((sch), op, locked_rq, task, ##args); \ - current->scx.kf_tasks[0] = NULL; \ -} while (0) - -#define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \ -({ \ - __typeof__((sch)->ops.op(task, ##args)) __ret; \ - WARN_ON_ONCE(current->scx.kf_tasks[0]); \ - current->scx.kf_tasks[0] = task; \ - __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \ - current->scx.kf_tasks[0] = NULL; \ - __ret; \ -}) - -#define SCX_CALL_OP_2TASKS_RET(sch, op, locked_rq, task0, task1, args...) \ -({ \ - __typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \ - WARN_ON_ONCE(current->scx.kf_tasks[0]); \ - current->scx.kf_tasks[0] = task0; \ - current->scx.kf_tasks[1] = task1; \ - __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task0, task1, ##args); \ - current->scx.kf_tasks[0] = NULL; \ - current->scx.kf_tasks[1] = NULL; \ - __ret; \ -}) - -/** - * scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task - * @sch: scx_sched being invoked - * @rq: rq to update as the currently-locked rq, or NULL - * @task: task whose affinity is changing - * @cpumask: new cpumask - * - * For cid-form schedulers, translate @cpumask to a cmask via the per-cpu - * scratch in ext_cid.c and dispatch through the ops_cid union view. Caller - * must hold @rq's rq lock so this_cpu_ptr is stable across the call. - */ -static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, - struct task_struct *task, - const struct cpumask *cpumask) -{ - WARN_ON_ONCE(current->scx.kf_tasks[0]); - current->scx.kf_tasks[0] = task; - if (rq) - update_locked_rq(rq); - - if (scx_is_cid_type()) { - struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); - /* - * Build the per-CPU arena cmask and hand BPF its arena address. - * Caller holds the rq lock with IRQs disabled, which makes us - * the sole user of the scratch area. - */ - scx_cpumask_to_cmask(cpumask, kern_va); - sch->ops_cid.set_cmask(task, scx_kaddr_to_arena(sch, kern_va)); - } else { - sch->ops.set_cpumask(task, cpumask); - } - - if (rq) - update_locked_rq(NULL); - current->scx.kf_tasks[0] = NULL; -} - -/* see SCX_CALL_OP_TASK() */ -static __always_inline bool scx_kf_arg_task_ok(struct scx_sched *sch, - struct task_struct *p) -{ - if (unlikely((p != current->scx.kf_tasks[0] && - p != current->scx.kf_tasks[1]))) { - scx_error(sch, "called on a task not being operated on"); - return false; - } - - return true; -} - -enum scx_dsq_iter_flags { - /* iterate in the reverse dispatch order */ - SCX_DSQ_ITER_REV = 1U << 16, - - __SCX_DSQ_ITER_HAS_SLICE = 1U << 30, - __SCX_DSQ_ITER_HAS_VTIME = 1U << 31, - - __SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV, - __SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS | - __SCX_DSQ_ITER_HAS_SLICE | - __SCX_DSQ_ITER_HAS_VTIME, -}; - -/** - * nldsq_next_task - Iterate to the next task in a non-local DSQ - * @dsq: non-local dsq being iterated - * @cur: current position, %NULL to start iteration - * @rev: walk backwards - * - * Returns %NULL when iteration is finished. - */ -static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq, - struct task_struct *cur, bool rev) -{ - struct list_head *list_node; - struct scx_dsq_list_node *dsq_lnode; - - lockdep_assert_held(&dsq->lock); - - if (cur) - list_node = &cur->scx.dsq_list.node; - else - list_node = &dsq->list; - - /* find the next task, need to skip BPF iteration cursors */ - do { - if (rev) - list_node = list_node->prev; - else - list_node = list_node->next; - - if (list_node == &dsq->list) - return NULL; - - dsq_lnode = container_of(list_node, struct scx_dsq_list_node, - node); - } while (dsq_lnode->flags & SCX_DSQ_LNODE_ITER_CURSOR); - - return container_of(dsq_lnode, struct task_struct, scx.dsq_list); -} - -#define nldsq_for_each_task(p, dsq) \ - for ((p) = nldsq_next_task((dsq), NULL, false); (p); \ - (p) = nldsq_next_task((dsq), (p), false)) - -/** - * nldsq_cursor_next_task - Iterate to the next task given a cursor in a non-local DSQ - * @cursor: scx_dsq_list_node initialized with INIT_DSQ_LIST_CURSOR() - * @dsq: non-local dsq being iterated - * - * Find the next task in a cursor based iteration. The caller must have - * initialized @cursor using INIT_DSQ_LIST_CURSOR() and can release the DSQ lock - * between the iteration steps. - * - * Only tasks which were queued before @cursor was initialized are visible. This - * bounds the iteration and guarantees that vtime never jumps in the other - * direction while iterating. - */ -static struct task_struct *nldsq_cursor_next_task(struct scx_dsq_list_node *cursor, - struct scx_dispatch_q *dsq) -{ - bool rev = cursor->flags & SCX_DSQ_ITER_REV; - struct task_struct *p; - - lockdep_assert_held(&dsq->lock); - BUG_ON(!(cursor->flags & SCX_DSQ_LNODE_ITER_CURSOR)); - - if (list_empty(&cursor->node)) - p = NULL; - else - p = container_of(cursor, struct task_struct, scx.dsq_list); - - /* skip cursors and tasks that were queued after @cursor init */ - do { - p = nldsq_next_task(dsq, p, rev); - } while (p && unlikely(u32_before(cursor->priv, p->scx.dsq_seq))); - - if (p) { - if (rev) - list_move_tail(&cursor->node, &p->scx.dsq_list.node); - else - list_move(&cursor->node, &p->scx.dsq_list.node); - } else { - list_del_init(&cursor->node); - } - - return p; -} - -/** - * nldsq_cursor_lost_task - Test whether someone else took the task since iteration - * @cursor: scx_dsq_list_node initialized with INIT_DSQ_LIST_CURSOR() - * @rq: rq @p was on - * @dsq: dsq @p was on - * @p: target task - * - * @p is a task returned by nldsq_cursor_next_task(). The locks may have been - * dropped and re-acquired inbetween. Verify that no one else took or is in the - * process of taking @p from @dsq. - * - * On %false return, the caller can assume full ownership of @p. - */ -static bool nldsq_cursor_lost_task(struct scx_dsq_list_node *cursor, - struct rq *rq, struct scx_dispatch_q *dsq, - struct task_struct *p) -{ - lockdep_assert_rq_held(rq); - lockdep_assert_held(&dsq->lock); - - /* - * @p could have already left $src_dsq, got re-enqueud, or be in the - * process of being consumed by someone else. - */ - if (unlikely(p->scx.dsq != dsq || - u32_before(cursor->priv, p->scx.dsq_seq) || - p->scx.holding_cpu >= 0)) - return true; - - /* if @p has stayed on @dsq, its rq couldn't have changed */ - if (WARN_ON_ONCE(rq != task_rq(p))) - return true; - - return false; -} - -/* - * BPF DSQ iterator. Tasks in a non-local DSQ can be iterated in [reverse] - * dispatch order. BPF-visible iterator is opaque and larger to allow future - * changes without breaking backward compatibility. Can be used with - * bpf_for_each(). See bpf_iter_scx_dsq_*(). - */ -struct bpf_iter_scx_dsq_kern { - struct scx_dsq_list_node cursor; - struct scx_dispatch_q *dsq; - u64 slice; - u64 vtime; -} __attribute__((aligned(8))); - -struct bpf_iter_scx_dsq { - u64 __opaque[6]; -} __attribute__((aligned(8))); - - -static u32 scx_get_task_state(const struct task_struct *p) -{ - return p->scx.flags & SCX_TASK_STATE_MASK; -} - -static void scx_set_task_state(struct task_struct *p, u32 state) -{ - u32 prev_state = scx_get_task_state(p); - bool warn = false; - - switch (state) { - case SCX_TASK_NONE: - warn = prev_state == SCX_TASK_DEAD; - break; - case SCX_TASK_INIT_BEGIN: - warn = prev_state != SCX_TASK_NONE; - break; - case SCX_TASK_INIT: - warn = prev_state != SCX_TASK_INIT_BEGIN; - p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; - break; - case SCX_TASK_READY: - warn = !(prev_state == SCX_TASK_INIT || - prev_state == SCX_TASK_ENABLED); - break; - case SCX_TASK_ENABLED: - warn = prev_state != SCX_TASK_READY; - break; - case SCX_TASK_DEAD: - warn = !(prev_state == SCX_TASK_NONE || - prev_state == SCX_TASK_INIT_BEGIN); - break; - default: - WARN_ONCE(1, "sched_ext: Invalid task state %d -> %d for %s[%d]", - prev_state, state, p->comm, p->pid); - return; - } - - WARN_ONCE(warn, "sched_ext: Invalid task state transition 0x%x -> 0x%x for %s[%d]", - prev_state, state, p->comm, p->pid); - - p->scx.flags &= ~SCX_TASK_STATE_MASK; - p->scx.flags |= state; -} - -/* - * SCX task iterator. - */ -struct scx_task_iter { - struct sched_ext_entity cursor; - struct task_struct *locked_task; - struct rq *rq; - struct rq_flags rf; - u32 cnt; - bool list_locked; -#ifdef CONFIG_EXT_SUB_SCHED - struct cgroup *cgrp; - struct cgroup_subsys_state *css_pos; - struct css_task_iter css_iter; -#endif -}; - -/** - * scx_task_iter_start - Lock scx_tasks_lock and start a task iteration - * @iter: iterator to init - * @cgrp: Optional root of cgroup subhierarchy to iterate - * - * Initialize @iter. Once initialized, @iter must eventually be stopped with - * scx_task_iter_stop(). - * - * If @cgrp is %NULL, scx_tasks is used for iteration and this function returns - * with scx_tasks_lock held and @iter->cursor inserted into scx_tasks. - * - * If @cgrp is not %NULL, @cgrp and its descendants' tasks are walked using - * @iter->css_iter. The caller must be holding cgroup_lock() to prevent cgroup - * task migrations. - * - * The two modes of iterations are largely independent and it's likely that - * scx_tasks can be removed in favor of always using cgroup iteration if - * CONFIG_SCHED_CLASS_EXT depends on CONFIG_CGROUPS. - * - * scx_tasks_lock and the rq lock may be released using scx_task_iter_unlock() - * between this and the first next() call or between any two next() calls. If - * the locks are released between two next() calls, the caller is responsible - * for ensuring that the task being iterated remains accessible either through - * RCU read lock or obtaining a reference count. - * - * All tasks which existed when the iteration started are guaranteed to be - * visited as long as they are not dead. - */ -static void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp) -{ - memset(iter, 0, sizeof(*iter)); - -#ifdef CONFIG_EXT_SUB_SCHED - if (cgrp) { - lockdep_assert_held(&cgroup_mutex); - iter->cgrp = cgrp; - iter->css_pos = css_next_descendant_pre(NULL, &iter->cgrp->self); - css_task_iter_start(iter->css_pos, CSS_TASK_ITER_WITH_DEAD, - &iter->css_iter); - return; - } -#endif - raw_spin_lock_irq(&scx_tasks_lock); - - iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; - list_add(&iter->cursor.tasks_node, &scx_tasks); - iter->list_locked = true; -} - -static void __scx_task_iter_rq_unlock(struct scx_task_iter *iter) -{ - if (iter->locked_task) { - __balance_callbacks(iter->rq, &iter->rf); - task_rq_unlock(iter->rq, iter->locked_task, &iter->rf); - iter->locked_task = NULL; - } -} - -/** - * scx_task_iter_unlock - Unlock rq and scx_tasks_lock held by a task iterator - * @iter: iterator to unlock - * - * If @iter is in the middle of a locked iteration, it may be locking the rq of - * the task currently being visited in addition to scx_tasks_lock. Unlock both. - * This function can be safely called anytime during an iteration. The next - * iterator operation will automatically restore the necessary locking. - */ -static void scx_task_iter_unlock(struct scx_task_iter *iter) -{ - __scx_task_iter_rq_unlock(iter); - if (iter->list_locked) { - iter->list_locked = false; - raw_spin_unlock_irq(&scx_tasks_lock); - } -} - -static void __scx_task_iter_maybe_relock(struct scx_task_iter *iter) -{ - if (!iter->list_locked) { - raw_spin_lock_irq(&scx_tasks_lock); - iter->list_locked = true; - } -} - -/** - * scx_task_iter_relock - Re-acquire scx_tasks_lock and, optionally, @p's rq - * @iter: iterator to relock - * @p: task whose rq to lock, or %NULL for scx_tasks_lock only - * - * Counterpart to scx_task_iter_unlock(). Locking @p's rq is optional. Once - * re-acquired, both locks are managed by the iterator from here on. - */ -static void scx_task_iter_relock(struct scx_task_iter *iter, - struct task_struct *p) -{ - __scx_task_iter_maybe_relock(iter); - if (p) { - iter->rq = task_rq_lock(p, &iter->rf); - iter->locked_task = p; - } -} - -/** - * scx_task_iter_stop - Stop a task iteration and unlock scx_tasks_lock - * @iter: iterator to exit - * - * Exit a previously initialized @iter. Must be called with scx_tasks_lock held - * which is released on return. If the iterator holds a task's rq lock, that rq - * lock is also released. See scx_task_iter_start() for details. - */ -static void scx_task_iter_stop(struct scx_task_iter *iter) -{ -#ifdef CONFIG_EXT_SUB_SCHED - if (iter->cgrp) { - if (iter->css_pos) - css_task_iter_end(&iter->css_iter); - __scx_task_iter_rq_unlock(iter); - return; - } -#endif - __scx_task_iter_maybe_relock(iter); - list_del_init(&iter->cursor.tasks_node); - scx_task_iter_unlock(iter); -} - -/** - * scx_task_iter_next - Next task - * @iter: iterator to walk - * - * Visit the next task. See scx_task_iter_start() for details. Locks are dropped - * and re-acquired every %SCX_TASK_ITER_BATCH iterations to avoid causing stalls - * by holding scx_tasks_lock for too long. - */ -static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) -{ - struct list_head *cursor = &iter->cursor.tasks_node; - struct sched_ext_entity *pos; - - if (!(++iter->cnt % SCX_TASK_ITER_BATCH)) { - scx_task_iter_unlock(iter); - cond_resched(); - } - -#ifdef CONFIG_EXT_SUB_SCHED - if (iter->cgrp) { - while (iter->css_pos) { - struct task_struct *p; - - p = css_task_iter_next(&iter->css_iter); - if (p) - return p; - - css_task_iter_end(&iter->css_iter); - iter->css_pos = css_next_descendant_pre(iter->css_pos, - &iter->cgrp->self); - if (iter->css_pos) - css_task_iter_start(iter->css_pos, CSS_TASK_ITER_WITH_DEAD, - &iter->css_iter); - } - return NULL; - } -#endif - __scx_task_iter_maybe_relock(iter); - - list_for_each_entry(pos, cursor, tasks_node) { - if (&pos->tasks_node == &scx_tasks) - return NULL; - if (!(pos->flags & SCX_TASK_CURSOR)) { - list_move(cursor, &pos->tasks_node); - return container_of(pos, struct task_struct, scx); - } - } - - /* can't happen, should always terminate at scx_tasks above */ - BUG(); -} - -/** - * scx_task_iter_next_locked - Next non-idle task with its rq locked - * @iter: iterator to walk - * - * Visit the non-idle task with its rq lock held. Allows callers to specify - * whether they would like to filter out dead tasks. See scx_task_iter_start() - * for details. - */ -static struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) -{ - struct task_struct *p; - - __scx_task_iter_rq_unlock(iter); - - while ((p = scx_task_iter_next(iter))) { - /* - * scx_task_iter is used to prepare and move tasks into SCX - * while loading the BPF scheduler and vice-versa while - * unloading. The init_tasks ("swappers") should be excluded - * from the iteration because: - * - * - It's unsafe to use __setschduler_prio() on an init_task to - * determine the sched_class to use as it won't preserve its - * idle_sched_class. - * - * - ops.init/exit_task() can easily be confused if called with - * init_tasks as they, e.g., share PID 0. - * - * As init_tasks are never scheduled through SCX, they can be - * skipped safely. Note that is_idle_task() which tests %PF_IDLE - * doesn't work here: - * - * - %PF_IDLE may not be set for an init_task whose CPU hasn't - * yet been onlined. - * - * - %PF_IDLE can be set on tasks that are not init_tasks. See - * play_idle_precise() used by CONFIG_IDLE_INJECT. - * - * Test for idle_sched_class as only init_tasks are on it. - */ - if (p->sched_class == &idle_sched_class) - continue; - - iter->rq = task_rq_lock(p, &iter->rf); - iter->locked_task = p; - - /* - * cgroup_task_dead() removes the dead tasks from cset->tasks - * after sched_ext_dead() and cgroup iteration may see tasks - * which already finished sched_ext_dead(). %SCX_TASK_DEAD is - * set by sched_ext_dead() under @p's rq lock. Test it to - * avoid visiting tasks which are already dead from SCX POV. - */ - if (scx_get_task_state(p) == SCX_TASK_DEAD) { - __scx_task_iter_rq_unlock(iter); - continue; - } - - return p; - } - return NULL; -} - -/** - * scx_add_event - Increase an event counter for 'name' by 'cnt' - * @sch: scx_sched to account events for - * @name: an event name defined in struct scx_event_stats - * @cnt: the number of the event occurred - * - * This can be used when preemption is not disabled. - */ -#define scx_add_event(sch, name, cnt) do { \ - this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ - trace_sched_ext_event(#name, (cnt)); \ -} while(0) - -/** - * __scx_add_event - Increase an event counter for 'name' by 'cnt' - * @sch: scx_sched to account events for - * @name: an event name defined in struct scx_event_stats - * @cnt: the number of the event occurred - * - * This should be used only when preemption is disabled. - */ -#define __scx_add_event(sch, name, cnt) do { \ - __this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ - trace_sched_ext_event(#name, cnt); \ -} while(0) - -/** - * scx_agg_event - Aggregate an event counter 'kind' from 'src_e' to 'dst_e' - * @dst_e: destination event stats - * @src_e: source event stats - * @kind: a kind of event to be aggregated - */ -#define scx_agg_event(dst_e, src_e, kind) do { \ - (dst_e)->kind += READ_ONCE((src_e)->kind); \ -} while(0) - -/** - * scx_dump_event - Dump an event 'kind' in 'events' to 's' - * @s: output seq_buf - * @events: event stats - * @kind: a kind of event to dump - */ -#define scx_dump_event(s, events, kind) do { \ - dump_line(&(s), "%40s: %16lld", #kind, (events)->kind); \ -} while (0) - - -static void scx_read_events(struct scx_sched *sch, - struct scx_event_stats *events); - -static enum scx_enable_state scx_enable_state(void) -{ - return atomic_read(&scx_enable_state_var); -} - -static enum scx_enable_state scx_set_enable_state(enum scx_enable_state to) -{ - return atomic_xchg(&scx_enable_state_var, to); -} - -static bool scx_tryset_enable_state(enum scx_enable_state to, - enum scx_enable_state from) -{ - int from_v = from; - - return atomic_try_cmpxchg(&scx_enable_state_var, &from_v, to); -} - -/** - * wait_ops_state - Busy-wait the specified ops state to end - * @p: target task - * @opss: state to wait the end of - * - * Busy-wait for @p to transition out of @opss. This can only be used when the - * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also - * has load_acquire semantics to ensure that the caller can see the updates made - * in the enqueueing and dispatching paths. - */ -static void wait_ops_state(struct task_struct *p, unsigned long opss) -{ - do { - cpu_relax(); - } while (atomic_long_read_acquire(&p->scx.ops_state) == opss); -} - -static inline bool __cpu_valid(s32 cpu) -{ - return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); -} - -/** - * scx_cpu_valid - Verify a cpu number, to be used on ops input args - * @sch: scx_sched to abort on error - * @cpu: cpu number which came from a BPF ops - * @where: extra information reported on error - * - * @cpu is a cpu number which came from the BPF scheduler and can be any value. - * Verify that it is in range and one of the possible cpus. If invalid, trigger - * an ops error. - */ -bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where) -{ - if (__cpu_valid(cpu)) { - return true; - } else { - scx_error(sch, "invalid CPU %d%s%s", cpu, where ? " " : "", where ?: ""); - return false; - } -} - -/** - * ops_sanitize_err - Sanitize a -errno value - * @sch: scx_sched to error out on error - * @ops_name: operation to blame on failure - * @err: -errno value to sanitize - * - * Verify @err is a valid -errno. If not, trigger scx_error() and return - * -%EPROTO. This is necessary because returning a rogue -errno up the chain can - * cause misbehaviors. For an example, a large negative return from - * ops.init_task() triggers an oops when passed up the call chain because the - * value fails IS_ERR() test after being encoded with ERR_PTR() and then is - * handled as a pointer. - */ -static int ops_sanitize_err(struct scx_sched *sch, const char *ops_name, s32 err) -{ - if (err < 0 && err >= -MAX_ERRNO) - return err; - - scx_error(sch, "ops.%s() returned an invalid errno %d", ops_name, err); - return -EPROTO; -} - -static void deferred_bal_cb_workfn(struct rq *rq) -{ - run_deferred(rq); -} - -static void deferred_irq_workfn(struct irq_work *irq_work) -{ - struct rq *rq = container_of(irq_work, struct rq, scx.deferred_irq_work); - - raw_spin_rq_lock(rq); - run_deferred(rq); - raw_spin_rq_unlock(rq); -} - -/** - * schedule_deferred - Schedule execution of deferred actions on an rq - * @rq: target rq - * - * Schedule execution of deferred actions on @rq. Deferred actions are executed - * with @rq locked but unpinned, and thus can unlock @rq to e.g. migrate tasks - * to other rqs. - */ -static void schedule_deferred(struct rq *rq) -{ - /* - * This is the fallback when schedule_deferred_locked() can't use - * the cheaper balance callback or wakeup hook paths (the target - * CPU is not in balance or wakeup). Currently, this is primarily - * hit by reenqueue operations targeting a remote CPU. - * - * Queue on the target CPU. The deferred work can run from any CPU - * correctly - the _locked() path already processes remote rqs from - * the calling CPU - but targeting the owning CPU allows IPI delivery - * without waiting for the calling CPU to re-enable IRQs and is - * cheaper as the reenqueue runs locally. - */ - irq_work_queue_on(&rq->scx.deferred_irq_work, cpu_of(rq)); -} - -/** - * schedule_deferred_locked - Schedule execution of deferred actions on an rq - * @rq: target rq - * - * Schedule execution of deferred actions on @rq. Equivalent to - * schedule_deferred() but requires @rq to be locked and can be more efficient. - */ -static void schedule_deferred_locked(struct rq *rq) -{ - lockdep_assert_rq_held(rq); - - /* - * If in the middle of waking up a task, task_woken_scx() will be called - * afterwards which will then run the deferred actions, no need to - * schedule anything. - */ - if (rq->scx.flags & SCX_RQ_IN_WAKEUP) - return; - - /* Don't do anything if there already is a deferred operation. */ - if (rq->scx.flags & SCX_RQ_BAL_CB_PENDING) - return; - - /* - * If in balance, the balance callbacks will be called before rq lock is - * released. Schedule one. - * - * - * We can't directly insert the callback into the - * rq's list: The call can drop its lock and make the pending balance - * callback visible to unrelated code paths that call rq_pin_lock(). - * - * Just let balance_one() know that it must do it itself. - */ - if (rq->scx.flags & SCX_RQ_IN_BALANCE) { - rq->scx.flags |= SCX_RQ_BAL_CB_PENDING; - return; - } - - /* - * No scheduler hooks available. Use the generic irq_work path. The - * above WAKEUP and BALANCE paths should cover most of the cases and the - * time to IRQ re-enable shouldn't be long. - */ - schedule_deferred(rq); -} - -static void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, - u64 reenq_flags, struct rq *locked_rq) -{ - struct rq *rq; - - /* - * Allowing reenqueues doesn't make sense while bypassing. This also - * blocks from new reenqueues to be scheduled on dead scheds. - */ - if (unlikely(READ_ONCE(sch->bypass_depth))) - return; - - if (dsq->id == SCX_DSQ_LOCAL) { - rq = container_of(dsq, struct rq, scx.local_dsq); - - struct scx_sched_pcpu *sch_pcpu = per_cpu_ptr(sch->pcpu, cpu_of(rq)); - struct scx_deferred_reenq_local *drl = &sch_pcpu->deferred_reenq_local; - - /* - * Pairs with smp_mb() in process_deferred_reenq_locals() and - * guarantees that there is a reenq_local() afterwards. - */ - smp_mb(); - - if (list_empty(&drl->node) || - (READ_ONCE(drl->flags) & reenq_flags) != reenq_flags) { - - guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); - - if (list_empty(&drl->node)) - list_move_tail(&drl->node, &rq->scx.deferred_reenq_locals); - WRITE_ONCE(drl->flags, drl->flags | reenq_flags); - } - } else if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) { - rq = this_rq(); - - struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu, cpu_of(rq)); - struct scx_deferred_reenq_user *dru = &dsq_pcpu->deferred_reenq_user; - - /* - * Pairs with smp_mb() in process_deferred_reenq_users() and - * guarantees that there is a reenq_user() afterwards. - */ - smp_mb(); - - if (list_empty(&dru->node) || - (READ_ONCE(dru->flags) & reenq_flags) != reenq_flags) { - - guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); - - if (list_empty(&dru->node)) - list_move_tail(&dru->node, &rq->scx.deferred_reenq_users); - WRITE_ONCE(dru->flags, dru->flags | reenq_flags); - } - } else { - scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id); - return; - } - - if (rq == locked_rq) - schedule_deferred_locked(rq); - else - schedule_deferred(rq); -} - -static void schedule_reenq_local(struct rq *rq, u64 reenq_flags) -{ - struct scx_sched *root = rcu_dereference_sched(scx_root); - - if (WARN_ON_ONCE(!root)) - return; - - schedule_dsq_reenq(root, &rq->scx.local_dsq, reenq_flags, rq); -} - -/** - * touch_core_sched - Update timestamp used for core-sched task ordering - * @rq: rq to read clock from, must be locked - * @p: task to update the timestamp for - * - * Update @p->scx.core_sched_at timestamp. This is used by scx_prio_less() to - * implement global or local-DSQ FIFO ordering for core-sched. Should be called - * when a task becomes runnable and its turn on the CPU ends (e.g. slice - * exhaustion). - */ -static void touch_core_sched(struct rq *rq, struct task_struct *p) -{ - lockdep_assert_rq_held(rq); - -#ifdef CONFIG_SCHED_CORE - /* - * It's okay to update the timestamp spuriously. Use - * sched_core_disabled() which is cheaper than enabled(). - * - * As this is used to determine ordering between tasks of sibling CPUs, - * it may be better to use per-core dispatch sequence instead. - */ - if (!sched_core_disabled()) - p->scx.core_sched_at = sched_clock_cpu(cpu_of(rq)); -#endif -} - -/** - * touch_core_sched_dispatch - Update core-sched timestamp on dispatch - * @rq: rq to read clock from, must be locked - * @p: task being dispatched - * - * If the BPF scheduler implements custom core-sched ordering via - * ops.core_sched_before(), @p->scx.core_sched_at is used to implement FIFO - * ordering within each local DSQ. This function is called from dispatch paths - * and updates @p->scx.core_sched_at if custom core-sched ordering is in effect. - */ -static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) -{ - lockdep_assert_rq_held(rq); - -#ifdef CONFIG_SCHED_CORE - if (unlikely(SCX_HAS_OP(scx_root, core_sched_before))) - touch_core_sched(rq, p); -#endif -} - -static void update_curr_scx(struct rq *rq) -{ - struct task_struct *curr = rq->curr; - s64 delta_exec; - - delta_exec = update_curr_common(rq); - if (unlikely(delta_exec <= 0)) - return; - - if (curr->scx.slice != SCX_SLICE_INF) { - curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec); - if (!curr->scx.slice) - touch_core_sched(rq, curr); - } - - dl_server_update(&rq->ext_server, delta_exec); -} - -static bool scx_dsq_priq_less(struct rb_node *node_a, - const struct rb_node *node_b) -{ - const struct task_struct *a = - container_of(node_a, struct task_struct, scx.dsq_priq); - const struct task_struct *b = - container_of(node_b, struct task_struct, scx.dsq_priq); - - return time_before64(a->scx.dsq_vtime, b->scx.dsq_vtime); -} - -static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags) -{ - /* scx_bpf_dsq_nr_queued() reads ->nr without locking, use WRITE_ONCE() */ - WRITE_ONCE(dsq->nr, dsq->nr + 1); - - /* - * Once @p reaches a local DSQ, it can only leave it by being dispatched - * to the CPU or dequeued. In both cases, the only way @p can go back to - * the BPF sched is through enqueueing. If being inserted into a local - * DSQ with IMMED, persist the state until the next enqueueing event in - * do_enqueue_task() so that we can maintain IMMED protection through - * e.g. SAVE/RESTORE cycles and slice extensions. - */ - if (enq_flags & SCX_ENQ_IMMED) { - if (unlikely(dsq->id != SCX_DSQ_LOCAL)) { - WARN_ON_ONCE(!(enq_flags & SCX_ENQ_GDSQ_FALLBACK)); - return; - } - p->scx.flags |= SCX_TASK_IMMED; - } - - if (p->scx.flags & SCX_TASK_IMMED) { - struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); - - if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) - return; - - rq->scx.nr_immed++; - - /* - * If @rq already had other tasks or the current task is not - * done yet, @p can't go on the CPU immediately. Re-enqueue. - */ - if (unlikely(dsq->nr > 1 || !rq_is_open(rq, enq_flags))) - schedule_reenq_local(rq, 0); - } -} - -static void dsq_dec_nr(struct scx_dispatch_q *dsq, struct task_struct *p) -{ - /* see dsq_inc_nr() */ - WRITE_ONCE(dsq->nr, dsq->nr - 1); - - if (p->scx.flags & SCX_TASK_IMMED) { - struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); - - if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL) || - WARN_ON_ONCE(rq->scx.nr_immed <= 0)) - return; - - rq->scx.nr_immed--; - } -} - -static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) -{ - p->scx.slice = READ_ONCE(sch->slice_dfl); - __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); -} - -/* - * Return true if @p is moving due to an internal SCX migration, false - * otherwise. - */ -static inline bool task_scx_migrating(struct task_struct *p) -{ - /* - * We only need to check sticky_cpu: it is set to the destination - * CPU in move_remote_task_to_local_dsq() before deactivate_task() - * and cleared when the task is enqueued on the destination, so it - * is only non-negative during an internal SCX migration. - */ - return p->scx.sticky_cpu >= 0; -} - -/* - * Call ops.dequeue() if the task is in BPF custody and not migrating. - * Clears %SCX_TASK_IN_CUSTODY when the callback is invoked. - */ -static void call_task_dequeue(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, u64 deq_flags) -{ - if (!(p->scx.flags & SCX_TASK_IN_CUSTODY) || task_scx_migrating(p)) - return; - - if (SCX_HAS_OP(sch, dequeue)) - SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags); - - p->scx.flags &= ~SCX_TASK_IN_CUSTODY; -} - -static void local_dsq_post_enq(struct scx_sched *sch, struct scx_dispatch_q *dsq, - struct task_struct *p, u64 enq_flags) -{ - struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); - - call_task_dequeue(sch, rq, p, 0); - - /* - * Note that @rq's lock may be dropped between this enqueue and @p - * actually getting on CPU. This gives higher-class tasks (e.g. RT) - * an opportunity to wake up on @rq and prevent @p from running. - * Here are some concrete examples: - * - * Example 1: - * - * We dispatch two tasks from a single ops.dispatch(): - * - First, a local task to this CPU's local DSQ; - * - Second, a local/remote task to a remote CPU's local DSQ. - * We must drop the local rq lock in order to finish the second - * dispatch. In that time, an RT task can wake up on the local rq. - * - * Example 2: - * - * We dispatch a local/remote task to a remote CPU's local DSQ. - * We must drop the remote rq lock before the dispatched task can run, - * which gives an RT task an opportunity to wake up on the remote rq. - * - * Both examples work the same if we replace dispatching with moving - * the tasks from a user-created DSQ. - * - * We must detect these wakeups so that we can re-enqueue IMMED tasks - * from @rq's local DSQ. scx_wakeup_preempt() serves exactly this - * purpose, but for it to be invoked, we must ensure that we bump - * @rq->next_class to &ext_sched_class if it's currently idle. - * - * wakeup_preempt() does the bumping, and since we only invoke it if - * @rq->next_class is below &ext_sched_class, it will also - * resched_curr(rq). - */ - if (sched_class_above(p->sched_class, rq->next_class)) - wakeup_preempt(rq, p, 0); - - /* - * If @rq is in balance, the CPU is already vacant and looking for the - * next task to run. No need to preempt or trigger resched after moving - * @p into its local DSQ. - * Note that the wakeup_preempt() above may have already triggered - * a resched if @rq->next_class was idle. It's harmless, since - * need_resched is cleared immediately after task pick. - */ - if (rq->scx.flags & SCX_RQ_IN_BALANCE) - return; - - if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && - rq->curr->sched_class == &ext_sched_class) { - rq->curr->scx.slice = 0; - resched_curr(rq); - } -} - -static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq, - struct scx_dispatch_q *dsq, struct task_struct *p, - u64 enq_flags) -{ - bool is_local = dsq->id == SCX_DSQ_LOCAL; - - WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); - WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || - !RB_EMPTY_NODE(&p->scx.dsq_priq)); - - if (!is_local) { - raw_spin_lock_nested(&dsq->lock, - (enq_flags & SCX_ENQ_NESTED) ? SINGLE_DEPTH_NESTING : 0); - - if (unlikely(dsq->id == SCX_DSQ_INVALID)) { - scx_error(sch, "attempting to dispatch to a destroyed dsq"); - /* fall back to the global dsq */ - raw_spin_unlock(&dsq->lock); - dsq = find_global_dsq(sch, task_cpu(p)); - raw_spin_lock(&dsq->lock); - } - } - - if (unlikely((dsq->id & SCX_DSQ_FLAG_BUILTIN) && - (enq_flags & SCX_ENQ_DSQ_PRIQ))) { - /* - * SCX_DSQ_LOCAL and SCX_DSQ_GLOBAL DSQs always consume from - * their FIFO queues. To avoid confusion and accidentally - * starving vtime-dispatched tasks by FIFO-dispatched tasks, we - * disallow any internal DSQ from doing vtime ordering of - * tasks. - */ - scx_error(sch, "cannot use vtime ordering for built-in DSQs"); - enq_flags &= ~SCX_ENQ_DSQ_PRIQ; - } - - if (enq_flags & SCX_ENQ_DSQ_PRIQ) { - struct rb_node *rbp; - - /* - * A PRIQ DSQ shouldn't be using FIFO enqueueing. As tasks are - * linked to both the rbtree and list on PRIQs, this can only be - * tested easily when adding the first task. - */ - if (unlikely(RB_EMPTY_ROOT(&dsq->priq) && - nldsq_next_task(dsq, NULL, false))) - scx_error(sch, "DSQ ID 0x%016llx already had FIFO-enqueued tasks", - dsq->id); - - p->scx.dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; - rb_add(&p->scx.dsq_priq, &dsq->priq, scx_dsq_priq_less); - - /* - * Find the previous task and insert after it on the list so - * that @dsq->list is vtime ordered. - */ - rbp = rb_prev(&p->scx.dsq_priq); - if (rbp) { - struct task_struct *prev = - container_of(rbp, struct task_struct, - scx.dsq_priq); - list_add(&p->scx.dsq_list.node, &prev->scx.dsq_list.node); - /* first task unchanged - no update needed */ - } else { - list_add(&p->scx.dsq_list.node, &dsq->list); - /* not builtin and new task is at head - use fastpath */ - rcu_assign_pointer(dsq->first_task, p); - } - } else { - /* a FIFO DSQ shouldn't be using PRIQ enqueuing */ - if (unlikely(!RB_EMPTY_ROOT(&dsq->priq))) - scx_error(sch, "DSQ ID 0x%016llx already had PRIQ-enqueued tasks", - dsq->id); - - if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) { - list_add(&p->scx.dsq_list.node, &dsq->list); - /* new task inserted at head - use fastpath */ - if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) - rcu_assign_pointer(dsq->first_task, p); - } else { - /* - * dsq->list can contain parked BPF iterator cursors, so - * list_empty() here isn't a reliable proxy for "no real - * task in the DSQ". Test dsq->first_task directly. - */ - list_add_tail(&p->scx.dsq_list.node, &dsq->list); - if (!dsq->first_task && !(dsq->id & SCX_DSQ_FLAG_BUILTIN)) - rcu_assign_pointer(dsq->first_task, p); - } - } - - /* seq records the order tasks are queued, used by BPF DSQ iterator */ - WRITE_ONCE(dsq->seq, dsq->seq + 1); - p->scx.dsq_seq = dsq->seq; - - dsq_inc_nr(dsq, p, enq_flags); - p->scx.dsq = dsq; - - /* - * Update custody and call ops.dequeue() before clearing ops_state: - * once ops_state is cleared, waiters in ops_dequeue() can proceed - * and dequeue_task_scx() will RMW p->scx.flags. If we clear - * ops_state first, both sides would modify p->scx.flags - * concurrently in a non-atomic way. - */ - if (is_local) { - local_dsq_post_enq(sch, dsq, p, enq_flags); - } else { - /* - * Task on global/bypass DSQ: leave custody, task on - * non-terminal DSQ: enter custody. - */ - if (dsq->id == SCX_DSQ_GLOBAL || dsq->id == SCX_DSQ_BYPASS) - call_task_dequeue(sch, rq, p, 0); - else - p->scx.flags |= SCX_TASK_IN_CUSTODY; - - raw_spin_unlock(&dsq->lock); - } - - /* - * We're transitioning out of QUEUEING or DISPATCHING. store_release to - * match waiters' load_acquire. - */ - if (enq_flags & SCX_ENQ_CLEAR_OPSS) - atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); -} - -static void task_unlink_from_dsq(struct task_struct *p, - struct scx_dispatch_q *dsq) -{ - WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node)); - - if (p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { - rb_erase(&p->scx.dsq_priq, &dsq->priq); - RB_CLEAR_NODE(&p->scx.dsq_priq); - p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; - } - - list_del_init(&p->scx.dsq_list.node); - dsq_dec_nr(dsq, p); - - if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN) && dsq->first_task == p) { - struct task_struct *first_task; - - first_task = nldsq_next_task(dsq, NULL, false); - rcu_assign_pointer(dsq->first_task, first_task); - } -} - -static void dispatch_dequeue(struct rq *rq, struct task_struct *p) -{ - struct scx_dispatch_q *dsq = p->scx.dsq; - bool is_local = dsq == &rq->scx.local_dsq; - - lockdep_assert_rq_held(rq); - - if (!dsq) { - /* - * If !dsq && on-list, @p is on @rq's ddsp_deferred_locals. - * Unlinking is all that's needed to cancel. - */ - if (unlikely(!list_empty(&p->scx.dsq_list.node))) - list_del_init(&p->scx.dsq_list.node); - - /* - * When dispatching directly from the BPF scheduler to a local - * DSQ, the task isn't associated with any DSQ but - * @p->scx.holding_cpu may be set under the protection of - * %SCX_OPSS_DISPATCHING. - */ - if (p->scx.holding_cpu >= 0) - p->scx.holding_cpu = -1; - - return; - } - - if (!is_local) - raw_spin_lock(&dsq->lock); - - /* - * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx.dsq_* can't - * change underneath us. - */ - if (p->scx.holding_cpu < 0) { - /* @p must still be on @dsq, dequeue */ - task_unlink_from_dsq(p, dsq); - } else { - /* - * We're racing against dispatch_to_local_dsq() which already - * removed @p from @dsq and set @p->scx.holding_cpu. Clear the - * holding_cpu which tells dispatch_to_local_dsq() that it lost - * the race. - */ - WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node)); - p->scx.holding_cpu = -1; - } - p->scx.dsq = NULL; - - if (!is_local) - raw_spin_unlock(&dsq->lock); -} - -/* - * Abbreviated version of dispatch_dequeue() that can be used when both @p's rq - * and dsq are locked. - */ -static void dispatch_dequeue_locked(struct task_struct *p, - struct scx_dispatch_q *dsq) -{ - lockdep_assert_rq_held(task_rq(p)); - lockdep_assert_held(&dsq->lock); - - task_unlink_from_dsq(p, dsq); - p->scx.dsq = NULL; -} - -static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch, - struct rq *rq, u64 dsq_id, - s32 tcpu) -{ - struct scx_dispatch_q *dsq; - - if (dsq_id == SCX_DSQ_LOCAL) - return &rq->scx.local_dsq; - - if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { - s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK); - - if (!scx_cpu_valid(sch, cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict")) - return find_global_dsq(sch, tcpu); - - return &cpu_rq(cpu)->scx.local_dsq; - } - - if (dsq_id == SCX_DSQ_GLOBAL) - dsq = find_global_dsq(sch, tcpu); - else - dsq = find_user_dsq(sch, dsq_id); - - if (unlikely(!dsq)) { - scx_error(sch, "non-existent DSQ 0x%llx", dsq_id); - return find_global_dsq(sch, tcpu); - } - - return dsq; -} - -static void mark_direct_dispatch(struct scx_sched *sch, - struct task_struct *ddsp_task, - struct task_struct *p, u64 dsq_id, - u64 enq_flags) -{ - /* - * Mark that dispatch already happened from ops.select_cpu() or - * ops.enqueue() by spoiling direct_dispatch_task with a non-NULL value - * which can never match a valid task pointer. - */ - __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); - - /* @p must match the task on the enqueue path */ - if (unlikely(p != ddsp_task)) { - if (IS_ERR(ddsp_task)) - scx_error(sch, "%s[%d] already direct-dispatched", - p->comm, p->pid); - else - scx_error(sch, "scheduling for %s[%d] but trying to direct-dispatch %s[%d]", - ddsp_task->comm, ddsp_task->pid, - p->comm, p->pid); - return; - } - - WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID); - WARN_ON_ONCE(p->scx.ddsp_enq_flags); - - p->scx.ddsp_dsq_id = dsq_id; - p->scx.ddsp_enq_flags = enq_flags; -} - -/* - * Clear @p direct dispatch state when leaving the scheduler. - * - * Direct dispatch state must be cleared in the following cases: - * - direct_dispatch(): cleared on the synchronous enqueue path, deferred - * dispatch keeps the state until consumed - * - process_ddsp_deferred_locals(): cleared after consuming deferred state, - * - do_enqueue_task(): cleared on enqueue fallbacks where the dispatch - * verdict is ignored (local/global/bypass) - * - dequeue_task_scx(): cleared after dispatch_dequeue(), covering deferred - * cancellation and holding_cpu races - * - scx_disable_task(): cleared for queued wakeup tasks, which are excluded by - * the scx_bypass() loop, so that stale state is not reused by a subsequent - * scheduler instance - */ -static inline void clear_direct_dispatch(struct task_struct *p) -{ - p->scx.ddsp_dsq_id = SCX_DSQ_INVALID; - p->scx.ddsp_enq_flags = 0; -} - -static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, - u64 enq_flags) -{ - struct rq *rq = task_rq(p); - struct scx_dispatch_q *dsq = - find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p)); - u64 ddsp_enq_flags; - - touch_core_sched_dispatch(rq, p); - - p->scx.ddsp_enq_flags |= enq_flags; - - /* - * We are in the enqueue path with @rq locked and pinned, and thus can't - * double lock a remote rq and enqueue to its local DSQ. For - * DSQ_LOCAL_ON verdicts targeting the local DSQ of a remote CPU, defer - * the enqueue so that it's executed when @rq can be unlocked. - */ - if (dsq->id == SCX_DSQ_LOCAL && dsq != &rq->scx.local_dsq) { - unsigned long opss; - - opss = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_STATE_MASK; - - switch (opss & SCX_OPSS_STATE_MASK) { - case SCX_OPSS_NONE: - break; - case SCX_OPSS_QUEUEING: - /* - * As @p was never passed to the BPF side, _release is - * not strictly necessary. Still do it for consistency. - */ - atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); - break; - default: - WARN_ONCE(true, "sched_ext: %s[%d] has invalid ops state 0x%lx in direct_dispatch()", - p->comm, p->pid, opss); - atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); - break; - } - - WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); - list_add_tail(&p->scx.dsq_list.node, - &rq->scx.ddsp_deferred_locals); - schedule_deferred_locked(rq); - return; - } - - ddsp_enq_flags = p->scx.ddsp_enq_flags; - clear_direct_dispatch(p); - - dispatch_enqueue(sch, rq, dsq, p, ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); -} - -static bool scx_rq_online(struct rq *rq) -{ - /* - * Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE indicates - * the online state as seen from the BPF scheduler. cpu_active() test - * guarantees that, if this function returns %true, %SCX_RQ_ONLINE will - * stay set until the current scheduling operation is complete even if - * we aren't locking @rq. - */ - return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq))); -} - -static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, - int sticky_cpu) -{ - struct scx_sched *sch = scx_task_sched(p); - struct task_struct **ddsp_taskp; - struct scx_dispatch_q *dsq; - unsigned long qseq; - - WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_QUEUED)); - - /* internal movements - rq migration / RESTORE */ - if (sticky_cpu == cpu_of(rq)) - goto local_norefill; - - /* - * Clear persistent TASK_IMMED for fresh enqueues, see dsq_inc_nr(). - * Note that exiting and migration-disabled tasks that skip - * ops.enqueue() below will lose IMMED protection unless - * %SCX_OPS_ENQ_EXITING / %SCX_OPS_ENQ_MIGRATION_DISABLED are set. - */ - p->scx.flags &= ~SCX_TASK_IMMED; - - /* - * If !scx_rq_online(), we already told the BPF scheduler that the CPU - * is offline and are just running the hotplug path. Don't bother the - * BPF scheduler. - */ - if (!scx_rq_online(rq)) - goto local; - - if (scx_bypassing(sch, cpu_of(rq))) { - __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); - goto bypass; - } - - if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) - goto direct; - - /* see %SCX_OPS_ENQ_EXITING */ - if (!(sch->ops.flags & SCX_OPS_ENQ_EXITING) && - unlikely(p->flags & PF_EXITING)) { - __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1); - goto local; - } - - /* see %SCX_OPS_ENQ_MIGRATION_DISABLED */ - if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) && - is_migration_disabled(p)) { - __scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1); - goto local; - } - - if (unlikely(!SCX_HAS_OP(sch, enqueue))) - goto global; - - /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ - qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; - - WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); - atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq); - - ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); - WARN_ON_ONCE(*ddsp_taskp); - *ddsp_taskp = p; - - SCX_CALL_OP_TASK(sch, enqueue, rq, p, enq_flags); - - *ddsp_taskp = NULL; - if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) - goto direct; - - /* - * Task is now in BPF scheduler's custody. Set %SCX_TASK_IN_CUSTODY - * so ops.dequeue() is called when it leaves custody. - */ - p->scx.flags |= SCX_TASK_IN_CUSTODY; - - /* - * If not directly dispatched, QUEUEING isn't clear yet and dispatch or - * dequeue may be waiting. The store_release matches their load_acquire. - */ - atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq); - return; - -direct: - direct_dispatch(sch, p, enq_flags); - return; -local_norefill: - dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, enq_flags); - return; -local: - dsq = &rq->scx.local_dsq; - goto enqueue; -global: - dsq = find_global_dsq(sch, task_cpu(p)); - goto enqueue; -bypass: - dsq = bypass_enq_target_dsq(sch, task_cpu(p)); - goto enqueue; - -enqueue: - /* - * For task-ordering, slice refill must be treated as implying the end - * of the current slice. Otherwise, the longer @p stays on the CPU, the - * higher priority it becomes from scx_prio_less()'s POV. - */ - touch_core_sched(rq, p); - refill_task_slice_dfl(sch, p); - clear_direct_dispatch(p); - dispatch_enqueue(sch, rq, dsq, p, enq_flags); -} - -static bool task_runnable(const struct task_struct *p) -{ - return !list_empty(&p->scx.runnable_node); -} - -static void set_task_runnable(struct rq *rq, struct task_struct *p) -{ - lockdep_assert_rq_held(rq); - - if (p->scx.flags & SCX_TASK_RESET_RUNNABLE_AT) { - p->scx.runnable_at = jiffies; - p->scx.flags &= ~SCX_TASK_RESET_RUNNABLE_AT; - } - - /* - * list_add_tail() must be used. scx_bypass() depends on tasks being - * appended to the runnable_list. - */ - list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list); -} - -static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at) -{ - list_del_init(&p->scx.runnable_node); - if (reset_runnable_at) - p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; -} - -static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_flags) -{ - struct scx_sched *sch = scx_task_sched(p); - int sticky_cpu = p->scx.sticky_cpu; - u64 enq_flags = core_enq_flags | rq->scx.extra_enq_flags; - - if (enq_flags & ENQUEUE_WAKEUP) - rq->scx.flags |= SCX_RQ_IN_WAKEUP; - - /* - * Restoring a running task will be immediately followed by - * set_next_task_scx() which expects the task to not be on the BPF - * scheduler as tasks can only start running through local DSQs. Force - * direct-dispatch into the local DSQ by setting the sticky_cpu. - */ - if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) - sticky_cpu = cpu_of(rq); - - if (p->scx.flags & SCX_TASK_QUEUED) { - WARN_ON_ONCE(!task_runnable(p)); - goto out; - } - - set_task_runnable(rq, p); - p->scx.flags |= SCX_TASK_QUEUED; - rq->scx.nr_running++; - add_nr_running(rq, 1); - - if (SCX_HAS_OP(sch, runnable) && !task_on_rq_migrating(p)) - SCX_CALL_OP_TASK(sch, runnable, rq, p, enq_flags); - - if (enq_flags & SCX_ENQ_WAKEUP) - touch_core_sched(rq, p); - - /* Start dl_server if this is the first task being enqueued */ - if (rq->scx.nr_running == 1) - dl_server_start(&rq->ext_server); - - do_enqueue_task(rq, p, enq_flags, sticky_cpu); - - if (sticky_cpu >= 0) - p->scx.sticky_cpu = -1; -out: - rq->scx.flags &= ~SCX_RQ_IN_WAKEUP; - - if ((enq_flags & SCX_ENQ_CPU_SELECTED) && - unlikely(cpu_of(rq) != p->scx.selected_cpu)) - __scx_add_event(sch, SCX_EV_SELECT_CPU_FALLBACK, 1); -} - -static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags) -{ - struct scx_sched *sch = scx_task_sched(p); - unsigned long opss; - - /* dequeue is always temporary, don't reset runnable_at */ - clr_task_runnable(p, false); - -retry: - /* acquire ensures that we see the preceding updates on QUEUED */ - opss = atomic_long_read_acquire(&p->scx.ops_state); - - switch (opss & SCX_OPSS_STATE_MASK) { - case SCX_OPSS_NONE: - break; - case SCX_OPSS_QUEUEING: - /* - * QUEUEING is started and finished while holding @p's rq lock. - * As we're holding the rq lock now, we shouldn't see QUEUEING. - */ - BUG(); - case SCX_OPSS_QUEUED: - /* - * A queued task must always be in BPF scheduler's custody. If - * SCX_TASK_IN_CUSTODY is clear, finish_dispatch() on another - * CPU has already passed call_task_dequeue() (which clears the - * flag), but has not yet written SCX_OPSS_NONE. That final - * store does not require this rq's lock, so retrying with - * cpu_relax() is bounded: we will observe NONE (or DISPATCHING, - * handled by the fallthrough) on a subsequent iteration. - */ - if (unlikely(!(READ_ONCE(p->scx.flags) & SCX_TASK_IN_CUSTODY))) { - cpu_relax(); - goto retry; - } - - if (atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, - SCX_OPSS_NONE)) - break; - fallthrough; - case SCX_OPSS_DISPATCHING: - /* - * If @p is being dispatched from the BPF scheduler to a DSQ, - * wait for the transfer to complete so that @p doesn't get - * added to its DSQ after dequeueing is complete. - * - * As we're waiting on DISPATCHING with the rq locked, the - * dispatching side shouldn't try to lock the rq while - * DISPATCHING is set. See dispatch_to_local_dsq(). - * - * DISPATCHING shouldn't have qseq set and control can reach - * here with NONE @opss from the above QUEUED case block. - * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. - */ - wait_ops_state(p, SCX_OPSS_DISPATCHING); - BUG_ON(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); - break; - } - - /* - * Call ops.dequeue() if the task is still in BPF custody. - * - * The code that clears ops_state to %SCX_OPSS_NONE does not always - * clear %SCX_TASK_IN_CUSTODY: in dispatch_to_local_dsq(), when - * we're moving a task that was in %SCX_OPSS_DISPATCHING to a - * remote CPU's local DSQ, we only set ops_state to %SCX_OPSS_NONE - * so that a concurrent dequeue can proceed, but we clear - * %SCX_TASK_IN_CUSTODY only when we later enqueue or move the - * task. So we can see NONE + IN_CUSTODY here and we must handle - * it. Similarly, after waiting on %SCX_OPSS_DISPATCHING we see - * NONE but the task may still have %SCX_TASK_IN_CUSTODY set until - * it is enqueued on the destination. - */ - call_task_dequeue(sch, rq, p, deq_flags); -} - -static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_flags) -{ - struct scx_sched *sch = scx_task_sched(p); - u64 deq_flags = core_deq_flags; - - /* - * Set %SCX_DEQ_SCHED_CHANGE when the dequeue is due to a property - * change (not sleep or core-sched pick). - */ - if (!(deq_flags & (DEQUEUE_SLEEP | SCX_DEQ_CORE_SCHED_EXEC))) - deq_flags |= SCX_DEQ_SCHED_CHANGE; - - if (!(p->scx.flags & SCX_TASK_QUEUED)) { - WARN_ON_ONCE(task_runnable(p)); - return true; - } - - ops_dequeue(rq, p, deq_flags); - - /* - * A currently running task which is going off @rq first gets dequeued - * and then stops running. As we want running <-> stopping transitions - * to be contained within runnable <-> quiescent transitions, trigger - * ->stopping() early here instead of in put_prev_task_scx(). - * - * @p may go through multiple stopping <-> running transitions between - * here and put_prev_task_scx() if task attribute changes occur while - * balance_one() leaves @rq unlocked. However, they don't contain any - * information meaningful to the BPF scheduler and can be suppressed by - * skipping the callbacks if the task is !QUEUED. - */ - if (SCX_HAS_OP(sch, stopping) && task_current(rq, p)) { - update_curr_scx(rq); - SCX_CALL_OP_TASK(sch, stopping, rq, p, false); - } - - if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p)) - SCX_CALL_OP_TASK(sch, quiescent, rq, p, deq_flags); - - if (deq_flags & SCX_DEQ_SLEEP) - p->scx.flags |= SCX_TASK_DEQD_FOR_SLEEP; - else - p->scx.flags &= ~SCX_TASK_DEQD_FOR_SLEEP; - - p->scx.flags &= ~SCX_TASK_QUEUED; - rq->scx.nr_running--; - sub_nr_running(rq, 1); - - dispatch_dequeue(rq, p); - clear_direct_dispatch(p); - return true; -} - -static void yield_task_scx(struct rq *rq) -{ - struct task_struct *p = rq->donor; - struct scx_sched *sch = scx_task_sched(p); - - if (SCX_HAS_OP(sch, yield)) - SCX_CALL_OP_2TASKS_RET(sch, yield, rq, p, NULL); - else - p->scx.slice = 0; -} - -static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) -{ - struct task_struct *from = rq->donor; - struct scx_sched *sch = scx_task_sched(from); - - if (SCX_HAS_OP(sch, yield) && sch == scx_task_sched(to)) - return SCX_CALL_OP_2TASKS_RET(sch, yield, rq, from, to); - else - return false; -} - -static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_flags) -{ - /* - * Preemption between SCX tasks is implemented by resetting the victim - * task's slice to 0 and triggering reschedule on the target CPU. - * Nothing to do. - */ - if (p->sched_class == &ext_sched_class) - return; - - /* - * Getting preempted by a higher-priority class. Reenqueue IMMED tasks. - * This captures all preemption cases including: - * - * - A SCX task is currently running. - * - * - @rq is waking from idle due to a SCX task waking to it. - * - * - A higher-priority wakes up while SCX dispatch is in progress. - */ - if (rq->scx.nr_immed) - schedule_reenq_local(rq, 0); -} - -static void move_local_task_to_local_dsq(struct scx_sched *sch, - struct task_struct *p, u64 enq_flags, - struct scx_dispatch_q *src_dsq, - struct rq *dst_rq) -{ - struct scx_dispatch_q *dst_dsq = &dst_rq->scx.local_dsq; - - /* @dsq is locked and @p is on @dst_rq */ - lockdep_assert_held(&src_dsq->lock); - lockdep_assert_rq_held(dst_rq); - - WARN_ON_ONCE(p->scx.holding_cpu >= 0); - - if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) - list_add(&p->scx.dsq_list.node, &dst_dsq->list); - else - list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list); - - dsq_inc_nr(dst_dsq, p, enq_flags); - p->scx.dsq = dst_dsq; - - local_dsq_post_enq(sch, dst_dsq, p, enq_flags); -} - -/** - * move_remote_task_to_local_dsq - Move a task from a foreign rq to a local DSQ - * @p: task to move - * @enq_flags: %SCX_ENQ_* - * @src_rq: rq to move the task from, locked on entry, released on return - * @dst_rq: rq to move the task into, locked on return - * - * Move @p which is currently on @src_rq to @dst_rq's local DSQ. - */ -static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, - struct rq *src_rq, struct rq *dst_rq) -{ - lockdep_assert_rq_held(src_rq); - - /* - * Set sticky_cpu before deactivate_task() to properly mark the - * beginning of an SCX-internal migration. - */ - p->scx.sticky_cpu = cpu_of(dst_rq); - deactivate_task(src_rq, p, 0); - set_task_cpu(p, cpu_of(dst_rq)); - - raw_spin_rq_unlock(src_rq); - raw_spin_rq_lock(dst_rq); - - /* - * We want to pass scx-specific enq_flags but activate_task() will - * truncate the upper 32 bit. As we own @rq, we can pass them through - * @rq->scx.extra_enq_flags instead. - */ - WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)); - WARN_ON_ONCE(dst_rq->scx.extra_enq_flags); - dst_rq->scx.extra_enq_flags = enq_flags; - activate_task(dst_rq, p, 0); - dst_rq->scx.extra_enq_flags = 0; -} - -/* - * Similar to kernel/sched/core.c::is_cpu_allowed(). However, there are two - * differences: - * - * - is_cpu_allowed() asks "Can this task run on this CPU?" while - * task_can_run_on_remote_rq() asks "Can the BPF scheduler migrate the task to - * this CPU?". - * - * While migration is disabled, is_cpu_allowed() has to say "yes" as the task - * must be allowed to finish on the CPU that it's currently on regardless of - * the CPU state. However, task_can_run_on_remote_rq() must say "no" as the - * BPF scheduler shouldn't attempt to migrate a task which has migration - * disabled. - * - * - The BPF scheduler is bypassed while the rq is offline and we can always say - * no to the BPF scheduler initiated migrations while offline. - * - * The caller must ensure that @p and @rq are on different CPUs. - */ -static bool task_can_run_on_remote_rq(struct scx_sched *sch, - struct task_struct *p, struct rq *rq, - bool enforce) -{ - s32 cpu = cpu_of(rq); - - WARN_ON_ONCE(task_cpu(p) == cpu); - - /* - * If @p has migration disabled, @p->cpus_ptr is updated to contain only - * the pinned CPU in migrate_disable_switch() while @p is being switched - * out. However, put_prev_task_scx() is called before @p->cpus_ptr is - * updated and thus another CPU may see @p on a DSQ inbetween leading to - * @p passing the below task_allowed_on_cpu() check while migration is - * disabled. - * - * Test the migration disabled state first as the race window is narrow - * and the BPF scheduler failing to check migration disabled state can - * easily be masked if task_allowed_on_cpu() is done first. - */ - if (unlikely(is_migration_disabled(p))) { - if (enforce) - scx_error(sch, "SCX_DSQ_LOCAL[_ON] cannot move migration disabled %s[%d] from CPU %d to %d", - p->comm, p->pid, task_cpu(p), cpu); - return false; - } - - /* - * We don't require the BPF scheduler to avoid dispatching to offline - * CPUs mostly for convenience but also because CPUs can go offline - * between scx_bpf_dsq_insert() calls and here. Trigger error iff the - * picked CPU is outside the allowed mask. - */ - if (!task_allowed_on_cpu(p, cpu)) { - if (enforce) - scx_error(sch, "SCX_DSQ_LOCAL[_ON] target CPU %d not allowed for %s[%d]", - cpu, p->comm, p->pid); - return false; - } - - if (!scx_rq_online(rq)) { - if (enforce) - __scx_add_event(sch, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE, 1); - return false; - } - - return true; -} - -/** - * unlink_dsq_and_lock_src_rq() - Unlink task from its DSQ and lock its task_rq - * @p: target task - * @dsq: locked DSQ @p is currently on - * @src_rq: rq @p is currently on, stable with @dsq locked - * - * Called with @dsq locked but no rq's locked. We want to move @p to a different - * DSQ, including any local DSQ, but are not locking @src_rq. Locking @src_rq is - * required when transferring into a local DSQ. Even when transferring into a - * non-local DSQ, it's better to use the same mechanism to protect against - * dequeues and maintain the invariant that @p->scx.dsq can only change while - * @src_rq is locked, which e.g. scx_dump_task() depends on. - * - * We want to grab @src_rq but that can deadlock if we try while locking @dsq, - * so we want to unlink @p from @dsq, drop its lock and then lock @src_rq. As - * this may race with dequeue, which can't drop the rq lock or fail, do a little - * dancing from our side. - * - * @p->scx.holding_cpu is set to this CPU before @dsq is unlocked. If @p gets - * dequeued after we unlock @dsq but before locking @src_rq, the holding_cpu - * would be cleared to -1. While other cpus may have updated it to different - * values afterwards, as this operation can't be preempted or recurse, the - * holding_cpu can never become this CPU again before we're done. Thus, we can - * tell whether we lost to dequeue by testing whether the holding_cpu still - * points to this CPU. See dispatch_dequeue() for the counterpart. - * - * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is - * still valid. %false if lost to dequeue. - */ -static bool unlink_dsq_and_lock_src_rq(struct task_struct *p, - struct scx_dispatch_q *dsq, - struct rq *src_rq) -{ - s32 cpu = raw_smp_processor_id(); - - lockdep_assert_held(&dsq->lock); - - WARN_ON_ONCE(p->scx.holding_cpu >= 0); - task_unlink_from_dsq(p, dsq); - p->scx.holding_cpu = cpu; - - raw_spin_unlock(&dsq->lock); - raw_spin_rq_lock(src_rq); - - /* task_rq couldn't have changed if we're still the holding cpu */ - return likely(p->scx.holding_cpu == cpu) && - !WARN_ON_ONCE(src_rq != task_rq(p)); -} - -static bool consume_remote_task(struct rq *this_rq, - struct task_struct *p, u64 enq_flags, - struct scx_dispatch_q *dsq, struct rq *src_rq) -{ - raw_spin_rq_unlock(this_rq); - - if (unlink_dsq_and_lock_src_rq(p, dsq, src_rq)) { - move_remote_task_to_local_dsq(p, enq_flags, src_rq, this_rq); - return true; - } else { - raw_spin_rq_unlock(src_rq); - raw_spin_rq_lock(this_rq); - return false; - } -} - -/** - * move_task_between_dsqs() - Move a task from one DSQ to another - * @sch: scx_sched being operated on - * @p: target task - * @enq_flags: %SCX_ENQ_* - * @src_dsq: DSQ @p is currently on, must not be a local DSQ - * @dst_dsq: DSQ @p is being moved to, can be any DSQ - * - * Must be called with @p's task_rq and @src_dsq locked. If @dst_dsq is a local - * DSQ and @p is on a different CPU, @p will be migrated and thus its task_rq - * will change. As @p's task_rq is locked, this function doesn't need to use the - * holding_cpu mechanism. - * - * On return, @src_dsq is unlocked and only @p's new task_rq, which is the - * return value, is locked. - */ -static struct rq *move_task_between_dsqs(struct scx_sched *sch, - struct task_struct *p, u64 enq_flags, - struct scx_dispatch_q *src_dsq, - struct scx_dispatch_q *dst_dsq) -{ - struct rq *src_rq = task_rq(p), *dst_rq; - - BUG_ON(src_dsq->id == SCX_DSQ_LOCAL); - lockdep_assert_held(&src_dsq->lock); - lockdep_assert_rq_held(src_rq); - - if (dst_dsq->id == SCX_DSQ_LOCAL) { - dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); - if (src_rq != dst_rq && - unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { - dst_dsq = find_global_dsq(sch, task_cpu(p)); - dst_rq = src_rq; - enq_flags |= SCX_ENQ_GDSQ_FALLBACK; - } - } else { - /* no need to migrate if destination is a non-local DSQ */ - dst_rq = src_rq; - } - - /* - * Move @p into $dst_dsq. If $dst_dsq is the local DSQ of a different - * CPU, @p will be migrated. - */ - if (dst_dsq->id == SCX_DSQ_LOCAL) { - /* @p is going from a non-local DSQ to a local DSQ */ - if (src_rq == dst_rq) { - task_unlink_from_dsq(p, src_dsq); - move_local_task_to_local_dsq(sch, p, enq_flags, - src_dsq, dst_rq); - raw_spin_unlock(&src_dsq->lock); - } else { - raw_spin_unlock(&src_dsq->lock); - move_remote_task_to_local_dsq(p, enq_flags, - src_rq, dst_rq); - } - } else { - /* - * @p is going from a non-local DSQ to a non-local DSQ. As - * $src_dsq is already locked, do an abbreviated dequeue. - */ - dispatch_dequeue_locked(p, src_dsq); - raw_spin_unlock(&src_dsq->lock); - - dispatch_enqueue(sch, dst_rq, dst_dsq, p, enq_flags); - } - - return dst_rq; -} - -static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq, - struct scx_dispatch_q *dsq, u64 enq_flags) -{ - struct task_struct *p; -retry: - /* - * The caller can't expect to successfully consume a task if the task's - * addition to @dsq isn't guaranteed to be visible somehow. Test - * @dsq->list without locking and skip if it seems empty. - */ - if (list_empty(&dsq->list)) - return false; - - raw_spin_lock(&dsq->lock); - - nldsq_for_each_task(p, dsq) { - struct rq *task_rq = task_rq(p); - - /* - * This loop can lead to multiple lockup scenarios, e.g. the BPF - * scheduler can put an enormous number of affinitized tasks into - * a contended DSQ, or the outer retry loop can repeatedly race - * against scx_bypass() dequeueing tasks from @dsq trying to put - * the system into the bypass mode. This can easily live-lock the - * machine. If aborting, exit from all non-bypass DSQs. - */ - if (unlikely(READ_ONCE(sch->aborting)) && dsq->id != SCX_DSQ_BYPASS) - break; - - if (rq == task_rq) { - task_unlink_from_dsq(p, dsq); - move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq); - raw_spin_unlock(&dsq->lock); - return true; - } - - if (task_can_run_on_remote_rq(sch, p, rq, false)) { - if (likely(consume_remote_task(rq, p, enq_flags, dsq, task_rq))) - return true; - goto retry; - } - } - - raw_spin_unlock(&dsq->lock); - return false; -} - -static bool consume_global_dsq(struct scx_sched *sch, struct rq *rq) -{ - int node = cpu_to_node(cpu_of(rq)); - - return consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq, 0); -} - -/** - * dispatch_to_local_dsq - Dispatch a task to a local dsq - * @sch: scx_sched being operated on - * @rq: current rq which is locked - * @dst_dsq: destination DSQ - * @p: task to dispatch - * @enq_flags: %SCX_ENQ_* - * - * We're holding @rq lock and want to dispatch @p to @dst_dsq which is a local - * DSQ. This function performs all the synchronization dancing needed because - * local DSQs are protected with rq locks. - * - * The caller must have exclusive ownership of @p (e.g. through - * %SCX_OPSS_DISPATCHING). - */ -static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, - struct scx_dispatch_q *dst_dsq, - struct task_struct *p, u64 enq_flags) -{ - struct rq *src_rq = task_rq(p); - struct rq *dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); - struct rq *locked_rq = rq; - - /* - * We're synchronized against dequeue through DISPATCHING. As @p can't - * be dequeued, its task_rq and cpus_allowed are stable too. - * - * If dispatching to @rq that @p is already on, no lock dancing needed. - */ - if (rq == src_rq && rq == dst_rq) { - dispatch_enqueue(sch, rq, dst_dsq, p, - enq_flags | SCX_ENQ_CLEAR_OPSS); - return; - } - - if (src_rq != dst_rq && - unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { - dispatch_enqueue(sch, rq, find_global_dsq(sch, task_cpu(p)), p, - enq_flags | SCX_ENQ_CLEAR_OPSS | SCX_ENQ_GDSQ_FALLBACK); - return; - } - - /* - * @p is on a possibly remote @src_rq which we need to lock to move the - * task. If dequeue is in progress, it'd be locking @src_rq and waiting - * on DISPATCHING, so we can't grab @src_rq lock while holding - * DISPATCHING. - * - * As DISPATCHING guarantees that @p is wholly ours, we can pretend that - * we're moving from a DSQ and use the same mechanism - mark the task - * under transfer with holding_cpu, release DISPATCHING and then follow - * the same protocol. See unlink_dsq_and_lock_src_rq(). - */ - p->scx.holding_cpu = raw_smp_processor_id(); - - /* store_release ensures that dequeue sees the above */ - atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); - - /* switch to @src_rq lock */ - if (locked_rq != src_rq) { - raw_spin_rq_unlock(locked_rq); - locked_rq = src_rq; - raw_spin_rq_lock(src_rq); - } - - /* task_rq couldn't have changed if we're still the holding cpu */ - if (likely(p->scx.holding_cpu == raw_smp_processor_id()) && - !WARN_ON_ONCE(src_rq != task_rq(p))) { - /* - * If @p is staying on the same rq, there's no need to go - * through the full deactivate/activate cycle. Optimize by - * abbreviating move_remote_task_to_local_dsq(). - */ - if (src_rq == dst_rq) { - p->scx.holding_cpu = -1; - dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, - enq_flags); - } else { - move_remote_task_to_local_dsq(p, enq_flags, - src_rq, dst_rq); - /* task has been moved to dst_rq, which is now locked */ - locked_rq = dst_rq; - } - - /* if the destination CPU is idle, wake it up */ - if (sched_class_above(p->sched_class, dst_rq->curr->sched_class)) - resched_curr(dst_rq); - } - - /* switch back to @rq lock */ - if (locked_rq != rq) { - raw_spin_rq_unlock(locked_rq); - raw_spin_rq_lock(rq); - } -} - -/** - * finish_dispatch - Asynchronously finish dispatching a task - * @rq: current rq which is locked - * @p: task to finish dispatching - * @qseq_at_dispatch: qseq when @p started getting dispatched - * @dsq_id: destination DSQ ID - * @enq_flags: %SCX_ENQ_* - * - * Dispatching to local DSQs may need to wait for queueing to complete or - * require rq lock dancing. As we don't wanna do either while inside - * ops.dispatch() to avoid locking order inversion, we split dispatching into - * two parts. scx_bpf_dsq_insert() which is called by ops.dispatch() records the - * task and its qseq. Once ops.dispatch() returns, this function is called to - * finish up. - * - * There is no guarantee that @p is still valid for dispatching or even that it - * was valid in the first place. Make sure that the task is still owned by the - * BPF scheduler and claim the ownership before dispatching. - */ -static void finish_dispatch(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, - unsigned long qseq_at_dispatch, - u64 dsq_id, u64 enq_flags) -{ - struct scx_dispatch_q *dsq; - unsigned long opss; - - touch_core_sched_dispatch(rq, p); -retry: - /* - * No need for _acquire here. @p is accessed only after a successful - * try_cmpxchg to DISPATCHING. - */ - opss = atomic_long_read(&p->scx.ops_state); - - switch (opss & SCX_OPSS_STATE_MASK) { - case SCX_OPSS_DISPATCHING: - case SCX_OPSS_NONE: - /* someone else already got to it */ - return; - case SCX_OPSS_QUEUED: - /* - * If qseq doesn't match, @p has gone through at least one - * dispatch/dequeue and re-enqueue cycle between - * scx_bpf_dsq_insert() and here and we have no claim on it. - */ - if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) - return; - - /* see SCX_EV_INSERT_NOT_OWNED definition */ - if (unlikely(!scx_task_on_sched(sch, p))) { - __scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1); - return; - } - - /* - * While we know @p is accessible, we don't yet have a claim on - * it - the BPF scheduler is allowed to dispatch tasks - * spuriously and there can be a racing dequeue attempt. Let's - * claim @p by atomically transitioning it from QUEUED to - * DISPATCHING. - */ - if (likely(atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, - SCX_OPSS_DISPATCHING))) - break; - goto retry; - case SCX_OPSS_QUEUEING: - /* - * do_enqueue_task() is in the process of transferring the task - * to the BPF scheduler while holding @p's rq lock. As we aren't - * holding any kernel or BPF resource that the enqueue path may - * depend upon, it's safe to wait. - */ - wait_ops_state(p, opss); - goto retry; - } - - BUG_ON(!(p->scx.flags & SCX_TASK_QUEUED)); - - dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, task_cpu(p)); - - if (dsq->id == SCX_DSQ_LOCAL) - dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); - else - dispatch_enqueue(sch, rq, dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -} - -static void flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) -{ - struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; - u32 u; - - for (u = 0; u < dspc->cursor; u++) { - struct scx_dsp_buf_ent *ent = &dspc->buf[u]; - - finish_dispatch(sch, rq, ent->task, ent->qseq, ent->dsq_id, - ent->enq_flags); - } - - dspc->nr_tasks += dspc->cursor; - dspc->cursor = 0; -} - -static inline void maybe_queue_balance_callback(struct rq *rq) -{ - lockdep_assert_rq_held(rq); - - if (!(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) - return; - - queue_balance_callback(rq, &rq->scx.deferred_bal_cb, - deferred_bal_cb_workfn); - - rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; -} - -/* - * One user of this function is scx_bpf_dispatch() which can be called - * recursively as sub-sched dispatches nest. Always inline to reduce stack usage - * from the call frame. - */ -static __always_inline bool -scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, - struct task_struct *prev, bool nested) -{ - struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; - int nr_loops = SCX_DSP_MAX_LOOPS; - s32 cpu = cpu_of(rq); - bool prev_on_sch = (prev->sched_class == &ext_sched_class) && - scx_task_on_sched(sch, prev); - - if (consume_global_dsq(sch, rq)) - return true; - - if (bypass_dsp_enabled(sch)) { - /* if @sch is bypassing, only the bypass DSQs are active */ - if (scx_bypassing(sch, cpu)) - return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0); - -#ifdef CONFIG_EXT_SUB_SCHED - /* - * If @sch isn't bypassing but its children are, @sch is - * responsible for making forward progress for both its own - * tasks that aren't bypassing and the bypassing descendants' - * tasks. The following implements a simple built-in behavior - - * let each CPU try to run the bypass DSQ every Nth time. - * - * Later, if necessary, we can add an ops flag to suppress the - * auto-consumption and a kfunc to consume the bypass DSQ and, - * so that the BPF scheduler can fully control scheduling of - * bypassed tasks. - */ - struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); - - if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && - consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0)) { - __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); - return true; - } -#endif /* CONFIG_EXT_SUB_SCHED */ - } - - if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq)) - return false; - - dspc->rq = rq; - - /* - * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, - * the local DSQ might still end up empty after a successful - * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() - * produced some tasks, retry. The BPF scheduler may depend on this - * looping behavior to simplify its implementation. - */ - do { - dspc->nr_tasks = 0; - - if (nested) { - SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), - prev_on_sch ? prev : NULL); - } else { - /* stash @prev so that nested invocations can access it */ - rq->scx.sub_dispatch_prev = prev; - SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), - prev_on_sch ? prev : NULL); - rq->scx.sub_dispatch_prev = NULL; - } - - flush_dispatch_buf(sch, rq); - - if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) { - rq->scx.flags |= SCX_RQ_BAL_KEEP; - return true; - } - if (rq->scx.local_dsq.nr) - return true; - if (consume_global_dsq(sch, rq)) - return true; - - /* - * ops.dispatch() can trap us in this loop by repeatedly - * dispatching ineligible tasks. Break out once in a while to - * allow the watchdog to run. As IRQ can't be enabled in - * balance(), we want to complete this scheduling cycle and then - * start a new one. IOW, we want to call resched_curr() on the - * next, most likely idle, task, not the current one. Use - * __scx_bpf_kick_cpu() for deferred kicking. - */ - if (unlikely(!--nr_loops)) { - scx_kick_cpu(sch, cpu, 0); - break; - } - } while (dspc->nr_tasks); - - /* - * Prevent the CPU from going idle while bypassed descendants have tasks - * queued. Without this fallback, bypassed tasks could stall if the host - * scheduler's ops.dispatch() doesn't yield any tasks. - */ - if (bypass_dsp_enabled(sch)) - return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0); - - return false; -} - -static int balance_one(struct rq *rq, struct task_struct *prev) -{ - struct scx_sched *sch = scx_root; - s32 cpu = cpu_of(rq); - - lockdep_assert_rq_held(rq); - rq->scx.flags |= SCX_RQ_IN_BALANCE; - rq->scx.flags &= ~SCX_RQ_BAL_KEEP; - - if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) && - unlikely(rq->scx.cpu_released)) { - /* - * If the previous sched_class for the current CPU was not SCX, - * notify the BPF scheduler that it again has control of the - * core. This callback complements ->cpu_release(), which is - * emitted in switch_class(). - */ - if (sch->ops.cpu_acquire) - SCX_CALL_OP(sch, cpu_acquire, rq, cpu, NULL); - rq->scx.cpu_released = false; - } - - if (prev->sched_class == &ext_sched_class) { - update_curr_scx(rq); - - /* - * If @prev is runnable & has slice left, it has priority and - * fetching more just increases latency for the fetched tasks. - * Tell pick_task_scx() to keep running @prev. If the BPF - * scheduler wants to handle this explicitly, it should - * implement ->cpu_release(). - * - * See scx_disable_workfn() for the explanation on the bypassing - * test. - */ - if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice && - !scx_bypassing(sch, cpu)) { - rq->scx.flags |= SCX_RQ_BAL_KEEP; - goto has_tasks; - } - } - - /* if there already are tasks to run, nothing to do */ - if (rq->scx.local_dsq.nr) - goto has_tasks; - - if (scx_dispatch_sched(sch, rq, prev, false)) - goto has_tasks; - - /* - * Didn't find another task to run. Keep running @prev unless - * %SCX_OPS_ENQ_LAST is in effect. - */ - if ((prev->scx.flags & SCX_TASK_QUEUED) && - (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu))) { - rq->scx.flags |= SCX_RQ_BAL_KEEP; - __scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1); - goto has_tasks; - } - rq->scx.flags &= ~SCX_RQ_IN_BALANCE; - return false; - -has_tasks: - /* - * @rq may have extra IMMED tasks without reenq scheduled: - * - * - rq_is_open() can't reliably tell when and how slice is going to be - * modified for $curr and allows IMMED tasks to be queued while - * dispatch is in progress. - * - * - A non-IMMED HEAD task can get queued in front of an IMMED task - * between the IMMED queueing and the subsequent scheduling event. - */ - if (unlikely(rq->scx.local_dsq.nr > 1 && rq->scx.nr_immed)) - schedule_reenq_local(rq, 0); - - rq->scx.flags &= ~SCX_RQ_IN_BALANCE; - return true; -} - -static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) -{ - struct scx_sched *sch = scx_task_sched(p); - - if (p->scx.flags & SCX_TASK_QUEUED) { - /* - * Core-sched might decide to execute @p before it is - * dispatched. Call ops_dequeue() to notify the BPF scheduler. - */ - ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC); - dispatch_dequeue(rq, p); - } - - p->se.exec_start = rq_clock_task(rq); - - /* see dequeue_task_scx() on why we skip when !QUEUED */ - if (SCX_HAS_OP(sch, running) && (p->scx.flags & SCX_TASK_QUEUED)) - SCX_CALL_OP_TASK(sch, running, rq, p); - - clr_task_runnable(p, true); - - /* - * @p is getting newly scheduled or got kicked after someone updated its - * slice. Refresh whether tick can be stopped. See scx_can_stop_tick(). - */ - if ((p->scx.slice == SCX_SLICE_INF) != - (bool)(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) { - if (p->scx.slice == SCX_SLICE_INF) - rq->scx.flags |= SCX_RQ_CAN_STOP_TICK; - else - rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK; - - sched_update_tick_dependency(rq); - - /* - * For now, let's refresh the load_avgs just when transitioning - * in and out of nohz. In the future, we might want to add a - * mechanism which calls the following periodically on - * tick-stopped CPUs. - */ - update_other_load_avgs(rq); - } -} - -static enum scx_cpu_preempt_reason -preempt_reason_from_class(const struct sched_class *class) -{ - if (class == &stop_sched_class) - return SCX_CPU_PREEMPT_STOP; - if (class == &dl_sched_class) - return SCX_CPU_PREEMPT_DL; - if (class == &rt_sched_class) - return SCX_CPU_PREEMPT_RT; - return SCX_CPU_PREEMPT_UNKNOWN; -} - -static void switch_class(struct rq *rq, struct task_struct *next) -{ - struct scx_sched *sch = scx_root; - const struct sched_class *next_class = next->sched_class; - - if (!(sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT)) - return; - - /* - * The callback is conceptually meant to convey that the CPU is no - * longer under the control of SCX. Therefore, don't invoke the callback - * if the next class is below SCX (in which case the BPF scheduler has - * actively decided not to schedule any tasks on the CPU). - */ - if (sched_class_above(&ext_sched_class, next_class)) - return; - - /* - * At this point we know that SCX was preempted by a higher priority - * sched_class, so invoke the ->cpu_release() callback if we have not - * done so already. We only send the callback once between SCX being - * preempted, and it regaining control of the CPU. - * - * ->cpu_release() complements ->cpu_acquire(), which is emitted the - * next time that balance_one() is invoked. - */ - if (!rq->scx.cpu_released) { - if (sch->ops.cpu_release) { - struct scx_cpu_release_args args = { - .reason = preempt_reason_from_class(next_class), - .task = next, - }; - - SCX_CALL_OP(sch, cpu_release, rq, cpu_of(rq), &args); - } - rq->scx.cpu_released = true; - } -} - -static void put_prev_task_scx(struct rq *rq, struct task_struct *p, - struct task_struct *next) -{ - struct scx_sched *sch = scx_task_sched(p); - - /* see kick_sync_wait_bal_cb() */ - smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); - - update_curr_scx(rq); - - /* see dequeue_task_scx() on why we skip when !QUEUED */ - if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED)) - SCX_CALL_OP_TASK(sch, stopping, rq, p, true); - - if (p->scx.flags & SCX_TASK_QUEUED) { - set_task_runnable(rq, p); - - /* - * If @p has slice left and is being put, @p is getting - * preempted by a higher priority scheduler class or core-sched - * forcing a different task. Leave it at the head of the local - * DSQ unless it was an IMMED task. IMMED tasks should not - * linger on a busy CPU, reenqueue them to the BPF scheduler. - */ - if (p->scx.slice && !scx_bypassing(sch, cpu_of(rq))) { - if (p->scx.flags & SCX_TASK_IMMED) { - p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; - do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - } else { - dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, SCX_ENQ_HEAD); - } - goto switch_class; - } - - /* - * If @p is runnable but we're about to enter a lower - * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell - * ops.enqueue() that @p is the only one available for this cpu, - * which should trigger an explicit follow-up scheduling event. - */ - if (next && sched_class_above(&ext_sched_class, next->sched_class)) { - WARN_ON_ONCE(!(sch->ops.flags & SCX_OPS_ENQ_LAST)); - do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); - } else { - do_enqueue_task(rq, p, 0, -1); - } - } - -switch_class: - if (next && next->sched_class != &ext_sched_class) - switch_class(rq, next); -} - -static void kick_sync_wait_bal_cb(struct rq *rq) -{ - struct scx_kick_syncs __rcu *ks = __this_cpu_read(scx_kick_syncs); - unsigned long *ksyncs = rcu_dereference_sched(ks)->syncs; - bool waited; - s32 cpu; - - /* - * Drop rq lock and enable IRQs while waiting. IRQs must be enabled - * — a target CPU may be waiting for us to process an IPI (e.g. TLB - * flush) while we wait for its kick_sync to advance. - * - * Also, keep advancing our own kick_sync so that new kick_sync waits - * targeting us, which can start after we drop the lock, cannot form - * cyclic dependencies. - */ -retry: - waited = false; - for_each_cpu(cpu, rq->scx.cpus_to_sync) { - /* - * smp_load_acquire() pairs with smp_store_release() on - * kick_sync updates on the target CPUs. - */ - if (cpu == cpu_of(rq) || - smp_load_acquire(&cpu_rq(cpu)->scx.kick_sync) != ksyncs[cpu]) { - cpumask_clear_cpu(cpu, rq->scx.cpus_to_sync); - continue; - } - - raw_spin_rq_unlock_irq(rq); - while (READ_ONCE(cpu_rq(cpu)->scx.kick_sync) == ksyncs[cpu]) { - smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); - cpu_relax(); - } - raw_spin_rq_lock_irq(rq); - waited = true; - } - - if (waited) - goto retry; -} - -static struct task_struct *first_local_task(struct rq *rq) -{ - return list_first_entry_or_null(&rq->scx.local_dsq.list, - struct task_struct, scx.dsq_list.node); -} - -static struct task_struct * -do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) -{ - struct task_struct *prev = rq->curr; - bool keep_prev; - struct task_struct *p; - - /* see kick_sync_wait_bal_cb() */ - smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); - - rq_modified_begin(rq, &ext_sched_class); - - rq_unpin_lock(rq, rf); - balance_one(rq, prev); - rq_repin_lock(rq, rf); - maybe_queue_balance_callback(rq); - - /* - * Defer to a balance callback which can drop rq lock and enable - * IRQs. Waiting directly in the pick path would deadlock against - * CPUs sending us IPIs (e.g. TLB flushes) while we wait for them. - */ - if (unlikely(rq->scx.kick_sync_pending)) { - rq->scx.kick_sync_pending = false; - queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, - kick_sync_wait_bal_cb); - } - - /* - * If any higher-priority sched class enqueued a runnable task on - * this rq during balance_one(), abort and return RETRY_TASK, so - * that the scheduler loop can restart. - * - * If @force_scx is true, always try to pick a SCHED_EXT task, - * regardless of any higher-priority sched classes activity. - */ - if (!force_scx && rq_modified_above(rq, &ext_sched_class)) - return RETRY_TASK; - - keep_prev = rq->scx.flags & SCX_RQ_BAL_KEEP; - if (unlikely(keep_prev && - prev->sched_class != &ext_sched_class)) { - WARN_ON_ONCE(scx_enable_state() == SCX_ENABLED); - keep_prev = false; - } - - /* - * If balance_one() is telling us to keep running @prev, replenish slice - * if necessary and keep running @prev. Otherwise, pop the first one - * from the local DSQ. - */ - if (keep_prev) { - p = prev; - if (!p->scx.slice) - refill_task_slice_dfl(scx_task_sched(p), p); - } else { - p = first_local_task(rq); - if (!p) - return NULL; - - if (unlikely(!p->scx.slice)) { - struct scx_sched *sch = scx_task_sched(p); - - if (!scx_bypassing(sch, cpu_of(rq)) && - !sch->warned_zero_slice) { - printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in %s()\n", - p->comm, p->pid, __func__); - sch->warned_zero_slice = true; - } - refill_task_slice_dfl(sch, p); - } - } - - return p; -} - -static struct task_struct *pick_task_scx(struct rq *rq, struct rq_flags *rf) -{ - return do_pick_task_scx(rq, rf, false); -} - -/* - * Select the next task to run from the ext scheduling class. - * - * Use do_pick_task_scx() directly with @force_scx enabled, since the - * dl_server must always select a sched_ext task. - */ -static struct task_struct * -ext_server_pick_task(struct sched_dl_entity *dl_se, struct rq_flags *rf) -{ - if (!scx_enabled()) - return NULL; - - return do_pick_task_scx(dl_se->rq, rf, true); -} - -/* - * Initialize the ext server deadline entity. - */ -void ext_server_init(struct rq *rq) -{ - struct sched_dl_entity *dl_se = &rq->ext_server; - - init_dl_entity(dl_se); - - dl_server_init(dl_se, rq, ext_server_pick_task); -} - -#ifdef CONFIG_SCHED_CORE -/** - * scx_prio_less - Task ordering for core-sched - * @a: task A - * @b: task B - * @in_fi: in forced idle state - * - * Core-sched is implemented as an additional scheduling layer on top of the - * usual sched_class'es and needs to find out the expected task ordering. For - * SCX, core-sched calls this function to interrogate the task ordering. - * - * Unless overridden by ops.core_sched_before(), @p->scx.core_sched_at is used - * to implement the default task ordering. The older the timestamp, the higher - * priority the task - the global FIFO ordering matching the default scheduling - * behavior. - * - * When ops.core_sched_before() is enabled, @p->scx.core_sched_at is used to - * implement FIFO ordering within each local DSQ. See pick_task_scx(). - */ -bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, - bool in_fi) -{ - struct scx_sched *sch_a = scx_task_sched(a); - struct scx_sched *sch_b = scx_task_sched(b); - - /* - * The const qualifiers are dropped from task_struct pointers when - * calling ops.core_sched_before(). Accesses are controlled by the - * verifier. - */ - if (sch_a == sch_b && SCX_HAS_OP(sch_a, core_sched_before) && - !scx_bypassing(sch_a, task_cpu(a))) - return SCX_CALL_OP_2TASKS_RET(sch_a, core_sched_before, - task_rq(a), - (struct task_struct *)a, - (struct task_struct *)b); - else - return time_after64(a->scx.core_sched_at, b->scx.core_sched_at); -} -#endif /* CONFIG_SCHED_CORE */ - -static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) -{ - struct scx_sched *sch = scx_task_sched(p); - bool bypassing; - - /* - * sched_exec() calls with %WF_EXEC when @p is about to exec(2) as it - * can be a good migration opportunity with low cache and memory - * footprint. Returning a CPU different than @prev_cpu triggers - * immediate rq migration. However, for SCX, as the current rq - * association doesn't dictate where the task is going to run, this - * doesn't fit well. If necessary, we can later add a dedicated method - * which can decide to preempt self to force it through the regular - * scheduling path. - */ - if (unlikely(wake_flags & WF_EXEC)) - return prev_cpu; - - bypassing = scx_bypassing(sch, task_cpu(p)); - if (likely(SCX_HAS_OP(sch, select_cpu)) && !bypassing) { - s32 cpu; - struct task_struct **ddsp_taskp; - - ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); - WARN_ON_ONCE(*ddsp_taskp); - *ddsp_taskp = p; - - this_rq()->scx.in_select_cpu = true; - cpu = SCX_CALL_OP_TASK_RET(sch, select_cpu, NULL, p, - scx_cpu_arg(prev_cpu), wake_flags); - cpu = scx_cpu_ret(sch, cpu); - this_rq()->scx.in_select_cpu = false; - p->scx.selected_cpu = cpu; - *ddsp_taskp = NULL; - if (scx_cpu_valid(sch, cpu, "from ops.select_cpu()")) - return cpu; - else - return prev_cpu; - } else { - s32 cpu; - - cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, NULL, 0); - if (cpu >= 0) { - refill_task_slice_dfl(sch, p); - p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL; - } else { - cpu = prev_cpu; - } - p->scx.selected_cpu = cpu; - - if (bypassing) - __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); - return cpu; - } -} - -static void task_woken_scx(struct rq *rq, struct task_struct *p) -{ - run_deferred(rq); -} - -static void set_cpus_allowed_scx(struct task_struct *p, - struct affinity_context *ac) -{ - struct scx_sched *sch = scx_task_sched(p); - - set_cpus_allowed_common(p, ac); - - if (task_dead_and_done(p)) - return; - - /* - * The effective cpumask is stored in @p->cpus_ptr which may temporarily - * differ from the configured one in @p->cpus_mask. Always tell the bpf - * scheduler the effective one. - * - * Fine-grained memory write control is enforced by BPF making the const - * designation pointless. Cast it away when calling the operation. - */ - if (SCX_HAS_OP(sch, set_cpumask)) - scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr); -} - -static void handle_hotplug(struct rq *rq, bool online) -{ - struct scx_sched *sch = scx_root; - s32 cpu = cpu_of(rq); - - atomic_long_inc(&scx_hotplug_seq); - - /* - * scx_root updates are protected by cpus_read_lock() and will stay - * stable here. Note that we can't depend on scx_enabled() test as the - * hotplug ops need to be enabled before __scx_enabled is set. - */ - if (unlikely(!sch)) - return; - - if (scx_enabled()) - scx_idle_update_selcpu_topology(&sch->ops); - - if (online && SCX_HAS_OP(sch, cpu_online)) - SCX_CALL_OP(sch, cpu_online, NULL, scx_cpu_arg(cpu)); - else if (!online && SCX_HAS_OP(sch, cpu_offline)) - SCX_CALL_OP(sch, cpu_offline, NULL, scx_cpu_arg(cpu)); - else - scx_exit(sch, SCX_EXIT_UNREG_KERN, - SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, - "cpu %d going %s, exiting scheduler", cpu, - online ? "online" : "offline"); -} - -void scx_rq_activate(struct rq *rq) -{ - handle_hotplug(rq, true); -} - -void scx_rq_deactivate(struct rq *rq) -{ - handle_hotplug(rq, false); -} - -static void rq_online_scx(struct rq *rq) -{ - rq->scx.flags |= SCX_RQ_ONLINE; -} - -static void rq_offline_scx(struct rq *rq) -{ - rq->scx.flags &= ~SCX_RQ_ONLINE; -} - -static bool check_rq_for_timeouts(struct rq *rq) -{ - struct scx_sched *sch; - struct task_struct *p; - struct rq_flags rf; - bool timed_out = false; - - rq_lock_irqsave(rq, &rf); - sch = rcu_dereference_bh(scx_root); - if (unlikely(!sch)) - goto out_unlock; - - list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) { - struct scx_sched *sch = scx_task_sched(p); - unsigned long last_runnable = p->scx.runnable_at; - - if (unlikely(time_after(jiffies, - last_runnable + READ_ONCE(sch->watchdog_timeout)))) { - u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); - - __scx_exit(sch, SCX_EXIT_ERROR_STALL, 0, cpu_of(rq), - "%s[%d] failed to run for %u.%03us", - p->comm, p->pid, dur_ms / 1000, - dur_ms % 1000); - timed_out = true; - break; - } - } -out_unlock: - rq_unlock_irqrestore(rq, &rf); - return timed_out; -} - -static void scx_watchdog_workfn(struct work_struct *work) -{ - unsigned long intv; - int cpu; - - WRITE_ONCE(scx_watchdog_timestamp, jiffies); - - for_each_online_cpu(cpu) { - if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) - break; - - cond_resched(); - } - - intv = READ_ONCE(scx_watchdog_interval); - if (intv < ULONG_MAX) - queue_delayed_work(system_dfl_wq, to_delayed_work(work), intv); -} - -void scx_tick(struct rq *rq) -{ - struct scx_sched *root; - unsigned long last_check; - - if (!scx_enabled()) - return; - - root = rcu_dereference_bh(scx_root); - if (unlikely(!root)) - return; - - last_check = READ_ONCE(scx_watchdog_timestamp); - if (unlikely(time_after(jiffies, - last_check + READ_ONCE(root->watchdog_timeout)))) { - u32 dur_ms = jiffies_to_msecs(jiffies - last_check); - - scx_exit(root, SCX_EXIT_ERROR_STALL, 0, - "watchdog failed to check in for %u.%03us", - dur_ms / 1000, dur_ms % 1000); - } - - update_other_load_avgs(rq); -} - -static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) -{ - struct scx_sched *sch = scx_task_sched(curr); - - update_curr_scx(rq); - - /* - * While disabling, always resched and refresh core-sched timestamp as - * we can't trust the slice management or ops.core_sched_before(). - */ - if (scx_bypassing(sch, cpu_of(rq))) { - curr->scx.slice = 0; - touch_core_sched(rq, curr); - } else if (SCX_HAS_OP(sch, tick)) { - SCX_CALL_OP_TASK(sch, tick, rq, curr); - } - - if (!curr->scx.slice) - resched_curr(rq); -} - -#ifdef CONFIG_EXT_GROUP_SCHED -static struct cgroup *tg_cgrp(struct task_group *tg) -{ - /* - * If CGROUP_SCHED is disabled, @tg is NULL. If @tg is an autogroup, - * @tg->css.cgroup is NULL. In both cases, @tg can be treated as the - * root cgroup. - */ - if (tg && tg->css.cgroup) - return tg->css.cgroup; - else - return &cgrp_dfl_root.cgrp; -} - -#define SCX_INIT_TASK_ARGS_CGROUP(tg) .cgroup = tg_cgrp(tg), - -#else /* CONFIG_EXT_GROUP_SCHED */ - -#define SCX_INIT_TASK_ARGS_CGROUP(tg) - -#endif /* CONFIG_EXT_GROUP_SCHED */ - -static int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork) -{ - int ret; - - p->scx.disallow = false; - - if (SCX_HAS_OP(sch, init_task)) { - struct scx_init_task_args args = { - SCX_INIT_TASK_ARGS_CGROUP(task_group(p)) - .fork = fork, - }; - - ret = SCX_CALL_OP_RET(sch, init_task, NULL, p, &args); - if (unlikely(ret)) { - ret = ops_sanitize_err(sch, "init_task", ret); - return ret; - } - } - - if (p->scx.disallow) { - if (unlikely(scx_parent(sch))) { - scx_error(sch, "non-root ops.init_task() set task->scx.disallow for %s[%d]", - p->comm, p->pid); - } else if (unlikely(fork)) { - scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] during fork", - p->comm, p->pid); - } else { - struct rq *rq; - struct rq_flags rf; - - rq = task_rq_lock(p, &rf); - - /* - * We're in the load path and @p->policy will be applied - * right after. Reverting @p->policy here and rejecting - * %SCHED_EXT transitions from scx_check_setscheduler() - * guarantees that if ops.init_task() sets @p->disallow, - * @p can never be in SCX. - */ - if (p->policy == SCHED_EXT) { - p->policy = SCHED_NORMAL; - atomic_long_inc(&scx_nr_rejected); - } - - task_rq_unlock(rq, p, &rf); - } - } - - return 0; -} - -static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p) -{ - struct rq *rq = task_rq(p); - u32 weight; - - lockdep_assert_rq_held(rq); - - /* - * Verify the task is not in BPF scheduler's custody. If flag - * transitions are consistent, the flag should always be clear - * here. - */ - WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY); - - /* - * Set the weight before calling ops.enable() so that the scheduler - * doesn't see a stale value if they inspect the task struct. - */ - if (task_has_idle_policy(p)) - weight = WEIGHT_IDLEPRIO; - else - weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; - - p->scx.weight = sched_weight_to_cgroup(weight); - - if (SCX_HAS_OP(sch, enable)) - SCX_CALL_OP_TASK(sch, enable, rq, p); - - if (SCX_HAS_OP(sch, set_weight)) - SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); -} - -static void scx_enable_task(struct scx_sched *sch, struct task_struct *p) -{ - __scx_enable_task(sch, p); - scx_set_task_state(p, SCX_TASK_ENABLED); -} - -static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) -{ - struct rq *rq = task_rq(p); - - lockdep_assert_rq_held(rq); - WARN_ON_ONCE(scx_get_task_state(p) != SCX_TASK_ENABLED); - - clear_direct_dispatch(p); - - if (SCX_HAS_OP(sch, disable)) - SCX_CALL_OP_TASK(sch, disable, rq, p); - scx_set_task_state(p, SCX_TASK_READY); - - /* - * Verify the task is not in BPF scheduler's custody. If flag - * transitions are consistent, the flag should always be clear - * here. - */ - WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY); -} - -static void __scx_disable_and_exit_task(struct scx_sched *sch, - struct task_struct *p) -{ - struct scx_exit_task_args args = { - .cancelled = false, - }; - - lockdep_assert_held(&p->pi_lock); - lockdep_assert_rq_held(task_rq(p)); - - switch (scx_get_task_state(p)) { - case SCX_TASK_NONE: - return; - case SCX_TASK_INIT: - args.cancelled = true; - break; - case SCX_TASK_READY: - break; - case SCX_TASK_ENABLED: - scx_disable_task(sch, p); - break; - default: - WARN_ON_ONCE(true); - return; - } - - if (SCX_HAS_OP(sch, exit_task)) - SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args); -} - -/* - * Undo a completed __scx_init_task(sch, p, false) when scx_enable_task() never - * ran. The task state has not been transitioned, so this mirrors the - * SCX_TASK_INIT branch in __scx_disable_and_exit_task(). - */ -static void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p) -{ - struct scx_exit_task_args args = { .cancelled = true }; - - lockdep_assert_held(&p->pi_lock); - lockdep_assert_rq_held(task_rq(p)); - - if (SCX_HAS_OP(sch, exit_task)) - SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args); -} - -static void scx_disable_and_exit_task(struct scx_sched *sch, - struct task_struct *p) -{ - __scx_disable_and_exit_task(sch, p); - - /* - * If set, @p exited between __scx_init_task() and scx_enable_task() in - * scx_sub_enable() and is initialized for both the associated sched and - * its parent. Exit for the child too - scx_enable_task() never ran for - * it, so undo only init_task. The flag is only set on the sub-enable - * path, so it's always clear when @p arrives here in %SCX_TASK_NONE. - */ - if (p->scx.flags & SCX_TASK_SUB_INIT) { - if (!WARN_ON_ONCE(!scx_enabling_sub_sched)) - scx_sub_init_cancel_task(scx_enabling_sub_sched, p); - p->scx.flags &= ~SCX_TASK_SUB_INIT; - } - - scx_set_task_sched(p, NULL); - scx_set_task_state(p, SCX_TASK_NONE); -} - -void init_scx_entity(struct sched_ext_entity *scx) -{ - memset(scx, 0, sizeof(*scx)); - INIT_LIST_HEAD(&scx->dsq_list.node); - RB_CLEAR_NODE(&scx->dsq_priq); - scx->sticky_cpu = -1; - scx->holding_cpu = -1; - INIT_LIST_HEAD(&scx->runnable_node); - scx->runnable_at = jiffies; - scx->ddsp_dsq_id = SCX_DSQ_INVALID; - scx->slice = SCX_SLICE_DFL; -} - -/* See scx_tid_alloc / scx_tid_cursor. */ -static u64 scx_alloc_tid(void) -{ - struct scx_tid_alloc *ta; - - guard(preempt)(); - ta = this_cpu_ptr(&scx_tid_alloc); - - if (unlikely(ta->next >= ta->end)) { - ta->next = atomic64_fetch_add(SCX_TID_CHUNK, &scx_tid_cursor); - ta->end = ta->next + SCX_TID_CHUNK; - } - return ta->next++; -} - -static void scx_tid_hash_insert(struct task_struct *p) -{ - int ret; - - lockdep_assert_held(&scx_tasks_lock); - - ret = rhashtable_lookup_insert_fast(&scx_tid_hash, - &p->scx.tid_hash_node, - scx_tid_hash_params); - WARN_ON_ONCE(ret); -} - -void scx_pre_fork(struct task_struct *p) -{ - /* - * BPF scheduler enable/disable paths want to be able to iterate and - * update all tasks which can become complex when racing forks. As - * enable/disable are very cold paths, let's use a percpu_rwsem to - * exclude forks. - */ - percpu_down_read(&scx_fork_rwsem); -} - -int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs) -{ - s32 ret; - - percpu_rwsem_assert_held(&scx_fork_rwsem); - - p->scx.tid = scx_alloc_tid(); - - if (scx_init_task_enabled) { -#ifdef CONFIG_EXT_SUB_SCHED - struct scx_sched *sch = kargs->cset->dfl_cgrp->scx_sched; -#else - struct scx_sched *sch = scx_root; -#endif - scx_set_task_state(p, SCX_TASK_INIT_BEGIN); - ret = __scx_init_task(sch, p, true); - if (unlikely(ret)) { - scx_set_task_state(p, SCX_TASK_NONE); - return ret; - } - scx_set_task_state(p, SCX_TASK_INIT); - scx_set_task_sched(p, sch); - } - - return 0; -} - -void scx_post_fork(struct task_struct *p) -{ - if (scx_init_task_enabled) { - scx_set_task_state(p, SCX_TASK_READY); - - /* - * Enable the task immediately if it's running on sched_ext. - * Otherwise, it'll be enabled in switching_to_scx() if and - * when it's ever configured to run with a SCHED_EXT policy. - */ - if (p->sched_class == &ext_sched_class) { - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); - scx_enable_task(scx_task_sched(p), p); - task_rq_unlock(rq, p, &rf); - } - } - - scoped_guard(raw_spinlock_irq, &scx_tasks_lock) { - list_add_tail(&p->scx.tasks_node, &scx_tasks); - if (scx_tid_to_task_enabled()) - scx_tid_hash_insert(p); - } - - percpu_up_read(&scx_fork_rwsem); -} - -void scx_cancel_fork(struct task_struct *p) -{ - if (scx_enabled()) { - struct rq *rq; - struct rq_flags rf; - - rq = task_rq_lock(p, &rf); - WARN_ON_ONCE(scx_get_task_state(p) >= SCX_TASK_READY); - scx_disable_and_exit_task(scx_task_sched(p), p); - task_rq_unlock(rq, p, &rf); - } - - percpu_up_read(&scx_fork_rwsem); -} - -/** - * task_dead_and_done - Is a task dead and done running? - * @p: target task - * - * Once sched_ext_dead() removes the dead task from scx_tasks and exits it, the - * task no longer exists from SCX's POV. However, certain sched_class ops may be - * invoked on these dead tasks leading to failures - e.g. sched_setscheduler() - * may try to switch a task which finished sched_ext_dead() back into SCX - * triggering invalid SCX task state transitions and worse. - * - * Once a task has finished the final switch, sched_ext_dead() is the only thing - * that needs to happen on the task. Use this test to short-circuit sched_class - * operations which may be called on dead tasks. - */ -static bool task_dead_and_done(struct task_struct *p) -{ - struct rq *rq = task_rq(p); - - lockdep_assert_rq_held(rq); - - /* - * In do_task_dead(), a dying task sets %TASK_DEAD with preemption - * disabled and __schedule(). If @p has %TASK_DEAD set and off CPU, @p - * won't ever run again. - */ - return unlikely(READ_ONCE(p->__state) == TASK_DEAD) && - !task_on_cpu(rq, p); -} - -void sched_ext_dead(struct task_struct *p) -{ - /* - * By the time control reaches here, @p has %TASK_DEAD set, switched out - * for the last time and then dropped the rq lock - task_dead_and_done() - * should be returning %true nullifying the straggling sched_class ops. - * Remove from scx_tasks and exit @p. - */ - scoped_guard(raw_spinlock_irqsave, &scx_tasks_lock) { - list_del_init(&p->scx.tasks_node); - if (scx_tid_to_task_enabled()) - rhashtable_remove_fast(&scx_tid_hash, - &p->scx.tid_hash_node, - scx_tid_hash_params); - } - - /* - * @p is off scx_tasks and wholly ours. scx_root_enable()'s READY -> - * ENABLED transitions can't race us. Disable ops for @p. - * - * %SCX_TASK_DEAD synchronizes against cgroup task iteration - see - * scx_task_iter_next_locked(). NONE tasks need no marking: cgroup - * iteration is only used from sub-sched paths, which require root - * enabled. Root enable transitions every live task to at least READY. - * - * %INIT_BEGIN means ops.init_task() is running for @p. Don't call - * into ops; transition to %DEAD so the post-init recheck unwinds - * via scx_sub_init_cancel_task(). - */ - if (scx_get_task_state(p) != SCX_TASK_NONE) { - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); - if (scx_get_task_state(p) != SCX_TASK_INIT_BEGIN) - scx_disable_and_exit_task(scx_task_sched(p), p); - scx_set_task_state(p, SCX_TASK_DEAD); - task_rq_unlock(rq, p, &rf); - } -} - -static void reweight_task_scx(struct rq *rq, struct task_struct *p, - const struct load_weight *lw) -{ - struct scx_sched *sch = scx_task_sched(p); - - lockdep_assert_rq_held(task_rq(p)); - - if (task_dead_and_done(p)) - return; - - p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight)); - if (SCX_HAS_OP(sch, set_weight)) - SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); -} - -static void prio_changed_scx(struct rq *rq, struct task_struct *p, u64 oldprio) -{ -} - -static void switching_to_scx(struct rq *rq, struct task_struct *p) -{ - struct scx_sched *sch = scx_task_sched(p); - - if (task_dead_and_done(p)) - return; - - scx_enable_task(sch, p); - - /* - * set_cpus_allowed_scx() is not called while @p is associated with a - * different scheduler class. Keep the BPF scheduler up-to-date. - */ - if (SCX_HAS_OP(sch, set_cpumask)) - scx_call_op_set_cpumask(sch, rq, p, (struct cpumask *)p->cpus_ptr); -} - -static void switched_from_scx(struct rq *rq, struct task_struct *p) -{ - if (task_dead_and_done(p)) - return; - - /* - * %NONE means SCX is no longer tracking @p at the task level (e.g. - * scx_fail_parent() handed @p back to the parent at NONE pending the - * parent's own teardown). There is nothing to disable; calling - * scx_disable_task() would WARN on the non-%ENABLED state and trigger a - * NONE -> READY validation failure. - */ - if (scx_get_task_state(p) == SCX_TASK_NONE) - return; - - scx_disable_task(scx_task_sched(p), p); -} - -static void switched_to_scx(struct rq *rq, struct task_struct *p) {} - -int scx_check_setscheduler(struct task_struct *p, int policy) -{ - lockdep_assert_rq_held(task_rq(p)); - - /* if disallow, reject transitioning into SCX */ - if (scx_enabled() && READ_ONCE(p->scx.disallow) && - p->policy != policy && policy == SCHED_EXT) - return -EACCES; - - return 0; -} - -static void process_ddsp_deferred_locals(struct rq *rq) -{ - struct task_struct *p; - - lockdep_assert_rq_held(rq); - - /* - * Now that @rq can be unlocked, execute the deferred enqueueing of - * tasks directly dispatched to the local DSQs of other CPUs. See - * direct_dispatch(). Keep popping from the head instead of using - * list_for_each_entry_safe() as dispatch_local_dsq() may unlock @rq - * temporarily. - */ - while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals, - struct task_struct, scx.dsq_list.node))) { - struct scx_sched *sch = scx_task_sched(p); - struct scx_dispatch_q *dsq; - u64 dsq_id = p->scx.ddsp_dsq_id; - u64 enq_flags = p->scx.ddsp_enq_flags; - - list_del_init(&p->scx.dsq_list.node); - clear_direct_dispatch(p); - - dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p)); - if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) - dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); - } -} - -/* - * Determine whether @p should be reenqueued from a local DSQ. - * - * @reenq_flags is mutable and accumulates state across the DSQ walk: - * - * - %SCX_REENQ_TSR_NOT_FIRST: Set after the first task is visited. "First" - * tracks position in the DSQ list, not among IMMED tasks. A non-IMMED task at - * the head consumes the first slot. - * - * - %SCX_REENQ_TSR_RQ_OPEN: Set by reenq_local() before the walk if - * rq_is_open() is true. - * - * An IMMED task is kept (returns %false) only if it's the first task in the DSQ - * AND the current task is done — i.e. it will execute immediately. All other - * IMMED tasks are reenqueued. This means if a non-IMMED task sits at the head, - * every IMMED task behind it gets reenqueued. - * - * Reenqueued tasks go through ops.enqueue() with %SCX_ENQ_REENQ | - * %SCX_TASK_REENQ_IMMED. If the BPF scheduler dispatches back to the same local - * DSQ with %SCX_ENQ_IMMED while the CPU is still unavailable, this triggers - * another reenq cycle. Repetitions are bounded by %SCX_REENQ_LOCAL_MAX_REPEAT - * in process_deferred_reenq_locals(). - */ -static bool local_task_should_reenq(struct task_struct *p, u64 *reenq_flags, u32 *reason) -{ - bool first; - - first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST); - *reenq_flags |= SCX_REENQ_TSR_NOT_FIRST; - - *reason = SCX_TASK_REENQ_KFUNC; - - if ((p->scx.flags & SCX_TASK_IMMED) && - (!first || !(*reenq_flags & SCX_REENQ_TSR_RQ_OPEN))) { - __scx_add_event(scx_task_sched(p), SCX_EV_REENQ_IMMED, 1); - *reason = SCX_TASK_REENQ_IMMED; - return true; - } - - return *reenq_flags & SCX_REENQ_ANY; -} - -static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) -{ - LIST_HEAD(tasks); - u32 nr_enqueued = 0; - struct task_struct *p, *n; - - lockdep_assert_rq_held(rq); - - if (WARN_ON_ONCE(reenq_flags & __SCX_REENQ_TSR_MASK)) - reenq_flags &= ~__SCX_REENQ_TSR_MASK; - if (rq_is_open(rq, 0)) - reenq_flags |= SCX_REENQ_TSR_RQ_OPEN; - - /* - * The BPF scheduler may choose to dispatch tasks back to - * @rq->scx.local_dsq. Move all candidate tasks off to a private list - * first to avoid processing the same tasks repeatedly. - */ - list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list, - scx.dsq_list.node) { - struct scx_sched *task_sch = scx_task_sched(p); - u32 reason; - - /* - * If @p is being migrated, @p's current CPU may not agree with - * its allowed CPUs and the migration_cpu_stop is about to - * deactivate and re-activate @p anyway. Skip re-enqueueing. - * - * While racing sched property changes may also dequeue and - * re-enqueue a migrating task while its current CPU and allowed - * CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to - * the current local DSQ for running tasks and thus are not - * visible to the BPF scheduler. - */ - if (p->migration_pending) - continue; - - if (!scx_is_descendant(task_sch, sch)) - continue; - - if (!local_task_should_reenq(p, &reenq_flags, &reason)) - continue; - - dispatch_dequeue(rq, p); - - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - p->scx.flags |= reason; - - list_add_tail(&p->scx.dsq_list.node, &tasks); - } - - list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { - list_del_init(&p->scx.dsq_list.node); - - do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - nr_enqueued++; - } - - return nr_enqueued; -} - -static void process_deferred_reenq_locals(struct rq *rq) -{ - u64 seq = ++rq->scx.deferred_reenq_locals_seq; - - lockdep_assert_rq_held(rq); - - while (true) { - struct scx_sched *sch; - u64 reenq_flags; - bool skip = false; - - scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { - struct scx_deferred_reenq_local *drl = - list_first_entry_or_null(&rq->scx.deferred_reenq_locals, - struct scx_deferred_reenq_local, - node); - struct scx_sched_pcpu *sch_pcpu; - - if (!drl) - return; - - sch_pcpu = container_of(drl, struct scx_sched_pcpu, - deferred_reenq_local); - sch = sch_pcpu->sch; - - reenq_flags = drl->flags; - WRITE_ONCE(drl->flags, 0); - list_del_init(&drl->node); - - if (likely(drl->seq != seq)) { - drl->seq = seq; - drl->cnt = 0; - } else { - if (unlikely(++drl->cnt > SCX_REENQ_LOCAL_MAX_REPEAT)) { - scx_error(sch, "SCX_ENQ_REENQ on SCX_DSQ_LOCAL repeated %u times", - drl->cnt); - skip = true; - } - - __scx_add_event(sch, SCX_EV_REENQ_LOCAL_REPEAT, 1); - } - } - - if (!skip) { - /* see schedule_dsq_reenq() */ - smp_mb(); - - reenq_local(sch, rq, reenq_flags); - } - } -} - -static bool user_task_should_reenq(struct task_struct *p, u64 reenq_flags, u32 *reason) -{ - *reason = SCX_TASK_REENQ_KFUNC; - return reenq_flags & SCX_REENQ_ANY; -} - -static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flags) -{ - struct rq *locked_rq = rq; - struct scx_sched *sch = dsq->sched; - struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, dsq, 0); - struct task_struct *p; - s32 nr_enqueued = 0; - - lockdep_assert_rq_held(rq); - - raw_spin_lock(&dsq->lock); - - while (likely(!READ_ONCE(sch->bypass_depth))) { - struct rq *task_rq; - u32 reason; - - p = nldsq_cursor_next_task(&cursor, dsq); - if (!p) - break; - - if (!user_task_should_reenq(p, reenq_flags, &reason)) - continue; - - task_rq = task_rq(p); - - if (locked_rq != task_rq) { - if (locked_rq) - raw_spin_rq_unlock(locked_rq); - if (unlikely(!raw_spin_rq_trylock(task_rq))) { - raw_spin_unlock(&dsq->lock); - raw_spin_rq_lock(task_rq); - raw_spin_lock(&dsq->lock); - } - locked_rq = task_rq; - - /* did we lose @p while switching locks? */ - if (nldsq_cursor_lost_task(&cursor, task_rq, dsq, p)) - continue; - } - - /* @p is on @dsq, its rq and @dsq are locked */ - dispatch_dequeue_locked(p, dsq); - raw_spin_unlock(&dsq->lock); - - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - p->scx.flags |= reason; - - do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1); - - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - - if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) { - raw_spin_rq_unlock(locked_rq); - locked_rq = NULL; - cpu_relax(); - } - - raw_spin_lock(&dsq->lock); - } - - list_del_init(&cursor.node); - raw_spin_unlock(&dsq->lock); - - if (locked_rq != rq) { - if (locked_rq) - raw_spin_rq_unlock(locked_rq); - raw_spin_rq_lock(rq); - } -} - -static void process_deferred_reenq_users(struct rq *rq) -{ - lockdep_assert_rq_held(rq); - - while (true) { - struct scx_dispatch_q *dsq; - u64 reenq_flags; - - scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { - struct scx_deferred_reenq_user *dru = - list_first_entry_or_null(&rq->scx.deferred_reenq_users, - struct scx_deferred_reenq_user, - node); - struct scx_dsq_pcpu *dsq_pcpu; - - if (!dru) - return; - - dsq_pcpu = container_of(dru, struct scx_dsq_pcpu, - deferred_reenq_user); - dsq = dsq_pcpu->dsq; - reenq_flags = dru->flags; - WRITE_ONCE(dru->flags, 0); - list_del_init(&dru->node); - } - - /* see schedule_dsq_reenq() */ - smp_mb(); - - BUG_ON(dsq->id & SCX_DSQ_FLAG_BUILTIN); - reenq_user(rq, dsq, reenq_flags); - } -} - -static void run_deferred(struct rq *rq) -{ - process_ddsp_deferred_locals(rq); - - if (!list_empty(&rq->scx.deferred_reenq_locals)) - process_deferred_reenq_locals(rq); - - if (!list_empty(&rq->scx.deferred_reenq_users)) - process_deferred_reenq_users(rq); -} - -#ifdef CONFIG_NO_HZ_FULL -bool scx_can_stop_tick(struct rq *rq) -{ - struct task_struct *p = rq->curr; - struct scx_sched *sch = scx_task_sched(p); - - if (p->sched_class != &ext_sched_class) - return true; - - if (scx_bypassing(sch, cpu_of(rq))) - return false; - - /* - * @rq can dispatch from different DSQs, so we can't tell whether it - * needs the tick or not by looking at nr_running. Allow stopping ticks - * iff the BPF scheduler indicated so. See set_next_task_scx(). - */ - return rq->scx.flags & SCX_RQ_CAN_STOP_TICK; -} -#endif - -#ifdef CONFIG_EXT_GROUP_SCHED - -DEFINE_STATIC_PERCPU_RWSEM(scx_cgroup_ops_rwsem); -static bool scx_cgroup_enabled; - -void scx_tg_init(struct task_group *tg) -{ - tg->scx.weight = CGROUP_WEIGHT_DFL; - tg->scx.bw_period_us = default_bw_period_us(); - tg->scx.bw_quota_us = RUNTIME_INF; - tg->scx.idle = false; -} - -int scx_tg_online(struct task_group *tg) -{ - struct scx_sched *sch = scx_root; - int ret = 0; - - WARN_ON_ONCE(tg->scx.flags & (SCX_TG_ONLINE | SCX_TG_INITED)); - - if (scx_cgroup_enabled) { - if (SCX_HAS_OP(sch, cgroup_init)) { - struct scx_cgroup_init_args args = - { .weight = tg->scx.weight, - .bw_period_us = tg->scx.bw_period_us, - .bw_quota_us = tg->scx.bw_quota_us, - .bw_burst_us = tg->scx.bw_burst_us }; - - ret = SCX_CALL_OP_RET(sch, cgroup_init, - NULL, tg->css.cgroup, &args); - if (ret) - ret = ops_sanitize_err(sch, "cgroup_init", ret); - } - if (ret == 0) - tg->scx.flags |= SCX_TG_ONLINE | SCX_TG_INITED; - } else { - tg->scx.flags |= SCX_TG_ONLINE; - } - - return ret; -} - -void scx_tg_offline(struct task_group *tg) -{ - struct scx_sched *sch = scx_root; - - WARN_ON_ONCE(!(tg->scx.flags & SCX_TG_ONLINE)); - - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_exit) && - (tg->scx.flags & SCX_TG_INITED)) - SCX_CALL_OP(sch, cgroup_exit, NULL, tg->css.cgroup); - tg->scx.flags &= ~(SCX_TG_ONLINE | SCX_TG_INITED); -} - -int scx_cgroup_can_attach(struct cgroup_taskset *tset) -{ - struct scx_sched *sch = scx_root; - struct cgroup_subsys_state *css; - struct task_struct *p; - int ret; - - if (!scx_cgroup_enabled) - return 0; - - cgroup_taskset_for_each(p, css, tset) { - struct cgroup *from = tg_cgrp(task_group(p)); - struct cgroup *to = tg_cgrp(css_tg(css)); - - WARN_ON_ONCE(p->scx.cgrp_moving_from); - - /* - * sched_move_task() omits identity migrations. Let's match the - * behavior so that ops.cgroup_prep_move() and ops.cgroup_move() - * always match one-to-one. - */ - if (from == to) - continue; - - if (SCX_HAS_OP(sch, cgroup_prep_move)) { - ret = SCX_CALL_OP_RET(sch, cgroup_prep_move, NULL, - p, from, css->cgroup); - if (ret) - goto err; - } - - p->scx.cgrp_moving_from = from; - } - - return 0; - -err: - cgroup_taskset_for_each(p, css, tset) { - if (SCX_HAS_OP(sch, cgroup_cancel_move) && - p->scx.cgrp_moving_from) - SCX_CALL_OP(sch, cgroup_cancel_move, NULL, - p, p->scx.cgrp_moving_from, css->cgroup); - p->scx.cgrp_moving_from = NULL; - } - - return ops_sanitize_err(sch, "cgroup_prep_move", ret); -} - -void scx_cgroup_move_task(struct task_struct *p) -{ - struct scx_sched *sch = scx_root; - - if (!scx_cgroup_enabled) - return; - - /* - * scx_cgroup_can_attach() sets cgrp_moving_from only when the task's - * cgroup changes. Migration keys off css rather than cgroup identity, - * so it can hand an unchanged-cgroup task here with cgrp_moving_from - * NULL. Nothing to report to the BPF scheduler then, so skip it and - * keep prep_move and move paired. - */ - if (SCX_HAS_OP(sch, cgroup_move) && p->scx.cgrp_moving_from) - SCX_CALL_OP_TASK(sch, cgroup_move, task_rq(p), - p, p->scx.cgrp_moving_from, - tg_cgrp(task_group(p))); - p->scx.cgrp_moving_from = NULL; -} - -void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) -{ - struct scx_sched *sch = scx_root; - struct cgroup_subsys_state *css; - struct task_struct *p; - - if (!scx_cgroup_enabled) - return; - - cgroup_taskset_for_each(p, css, tset) { - if (SCX_HAS_OP(sch, cgroup_cancel_move) && - p->scx.cgrp_moving_from) - SCX_CALL_OP(sch, cgroup_cancel_move, NULL, - p, p->scx.cgrp_moving_from, css->cgroup); - p->scx.cgrp_moving_from = NULL; - } -} - -void scx_group_set_weight(struct task_group *tg, unsigned long weight) -{ - struct scx_sched *sch; - - percpu_down_read(&scx_cgroup_ops_rwsem); - sch = scx_root; - - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_weight) && - tg->scx.weight != weight) - SCX_CALL_OP(sch, cgroup_set_weight, NULL, tg_cgrp(tg), weight); - - tg->scx.weight = weight; - - percpu_up_read(&scx_cgroup_ops_rwsem); -} - -void scx_group_set_idle(struct task_group *tg, bool idle) -{ - struct scx_sched *sch; - - percpu_down_read(&scx_cgroup_ops_rwsem); - sch = scx_root; - - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_idle)) - SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle); - - /* Update the task group's idle state */ - tg->scx.idle = idle; - - percpu_up_read(&scx_cgroup_ops_rwsem); -} - -void scx_group_set_bandwidth(struct task_group *tg, - u64 period_us, u64 quota_us, u64 burst_us) -{ - struct scx_sched *sch; - - percpu_down_read(&scx_cgroup_ops_rwsem); - sch = scx_root; - - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_bandwidth) && - (tg->scx.bw_period_us != period_us || - tg->scx.bw_quota_us != quota_us || - tg->scx.bw_burst_us != burst_us)) - SCX_CALL_OP(sch, cgroup_set_bandwidth, NULL, - tg_cgrp(tg), period_us, quota_us, burst_us); - - tg->scx.bw_period_us = period_us; - tg->scx.bw_quota_us = quota_us; - tg->scx.bw_burst_us = burst_us; - - percpu_up_read(&scx_cgroup_ops_rwsem); -} -#endif /* CONFIG_EXT_GROUP_SCHED */ - -#if defined(CONFIG_EXT_GROUP_SCHED) || defined(CONFIG_EXT_SUB_SCHED) -static struct cgroup *root_cgroup(void) -{ - return &cgrp_dfl_root.cgrp; -} - -static void scx_cgroup_lock(void) -{ -#ifdef CONFIG_EXT_GROUP_SCHED - percpu_down_write(&scx_cgroup_ops_rwsem); -#endif - cgroup_lock(); -} - -static void scx_cgroup_unlock(void) -{ - cgroup_unlock(); -#ifdef CONFIG_EXT_GROUP_SCHED - percpu_up_write(&scx_cgroup_ops_rwsem); -#endif -} -#else /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */ -static inline struct cgroup *root_cgroup(void) { return NULL; } -static inline void scx_cgroup_lock(void) {} -static inline void scx_cgroup_unlock(void) {} -#endif /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */ - -#ifdef CONFIG_EXT_SUB_SCHED -static struct cgroup *sch_cgroup(struct scx_sched *sch) -{ - return sch->cgrp; -} - -/* for each descendant of @cgrp including self, set ->scx_sched to @sch */ -static void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) -{ - struct cgroup *pos; - struct cgroup_subsys_state *css; - - cgroup_for_each_live_descendant_pre(pos, css, cgrp) - rcu_assign_pointer(pos->scx_sched, sch); -} -#else /* CONFIG_EXT_SUB_SCHED */ -static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; } -static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {} -#endif /* CONFIG_EXT_SUB_SCHED */ - -/* - * Omitted operations: - * - * - migrate_task_rq: Unnecessary as task to cpu mapping is transient. - * - * - task_fork/dead: We need fork/dead notifications for all tasks regardless of - * their current sched_class. Call them directly from sched core instead. - */ -DEFINE_SCHED_CLASS(ext) = { - .enqueue_task = enqueue_task_scx, - .dequeue_task = dequeue_task_scx, - .yield_task = yield_task_scx, - .yield_to_task = yield_to_task_scx, - - .wakeup_preempt = wakeup_preempt_scx, - - .pick_task = pick_task_scx, - - .put_prev_task = put_prev_task_scx, - .set_next_task = set_next_task_scx, - - .select_task_rq = select_task_rq_scx, - .task_woken = task_woken_scx, - .set_cpus_allowed = set_cpus_allowed_scx, - - .rq_online = rq_online_scx, - .rq_offline = rq_offline_scx, - - .task_tick = task_tick_scx, - - .switching_to = switching_to_scx, - .switched_from = switched_from_scx, - .switched_to = switched_to_scx, - .reweight_task = reweight_task_scx, - .prio_changed = prio_changed_scx, - - .update_curr = update_curr_scx, - -#ifdef CONFIG_UCLAMP_TASK - .uclamp_enabled = 1, -#endif -}; - -static s32 init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, - struct scx_sched *sch) -{ - s32 cpu; - - memset(dsq, 0, sizeof(*dsq)); - - raw_spin_lock_init(&dsq->lock); - INIT_LIST_HEAD(&dsq->list); - dsq->id = dsq_id; - dsq->sched = sch; - - dsq->pcpu = alloc_percpu(struct scx_dsq_pcpu); - if (!dsq->pcpu) - return -ENOMEM; - - for_each_possible_cpu(cpu) { - struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); - - pcpu->dsq = dsq; - INIT_LIST_HEAD(&pcpu->deferred_reenq_user.node); - } - - return 0; -} - -static void exit_dsq(struct scx_dispatch_q *dsq) -{ - s32 cpu; - - for_each_possible_cpu(cpu) { - struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); - struct scx_deferred_reenq_user *dru = &pcpu->deferred_reenq_user; - struct rq *rq = cpu_rq(cpu); - - /* - * There must have been a RCU grace period since the last - * insertion and @dsq should be off the deferred list by now. - */ - if (WARN_ON_ONCE(!list_empty(&dru->node))) { - guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); - list_del_init(&dru->node); - } - } - - free_percpu(dsq->pcpu); -} - -static void free_dsq_rcufn(struct rcu_head *rcu) -{ - struct scx_dispatch_q *dsq = container_of(rcu, struct scx_dispatch_q, rcu); - - exit_dsq(dsq); - kfree(dsq); -} - -static void free_dsq_irq_workfn(struct irq_work *irq_work) -{ - struct llist_node *to_free = llist_del_all(&dsqs_to_free); - struct scx_dispatch_q *dsq, *tmp_dsq; - - llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) - call_rcu(&dsq->rcu, free_dsq_rcufn); -} - -static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); - -static void destroy_dsq(struct scx_sched *sch, u64 dsq_id) -{ - struct scx_dispatch_q *dsq; - unsigned long flags; - - rcu_read_lock(); - - dsq = find_user_dsq(sch, dsq_id); - if (!dsq) - goto out_unlock_rcu; - - raw_spin_lock_irqsave(&dsq->lock, flags); - - if (dsq->nr) { - scx_error(sch, "attempting to destroy in-use dsq 0x%016llx (nr=%u)", - dsq->id, dsq->nr); - goto out_unlock_dsq; - } - - if (rhashtable_remove_fast(&sch->dsq_hash, &dsq->hash_node, - dsq_hash_params)) - goto out_unlock_dsq; - - /* - * Mark dead by invalidating ->id to prevent dispatch_enqueue() from - * queueing more tasks. As this function can be called from anywhere, - * freeing is bounced through an irq work to avoid nesting RCU - * operations inside scheduler locks. - */ - dsq->id = SCX_DSQ_INVALID; - if (llist_add(&dsq->free_node, &dsqs_to_free)) - irq_work_queue(&free_dsq_irq_work); - -out_unlock_dsq: - raw_spin_unlock_irqrestore(&dsq->lock, flags); -out_unlock_rcu: - rcu_read_unlock(); -} - -#ifdef CONFIG_EXT_GROUP_SCHED -static void scx_cgroup_exit(struct scx_sched *sch) -{ - struct cgroup_subsys_state *css; - - scx_cgroup_enabled = false; - - /* - * scx_tg_on/offline() are excluded through cgroup_lock(). If we walk - * cgroups and exit all the inited ones, all online cgroups are exited. - */ - css_for_each_descendant_post(css, &root_task_group.css) { - struct task_group *tg = css_tg(css); - - if (!(tg->scx.flags & SCX_TG_INITED)) - continue; - tg->scx.flags &= ~SCX_TG_INITED; - - if (!sch->ops.cgroup_exit) - continue; - - SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); - } -} - -static int scx_cgroup_init(struct scx_sched *sch) -{ - struct cgroup_subsys_state *css; - int ret; - - /* - * scx_tg_on/offline() are excluded through cgroup_lock(). If we walk - * cgroups and init, all online cgroups are initialized. - */ - css_for_each_descendant_pre(css, &root_task_group.css) { - struct task_group *tg = css_tg(css); - struct scx_cgroup_init_args args = { - .weight = tg->scx.weight, - .bw_period_us = tg->scx.bw_period_us, - .bw_quota_us = tg->scx.bw_quota_us, - .bw_burst_us = tg->scx.bw_burst_us, - }; - - if ((tg->scx.flags & - (SCX_TG_ONLINE | SCX_TG_INITED)) != SCX_TG_ONLINE) - continue; - - if (!sch->ops.cgroup_init) { - tg->scx.flags |= SCX_TG_INITED; - continue; - } - - ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, - css->cgroup, &args); - if (ret) { - scx_error(sch, "ops.cgroup_init() failed (%d)", ret); - return ret; - } - tg->scx.flags |= SCX_TG_INITED; - } - - WARN_ON_ONCE(scx_cgroup_enabled); - scx_cgroup_enabled = true; - - return 0; -} - -#else -static void scx_cgroup_exit(struct scx_sched *sch) {} -static int scx_cgroup_init(struct scx_sched *sch) { return 0; } -#endif - - -/******************************************************************************** - * Sysfs interface and ops enable/disable. - */ - -#define SCX_ATTR(_name) \ - static struct kobj_attribute scx_attr_##_name = { \ - .attr = { .name = __stringify(_name), .mode = 0444 }, \ - .show = scx_attr_##_name##_show, \ - } - -static ssize_t scx_attr_state_show(struct kobject *kobj, - struct kobj_attribute *ka, char *buf) -{ - return sysfs_emit(buf, "%s\n", scx_enable_state_str[scx_enable_state()]); -} -SCX_ATTR(state); - -static ssize_t scx_attr_switch_all_show(struct kobject *kobj, - struct kobj_attribute *ka, char *buf) -{ - return sysfs_emit(buf, "%d\n", READ_ONCE(scx_switching_all)); -} -SCX_ATTR(switch_all); - -static ssize_t scx_attr_nr_rejected_show(struct kobject *kobj, - struct kobj_attribute *ka, char *buf) -{ - return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_nr_rejected)); -} -SCX_ATTR(nr_rejected); - -static ssize_t scx_attr_hotplug_seq_show(struct kobject *kobj, - struct kobj_attribute *ka, char *buf) -{ - return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_hotplug_seq)); -} -SCX_ATTR(hotplug_seq); - -static ssize_t scx_attr_enable_seq_show(struct kobject *kobj, - struct kobj_attribute *ka, char *buf) -{ - return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_enable_seq)); -} -SCX_ATTR(enable_seq); - -static struct attribute *scx_global_attrs[] = { - &scx_attr_state.attr, - &scx_attr_switch_all.attr, - &scx_attr_nr_rejected.attr, - &scx_attr_hotplug_seq.attr, - &scx_attr_enable_seq.attr, - NULL, -}; - -static const struct attribute_group scx_global_attr_group = { - .attrs = scx_global_attrs, -}; - -static void free_pnode(struct scx_sched_pnode *pnode); -static void free_exit_info(struct scx_exit_info *ei); - -static s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) -{ - size_t size = struct_size_t(struct scx_cmask, bits, - SCX_CMASK_NR_WORDS(num_possible_cpus())); - int cpu; - - if (!sch->is_cid_type || !sch->arena_pool) - return 0; - - sch->set_cmask_scratch = alloc_percpu(struct scx_cmask *); - if (!sch->set_cmask_scratch) - return -ENOMEM; - - for_each_possible_cpu(cpu) { - struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu); - - *slot = scx_arena_alloc(sch, size); - if (!*slot) - return -ENOMEM; - scx_cmask_init(*slot, 0, num_possible_cpus()); - } - return 0; -} - -static void scx_set_cmask_scratch_free(struct scx_sched *sch) -{ - size_t size = struct_size_t(struct scx_cmask, bits, - SCX_CMASK_NR_WORDS(num_possible_cpus())); - int cpu; - - if (!sch->set_cmask_scratch) - return; - - for_each_possible_cpu(cpu) { - struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu); - - scx_arena_free(sch, *slot, size); - } - free_percpu(sch->set_cmask_scratch); - sch->set_cmask_scratch = NULL; -} - -static void scx_sched_free_rcu_work(struct work_struct *work) -{ - struct rcu_work *rcu_work = to_rcu_work(work); - struct scx_sched *sch = container_of(rcu_work, struct scx_sched, rcu_work); - struct rhashtable_iter rht_iter; - struct scx_dispatch_q *dsq; - int cpu, node; - - irq_work_sync(&sch->disable_irq_work); - kthread_destroy_worker(sch->helper); - timer_shutdown_sync(&sch->bypass_lb_timer); - free_cpumask_var(sch->bypass_lb_donee_cpumask); - free_cpumask_var(sch->bypass_lb_resched_cpumask); - -#ifdef CONFIG_EXT_SUB_SCHED - kfree(sch->cgrp_path); - if (sch_cgroup(sch)) - cgroup_put(sch_cgroup(sch)); - if (sch->sub_kset) - kobject_put(&sch->sub_kset->kobj); -#endif /* CONFIG_EXT_SUB_SCHED */ - - for_each_possible_cpu(cpu) { - struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); - - /* - * $sch would have entered bypass mode before the RCU grace - * period. As that blocks new deferrals, all - * deferred_reenq_local_node's must be off-list by now. - */ - WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node)); - - exit_dsq(bypass_dsq(sch, cpu)); - } - - free_percpu(sch->pcpu); - - for_each_node_state(node, N_POSSIBLE) - free_pnode(sch->pnode[node]); - kfree(sch->pnode); - - rhashtable_walk_enter(&sch->dsq_hash, &rht_iter); - do { - rhashtable_walk_start(&rht_iter); - - while (!IS_ERR_OR_NULL((dsq = rhashtable_walk_next(&rht_iter)))) - destroy_dsq(sch, dsq->id); - - rhashtable_walk_stop(&rht_iter); - } while (dsq == ERR_PTR(-EAGAIN)); - rhashtable_walk_exit(&rht_iter); - - rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); - free_exit_info(sch->exit_info); - scx_set_cmask_scratch_free(sch); - scx_arena_pool_destroy(sch); - if (sch->arena_map) - bpf_map_put(sch->arena_map); - kfree(sch); -} - -static void scx_kobj_release(struct kobject *kobj) -{ - struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); - - INIT_RCU_WORK(&sch->rcu_work, scx_sched_free_rcu_work); - queue_rcu_work(system_dfl_wq, &sch->rcu_work); -} - -static ssize_t scx_attr_ops_show(struct kobject *kobj, - struct kobj_attribute *ka, char *buf) -{ - struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); - - return sysfs_emit(buf, "%s\n", sch->ops.name); -} -SCX_ATTR(ops); - -#define scx_attr_event_show(buf, at, events, kind) ({ \ - sysfs_emit_at(buf, at, "%s %llu\n", #kind, (events)->kind); \ -}) - -static ssize_t scx_attr_events_show(struct kobject *kobj, - struct kobj_attribute *ka, char *buf) -{ - struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); - struct scx_event_stats events; - int at = 0; - - scx_read_events(sch, &events); - at += scx_attr_event_show(buf, at, &events, SCX_EV_SELECT_CPU_FALLBACK); - at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); - at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_KEEP_LAST); - at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_EXITING); - at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); - at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_IMMED); - at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_LOCAL_REPEAT); - at += scx_attr_event_show(buf, at, &events, SCX_EV_REFILL_SLICE_DFL); - at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DURATION); - at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DISPATCH); - at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_ACTIVATE); - at += scx_attr_event_show(buf, at, &events, SCX_EV_INSERT_NOT_OWNED); - at += scx_attr_event_show(buf, at, &events, SCX_EV_SUB_BYPASS_DISPATCH); - return at; -} -SCX_ATTR(events); - -static struct attribute *scx_sched_attrs[] = { - &scx_attr_ops.attr, - &scx_attr_events.attr, - NULL, -}; -ATTRIBUTE_GROUPS(scx_sched); - -static const struct kobj_type scx_ktype = { - .release = scx_kobj_release, - .sysfs_ops = &kobj_sysfs_ops, - .default_groups = scx_sched_groups, -}; - -static int scx_uevent(const struct kobject *kobj, struct kobj_uevent_env *env) -{ - const struct scx_sched *sch; - - /* - * scx_uevent() can be reached by both scx_sched kobjects (scx_ktype) - * and sub-scheduler kset kobjects (kset_ktype) through the parent - * chain walk. Filter out the latter to avoid invalid casts. - */ - if (kobj->ktype != &scx_ktype) - return 0; - - sch = container_of(kobj, struct scx_sched, kobj); - - return add_uevent_var(env, "SCXOPS=%s", sch->ops.name); -} - -static const struct kset_uevent_ops scx_uevent_ops = { - .uevent = scx_uevent, -}; - -/* - * Used by sched_fork() and __setscheduler_prio() to pick the matching - * sched_class. dl/rt are already handled. - */ -bool task_should_scx(int policy) -{ - /* if disabled, nothing should be on it */ - if (!scx_enabled()) - return false; - - /* scx is taking over all SCHED_OTHER and SCHED_EXT tasks */ - if (READ_ONCE(scx_switching_all)) - return true; - - /* - * scx is tearing down - keep new SCHED_EXT tasks out. - * - * Must come after scx_switching_all test, which serves as a proxy - * for __scx_switched_all. While __scx_switched_all is set, we must - * return true via the branch above: a fork routed to fair would - * stall because next_active_class() skips fair. - * - * This can develop into a deadlock - scx holds scx_enable_mutex across - * kthread_create() in scx_alloc_and_add_sched(); if the new kthread is - * the stalled task, the disable path can never grab the mutex to clear - * scx_switching_all. - */ - if (unlikely(scx_enable_state() == SCX_DISABLING)) - return false; - - return policy == SCHED_EXT; -} - -bool scx_allow_ttwu_queue(const struct task_struct *p) -{ - struct scx_sched *sch; - - if (!scx_enabled()) - return true; - - sch = scx_task_sched(p); - if (unlikely(!sch)) - return true; - - if (sch->ops.flags & SCX_OPS_ALLOW_QUEUED_WAKEUP) - return true; - - if (unlikely(p->sched_class != &ext_sched_class)) - return true; - - return false; -} - -/** - * handle_lockup - sched_ext common lockup handler - * @fmt: format string - * - * Called on system stall or lockup condition and initiates abort of sched_ext - * if enabled, which may resolve the reported lockup. - * - * Returns %true if sched_ext is enabled and abort was initiated, which may - * resolve the lockup. %false if sched_ext is not enabled or abort was already - * initiated by someone else. - */ -static __printf(1, 2) bool handle_lockup(const char *fmt, ...) -{ - struct scx_sched *sch; - va_list args; - bool ret; - - guard(rcu)(); - - sch = rcu_dereference(scx_root); - if (unlikely(!sch)) - return false; - - switch (scx_enable_state()) { - case SCX_ENABLING: - case SCX_ENABLED: - va_start(args, fmt); - ret = scx_verror(sch, fmt, args); - va_end(args); - return ret; - default: - return false; - } -} - -/** - * scx_rcu_cpu_stall - sched_ext RCU CPU stall handler - * - * While there are various reasons why RCU CPU stalls can occur on a system - * that may not be caused by the current BPF scheduler, try kicking out the - * current scheduler in an attempt to recover the system to a good state before - * issuing panics. - * - * Returns %true if sched_ext is enabled and abort was initiated, which may - * resolve the reported RCU stall. %false if sched_ext is not enabled or someone - * else already initiated abort. - */ -bool scx_rcu_cpu_stall(void) -{ - return handle_lockup("RCU CPU stall detected!"); -} - -/** - * scx_softlockup - sched_ext softlockup handler - * @dur_s: number of seconds of CPU stuck due to soft lockup - * - * On some multi-socket setups (e.g. 2x Intel 8480c), the BPF scheduler can - * live-lock the system by making many CPUs target the same DSQ to the point - * where soft-lockup detection triggers. This function is called from - * soft-lockup watchdog when the triggering point is close and tries to unjam - * the system and aborting the BPF scheduler. - */ -void scx_softlockup(u32 dur_s) -{ - if (!handle_lockup("soft lockup - CPU %d stuck for %us", smp_processor_id(), dur_s)) - return; - - printk_deferred(KERN_ERR "sched_ext: Soft lockup - CPU %d stuck for %us, disabling BPF scheduler\n", - smp_processor_id(), dur_s); -} - -/* - * scx_hardlockup() runs from NMI and eventually calls scx_claim_exit(), - * which takes scx_sched_lock. scx_sched_lock isn't NMI-safe and grabbing - * it from NMI context can lead to deadlocks. Defer via irq_work; the - * disable path runs off irq_work anyway. - */ -static atomic_t scx_hardlockup_cpu = ATOMIC_INIT(-1); - -static void scx_hardlockup_irq_workfn(struct irq_work *work) -{ - int cpu = atomic_xchg(&scx_hardlockup_cpu, -1); - - if (cpu >= 0 && handle_lockup("hard lockup - CPU %d", cpu)) - printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n", - cpu); -} - -static DEFINE_IRQ_WORK(scx_hardlockup_irq_work, scx_hardlockup_irq_workfn); - -/** - * scx_hardlockup - sched_ext hardlockup handler - * - * A poorly behaving BPF scheduler can trigger hard lockup by e.g. putting - * numerous affinitized tasks in a single queue and directing all CPUs at it. - * Try kicking out the current scheduler in an attempt to recover the system to - * a good state before taking more drastic actions. - * - * Queues an irq_work; the handle_lockup() call happens in IRQ context (see - * scx_hardlockup_irq_workfn). - * - * Returns %true if sched_ext is enabled and the work was queued, %false - * otherwise. - */ -bool scx_hardlockup(int cpu) -{ - if (!rcu_access_pointer(scx_root)) - return false; - - atomic_cmpxchg(&scx_hardlockup_cpu, -1, cpu); - irq_work_queue(&scx_hardlockup_irq_work); - return true; -} - -static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, - struct cpumask *donee_mask, struct cpumask *resched_mask, - u32 nr_donor_target, u32 nr_donee_target) -{ - struct rq *donor_rq = cpu_rq(donor); - struct scx_dispatch_q *donor_dsq = bypass_dsq(sch, donor); - struct task_struct *p, *n; - struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, donor_dsq, 0); - s32 delta = READ_ONCE(donor_dsq->nr) - nr_donor_target; - u32 nr_balanced = 0, min_delta_us; - - /* - * All we want to guarantee is reasonable forward progress. No reason to - * fine tune. Assuming every task on @donor_dsq runs their full slice, - * consider offloading iff the total queued duration is over the - * threshold. - */ - min_delta_us = READ_ONCE(scx_bypass_lb_intv_us) / SCX_BYPASS_LB_MIN_DELTA_DIV; - if (delta < DIV_ROUND_UP(min_delta_us, READ_ONCE(scx_slice_bypass_us))) - return 0; - - raw_spin_rq_lock_irq(donor_rq); - raw_spin_lock(&donor_dsq->lock); - list_add(&cursor.node, &donor_dsq->list); -resume: - n = container_of(&cursor, struct task_struct, scx.dsq_list); - n = nldsq_next_task(donor_dsq, n, false); - - while ((p = n)) { - struct scx_dispatch_q *donee_dsq; - int donee; - - n = nldsq_next_task(donor_dsq, n, false); - - if (donor_dsq->nr <= nr_donor_target) - break; - - if (cpumask_empty(donee_mask)) - break; - - /* - * If an earlier pass placed @p on @donor_dsq from a different - * CPU and the donee hasn't consumed it yet, @p is still on the - * previous CPU and task_rq(@p) != @donor_rq. @p can't be moved - * without its rq locked. Skip. - */ - if (task_rq(p) != donor_rq) - continue; - - donee = cpumask_any_and_distribute(donee_mask, p->cpus_ptr); - if (donee >= nr_cpu_ids) - continue; - - donee_dsq = bypass_dsq(sch, donee); - - /* - * $p's rq is not locked but $p's DSQ lock protects its - * scheduling properties making this test safe. - */ - if (!task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false)) - continue; - - /* - * Moving $p from one non-local DSQ to another. The source rq - * and DSQ are already locked. Do an abbreviated dequeue and - * then perform enqueue without unlocking $donor_dsq. - * - * We don't want to drop and reacquire the lock on each - * iteration as @donor_dsq can be very long and potentially - * highly contended. Donee DSQs are less likely to be contended. - * The nested locking is safe as only this LB moves tasks - * between bypass DSQs. - */ - dispatch_dequeue_locked(p, donor_dsq); - dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, SCX_ENQ_NESTED); - - /* - * $donee might have been idle and need to be woken up. No need - * to be clever. Kick every CPU that receives tasks. - */ - cpumask_set_cpu(donee, resched_mask); - - if (READ_ONCE(donee_dsq->nr) >= nr_donee_target) - cpumask_clear_cpu(donee, donee_mask); - - nr_balanced++; - if (!(nr_balanced % SCX_BYPASS_LB_BATCH) && n) { - list_move_tail(&cursor.node, &n->scx.dsq_list.node); - raw_spin_unlock(&donor_dsq->lock); - raw_spin_rq_unlock_irq(donor_rq); - cpu_relax(); - raw_spin_rq_lock_irq(donor_rq); - raw_spin_lock(&donor_dsq->lock); - goto resume; - } - } - - list_del_init(&cursor.node); - raw_spin_unlock(&donor_dsq->lock); - raw_spin_rq_unlock_irq(donor_rq); - - return nr_balanced; -} - -static void bypass_lb_node(struct scx_sched *sch, int node) -{ - const struct cpumask *node_mask = cpumask_of_node(node); - struct cpumask *donee_mask = sch->bypass_lb_donee_cpumask; - struct cpumask *resched_mask = sch->bypass_lb_resched_cpumask; - u32 nr_tasks = 0, nr_cpus = 0, nr_balanced = 0; - u32 nr_target, nr_donor_target; - u32 before_min = U32_MAX, before_max = 0; - u32 after_min = U32_MAX, after_max = 0; - int cpu; - - /* count the target tasks and CPUs */ - for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr); - - nr_tasks += nr; - nr_cpus++; - - before_min = min(nr, before_min); - before_max = max(nr, before_max); - } - - if (!nr_cpus) - return; - - /* - * We don't want CPUs to have more than $nr_donor_target tasks and - * balancing to fill donee CPUs upto $nr_target. Once targets are - * calculated, find the donee CPUs. - */ - nr_target = DIV_ROUND_UP(nr_tasks, nr_cpus); - nr_donor_target = DIV_ROUND_UP(nr_target * SCX_BYPASS_LB_DONOR_PCT, 100); - - cpumask_clear(donee_mask); - for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - if (READ_ONCE(bypass_dsq(sch, cpu)->nr) < nr_target) - cpumask_set_cpu(cpu, donee_mask); - } - - /* iterate !donee CPUs and see if they should be offloaded */ - cpumask_clear(resched_mask); - for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - if (cpumask_empty(donee_mask)) - break; - if (cpumask_test_cpu(cpu, donee_mask)) - continue; - if (READ_ONCE(bypass_dsq(sch, cpu)->nr) <= nr_donor_target) - continue; - - nr_balanced += bypass_lb_cpu(sch, cpu, donee_mask, resched_mask, - nr_donor_target, nr_target); - } - - for_each_cpu(cpu, resched_mask) - resched_cpu(cpu); - - for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr); - - after_min = min(nr, after_min); - after_max = max(nr, after_max); - - } - - trace_sched_ext_bypass_lb(node, nr_cpus, nr_tasks, nr_balanced, - before_min, before_max, after_min, after_max); -} - -/* - * In bypass mode, all tasks are put on the per-CPU bypass DSQs. If the machine - * is over-saturated and the BPF scheduler skewed tasks into few CPUs, some - * bypass DSQs can be overloaded. If there are enough tasks to saturate other - * lightly loaded CPUs, such imbalance can lead to very high execution latency - * on the overloaded CPUs and thus to hung tasks and RCU stalls. To avoid such - * outcomes, a simple load balancing mechanism is implemented by the following - * timer which runs periodically while bypass mode is in effect. - */ -static void scx_bypass_lb_timerfn(struct timer_list *timer) -{ - struct scx_sched *sch = container_of(timer, struct scx_sched, bypass_lb_timer); - int node; - u32 intv_us; - - if (!bypass_dsp_enabled(sch)) - return; - - for_each_node_with_cpus(node) - bypass_lb_node(sch, node); - - intv_us = READ_ONCE(scx_bypass_lb_intv_us); - if (intv_us) - mod_timer(timer, jiffies + usecs_to_jiffies(intv_us)); -} - -static bool inc_bypass_depth(struct scx_sched *sch) -{ - lockdep_assert_held(&scx_bypass_lock); - - WARN_ON_ONCE(sch->bypass_depth < 0); - WRITE_ONCE(sch->bypass_depth, sch->bypass_depth + 1); - if (sch->bypass_depth != 1) - return false; - - WRITE_ONCE(sch->slice_dfl, READ_ONCE(scx_slice_bypass_us) * NSEC_PER_USEC); - sch->bypass_timestamp = ktime_get_ns(); - scx_add_event(sch, SCX_EV_BYPASS_ACTIVATE, 1); - return true; -} - -static bool dec_bypass_depth(struct scx_sched *sch) -{ - lockdep_assert_held(&scx_bypass_lock); - - WARN_ON_ONCE(sch->bypass_depth < 1); - WRITE_ONCE(sch->bypass_depth, sch->bypass_depth - 1); - if (sch->bypass_depth != 0) - return false; - - WRITE_ONCE(sch->slice_dfl, SCX_SLICE_DFL); - scx_add_event(sch, SCX_EV_BYPASS_DURATION, - ktime_get_ns() - sch->bypass_timestamp); - return true; -} - -static void enable_bypass_dsp(struct scx_sched *sch) -{ - struct scx_sched *host = scx_parent(sch) ?: sch; - u32 intv_us = READ_ONCE(scx_bypass_lb_intv_us); - s32 ret; - - /* - * @sch->bypass_depth transitioning from 0 to 1 triggers enabling. - * Shouldn't stagger. - */ - if (WARN_ON_ONCE(test_and_set_bit(0, &sch->bypass_dsp_claim))) - return; - - /* - * When a sub-sched bypasses, its tasks are queued on the bypass DSQs of - * the nearest non-bypassing ancestor or root. As enable_bypass_dsp() is - * called iff @sch is not already bypassed due to an ancestor bypassing, - * we can assume that the parent is not bypassing and thus will be the - * host of the bypass DSQs. - * - * While the situation may change in the future, the following - * guarantees that the nearest non-bypassing ancestor or root has bypass - * dispatch enabled while a descendant is bypassing, which is all that's - * required. - * - * bypass_dsp_enabled() test is used to determine whether to enter the - * bypass dispatch handling path from both bypassing and hosting scheds. - * Bump enable depth on both @sch and bypass dispatch host. - */ - ret = atomic_inc_return(&sch->bypass_dsp_enable_depth); - WARN_ON_ONCE(ret <= 0); - - if (host != sch) { - ret = atomic_inc_return(&host->bypass_dsp_enable_depth); - WARN_ON_ONCE(ret <= 0); - } - - /* - * The LB timer will stop running if bypass dispatch is disabled. Start - * after enabling bypass dispatch. - */ - if (intv_us && !timer_pending(&host->bypass_lb_timer)) - mod_timer(&host->bypass_lb_timer, - jiffies + usecs_to_jiffies(intv_us)); -} - -/* may be called without holding scx_bypass_lock */ -static void disable_bypass_dsp(struct scx_sched *sch) -{ - s32 ret; - - if (!test_and_clear_bit(0, &sch->bypass_dsp_claim)) - return; - - ret = atomic_dec_return(&sch->bypass_dsp_enable_depth); - WARN_ON_ONCE(ret < 0); - - if (scx_parent(sch)) { - ret = atomic_dec_return(&scx_parent(sch)->bypass_dsp_enable_depth); - WARN_ON_ONCE(ret < 0); - } -} - -/** - * scx_bypass - [Un]bypass scx_ops and guarantee forward progress - * @sch: sched to bypass - * @bypass: true for bypass, false for unbypass - * - * Bypassing guarantees that all runnable tasks make forward progress without - * trusting the BPF scheduler. We can't grab any mutexes or rwsems as they might - * be held by tasks that the BPF scheduler is forgetting to run, which - * unfortunately also excludes toggling the static branches. - * - * Let's work around by overriding a couple ops and modifying behaviors based on - * the DISABLING state and then cycling the queued tasks through dequeue/enqueue - * to force global FIFO scheduling. - * - * - ops.select_cpu() is ignored and the default select_cpu() is used. - * - * - ops.enqueue() is ignored and tasks are queued in simple global FIFO order. - * %SCX_OPS_ENQ_LAST is also ignored. - * - * - ops.dispatch() is ignored. - * - * - balance_one() does not set %SCX_RQ_BAL_KEEP on non-zero slice as slice - * can't be trusted. Whenever a tick triggers, the running task is rotated to - * the tail of the queue with core_sched_at touched. - * - * - pick_next_task() suppresses zero slice warning. - * - * - scx_kick_cpu() is disabled to avoid irq_work malfunction during PM - * operations. - * - * - scx_prio_less() reverts to the default core_sched_at order. - */ -static void scx_bypass(struct scx_sched *sch, bool bypass) -{ - struct scx_sched *pos; - unsigned long flags; - int cpu; - - raw_spin_lock_irqsave(&scx_bypass_lock, flags); - - if (bypass) { - if (!inc_bypass_depth(sch)) - goto unlock; - - enable_bypass_dsp(sch); - } else { - if (!dec_bypass_depth(sch)) - goto unlock; - } - - /* - * Bypass state is propagated to all descendants - an scx_sched bypasses - * if itself or any of its ancestors are in bypass mode. - */ - raw_spin_lock(&scx_sched_lock); - scx_for_each_descendant_pre(pos, sch) { - if (pos == sch) - continue; - if (bypass) - inc_bypass_depth(pos); - else - dec_bypass_depth(pos); - } - raw_spin_unlock(&scx_sched_lock); - - /* - * No task property is changing. We just need to make sure all currently - * queued tasks are re-queued according to the new scx_bypassing() - * state. As an optimization, walk each rq's runnable_list instead of - * the scx_tasks list. - * - * This function can't trust the scheduler and thus can't use - * cpus_read_lock(). Walk all possible CPUs instead of online. - */ - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); - struct task_struct *p, *n; - - raw_spin_rq_lock(rq); - raw_spin_lock(&scx_sched_lock); - - scx_for_each_descendant_pre(pos, sch) { - struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu); - - if (pos->bypass_depth) - pcpu->flags |= SCX_SCHED_PCPU_BYPASSING; - else - pcpu->flags &= ~SCX_SCHED_PCPU_BYPASSING; - } - - raw_spin_unlock(&scx_sched_lock); - - /* - * We need to guarantee that no tasks are on the BPF scheduler - * while bypassing. Either we see enabled or the enable path - * sees scx_bypassing() before moving tasks to SCX. - */ - if (!scx_enabled()) { - raw_spin_rq_unlock(rq); - continue; - } - - /* - * The use of list_for_each_entry_safe_reverse() is required - * because each task is going to be removed from and added back - * to the runnable_list during iteration. Because they're added - * to the tail of the list, safe reverse iteration can still - * visit all nodes. - */ - list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, - scx.runnable_node) { - if (!scx_is_descendant(scx_task_sched(p), sch)) - continue; - - /* cycling deq/enq is enough, see the function comment */ - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - /* nothing */ ; - } - } - - /* resched to restore ticks and idle state */ - if (cpu_online(cpu) || cpu == smp_processor_id()) - resched_curr(rq); - - raw_spin_rq_unlock(rq); - } - - /* disarming must come after moving all tasks out of the bypass DSQs */ - if (!bypass) - disable_bypass_dsp(sch); -unlock: - raw_spin_unlock_irqrestore(&scx_bypass_lock, flags); -} - -static void free_exit_info(struct scx_exit_info *ei) -{ - kvfree(ei->dump); - kfree(ei->msg); - kfree(ei->bt); - kfree(ei); -} - -static struct scx_exit_info *alloc_exit_info(size_t exit_dump_len) -{ - struct scx_exit_info *ei; - - ei = kzalloc_obj(*ei); - if (!ei) - return NULL; - - ei->exit_cpu = -1; - ei->bt = kzalloc_objs(ei->bt[0], SCX_EXIT_BT_LEN); - ei->msg = kzalloc(SCX_EXIT_MSG_LEN, GFP_KERNEL); - ei->dump = kvzalloc(exit_dump_len, GFP_KERNEL); - - if (!ei->bt || !ei->msg || !ei->dump) { - free_exit_info(ei); - return NULL; - } - - return ei; -} - -static const char *scx_exit_reason(enum scx_exit_kind kind) -{ - switch (kind) { - case SCX_EXIT_UNREG: - return "unregistered from user space"; - case SCX_EXIT_UNREG_BPF: - return "unregistered from BPF"; - case SCX_EXIT_UNREG_KERN: - return "unregistered from the main kernel"; - case SCX_EXIT_SYSRQ: - return "disabled by sysrq-S"; - case SCX_EXIT_PARENT: - return "parent exiting"; - case SCX_EXIT_ERROR: - return "runtime error"; - case SCX_EXIT_ERROR_BPF: - return "scx_bpf_error"; - case SCX_EXIT_ERROR_STALL: - return "runnable task stall"; - default: - return ""; - } -} - -static void free_kick_syncs(void) -{ - int cpu; - - for_each_possible_cpu(cpu) { - struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); - struct scx_kick_syncs *to_free; - - to_free = rcu_replace_pointer(*ksyncs, NULL, true); - if (to_free) - kvfree_rcu(to_free, rcu); - } -} - -static void refresh_watchdog(void) -{ - struct scx_sched *sch; - unsigned long intv = ULONG_MAX; - - /* take the shortest timeout and use its half for watchdog interval */ - rcu_read_lock(); - list_for_each_entry_rcu(sch, &scx_sched_all, all) - intv = max(min(intv, sch->watchdog_timeout / 2), 1); - rcu_read_unlock(); - - WRITE_ONCE(scx_watchdog_timestamp, jiffies); - WRITE_ONCE(scx_watchdog_interval, intv); - - if (intv < ULONG_MAX) - mod_delayed_work(system_dfl_wq, &scx_watchdog_work, intv); - else - cancel_delayed_work_sync(&scx_watchdog_work); -} - -static s32 scx_link_sched(struct scx_sched *sch) -{ - const char *err_msg = ""; - s32 ret = 0; - - scoped_guard(raw_spinlock_irq, &scx_sched_lock) { -#ifdef CONFIG_EXT_SUB_SCHED - struct scx_sched *parent = scx_parent(sch); - - if (parent) { - /* - * scx_claim_exit() propagates exit_kind transition to - * its sub-scheds while holding scx_sched_lock - either - * we can see the parent's non-NONE exit_kind or the - * parent can shoot us down. - */ - if (atomic_read(&parent->exit_kind) != SCX_EXIT_NONE) { - err_msg = "parent disabled"; - ret = -ENOENT; - break; - } - - ret = rhashtable_lookup_insert_fast(&scx_sched_hash, - &sch->hash_node, scx_sched_hash_params); - if (ret) { - err_msg = "failed to insert into scx_sched_hash"; - break; - } - - list_add_tail(&sch->sibling, &parent->children); - } -#endif /* CONFIG_EXT_SUB_SCHED */ - - list_add_tail_rcu(&sch->all, &scx_sched_all); - } - - /* - * scx_error() takes scx_sched_lock via scx_claim_exit(), so it must run after - * the guard above is released. - */ - if (ret) { - scx_error(sch, "%s (%d)", err_msg, ret); - return ret; - } - - refresh_watchdog(); - return 0; -} - -static void scx_unlink_sched(struct scx_sched *sch) -{ - scoped_guard(raw_spinlock_irq, &scx_sched_lock) { -#ifdef CONFIG_EXT_SUB_SCHED - if (scx_parent(sch)) { - rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node, - scx_sched_hash_params); - list_del_init(&sch->sibling); - } -#endif /* CONFIG_EXT_SUB_SCHED */ - list_del_rcu(&sch->all); - } - - refresh_watchdog(); -} - -/* - * Called to disable future dumps and wait for in-progress one while disabling - * @sch. Once @sch becomes empty during disable, there's no point in dumping it. - * This prevents calling dump ops on a dead sch. - */ -static void scx_disable_dump(struct scx_sched *sch) -{ - guard(raw_spinlock_irqsave)(&scx_dump_lock); - sch->dump_disabled = true; -} - -static void scx_log_sched_disable(struct scx_sched *sch) -{ - struct scx_exit_info *ei = sch->exit_info; - const char *type = scx_parent(sch) ? "sub-scheduler" : "scheduler"; - - if (ei->kind >= SCX_EXIT_ERROR) { - pr_err("sched_ext: BPF %s \"%s\" disabled (%s)\n", type, - sch->ops.name, ei->reason); - - if (ei->msg[0] != '\0') - pr_err("sched_ext: %s: %s\n", sch->ops.name, ei->msg); -#ifdef CONFIG_STACKTRACE - stack_trace_print(ei->bt, ei->bt_len, 2); -#endif - } else { - pr_info("sched_ext: BPF %s \"%s\" disabled (%s)\n", type, - sch->ops.name, ei->reason); - } -} - -#ifdef CONFIG_EXT_SUB_SCHED -static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq); - -static void drain_descendants(struct scx_sched *sch) -{ - /* - * Child scheds that finished the critical part of disabling will take - * themselves off @sch->children. Wait for it to drain. As propagation - * is recursive, empty @sch->children means that all proper descendant - * scheds reached unlinking stage. - */ - wait_event(scx_unlink_waitq, list_empty(&sch->children)); -} - -static void scx_fail_parent(struct scx_sched *sch, - struct task_struct *failed, s32 fail_code) -{ - struct scx_sched *parent = scx_parent(sch); - struct scx_task_iter sti; - struct task_struct *p; - - scx_error(parent, "ops.init_task() failed (%d) for %s[%d] while disabling a sub-scheduler", - fail_code, failed->comm, failed->pid); - - /* - * Once $parent is bypassed, it's safe to put SCX_TASK_NONE tasks into - * it. This may cause downstream failures on the BPF side but $parent is - * dying anyway. - */ - scx_bypass(parent, true); - - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - if (scx_task_on_sched(parent, p)) - continue; - - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - scx_disable_and_exit_task(sch, p); - scx_set_task_sched(p, parent); - } - } - scx_task_iter_stop(&sti); -} - -static void scx_sub_disable(struct scx_sched *sch) -{ - struct scx_sched *parent = scx_parent(sch); - struct scx_task_iter sti; - struct task_struct *p; - int ret; - - /* - * Guarantee forward progress and wait for descendants to be disabled. - * To limit disruptions, $parent is not bypassed. Tasks are fully - * prepped and then inserted back into $parent. - */ - scx_bypass(sch, true); - drain_descendants(sch); - - /* - * Here, every runnable task is guaranteed to make forward progress and - * we can safely use blocking synchronization constructs. Actually - * disable ops. - */ - mutex_lock(&scx_enable_mutex); - percpu_down_write(&scx_fork_rwsem); - scx_cgroup_lock(); - - set_cgroup_sched(sch_cgroup(sch), parent); - - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - struct rq *rq; - struct rq_flags rf; - - /* filter out duplicate visits */ - if (scx_task_on_sched(parent, p)) - continue; - - /* - * By the time control reaches here, all descendant schedulers - * should already have been disabled. - */ - WARN_ON_ONCE(!scx_task_on_sched(sch, p)); - - /* - * @p is pinned by the iter: css_task_iter_next() takes a - * reference and holds it until the next iter_next() call, so - * @p->usage is guaranteed > 0. - */ - get_task_struct(p); - - scx_task_iter_unlock(&sti); - - /* - * $p is READY or ENABLED on @sch. Initialize for $parent, - * disable and exit from @sch, and then switch over to $parent. - * - * If a task fails to initialize for $parent, the only available - * action is disabling $parent too. While this allows disabling - * of a child sched to cause the parent scheduler to fail, the - * failure can only originate from ops.init_task() of the - * parent. A child can't directly affect the parent through its - * own failures. - */ - ret = __scx_init_task(parent, p, false); - if (ret) { - scx_fail_parent(sch, p, ret); - put_task_struct(p); - break; - } - - rq = task_rq_lock(p, &rf); - - if (scx_get_task_state(p) == SCX_TASK_DEAD) { - /* - * sched_ext_dead() raced us between __scx_init_task() - * and this rq lock and ran exit_task() on @sch (the - * sched @p was on at that point), not on $parent. - * $parent's just-completed init is owed an exit_task() - * and we issue it here. - */ - scx_sub_init_cancel_task(parent, p); - task_rq_unlock(rq, p, &rf); - put_task_struct(p); - continue; - } - - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - /* - * $p is initialized for $parent and still attached to - * @sch. Disable and exit for @sch, switch over to - * $parent, override the state to READY to account for - * $p having already been initialized, and then enable. - */ - scx_disable_and_exit_task(sch, p); - scx_set_task_state(p, SCX_TASK_INIT_BEGIN); - scx_set_task_state(p, SCX_TASK_INIT); - scx_set_task_sched(p, parent); - scx_set_task_state(p, SCX_TASK_READY); - scx_enable_task(parent, p); - } - - task_rq_unlock(rq, p, &rf); - put_task_struct(p); - } - scx_task_iter_stop(&sti); - - scx_disable_dump(sch); - - scx_cgroup_unlock(); - percpu_up_write(&scx_fork_rwsem); - - /* - * All tasks are moved off of @sch but there may still be on-going - * operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use - * the expedited version as ancestors may be waiting in bypass mode. - * Also, tell the parent that there is no need to keep running bypass - * DSQs for us. - */ - synchronize_rcu_expedited(); - disable_bypass_dsp(sch); - - scx_unlink_sched(sch); - - mutex_unlock(&scx_enable_mutex); - - /* - * @sch is now unlinked from the parent's children list. Notify and call - * ops.sub_detach/exit(). Note that ops.sub_detach/exit() must be called - * after unlinking and releasing all locks. See scx_claim_exit(). - */ - wake_up_all(&scx_unlink_waitq); - - if (parent->ops.sub_detach && sch->sub_attached) { - struct scx_sub_detach_args sub_detach_args = { - .ops = &sch->ops, - .cgroup_path = sch->cgrp_path, - }; - SCX_CALL_OP(parent, sub_detach, NULL, - &sub_detach_args); - } - - scx_log_sched_disable(sch); - - if (sch->ops.exit) - SCX_CALL_OP(sch, exit, NULL, sch->exit_info); - if (sch->sub_kset) - kobject_del(&sch->sub_kset->kobj); - kobject_del(&sch->kobj); -} -#else /* CONFIG_EXT_SUB_SCHED */ -static inline void drain_descendants(struct scx_sched *sch) { } -static inline void scx_sub_disable(struct scx_sched *sch) { } -#endif /* CONFIG_EXT_SUB_SCHED */ - -static void scx_root_disable(struct scx_sched *sch) -{ - struct scx_task_iter sti; - struct task_struct *p; - bool was_switched_all; - int cpu; - - /* guarantee forward progress and wait for descendants to be disabled */ - scx_bypass(sch, true); - drain_descendants(sch); - - switch (scx_set_enable_state(SCX_DISABLING)) { - case SCX_DISABLING: - WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); - break; - case SCX_DISABLED: - pr_warn("sched_ext: ops error detected without ops (%s)\n", - sch->exit_info->msg); - WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING); - goto done; - default: - break; - } - - /* - * Here, every runnable task is guaranteed to make forward progress and - * we can safely use blocking synchronization constructs. Actually - * disable ops. - */ - mutex_lock(&scx_enable_mutex); - - was_switched_all = scx_switched_all(); - - static_branch_disable(&__scx_switched_all); - WRITE_ONCE(scx_switching_all, false); - - /* - * Shut down cgroup support before tasks so that the cgroup attach path - * doesn't race against scx_disable_and_exit_task(). - */ - scx_cgroup_lock(); - scx_cgroup_exit(sch); - scx_cgroup_unlock(); - - /* - * The BPF scheduler is going away. All tasks including %TASK_DEAD ones - * must be switched out and exited synchronously. - */ - percpu_down_write(&scx_fork_rwsem); - - scx_init_task_enabled = false; - - scx_task_iter_start(&sti, NULL); - while ((p = scx_task_iter_next_locked(&sti))) { - unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - const struct sched_class *old_class = p->sched_class; - const struct sched_class *new_class = scx_setscheduler_class(p); - - update_rq_clock(task_rq(p)); - - if (old_class != new_class) - queue_flags |= DEQUEUE_CLASS; - - scoped_guard (sched_change, p, queue_flags) { - p->sched_class = new_class; - } - - scx_disable_and_exit_task(scx_task_sched(p), p); - } - scx_task_iter_stop(&sti); - - scx_disable_dump(sch); - - scx_cgroup_lock(); - set_cgroup_sched(sch_cgroup(sch), NULL); - scx_cgroup_unlock(); - - percpu_up_write(&scx_fork_rwsem); - - /* - * Invalidate all the rq clocks to prevent getting outdated - * rq clocks from a previous scx scheduler. - * - * Also re-balance the dl_server bandwidth reservations: detach - * ext_server (no more sched_ext tasks) and reinstate fair_server if it - * was previously detached because we were running in full mode. - * - * Unlike the enable path, this runs on a recovery path that cannot - * fail, so we use dl_server_swap_bw() to atomically free ext_server's - * bandwidth and reclaim it for fair_server under the same dl_b lock. - * - * The swap can still fail with -EBUSY if someone bumped ext_server's - * runtime via debugfs between enable and disable; in that narrow case - * both servers end up detached and we just WARN. - */ - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); - - scx_rq_clock_invalidate(rq); - - scoped_guard(rq_lock_irqsave, rq) { - update_rq_clock(rq); - if (was_switched_all) { - if (WARN_ON_ONCE(dl_server_swap_bw(&rq->ext_server, - &rq->fair_server))) - pr_warn("failed to re-attach fair_server on CPU %d\n", cpu); - } else { - dl_server_detach_bw(&rq->ext_server); - } - } - } - - /* no task is on scx, turn off all the switches and flush in-progress calls */ - static_branch_disable(&__scx_enabled); - static_branch_disable(&__scx_is_cid_type); - if (sch->ops.flags & SCX_OPS_TID_TO_TASK) - static_branch_disable(&__scx_tid_to_task_enabled); - bitmap_zero(sch->has_op, SCX_OPI_END); - scx_idle_disable(); - synchronize_rcu(); - if (sch->ops.flags & SCX_OPS_TID_TO_TASK) - rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL); - - scx_log_sched_disable(sch); - - if (sch->ops.exit) - SCX_CALL_OP(sch, exit, NULL, sch->exit_info); - - scx_unlink_sched(sch); - - /* - * scx_root clearing must be inside cpus_read_lock(). See - * handle_hotplug(). - */ - cpus_read_lock(); - RCU_INIT_POINTER(scx_root, NULL); - cpus_read_unlock(); - - /* - * Delete the kobject from the hierarchy synchronously. Otherwise, sysfs - * could observe an object of the same name still in the hierarchy when - * the next scheduler is loaded. - */ -#ifdef CONFIG_EXT_SUB_SCHED - if (sch->sub_kset) - kobject_del(&sch->sub_kset->kobj); -#endif - kobject_del(&sch->kobj); - - free_kick_syncs(); - - mutex_unlock(&scx_enable_mutex); - - WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING); -done: - scx_bypass(sch, false); -} - -/* - * Claim the exit on @sch. The caller must ensure that the helper kthread work - * is kicked before the current task can be preempted. Once exit_kind is - * claimed, scx_error() can no longer trigger, so if the current task gets - * preempted and the BPF scheduler fails to schedule it back, the helper work - * will never be kicked and the whole system can wedge. - */ -static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind) -{ - int none = SCX_EXIT_NONE; - - lockdep_assert_preemption_disabled(); - - if (WARN_ON_ONCE(kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE)) - kind = SCX_EXIT_ERROR; - - if (!atomic_try_cmpxchg(&sch->exit_kind, &none, kind)) - return false; - - /* - * Some CPUs may be trapped in the dispatch paths. Set the aborting - * flag to break potential live-lock scenarios, ensuring we can - * successfully reach scx_bypass(). - */ - WRITE_ONCE(sch->aborting, true); - - /* - * Propagate exits to descendants immediately. Each has a dedicated - * helper kthread and can run in parallel. While most of disabling is - * serialized, running them in separate threads allows parallelizing - * ops.exit(), which can take arbitrarily long prolonging bypass mode. - * - * To guarantee forward progress, this propagation must be in-line so - * that ->aborting is synchronously asserted for all sub-scheds. The - * propagation is also the interlocking point against sub-sched - * attachment. See scx_link_sched(). - * - * This doesn't cause recursions as propagation only takes place for - * non-propagation exits. - */ - if (kind != SCX_EXIT_PARENT) { - scoped_guard (raw_spinlock_irqsave, &scx_sched_lock) { - struct scx_sched *pos; - scx_for_each_descendant_pre(pos, sch) - scx_disable(pos, SCX_EXIT_PARENT); - } - } - - return true; -} - -static void scx_disable_workfn(struct kthread_work *work) -{ - struct scx_sched *sch = container_of(work, struct scx_sched, disable_work); - struct scx_exit_info *ei = sch->exit_info; - int kind; - - kind = atomic_read(&sch->exit_kind); - while (true) { - if (kind == SCX_EXIT_DONE) /* already disabled? */ - return; - WARN_ON_ONCE(kind == SCX_EXIT_NONE); - if (atomic_try_cmpxchg(&sch->exit_kind, &kind, SCX_EXIT_DONE)) - break; - } - ei->kind = kind; - ei->reason = scx_exit_reason(ei->kind); - - if (scx_parent(sch)) - scx_sub_disable(sch); - else - scx_root_disable(sch); -} - -static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind) -{ - guard(preempt)(); - if (scx_claim_exit(sch, kind)) - irq_work_queue(&sch->disable_irq_work); -} - -/** - * scx_flush_disable_work - flush the disable work and wait for it to finish - * @sch: the scheduler - * - * sch->disable_work might still not queued, causing kthread_flush_work() - * as a noop. Syncing the irq_work first is required to guarantee the - * kthread work has been queued before waiting for it. - */ -static void scx_flush_disable_work(struct scx_sched *sch) -{ - int kind; - - do { - irq_work_sync(&sch->disable_irq_work); - kthread_flush_work(&sch->disable_work); - kind = atomic_read(&sch->exit_kind); - } while (kind != SCX_EXIT_NONE && kind != SCX_EXIT_DONE); -} - -static void dump_newline(struct seq_buf *s) -{ - trace_sched_ext_dump(""); - - /* @s may be zero sized and seq_buf triggers WARN if so */ - if (s->size) - seq_buf_putc(s, '\n'); -} - -static __printf(2, 3) void dump_line(struct seq_buf *s, const char *fmt, ...) -{ - va_list args; - -#ifdef CONFIG_TRACEPOINTS - if (trace_sched_ext_dump_enabled()) { - /* protected by scx_dump_lock */ - static char line_buf[SCX_EXIT_MSG_LEN]; - - va_start(args, fmt); - vscnprintf(line_buf, sizeof(line_buf), fmt, args); - va_end(args); - - trace_call__sched_ext_dump(line_buf); - } -#endif - /* @s may be zero sized and seq_buf triggers WARN if so */ - if (s->size) { - va_start(args, fmt); - seq_buf_vprintf(s, fmt, args); - va_end(args); - - seq_buf_putc(s, '\n'); - } -} - -static void dump_stack_trace(struct seq_buf *s, const char *prefix, - const unsigned long *bt, unsigned int len) -{ - unsigned int i; - - for (i = 0; i < len; i++) - dump_line(s, "%s%pS", prefix, (void *)bt[i]); -} - -static void ops_dump_init(struct seq_buf *s, const char *prefix) -{ - struct scx_dump_data *dd = &scx_dump_data; - - lockdep_assert_irqs_disabled(); - - dd->cpu = smp_processor_id(); /* allow scx_bpf_dump() */ - dd->first = true; - dd->cursor = 0; - dd->s = s; - dd->prefix = prefix; -} - -static void ops_dump_flush(void) -{ - struct scx_dump_data *dd = &scx_dump_data; - char *line = dd->buf.line; - - if (!dd->cursor) - return; - - /* - * There's something to flush and this is the first line. Insert a blank - * line to distinguish ops dump. - */ - if (dd->first) { - dump_newline(dd->s); - dd->first = false; - } - - /* - * There may be multiple lines in $line. Scan and emit each line - * separately. - */ - while (true) { - char *end = line; - char c; - - while (*end != '\n' && *end != '\0') - end++; - - /* - * If $line overflowed, it may not have newline at the end. - * Always emit with a newline. - */ - c = *end; - *end = '\0'; - dump_line(dd->s, "%s%s", dd->prefix, line); - if (c == '\0') - break; - - /* move to the next line */ - end++; - if (*end == '\0') - break; - line = end; - } - - dd->cursor = 0; -} - -static void ops_dump_exit(void) -{ - ops_dump_flush(); - scx_dump_data.cpu = -1; -} - -static void scx_dump_task(struct scx_sched *sch, struct seq_buf *s, struct scx_dump_ctx *dctx, - struct rq *rq, struct task_struct *p, char marker) -{ - static unsigned long bt[SCX_EXIT_BT_LEN]; - struct scx_sched *task_sch = scx_task_sched(p); - const char *own_marker; - char sch_id_buf[32]; - char dsq_id_buf[19] = "(n/a)"; - unsigned long ops_state = atomic_long_read(&p->scx.ops_state); - unsigned int bt_len = 0; - - own_marker = task_sch == sch ? "*" : ""; - - if (task_sch->level == 0) - scnprintf(sch_id_buf, sizeof(sch_id_buf), "root"); - else - scnprintf(sch_id_buf, sizeof(sch_id_buf), "sub%d-%llu", - task_sch->level, task_sch->ops.sub_cgroup_id); - - if (p->scx.dsq) - scnprintf(dsq_id_buf, sizeof(dsq_id_buf), "0x%llx", - (unsigned long long)p->scx.dsq->id); - - dump_newline(s); - dump_line(s, " %c%c %s[%d] %s%s %+ldms", - marker, task_state_to_char(p), p->comm, p->pid, - own_marker, sch_id_buf, - jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies)); - dump_line(s, " scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu", - scx_get_task_state(p) >> SCX_TASK_STATE_SHIFT, - p->scx.flags & ~SCX_TASK_STATE_MASK, - p->scx.dsq_flags, ops_state & SCX_OPSS_STATE_MASK, - ops_state >> SCX_OPSS_QSEQ_SHIFT); - dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s", - p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf); - dump_line(s, " dsq_vtime=%llu slice=%llu weight=%u", - p->scx.dsq_vtime, p->scx.slice, p->scx.weight); - dump_line(s, " cpus=%*pb no_mig=%u", cpumask_pr_args(p->cpus_ptr), - p->migration_disabled); - - if (SCX_HAS_OP(sch, dump_task)) { - ops_dump_init(s, " "); - SCX_CALL_OP(sch, dump_task, rq, dctx, p); - ops_dump_exit(); - } - -#ifdef CONFIG_STACKTRACE - bt_len = stack_trace_save_tsk(p, bt, SCX_EXIT_BT_LEN, 1); -#endif - if (bt_len) { - dump_newline(s); - dump_stack_trace(s, " ", bt, bt_len); - } -} - -static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, - struct scx_dump_ctx *dctx, int cpu, - bool dump_all_tasks) -{ - struct rq *rq = cpu_rq(cpu); - struct rq_flags rf; - struct task_struct *p; - struct seq_buf ns; - size_t avail, used; - char *buf; - bool idle; - - rq_lock_irqsave(rq, &rf); - - idle = list_empty(&rq->scx.runnable_list) && - rq->curr->sched_class == &idle_sched_class; - - if (idle && !SCX_HAS_OP(sch, dump_cpu)) - goto next; - - /* - * We don't yet know whether ops.dump_cpu() will produce output - * and we may want to skip the default CPU dump if it doesn't. - * Use a nested seq_buf to generate the standard dump so that we - * can decide whether to commit later. - */ - avail = seq_buf_get_buf(s, &buf); - seq_buf_init(&ns, buf, avail); - - dump_newline(&ns); - dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu", - cpu, rq->scx.nr_running, rq->scx.flags, - rq->scx.cpu_released, rq->scx.ops_qseq, - rq->scx.kick_sync); - dump_line(&ns, " curr=%s[%d] class=%ps", - rq->curr->comm, rq->curr->pid, - rq->curr->sched_class); - if (!cpumask_empty(rq->scx.cpus_to_kick)) - dump_line(&ns, " cpus_to_kick : %*pb", - cpumask_pr_args(rq->scx.cpus_to_kick)); - if (!cpumask_empty(rq->scx.cpus_to_kick_if_idle)) - dump_line(&ns, " idle_to_kick : %*pb", - cpumask_pr_args(rq->scx.cpus_to_kick_if_idle)); - if (!cpumask_empty(rq->scx.cpus_to_preempt)) - dump_line(&ns, " cpus_to_preempt: %*pb", - cpumask_pr_args(rq->scx.cpus_to_preempt)); - if (!cpumask_empty(rq->scx.cpus_to_wait)) - dump_line(&ns, " cpus_to_wait : %*pb", - cpumask_pr_args(rq->scx.cpus_to_wait)); - if (!cpumask_empty(rq->scx.cpus_to_sync)) - dump_line(&ns, " cpus_to_sync : %*pb", - cpumask_pr_args(rq->scx.cpus_to_sync)); - - used = seq_buf_used(&ns); - if (SCX_HAS_OP(sch, dump_cpu)) { - ops_dump_init(&ns, " "); - SCX_CALL_OP(sch, dump_cpu, rq, dctx, scx_cpu_arg(cpu), idle); - ops_dump_exit(); - } - - /* - * If idle && nothing generated by ops.dump_cpu(), there's - * nothing interesting. Skip. - */ - if (idle && used == seq_buf_used(&ns)) - goto next; - - /* - * $s may already have overflowed when $ns was created. If so, - * calling commit on it will trigger BUG. - */ - if (avail) { - seq_buf_commit(s, seq_buf_used(&ns)); - if (seq_buf_has_overflowed(&ns)) - seq_buf_set_overflow(s); - } - - if (rq->curr->sched_class == &ext_sched_class && - (dump_all_tasks || scx_task_on_sched(sch, rq->curr))) - scx_dump_task(sch, s, dctx, rq, rq->curr, '*'); - - list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) - if (dump_all_tasks || scx_task_on_sched(sch, p)) - scx_dump_task(sch, s, dctx, rq, p, ' '); -next: - rq_unlock_irqrestore(rq, &rf); -} - -/* - * Dump scheduler state. If @dump_all_tasks is true, dump all tasks regardless - * of which scheduler they belong to. If false, only dump tasks owned by @sch. - * For SysRq-D dumps, @dump_all_tasks=false since all schedulers are dumped - * separately. For error dumps, @dump_all_tasks=true since only the failing - * scheduler is dumped. - */ -static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, - size_t dump_len, bool dump_all_tasks) -{ - static const char trunc_marker[] = "\n\n~~~~ TRUNCATED ~~~~\n"; - struct scx_dump_ctx dctx = { - .kind = ei->kind, - .exit_code = ei->exit_code, - .reason = ei->reason, - .at_ns = ktime_get_ns(), - .at_jiffies = jiffies, - }; - struct seq_buf s; - struct scx_event_stats events; - int cpu; - - guard(raw_spinlock_irqsave)(&scx_dump_lock); - - if (sch->dump_disabled) - return; - - seq_buf_init(&s, ei->dump, dump_len); - -#ifdef CONFIG_EXT_SUB_SCHED - if (sch->level == 0) - dump_line(&s, "%s: root", sch->ops.name); - else - dump_line(&s, "%s: sub%d-%llu %s", - sch->ops.name, sch->level, sch->ops.sub_cgroup_id, - sch->cgrp_path); -#endif - if (ei->kind == SCX_EXIT_NONE) { - dump_line(&s, "Debug dump triggered by %s", ei->reason); - } else { - if (ei->exit_cpu >= 0) - dump_line(&s, "%s[%d] triggered exit kind %d on CPU %d:", - current->comm, current->pid, ei->kind, - ei->exit_cpu); - else - dump_line(&s, "%s[%d] triggered exit kind %d:", - current->comm, current->pid, ei->kind); - dump_line(&s, " %s (%s)", ei->reason, ei->msg); - dump_newline(&s); - dump_line(&s, "Backtrace:"); - dump_stack_trace(&s, " ", ei->bt, ei->bt_len); - } - - if (SCX_HAS_OP(sch, dump)) { - ops_dump_init(&s, ""); - SCX_CALL_OP(sch, dump, NULL, &dctx); - ops_dump_exit(); - } - - dump_newline(&s); - dump_line(&s, "CPU states"); - dump_line(&s, "----------"); - - /* - * Dump the exit CPU first so it isn't lost to dump truncation, then - * walk the rest in order, skipping the one already dumped. - */ - if (ei->exit_cpu >= 0) - scx_dump_cpu(sch, &s, &dctx, ei->exit_cpu, dump_all_tasks); - for_each_possible_cpu(cpu) { - if (cpu != ei->exit_cpu) - scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); - } - - dump_newline(&s); - dump_line(&s, "Event counters"); - dump_line(&s, "--------------"); - - scx_read_events(sch, &events); - scx_dump_event(s, &events, SCX_EV_SELECT_CPU_FALLBACK); - scx_dump_event(s, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); - scx_dump_event(s, &events, SCX_EV_DISPATCH_KEEP_LAST); - scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_EXITING); - scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); - scx_dump_event(s, &events, SCX_EV_REENQ_IMMED); - scx_dump_event(s, &events, SCX_EV_REENQ_LOCAL_REPEAT); - scx_dump_event(s, &events, SCX_EV_REFILL_SLICE_DFL); - scx_dump_event(s, &events, SCX_EV_BYPASS_DURATION); - scx_dump_event(s, &events, SCX_EV_BYPASS_DISPATCH); - scx_dump_event(s, &events, SCX_EV_BYPASS_ACTIVATE); - scx_dump_event(s, &events, SCX_EV_INSERT_NOT_OWNED); - scx_dump_event(s, &events, SCX_EV_SUB_BYPASS_DISPATCH); - - if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker)) - memcpy(ei->dump + dump_len - sizeof(trunc_marker), - trunc_marker, sizeof(trunc_marker)); -} - -static void scx_disable_irq_workfn(struct irq_work *irq_work) -{ - struct scx_sched *sch = container_of(irq_work, struct scx_sched, disable_irq_work); - struct scx_exit_info *ei = sch->exit_info; - - if (ei->kind >= SCX_EXIT_ERROR) - scx_dump_state(sch, ei, sch->ops.exit_dump_len, true); - - kthread_queue_work(sch->helper, &sch->disable_work); -} - -bool scx_vexit(struct scx_sched *sch, - enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu, - const char *fmt, va_list args) -{ - struct scx_exit_info *ei = sch->exit_info; - - guard(preempt)(); - - if (!scx_claim_exit(sch, kind)) - return false; - - ei->exit_code = exit_code; -#ifdef CONFIG_STACKTRACE - if (kind >= SCX_EXIT_ERROR) - ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1); -#endif - vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args); - - /* - * Set ei->kind and ->reason for scx_dump_state(). They'll be set again - * in scx_disable_workfn(). - */ - ei->kind = kind; - ei->reason = scx_exit_reason(ei->kind); - ei->exit_cpu = exit_cpu; - - irq_work_queue(&sch->disable_irq_work); - return true; -} - -static int alloc_kick_syncs(void) -{ - int cpu; - - /* - * Allocate per-CPU arrays sized by nr_cpu_ids. Use kvzalloc as size - * can exceed percpu allocator limits on large machines. - */ - for_each_possible_cpu(cpu) { - struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); - struct scx_kick_syncs *new_ksyncs; - - WARN_ON_ONCE(rcu_access_pointer(*ksyncs)); - - new_ksyncs = kvzalloc_node(struct_size(new_ksyncs, syncs, nr_cpu_ids), - GFP_KERNEL, cpu_to_node(cpu)); - if (!new_ksyncs) { - free_kick_syncs(); - return -ENOMEM; - } - - rcu_assign_pointer(*ksyncs, new_ksyncs); - } - - return 0; -} - -static void free_pnode(struct scx_sched_pnode *pnode) -{ - if (!pnode) - return; - exit_dsq(&pnode->global_dsq); - kfree(pnode); -} - -static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node) -{ - struct scx_sched_pnode *pnode; - - pnode = kzalloc_node(sizeof(*pnode), GFP_KERNEL, node); - if (!pnode) - return NULL; - - if (init_dsq(&pnode->global_dsq, SCX_DSQ_GLOBAL, sch)) { - kfree(pnode); - return NULL; - } - - return pnode; -} - -/* - * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid - * starvation. During the READY -> ENABLED task switching loop, the calling - * thread's sched_class gets switched from fair to ext. As fair has higher - * priority than ext, the calling thread can be indefinitely starved under - * fair-class saturation, leading to a system hang. - */ -struct scx_enable_cmd { - struct kthread_work work; - union { - struct sched_ext_ops *ops; - struct sched_ext_ops_cid *ops_cid; - }; - bool is_cid_type; - struct bpf_map *arena_map; /* arena ref to transfer to sch */ - int ret; -}; - -/* - * Allocate and initialize a new scx_sched. @cgrp's reference is always - * consumed whether the function succeeds or fails. - */ -static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, - struct cgroup *cgrp, - struct scx_sched *parent) -{ - struct sched_ext_ops *ops = cmd->ops; - struct scx_sched *sch; - s32 level = parent ? parent->level + 1 : 0; - s32 node, cpu, ret, bypass_fail_cpu = nr_cpu_ids; - - sch = kzalloc_flex(*sch, ancestors, level + 1); - if (!sch) { - ret = -ENOMEM; - goto err_put_cgrp; - } - - sch->exit_info = alloc_exit_info(ops->exit_dump_len); - if (!sch->exit_info) { - ret = -ENOMEM; - goto err_free_sch; - } - - ret = rhashtable_init(&sch->dsq_hash, &dsq_hash_params); - if (ret < 0) - goto err_free_ei; - - sch->pnode = kzalloc_objs(sch->pnode[0], nr_node_ids); - if (!sch->pnode) { - ret = -ENOMEM; - goto err_free_hash; - } - - for_each_node_state(node, N_POSSIBLE) { - sch->pnode[node] = alloc_pnode(sch, node); - if (!sch->pnode[node]) { - ret = -ENOMEM; - goto err_free_pnode; - } - } - - sch->dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; - sch->pcpu = __alloc_percpu(struct_size_t(struct scx_sched_pcpu, - dsp_ctx.buf, sch->dsp_max_batch), - __alignof__(struct scx_sched_pcpu)); - if (!sch->pcpu) { - ret = -ENOMEM; - goto err_free_pnode; - } - - for_each_possible_cpu(cpu) { - ret = init_dsq(bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch); - if (ret) { - bypass_fail_cpu = cpu; - goto err_free_pcpu; - } - } - - for_each_possible_cpu(cpu) { - struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); - - pcpu->sch = sch; - INIT_LIST_HEAD(&pcpu->deferred_reenq_local.node); - } - - sch->helper = kthread_run_worker(0, "sched_ext_helper"); - if (IS_ERR(sch->helper)) { - ret = PTR_ERR(sch->helper); - goto err_free_pcpu; - } - - sched_set_fifo(sch->helper->task); - - if (parent) - memcpy(sch->ancestors, parent->ancestors, - level * sizeof(parent->ancestors[0])); - sch->ancestors[level] = sch; - sch->level = level; - - if (ops->timeout_ms) - sch->watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); - else - sch->watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; - - sch->slice_dfl = SCX_SLICE_DFL; - atomic_set(&sch->exit_kind, SCX_EXIT_NONE); - sch->disable_irq_work = IRQ_WORK_INIT_HARD(scx_disable_irq_workfn); - kthread_init_work(&sch->disable_work, scx_disable_workfn); - timer_setup(&sch->bypass_lb_timer, scx_bypass_lb_timerfn, 0); - - if (!alloc_cpumask_var(&sch->bypass_lb_donee_cpumask, GFP_KERNEL)) { - ret = -ENOMEM; - goto err_stop_helper; - } - if (!alloc_cpumask_var(&sch->bypass_lb_resched_cpumask, GFP_KERNEL)) { - ret = -ENOMEM; - goto err_free_lb_cpumask; - } - /* - * Copy ops through the right union view. For cid-form the source is - * struct sched_ext_ops_cid which lacks the trailing cpu_acquire/ - * cpu_release; those stay zero from kzalloc. - */ - if (cmd->is_cid_type) { - sch->ops_cid = *cmd->ops_cid; - sch->is_cid_type = true; - } else { - sch->ops = *cmd->ops; - } - - rcu_assign_pointer(ops->priv, sch); - - sch->kobj.kset = scx_kset; - INIT_LIST_HEAD(&sch->all); - -#ifdef CONFIG_EXT_SUB_SCHED - char *buf = kzalloc(PATH_MAX, GFP_KERNEL); - if (!buf) { - ret = -ENOMEM; - goto err_free_lb_resched; - } - cgroup_path(cgrp, buf, PATH_MAX); - sch->cgrp_path = kstrdup(buf, GFP_KERNEL); - kfree(buf); - if (!sch->cgrp_path) { - ret = -ENOMEM; - goto err_free_lb_resched; - } - - sch->cgrp = cgrp; - INIT_LIST_HEAD(&sch->children); - INIT_LIST_HEAD(&sch->sibling); - - if (parent) - ret = kobject_init_and_add(&sch->kobj, &scx_ktype, - &parent->sub_kset->kobj, - "sub-%llu", cgroup_id(cgrp)); - else - ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root"); - - if (ret < 0) { - RCU_INIT_POINTER(ops->priv, NULL); - kobject_put(&sch->kobj); - return ERR_PTR(ret); - } - - if (ops->sub_attach) { - sch->sub_kset = kset_create_and_add("sub", NULL, &sch->kobj); - if (!sch->sub_kset) { - RCU_INIT_POINTER(ops->priv, NULL); - kobject_put(&sch->kobj); - return ERR_PTR(-ENOMEM); - } - } -#else /* CONFIG_EXT_SUB_SCHED */ - ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root"); - if (ret < 0) { - RCU_INIT_POINTER(ops->priv, NULL); - kobject_put(&sch->kobj); - return ERR_PTR(ret); - } -#endif /* CONFIG_EXT_SUB_SCHED */ - - /* - * Consume the arena_map ref bpf_scx_reg_cid() took. Defer to here so - * earlier failure paths leave cmd->arena_map set and bpf_scx_reg_cid - * drops the ref. After this point, sch owns the ref and any cleanup - * runs through scx_sched_free_rcu_work() which puts it. - */ - sch->arena_map = cmd->arena_map; - /* BPF arena is only available on MMU && 64BIT */ -#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) - if (sch->arena_map) - sch->arena_kern_base = bpf_arena_map_kern_vm_start(sch->arena_map); -#endif - cmd->arena_map = NULL; - return sch; - -#ifdef CONFIG_EXT_SUB_SCHED -err_free_lb_resched: - RCU_INIT_POINTER(ops->priv, NULL); - free_cpumask_var(sch->bypass_lb_resched_cpumask); -#endif -err_free_lb_cpumask: - free_cpumask_var(sch->bypass_lb_donee_cpumask); -err_stop_helper: - kthread_destroy_worker(sch->helper); -err_free_pcpu: - for_each_possible_cpu(cpu) { - if (cpu == bypass_fail_cpu) - break; - exit_dsq(bypass_dsq(sch, cpu)); - } - free_percpu(sch->pcpu); -err_free_pnode: - for_each_node_state(node, N_POSSIBLE) - free_pnode(sch->pnode[node]); - kfree(sch->pnode); -err_free_hash: - rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); -err_free_ei: - free_exit_info(sch->exit_info); -err_free_sch: - kfree(sch); -err_put_cgrp: -#ifdef CONFIG_EXT_SUB_SCHED - cgroup_put(cgrp); -#endif - return ERR_PTR(ret); -} - -static int check_hotplug_seq(struct scx_sched *sch, - const struct sched_ext_ops *ops) -{ - unsigned long long global_hotplug_seq; - - /* - * If a hotplug event has occurred between when a scheduler was - * initialized, and when we were able to attach, exit and notify user - * space about it. - */ - if (ops->hotplug_seq) { - global_hotplug_seq = atomic_long_read(&scx_hotplug_seq); - if (ops->hotplug_seq != global_hotplug_seq) { - scx_exit(sch, SCX_EXIT_UNREG_KERN, - SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, - "expected hotplug seq %llu did not match actual %llu", - ops->hotplug_seq, global_hotplug_seq); - return -EBUSY; - } - } - - return 0; -} - -static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) -{ - /* - * It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the - * ops.enqueue() callback isn't implemented. - */ - if ((ops->flags & SCX_OPS_ENQ_LAST) && !ops->enqueue) { - scx_error(sch, "SCX_OPS_ENQ_LAST requires ops.enqueue() to be implemented"); - return -EINVAL; - } - - /* - * SCX_OPS_TID_TO_TASK is enabled by the root scheduler. A sub-sched - * may set it to declare a dependency; reject if the root hasn't - * enabled it. - */ - if ((ops->flags & SCX_OPS_TID_TO_TASK) && scx_parent(sch) && - !(scx_root->ops.flags & SCX_OPS_TID_TO_TASK)) { - scx_error(sch, "SCX_OPS_TID_TO_TASK requires root scheduler to enable it"); - return -EINVAL; - } - - /* - * SCX_OPS_BUILTIN_IDLE_PER_NODE requires built-in CPU idle - * selection policy to be enabled. - */ - if ((ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE) && - (ops->update_idle && !(ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE))) { - scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE requires CPU idle selection enabled"); - return -EINVAL; - } - - /* - * cid-form's struct is shorter and doesn't include the cpu_acquire / - * cpu_release tail; reading those fields off a cid-form @ops would - * run past the BPF allocation. Skip for cid-form. - */ - if (!sch->is_cid_type && (ops->cpu_acquire || ops->cpu_release)) - pr_warn("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n"); - - /* - * Sub-scheduler support is tied to the cid-form struct_ops. A sub-sched - * attaches through a cid-form-only interface (sub_attach/sub_detach), - * and a root that accepts sub-scheds must expose cid-form state to - * them. Reject cpu-form schedulers on either side. - */ - if (!sch->is_cid_type) { - if (scx_parent(sch)) { - scx_error(sch, "sub-sched requires cid-form struct_ops"); - return -EINVAL; - } - if (ops->sub_attach || ops->sub_detach) { - scx_error(sch, "sub_attach/sub_detach requires cid-form struct_ops"); - return -EINVAL; - } - } - - return 0; -} - -static void scx_root_enable_workfn(struct kthread_work *work) -{ - struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work); - struct sched_ext_ops *ops = cmd->ops; - struct cgroup *cgrp = root_cgroup(); - struct scx_sched *sch; - struct scx_task_iter sti; - struct task_struct *p; - int i, cpu, ret; - - mutex_lock(&scx_enable_mutex); - - if (scx_enable_state() != SCX_DISABLED) { - ret = -EBUSY; - goto err_unlock; - } - - /* - * @ops->priv binds @ops to its scx_sched instance. It is set here by - * scx_alloc_and_add_sched() and cleared at the tail of bpf_scx_unreg(), - * which runs after scx_root_disable() has dropped scx_enable_mutex. If - * it's still non-NULL here, a previous attachment on @ops has not - * finished tearing down; proceeding would let the in-flight unreg's - * RCU_INIT_POINTER(NULL) clobber the @ops->priv we are about to assign. - */ - if (rcu_access_pointer(ops->priv)) { - ret = -EBUSY; - goto err_unlock; - } - - ret = alloc_kick_syncs(); - if (ret) - goto err_unlock; - - if (ops->flags & SCX_OPS_TID_TO_TASK) { - ret = rhashtable_init(&scx_tid_hash, &scx_tid_hash_params); - if (ret) - goto err_free_ksyncs; - } - -#ifdef CONFIG_EXT_SUB_SCHED - cgroup_get(cgrp); -#endif - sch = scx_alloc_and_add_sched(cmd, cgrp, NULL); - if (IS_ERR(sch)) { - ret = PTR_ERR(sch); - goto err_free_tid_hash; - } - - if (sch->is_cid_type) - static_branch_enable(&__scx_is_cid_type); - - /* - * Transition to ENABLING and clear exit info to arm the disable path. - * Failure triggers full disabling from here on. - */ - WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED); - WARN_ON_ONCE(scx_root); - - atomic_long_set(&scx_nr_rejected, 0); - - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); - - rq->scx.local_dsq.sched = sch; - rq->scx.cpuperf_target = SCX_CPUPERF_ONE; - } - - /* - * Keep CPUs stable during enable so that the BPF scheduler can track - * online CPUs by watching ->on/offline_cpu() after ->init(). - */ - cpus_read_lock(); - - /* - * Build the cid mapping before publishing scx_root. The cid kfuncs - * dereference the cid arrays unconditionally once scx_prog_sched() - * returns non-NULL; the rcu_assign_pointer() below pairs with their - * rcu_dereference() to make the populated arrays visible. - */ - ret = scx_cid_init(sch); - if (ret) { - cpus_read_unlock(); - goto err_disable; - } - - /* - * Make the scheduler instance visible. Must be inside cpus_read_lock(). - * See handle_hotplug(). - */ - rcu_assign_pointer(scx_root, sch); - - ret = scx_link_sched(sch); - if (ret) { - cpus_read_unlock(); - goto err_disable; - } - - scx_idle_enable(ops); - - if (sch->ops.init) { - ret = SCX_CALL_OP_RET(sch, init, NULL); - if (ret) { - ret = ops_sanitize_err(sch, "init", ret); - cpus_read_unlock(); - scx_error(sch, "ops.init() failed (%d)", ret); - goto err_disable; - } - sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; - } - - ret = scx_arena_pool_init(sch); - if (ret) { - cpus_read_unlock(); - goto err_disable; - } - - ret = scx_set_cmask_scratch_alloc(sch); - if (ret) { - cpus_read_unlock(); - goto err_disable; - } - - for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++) - if (((void (**)(void))ops)[i]) - set_bit(i, sch->has_op); - - ret = check_hotplug_seq(sch, ops); - if (ret) { - cpus_read_unlock(); - goto err_disable; - } - scx_idle_update_selcpu_topology(ops); - - cpus_read_unlock(); - - ret = validate_ops(sch, ops); - if (ret) - goto err_disable; - - /* - * Attach the ext_server bandwidth reservation before anything is - * committed so that we can fail the enable if the root domain cannot - * accommodate it. The matching fair_server detach is deferred to the - * tail of this function, after the switch is fully committed and can no - * longer fail. - * - * On failure, err_disable funnels into scx_root_disable() which - * detaches ext_server, so partially-attached state is cleaned up - * automatically. - */ - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); - - scoped_guard(rq_lock_irqsave, rq) { - update_rq_clock(rq); - ret = dl_server_attach_bw(&rq->ext_server); - } - if (ret) { - pr_warn("sched_ext: failed to attach ext_server on CPU %d (%d)\n", - cpu, ret); - goto err_disable; - } - } - - /* - * Once __scx_enabled is set, %current can be switched to SCX anytime. - * This can lead to stalls as some BPF schedulers (e.g. userspace - * scheduling) may not function correctly before all tasks are switched. - * Init in bypass mode to guarantee forward progress. - */ - scx_bypass(sch, true); - - for (i = SCX_OPI_NORMAL_BEGIN; i < SCX_OPI_NORMAL_END; i++) - if (((void (**)(void))ops)[i]) - set_bit(i, sch->has_op); - - if (sch->ops.cpu_acquire || sch->ops.cpu_release) - sch->ops.flags |= SCX_OPS_HAS_CPU_PREEMPT; - - /* - * Lock out forks, cgroup on/offlining and moves before opening the - * floodgate so that they don't wander into the operations prematurely. - */ - percpu_down_write(&scx_fork_rwsem); - - WARN_ON_ONCE(scx_init_task_enabled); - scx_init_task_enabled = true; - - /* flip under fork_rwsem; the iter below covers existing tasks */ - if (ops->flags & SCX_OPS_TID_TO_TASK) - static_branch_enable(&__scx_tid_to_task_enabled); - - /* - * Enable ops for every task. Fork is excluded by scx_fork_rwsem - * preventing new tasks from being added. No need to exclude tasks - * leaving as sched_ext_free() can handle both prepped and enabled - * tasks. Prep all tasks first and then enable them with preemption - * disabled. - * - * All cgroups should be initialized before scx_init_task() so that the - * BPF scheduler can reliably track each task's cgroup membership from - * scx_init_task(). Lock out cgroup on/offlining and task migrations - * while tasks are being initialized so that scx_cgroup_can_attach() - * never sees uninitialized tasks. - */ - scx_cgroup_lock(); - set_cgroup_sched(sch_cgroup(sch), sch); - ret = scx_cgroup_init(sch); - if (ret) - goto err_disable_unlock_all; - - scx_task_iter_start(&sti, NULL); - while ((p = scx_task_iter_next_locked(&sti))) { - /* - * @p is in scx_tasks under scx_tasks_lock, and SCX_TASK_DEAD - * tasks are filtered by scx_task_iter_next_locked(). - * sched_ext_dead() removes @p from scx_tasks under the same - * lock before put_task_struct_rcu_user() runs, so @p->usage - * is guaranteed > 0 here. - */ - get_task_struct(p); - - /* - * Set %INIT_BEGIN under the iter's rq lock so that a concurrent - * sched_ext_dead() does not call ops.exit_task() on @p while - * ops.init_task() is running. If sched_ext_dead() runs before - * this store, it has already removed @p from scx_tasks and the - * iter won't visit @p; if it runs after, it observes - * %INIT_BEGIN and transitions to %DEAD without calling ops, - * leaving the post-init recheck below to unwind. - */ - scx_set_task_state(p, SCX_TASK_INIT_BEGIN); - scx_task_iter_unlock(&sti); - - ret = __scx_init_task(sch, p, false); - - scx_task_iter_relock(&sti, p); - - if (unlikely(ret)) { - if (scx_get_task_state(p) != SCX_TASK_DEAD) - scx_set_task_state(p, SCX_TASK_NONE); - scx_task_iter_stop(&sti); - scx_error(sch, "ops.init_task() failed (%d) for %s[%d]", - ret, p->comm, p->pid); - put_task_struct(p); - goto err_disable_unlock_all; - } - - if (scx_get_task_state(p) == SCX_TASK_DEAD) { - /* - * sched_ext_dead() observed %INIT_BEGIN and set %DEAD. - * ops.exit_task() is owed to the sched __scx_init_task() - * ran against; call it now. - */ - scx_sub_init_cancel_task(sch, p); - } else { - scx_set_task_state(p, SCX_TASK_INIT); - scx_set_task_sched(p, sch); - scx_set_task_state(p, SCX_TASK_READY); - } - - /* - * Insert into the tid hash. scx_tasks_lock is held by the iter; - * list_empty() guards against sched_ext_dead() having taken @p - * off the list while init ran unlocked. - */ - if (scx_tid_to_task_enabled() && !list_empty(&p->scx.tasks_node)) - scx_tid_hash_insert(p); - - put_task_struct(p); - } - scx_task_iter_stop(&sti); - scx_cgroup_unlock(); - percpu_up_write(&scx_fork_rwsem); - - /* - * All tasks are READY. It's safe to turn on scx_enabled() and switch - * all eligible tasks. - */ - WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL)); - static_branch_enable(&__scx_enabled); - - /* - * We're fully committed and can't fail. The task READY -> ENABLED - * transitions here are synchronized against sched_ext_free() through - * scx_tasks_lock. - */ - percpu_down_write(&scx_fork_rwsem); - scx_task_iter_start(&sti, NULL); - while ((p = scx_task_iter_next_locked(&sti))) { - unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE; - const struct sched_class *old_class = p->sched_class; - const struct sched_class *new_class = scx_setscheduler_class(p); - - if (scx_get_task_state(p) != SCX_TASK_READY) - continue; - - if (old_class != new_class) - queue_flags |= DEQUEUE_CLASS; - - scoped_guard (sched_change, p, queue_flags) { - p->scx.slice = READ_ONCE(sch->slice_dfl); - p->sched_class = new_class; - } - } - scx_task_iter_stop(&sti); - percpu_up_write(&scx_fork_rwsem); - - scx_bypass(sch, false); - - if (!scx_tryset_enable_state(SCX_ENABLED, SCX_ENABLING)) { - WARN_ON_ONCE(atomic_read(&sch->exit_kind) == SCX_EXIT_NONE); - goto err_disable; - } - - if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL)) - static_branch_enable(&__scx_switched_all); - - /* - * Detach the fair_server bandwidth reservation now that the switch - * is fully committed. In full mode (!SCX_OPS_SWITCH_PARTIAL) no - * task will ever run in the fair class, so give that bandwidth - * back to the RT class. The matching ext_server attach already - * happened earlier; this only releases bandwidth and cannot fail. - * - * In partial mode keep fair_server attached. - */ - if (scx_switched_all()) { - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); - - guard(rq_lock_irqsave)(rq); - update_rq_clock(rq); - dl_server_detach_bw(&rq->fair_server); - } - } - - pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n", - sch->ops.name, scx_switched_all() ? "" : " (partial)"); - kobject_uevent(&sch->kobj, KOBJ_ADD); - mutex_unlock(&scx_enable_mutex); - - atomic_long_inc(&scx_enable_seq); - - cmd->ret = 0; - return; - -err_free_tid_hash: - if (ops->flags & SCX_OPS_TID_TO_TASK) - rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL); -err_free_ksyncs: - free_kick_syncs(); -err_unlock: - mutex_unlock(&scx_enable_mutex); - cmd->ret = ret; - return; - -err_disable_unlock_all: - scx_cgroup_unlock(); - percpu_up_write(&scx_fork_rwsem); - /* we'll soon enter disable path, keep bypass on */ -err_disable: - mutex_unlock(&scx_enable_mutex); - /* - * Returning an error code here would not pass all the error information - * to userspace. Record errno using scx_error() for cases scx_error() - * wasn't already invoked and exit indicating success so that the error - * is notified through ops.exit() with all the details. - * - * Flush scx_disable_work to ensure that error is reported before init - * completion. sch's base reference will be put by bpf_scx_unreg(). - */ - scx_error(sch, "scx_root_enable() failed (%d)", ret); - scx_flush_disable_work(sch); - cmd->ret = 0; -} - -#ifdef CONFIG_EXT_SUB_SCHED -/* verify that a scheduler can be attached to @cgrp and return the parent */ -static struct scx_sched *find_parent_sched(struct cgroup *cgrp) -{ - struct scx_sched *parent = cgrp->scx_sched; - struct scx_sched *pos; - - lockdep_assert_held(&scx_sched_lock); - - /* can't attach twice to the same cgroup */ - if (parent->cgrp == cgrp) - return ERR_PTR(-EBUSY); - - /* does $parent allow sub-scheds? */ - if (!parent->ops.sub_attach) - return ERR_PTR(-EOPNOTSUPP); - - /* can't insert between $parent and its exiting children */ - list_for_each_entry(pos, &parent->children, sibling) - if (cgroup_is_descendant(pos->cgrp, cgrp)) - return ERR_PTR(-EBUSY); - - return parent; -} - -static bool assert_task_ready_or_enabled(struct task_struct *p) -{ - u32 state = scx_get_task_state(p); - - switch (state) { - case SCX_TASK_READY: - case SCX_TASK_ENABLED: - return true; - default: - WARN_ONCE(true, "sched_ext: Invalid task state %d for %s[%d] during enabling sub sched", - state, p->comm, p->pid); - return false; - } -} - -static void scx_sub_enable_workfn(struct kthread_work *work) -{ - struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work); - struct sched_ext_ops *ops = cmd->ops; - struct cgroup *cgrp; - struct scx_sched *parent, *sch; - struct scx_task_iter sti; - struct task_struct *p; - s32 i, ret; - - mutex_lock(&scx_enable_mutex); - - if (!scx_enabled()) { - ret = -ENODEV; - goto out_unlock; - } - - /* See scx_root_enable_workfn() for the @ops->priv check. */ - if (rcu_access_pointer(ops->priv)) { - ret = -EBUSY; - goto out_unlock; - } - - cgrp = cgroup_get_from_id(ops->sub_cgroup_id); - if (IS_ERR(cgrp)) { - ret = PTR_ERR(cgrp); - goto out_unlock; - } - - raw_spin_lock_irq(&scx_sched_lock); - parent = find_parent_sched(cgrp); - if (IS_ERR(parent)) { - raw_spin_unlock_irq(&scx_sched_lock); - ret = PTR_ERR(parent); - goto out_put_cgrp; - } - kobject_get(&parent->kobj); - raw_spin_unlock_irq(&scx_sched_lock); - - /* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */ - sch = scx_alloc_and_add_sched(cmd, cgrp, parent); - kobject_put(&parent->kobj); - if (IS_ERR(sch)) { - ret = PTR_ERR(sch); - goto out_unlock; - } - - ret = scx_link_sched(sch); - if (ret) - goto err_disable; - - if (sch->level >= SCX_SUB_MAX_DEPTH) { - scx_error(sch, "max nesting depth %d violated", - SCX_SUB_MAX_DEPTH); - goto err_disable; - } - - if (sch->ops.init) { - ret = SCX_CALL_OP_RET(sch, init, NULL); - if (ret) { - ret = ops_sanitize_err(sch, "init", ret); - scx_error(sch, "ops.init() failed (%d)", ret); - goto err_disable; - } - sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; - } - - ret = scx_arena_pool_init(sch); - if (ret) - goto err_disable; - - ret = scx_set_cmask_scratch_alloc(sch); - if (ret) - goto err_disable; - - if (validate_ops(sch, ops)) - goto err_disable; - - struct scx_sub_attach_args sub_attach_args = { - .ops = &sch->ops, - .cgroup_path = sch->cgrp_path, - }; - - ret = SCX_CALL_OP_RET(parent, sub_attach, NULL, - &sub_attach_args); - if (ret) { - ret = ops_sanitize_err(sch, "sub_attach", ret); - scx_error(sch, "parent rejected (%d)", ret); - goto err_disable; - } - sch->sub_attached = true; - - scx_bypass(sch, true); - - for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) - if (((void (**)(void))ops)[i]) - set_bit(i, sch->has_op); - - percpu_down_write(&scx_fork_rwsem); - scx_cgroup_lock(); - - /* - * Set cgroup->scx_sched's and check CSS_ONLINE. Either we see - * !CSS_ONLINE or scx_cgroup_lifetime_notify() sees and shoots us down. - */ - set_cgroup_sched(sch_cgroup(sch), sch); - if (!(cgrp->self.flags & CSS_ONLINE)) { - scx_error(sch, "cgroup is not online"); - goto err_unlock_and_disable; - } - - /* - * Initialize tasks for the new child $sch without exiting them for - * $parent so that the tasks can always be reverted back to $parent - * sched on child init failure. - */ - WARN_ON_ONCE(scx_enabling_sub_sched); - scx_enabling_sub_sched = sch; - - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - struct rq *rq; - struct rq_flags rf; - - /* - * Task iteration may visit the same task twice when racing - * against exiting. Use %SCX_TASK_SUB_INIT to mark tasks which - * finished __scx_init_task() and skip if set. - * - * A task may exit and get freed between __scx_init_task() - * completion and scx_enable_task(). In such cases, - * scx_disable_and_exit_task() must exit the task for both the - * parent and child scheds. - */ - if (p->scx.flags & SCX_TASK_SUB_INIT) - continue; - - /* @p is pinned by the iter; see scx_sub_disable() */ - get_task_struct(p); - - if (!assert_task_ready_or_enabled(p)) { - ret = -EINVAL; - goto abort; - } - - scx_task_iter_unlock(&sti); - - /* - * As $p is still on $parent, it can't be transitioned to INIT. - * Let's worry about task state later. Use __scx_init_task(). - */ - ret = __scx_init_task(sch, p, false); - if (ret) - goto abort; - - rq = task_rq_lock(p, &rf); - - if (scx_get_task_state(p) == SCX_TASK_DEAD) { - /* - * sched_ext_dead() raced us between __scx_init_task() - * and this rq lock and ran exit_task() on $parent (the - * sched @p was on at that point), not on @sch. @sch's - * just-completed init is owed an exit_task() and we - * issue it here. - */ - scx_sub_init_cancel_task(sch, p); - task_rq_unlock(rq, p, &rf); - put_task_struct(p); - continue; - } - - p->scx.flags |= SCX_TASK_SUB_INIT; - task_rq_unlock(rq, p, &rf); - - put_task_struct(p); - } - scx_task_iter_stop(&sti); - - /* - * All tasks are prepped. Disable/exit tasks for $parent and enable for - * the new @sch. - */ - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - /* - * Use clearing of %SCX_TASK_SUB_INIT to detect and skip - * duplicate iterations. - */ - if (!(p->scx.flags & SCX_TASK_SUB_INIT)) - continue; - - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - /* - * $p must be either READY or ENABLED. If ENABLED, - * __scx_disabled_and_exit_task() first disables and - * makes it READY. However, after exiting $p, it will - * leave $p as READY. - */ - assert_task_ready_or_enabled(p); - __scx_disable_and_exit_task(parent, p); - - /* - * $p is now only initialized for @sch and READY, which - * is what we want. Assign it to @sch and enable. - */ - scx_set_task_sched(p, sch); - scx_enable_task(sch, p); - - p->scx.flags &= ~SCX_TASK_SUB_INIT; - } - } - scx_task_iter_stop(&sti); - - scx_enabling_sub_sched = NULL; - - scx_cgroup_unlock(); - percpu_up_write(&scx_fork_rwsem); - - scx_bypass(sch, false); - - pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name); - kobject_uevent(&sch->kobj, KOBJ_ADD); - ret = 0; - goto out_unlock; - -out_put_cgrp: - cgroup_put(cgrp); -out_unlock: - mutex_unlock(&scx_enable_mutex); - cmd->ret = ret; - return; - -abort: - put_task_struct(p); - scx_task_iter_stop(&sti); - - /* - * Undo __scx_init_task() for tasks we marked. scx_enable_task() never - * ran for @sch on them, so calling scx_disable_task() here would invoke - * ops.disable() without a matching ops.enable(). scx_enabling_sub_sched - * must stay set until SUB_INIT is cleared from every marked task - - * scx_disable_and_exit_task() reads it when a task exits concurrently. - */ - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - if (p->scx.flags & SCX_TASK_SUB_INIT) { - scx_sub_init_cancel_task(sch, p); - p->scx.flags &= ~SCX_TASK_SUB_INIT; - } - } - scx_task_iter_stop(&sti); - scx_enabling_sub_sched = NULL; -err_unlock_and_disable: - /* we'll soon enter disable path, keep bypass on */ - scx_cgroup_unlock(); - percpu_up_write(&scx_fork_rwsem); -err_disable: - mutex_unlock(&scx_enable_mutex); - scx_flush_disable_work(sch); - cmd->ret = 0; -} - -static s32 scx_cgroup_lifetime_notify(struct notifier_block *nb, - unsigned long action, void *data) -{ - struct cgroup *cgrp = data; - struct cgroup *parent = cgroup_parent(cgrp); - - if (!cgroup_on_dfl(cgrp)) - return NOTIFY_OK; - - switch (action) { - case CGROUP_LIFETIME_ONLINE: - /* inherit ->scx_sched from $parent */ - if (parent) - rcu_assign_pointer(cgrp->scx_sched, parent->scx_sched); - break; - case CGROUP_LIFETIME_OFFLINE: - /* if there is a sched attached, shoot it down */ - if (cgrp->scx_sched && cgrp->scx_sched->cgrp == cgrp) - scx_exit(cgrp->scx_sched, SCX_EXIT_UNREG_KERN, - SCX_ECODE_RSN_CGROUP_OFFLINE, - "cgroup %llu going offline", cgroup_id(cgrp)); - break; - } - - return NOTIFY_OK; -} - -static struct notifier_block scx_cgroup_lifetime_nb = { - .notifier_call = scx_cgroup_lifetime_notify, -}; - -static s32 __init scx_cgroup_lifetime_notifier_init(void) -{ - return blocking_notifier_chain_register(&cgroup_lifetime_notifier, - &scx_cgroup_lifetime_nb); -} -core_initcall(scx_cgroup_lifetime_notifier_init); -#endif /* CONFIG_EXT_SUB_SCHED */ - -static s32 scx_enable(struct scx_enable_cmd *cmd, struct bpf_link *link) -{ - static struct kthread_worker *helper; - static DEFINE_MUTEX(helper_mutex); - - if (housekeeping_enabled(HK_TYPE_DOMAIN_BOOT)) { - pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation\n"); - return -EINVAL; - } - - if (!READ_ONCE(helper)) { - mutex_lock(&helper_mutex); - if (!helper) { - struct kthread_worker *w = - kthread_run_worker(0, "scx_enable_helper"); - if (IS_ERR_OR_NULL(w)) { - mutex_unlock(&helper_mutex); - return -ENOMEM; - } - sched_set_fifo(w->task); - WRITE_ONCE(helper, w); - } - mutex_unlock(&helper_mutex); - } - -#ifdef CONFIG_EXT_SUB_SCHED - if (cmd->ops->sub_cgroup_id > 1) - kthread_init_work(&cmd->work, scx_sub_enable_workfn); - else -#endif /* CONFIG_EXT_SUB_SCHED */ - kthread_init_work(&cmd->work, scx_root_enable_workfn); - - kthread_queue_work(READ_ONCE(helper), &cmd->work); - kthread_flush_work(&cmd->work); - return cmd->ret; -} - - -/******************************************************************************** - * bpf_struct_ops plumbing. - */ -#include -#include -#include - -static const struct btf_type *task_struct_type; - -static bool bpf_scx_is_valid_access(int off, int size, - enum bpf_access_type type, - const struct bpf_prog *prog, - struct bpf_insn_access_aux *info) -{ - if (type != BPF_READ) - return false; - if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) - return false; - if (off % size != 0) - return false; - - return btf_ctx_access(off, size, type, prog, info); -} - -static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, - const struct bpf_reg_state *reg, int off, - int size) -{ - const struct btf_type *t; - - t = btf_type_by_id(reg->btf, reg->btf_id); - if (t == task_struct_type) { - /* - * COMPAT: Will be removed in v6.23. - */ - if ((off >= offsetof(struct task_struct, scx.slice) && - off + size <= offsetofend(struct task_struct, scx.slice)) || - (off >= offsetof(struct task_struct, scx.dsq_vtime) && - off + size <= offsetofend(struct task_struct, scx.dsq_vtime))) { - pr_warn("sched_ext: Writing directly to p->scx.slice/dsq_vtime is deprecated, use scx_bpf_task_set_slice/dsq_vtime()"); - return SCALAR_VALUE; - } - - if (off >= offsetof(struct task_struct, scx.disallow) && - off + size <= offsetofend(struct task_struct, scx.disallow)) - return SCALAR_VALUE; - } - - return -EACCES; -} - -static const struct bpf_verifier_ops bpf_scx_verifier_ops = { - .get_func_proto = bpf_base_func_proto, - .is_valid_access = bpf_scx_is_valid_access, - .btf_struct_access = bpf_scx_btf_struct_access, -}; - -static int bpf_scx_init_member(const struct btf_type *t, - const struct btf_member *member, - void *kdata, const void *udata) -{ - const struct sched_ext_ops *uops = udata; - struct sched_ext_ops *ops = kdata; - u32 moff = __btf_member_bit_offset(t, member) / 8; - int ret; - - switch (moff) { - case offsetof(struct sched_ext_ops, dispatch_max_batch): - if (*(u32 *)(udata + moff) > INT_MAX) - return -E2BIG; - ops->dispatch_max_batch = *(u32 *)(udata + moff); - return 1; - case offsetof(struct sched_ext_ops, flags): - if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) - return -EINVAL; - ops->flags = *(u64 *)(udata + moff); - return 1; - case offsetof(struct sched_ext_ops, name): - ret = bpf_obj_name_cpy(ops->name, uops->name, - sizeof(ops->name)); - if (ret < 0) - return ret; - if (ret == 0) - return -EINVAL; - return 1; - case offsetof(struct sched_ext_ops, timeout_ms): - if (msecs_to_jiffies(*(u32 *)(udata + moff)) > - SCX_WATCHDOG_MAX_TIMEOUT) - return -E2BIG; - ops->timeout_ms = *(u32 *)(udata + moff); - return 1; - case offsetof(struct sched_ext_ops, exit_dump_len): - ops->exit_dump_len = - *(u32 *)(udata + moff) ?: SCX_EXIT_DUMP_DFL_LEN; - return 1; - case offsetof(struct sched_ext_ops, hotplug_seq): - ops->hotplug_seq = *(u64 *)(udata + moff); - return 1; -#ifdef CONFIG_EXT_SUB_SCHED - case offsetof(struct sched_ext_ops, sub_cgroup_id): - ops->sub_cgroup_id = *(u64 *)(udata + moff); - return 1; -#endif /* CONFIG_EXT_SUB_SCHED */ - } - - return 0; -} - -#ifdef CONFIG_EXT_SUB_SCHED -static void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog) -{ - struct scx_sched *sch; - - guard(rcu)(); - sch = scx_prog_sched(prog->aux); - if (unlikely(!sch)) - return; - - scx_error(sch, "dispatch recursion detected"); -} -#endif /* CONFIG_EXT_SUB_SCHED */ - -static int bpf_scx_check_member(const struct btf_type *t, - const struct btf_member *member, - const struct bpf_prog *prog) -{ - u32 moff = __btf_member_bit_offset(t, member) / 8; - - switch (moff) { - case offsetof(struct sched_ext_ops, init_task): -#ifdef CONFIG_EXT_GROUP_SCHED - case offsetof(struct sched_ext_ops, cgroup_init): - case offsetof(struct sched_ext_ops, cgroup_exit): - case offsetof(struct sched_ext_ops, cgroup_prep_move): -#endif - case offsetof(struct sched_ext_ops, cpu_online): - case offsetof(struct sched_ext_ops, cpu_offline): - case offsetof(struct sched_ext_ops, init): - case offsetof(struct sched_ext_ops, exit): - case offsetof(struct sched_ext_ops, sub_attach): - case offsetof(struct sched_ext_ops, sub_detach): - break; - default: - if (prog->sleepable) - return -EINVAL; - } - -#ifdef CONFIG_EXT_SUB_SCHED - /* - * Enable private stack for operations that can nest along the - * hierarchy. - * - * XXX - Ideally, we should only do this for scheds that allow - * sub-scheds and sub-scheds themselves but I don't know how to access - * struct_ops from here. - */ - switch (moff) { - case offsetof(struct sched_ext_ops, dispatch): - prog->aux->priv_stack_requested = true; - prog->aux->recursion_detected = scx_pstack_recursion_on_dispatch; - } -#endif /* CONFIG_EXT_SUB_SCHED */ - - return 0; -} - -static int bpf_scx_reg(void *kdata, struct bpf_link *link) -{ - struct scx_enable_cmd cmd = { .ops = kdata }; - - return scx_enable(&cmd, link); -} - -struct scx_arena_scan { - struct bpf_map *arena; - int err; -}; - -/* - * The verifier enforces one arena per BPF program, so each struct_ops - * member prog contributes at most one arena via bpf_prog_arena(). - * Require all non-NULL contributions to match. - */ -static int scx_arena_scan_prog(struct bpf_prog *prog, void *data) -{ - struct scx_arena_scan *s = data; - struct bpf_map *arena = NULL; - - /* arena.o, which defines these, is built only on MMU && 64BIT */ -#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) - arena = bpf_prog_arena(prog); -#endif - if (!arena) - return 0; - if (s->arena && s->arena != arena) { - s->err = -EINVAL; - return 1; - } - s->arena = arena; - return 0; -} - -static int bpf_scx_reg_cid(void *kdata, struct bpf_link *link) -{ - struct scx_enable_cmd cmd = { .ops_cid = kdata, .is_cid_type = true }; - struct scx_arena_scan scan = {}; - int ret; - - bpf_struct_ops_for_each_prog(kdata, scx_arena_scan_prog, &scan); - if (scan.err) { - pr_err("sched_ext: cid-form scheduler uses multiple arena maps\n"); - return scan.err; - } - if (!scan.arena) { - pr_err("sched_ext: cid-form scheduler must use a BPF arena map\n"); - return -EINVAL; - } - - bpf_map_inc(scan.arena); - cmd.arena_map = scan.arena; - ret = scx_enable(&cmd, link); - if (cmd.arena_map) /* not consumed by scx_alloc_and_add_sched() */ - bpf_map_put(cmd.arena_map); - return ret; -} - -static void bpf_scx_unreg(void *kdata, struct bpf_link *link) -{ - struct sched_ext_ops *ops = kdata; - struct scx_sched *sch = rcu_dereference_protected(ops->priv, true); - - scx_disable(sch, SCX_EXIT_UNREG); - scx_flush_disable_work(sch); - RCU_INIT_POINTER(ops->priv, NULL); - kobject_put(&sch->kobj); -} - -static int bpf_scx_init(struct btf *btf) -{ - task_struct_type = btf_type_by_id(btf, btf_tracing_ids[BTF_TRACING_TYPE_TASK]); - - return 0; -} - -static int bpf_scx_update(void *kdata, void *old_kdata, struct bpf_link *link) -{ - /* - * sched_ext does not support updating the actively-loaded BPF - * scheduler, as registering a BPF scheduler can always fail if the - * scheduler returns an error code for e.g. ops.init(), ops.init_task(), - * etc. Similarly, we can always race with unregistration happening - * elsewhere, such as with sysrq. - */ - return -EOPNOTSUPP; -} - -static int bpf_scx_validate(void *kdata) -{ - return 0; -} - -static s32 sched_ext_ops__select_cpu(struct task_struct *p, s32 prev_cpu, u64 wake_flags) { return -EINVAL; } -static void sched_ext_ops__enqueue(struct task_struct *p, u64 enq_flags) {} -static void sched_ext_ops__dequeue(struct task_struct *p, u64 enq_flags) {} -static void sched_ext_ops__dispatch(s32 prev_cpu, struct task_struct *prev__nullable) {} -static void sched_ext_ops__tick(struct task_struct *p) {} -static void sched_ext_ops__runnable(struct task_struct *p, u64 enq_flags) {} -static void sched_ext_ops__running(struct task_struct *p) {} -static void sched_ext_ops__stopping(struct task_struct *p, bool runnable) {} -static void sched_ext_ops__quiescent(struct task_struct *p, u64 deq_flags) {} -static bool sched_ext_ops__yield(struct task_struct *from, struct task_struct *to__nullable) { return false; } -static bool sched_ext_ops__core_sched_before(struct task_struct *a, struct task_struct *b) { return false; } -static void sched_ext_ops__set_weight(struct task_struct *p, u32 weight) {} -static void sched_ext_ops__set_cpumask(struct task_struct *p, const struct cpumask *mask) {} -static void sched_ext_ops__update_idle(s32 cpu, bool idle) {} -static void sched_ext_ops__cpu_acquire(s32 cpu, struct scx_cpu_acquire_args *args) {} -static void sched_ext_ops__cpu_release(s32 cpu, struct scx_cpu_release_args *args) {} -static s32 sched_ext_ops__init_task(struct task_struct *p, struct scx_init_task_args *args) { return -EINVAL; } -static void sched_ext_ops__exit_task(struct task_struct *p, struct scx_exit_task_args *args) {} -static void sched_ext_ops__enable(struct task_struct *p) {} -static void sched_ext_ops__disable(struct task_struct *p) {} -#ifdef CONFIG_EXT_GROUP_SCHED -static s32 sched_ext_ops__cgroup_init(struct cgroup *cgrp, struct scx_cgroup_init_args *args) { return -EINVAL; } -static void sched_ext_ops__cgroup_exit(struct cgroup *cgrp) {} -static s32 sched_ext_ops__cgroup_prep_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) { return -EINVAL; } -static void sched_ext_ops__cgroup_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {} -static void sched_ext_ops__cgroup_cancel_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {} -static void sched_ext_ops__cgroup_set_weight(struct cgroup *cgrp, u32 weight) {} -static void sched_ext_ops__cgroup_set_bandwidth(struct cgroup *cgrp, u64 period_us, u64 quota_us, u64 burst_us) {} -static void sched_ext_ops__cgroup_set_idle(struct cgroup *cgrp, bool idle) {} -#endif /* CONFIG_EXT_GROUP_SCHED */ -static s32 sched_ext_ops__sub_attach(struct scx_sub_attach_args *args) { return -EINVAL; } -static void sched_ext_ops__sub_detach(struct scx_sub_detach_args *args) {} -static void sched_ext_ops__cpu_online(s32 cpu) {} -static void sched_ext_ops__cpu_offline(s32 cpu) {} -static s32 sched_ext_ops__init(void) { return -EINVAL; } -static void sched_ext_ops__exit(struct scx_exit_info *info) {} -static void sched_ext_ops__dump(struct scx_dump_ctx *ctx) {} -static void sched_ext_ops__dump_cpu(struct scx_dump_ctx *ctx, s32 cpu, bool idle) {} -static void sched_ext_ops__dump_task(struct scx_dump_ctx *ctx, struct task_struct *p) {} - -static struct sched_ext_ops __bpf_ops_sched_ext_ops = { - .select_cpu = sched_ext_ops__select_cpu, - .enqueue = sched_ext_ops__enqueue, - .dequeue = sched_ext_ops__dequeue, - .dispatch = sched_ext_ops__dispatch, - .tick = sched_ext_ops__tick, - .runnable = sched_ext_ops__runnable, - .running = sched_ext_ops__running, - .stopping = sched_ext_ops__stopping, - .quiescent = sched_ext_ops__quiescent, - .yield = sched_ext_ops__yield, - .core_sched_before = sched_ext_ops__core_sched_before, - .set_weight = sched_ext_ops__set_weight, - .set_cpumask = sched_ext_ops__set_cpumask, - .update_idle = sched_ext_ops__update_idle, - .cpu_acquire = sched_ext_ops__cpu_acquire, - .cpu_release = sched_ext_ops__cpu_release, - .init_task = sched_ext_ops__init_task, - .exit_task = sched_ext_ops__exit_task, - .enable = sched_ext_ops__enable, - .disable = sched_ext_ops__disable, -#ifdef CONFIG_EXT_GROUP_SCHED - .cgroup_init = sched_ext_ops__cgroup_init, - .cgroup_exit = sched_ext_ops__cgroup_exit, - .cgroup_prep_move = sched_ext_ops__cgroup_prep_move, - .cgroup_move = sched_ext_ops__cgroup_move, - .cgroup_cancel_move = sched_ext_ops__cgroup_cancel_move, - .cgroup_set_weight = sched_ext_ops__cgroup_set_weight, - .cgroup_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth, - .cgroup_set_idle = sched_ext_ops__cgroup_set_idle, -#endif - .sub_attach = sched_ext_ops__sub_attach, - .sub_detach = sched_ext_ops__sub_detach, - .cpu_online = sched_ext_ops__cpu_online, - .cpu_offline = sched_ext_ops__cpu_offline, - .init = sched_ext_ops__init, - .exit = sched_ext_ops__exit, - .dump = sched_ext_ops__dump, - .dump_cpu = sched_ext_ops__dump_cpu, - .dump_task = sched_ext_ops__dump_task, -}; - -static struct bpf_struct_ops bpf_sched_ext_ops = { - .verifier_ops = &bpf_scx_verifier_ops, - .reg = bpf_scx_reg, - .unreg = bpf_scx_unreg, - .check_member = bpf_scx_check_member, - .init_member = bpf_scx_init_member, - .init = bpf_scx_init, - .update = bpf_scx_update, - .validate = bpf_scx_validate, - .name = "sched_ext_ops", - .owner = THIS_MODULE, - .cfi_stubs = &__bpf_ops_sched_ext_ops -}; - -/* - * cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types - * identical, only param names differ across structs) are reused; only - * set_cmask needs a fresh stub since the second argument type differs. - */ -static void sched_ext_ops_cid__set_cmask(struct task_struct *p, - const struct scx_cmask *cmask) {} - -static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { - .select_cid = sched_ext_ops__select_cpu, - .enqueue = sched_ext_ops__enqueue, - .dequeue = sched_ext_ops__dequeue, - .dispatch = sched_ext_ops__dispatch, - .tick = sched_ext_ops__tick, - .runnable = sched_ext_ops__runnable, - .running = sched_ext_ops__running, - .stopping = sched_ext_ops__stopping, - .quiescent = sched_ext_ops__quiescent, - .yield = sched_ext_ops__yield, - .core_sched_before = sched_ext_ops__core_sched_before, - .set_weight = sched_ext_ops__set_weight, - .set_cmask = sched_ext_ops_cid__set_cmask, - .update_idle = sched_ext_ops__update_idle, - .init_task = sched_ext_ops__init_task, - .exit_task = sched_ext_ops__exit_task, - .enable = sched_ext_ops__enable, - .disable = sched_ext_ops__disable, -#ifdef CONFIG_EXT_GROUP_SCHED - .cgroup_init = sched_ext_ops__cgroup_init, - .cgroup_exit = sched_ext_ops__cgroup_exit, - .cgroup_prep_move = sched_ext_ops__cgroup_prep_move, - .cgroup_move = sched_ext_ops__cgroup_move, - .cgroup_cancel_move = sched_ext_ops__cgroup_cancel_move, - .cgroup_set_weight = sched_ext_ops__cgroup_set_weight, - .cgroup_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth, - .cgroup_set_idle = sched_ext_ops__cgroup_set_idle, -#endif - .sub_attach = sched_ext_ops__sub_attach, - .sub_detach = sched_ext_ops__sub_detach, - .cid_online = sched_ext_ops__cpu_online, - .cid_offline = sched_ext_ops__cpu_offline, - .init = sched_ext_ops__init, - .exit = sched_ext_ops__exit, - .dump = sched_ext_ops__dump, - .dump_cid = sched_ext_ops__dump_cpu, - .dump_task = sched_ext_ops__dump_task, -}; - -/* - * The cid-form struct_ops shares all bpf_struct_ops hooks with the cpu form. - * init_member, check_member, reg, unreg, etc. process kdata as the byte block - * verified to match by the BUILD_BUG_ON checks in scx_init(). - */ -static struct bpf_struct_ops bpf_sched_ext_ops_cid = { - .verifier_ops = &bpf_scx_verifier_ops, - .reg = bpf_scx_reg_cid, - .unreg = bpf_scx_unreg, - .check_member = bpf_scx_check_member, - .init_member = bpf_scx_init_member, - .init = bpf_scx_init, - .update = bpf_scx_update, - .validate = bpf_scx_validate, - .name = "sched_ext_ops_cid", - .owner = THIS_MODULE, - .cfi_stubs = &__bpf_ops_sched_ext_ops_cid -}; - - -/******************************************************************************** - * System integration and init. - */ - -static void sysrq_handle_sched_ext_reset(u8 key) -{ - struct scx_sched *sch; - - sch = rcu_dereference(scx_root); - if (likely(sch)) - scx_disable(sch, SCX_EXIT_SYSRQ); - else - pr_info("sched_ext: BPF schedulers not loaded\n"); -} - -static const struct sysrq_key_op sysrq_sched_ext_reset_op = { - .handler = sysrq_handle_sched_ext_reset, - .help_msg = "reset-sched-ext(S)", - .action_msg = "Disable sched_ext and revert all tasks to CFS", - .enable_mask = SYSRQ_ENABLE_RTNICE, -}; - -static void sysrq_handle_sched_ext_dump(u8 key) -{ - struct scx_exit_info ei = { - .kind = SCX_EXIT_NONE, - .exit_cpu = -1, - .reason = "SysRq-D", - }; - struct scx_sched *sch; - - list_for_each_entry_rcu(sch, &scx_sched_all, all) - scx_dump_state(sch, &ei, 0, false); -} - -static const struct sysrq_key_op sysrq_sched_ext_dump_op = { - .handler = sysrq_handle_sched_ext_dump, - .help_msg = "dump-sched-ext(D)", - .action_msg = "Trigger sched_ext debug dump", - .enable_mask = SYSRQ_ENABLE_RTNICE, -}; - -static bool can_skip_idle_kick(struct rq *rq) -{ - lockdep_assert_rq_held(rq); - - /* - * We can skip idle kicking if @rq is going to go through at least one - * full SCX scheduling cycle before going idle. Just checking whether - * curr is not idle is insufficient because we could be racing - * balance_one() trying to pull the next task from a remote rq, which - * may fail, and @rq may become idle afterwards. - * - * The race window is small and we don't and can't guarantee that @rq is - * only kicked while idle anyway. Skip only when sure. - */ - return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_BALANCE); -} - -static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *ksyncs) -{ - struct rq *rq = cpu_rq(cpu); - struct scx_rq *this_scx = &this_rq->scx; - const struct sched_class *cur_class; - bool should_wait = false; - unsigned long flags; - - raw_spin_rq_lock_irqsave(rq, flags); - cur_class = rq->curr->sched_class; - - /* - * During CPU hotplug, a CPU may depend on kicking itself to make - * forward progress. Allow kicking self regardless of online state. If - * @cpu is running a higher class task, we have no control over @cpu. - * Skip kicking. - */ - if ((cpu_online(cpu) || cpu == cpu_of(this_rq)) && - !sched_class_above(cur_class, &ext_sched_class)) { - if (cpumask_test_cpu(cpu, this_scx->cpus_to_preempt)) { - if (cur_class == &ext_sched_class) - rq->curr->scx.slice = 0; - cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); - } - - if (cpumask_test_cpu(cpu, this_scx->cpus_to_wait)) { - if (cur_class == &ext_sched_class) { - cpumask_set_cpu(cpu, this_scx->cpus_to_sync); - ksyncs[cpu] = rq->scx.kick_sync; - should_wait = true; - } - cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); - } - - resched_curr(rq); - } else { - cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); - cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); - } - - raw_spin_rq_unlock_irqrestore(rq, flags); - - return should_wait; -} - -static void kick_one_cpu_if_idle(s32 cpu, struct rq *this_rq) -{ - struct rq *rq = cpu_rq(cpu); - unsigned long flags; - - raw_spin_rq_lock_irqsave(rq, flags); - - if (!can_skip_idle_kick(rq) && - (cpu_online(cpu) || cpu == cpu_of(this_rq))) - resched_curr(rq); - - raw_spin_rq_unlock_irqrestore(rq, flags); -} - -static void kick_cpus_irq_workfn(struct irq_work *irq_work) -{ - struct rq *this_rq = this_rq(); - struct scx_rq *this_scx = &this_rq->scx; - struct scx_kick_syncs __rcu *ksyncs_pcpu = __this_cpu_read(scx_kick_syncs); - bool should_wait = false; - unsigned long *ksyncs; - s32 cpu; - - /* can race with free_kick_syncs() during scheduler disable */ - if (unlikely(!ksyncs_pcpu)) - return; - - ksyncs = rcu_dereference_bh(ksyncs_pcpu)->syncs; - - for_each_cpu(cpu, this_scx->cpus_to_kick) { - should_wait |= kick_one_cpu(cpu, this_rq, ksyncs); - cpumask_clear_cpu(cpu, this_scx->cpus_to_kick); - cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); - } - - for_each_cpu(cpu, this_scx->cpus_to_kick_if_idle) { - kick_one_cpu_if_idle(cpu, this_rq); - cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); - } - - /* - * Can't wait in hardirq — kick_sync can't advance, deadlocking if - * CPUs wait for each other. Defer to kick_sync_wait_bal_cb(). - */ - if (should_wait) { - raw_spin_rq_lock(this_rq); - this_scx->kick_sync_pending = true; - resched_curr(this_rq); - raw_spin_rq_unlock(this_rq); - } -} - -/** - * print_scx_info - print out sched_ext scheduler state - * @log_lvl: the log level to use when printing - * @p: target task - * - * If a sched_ext scheduler is enabled, print the name and state of the - * scheduler. If @p is on sched_ext, print further information about the task. - * - * This function can be safely called on any task as long as the task_struct - * itself is accessible. While safe, this function isn't synchronized and may - * print out mixups or garbages of limited length. - */ -void print_scx_info(const char *log_lvl, struct task_struct *p) -{ - struct scx_sched *sch; - enum scx_enable_state state = scx_enable_state(); - const char *all = READ_ONCE(scx_switching_all) ? "+all" : ""; - char runnable_at_buf[22] = "?"; - struct sched_class *class; - unsigned long runnable_at; - - guard(rcu)(); - - sch = scx_task_sched_rcu(p); - - if (!sch) - return; - - /* - * Carefully check if the task was running on sched_ext, and then - * carefully copy the time it's been runnable, and its state. - */ - if (copy_from_kernel_nofault(&class, &p->sched_class, sizeof(class)) || - class != &ext_sched_class) { - printk("%sSched_ext: %s (%s%s)", log_lvl, sch->ops.name, - scx_enable_state_str[state], all); - return; - } - - if (!copy_from_kernel_nofault(&runnable_at, &p->scx.runnable_at, - sizeof(runnable_at))) - scnprintf(runnable_at_buf, sizeof(runnable_at_buf), "%+ldms", - jiffies_delta_msecs(runnable_at, jiffies)); - - /* print everything onto one line to conserve console space */ - printk("%sSched_ext: %s (%s%s), task: runnable_at=%s", - log_lvl, sch->ops.name, scx_enable_state_str[state], all, - runnable_at_buf); -} - -static int scx_pm_handler(struct notifier_block *nb, unsigned long event, void *ptr) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = rcu_dereference(scx_root); - if (!sch) - return NOTIFY_OK; - - /* - * SCX schedulers often have userspace components which are sometimes - * involved in critial scheduling paths. PM operations involve freezing - * userspace which can lead to scheduling misbehaviors including stalls. - * Let's bypass while PM operations are in progress. - */ - switch (event) { - case PM_HIBERNATION_PREPARE: - case PM_SUSPEND_PREPARE: - case PM_RESTORE_PREPARE: - scx_bypass(sch, true); - break; - case PM_POST_HIBERNATION: - case PM_POST_SUSPEND: - case PM_POST_RESTORE: - scx_bypass(sch, false); - break; - } - - return NOTIFY_OK; -} - -static struct notifier_block scx_pm_notifier = { - .notifier_call = scx_pm_handler, -}; - -void __init init_sched_ext_class(void) -{ - s32 cpu, v; - - /* - * The following is to prevent the compiler from optimizing out the enum - * definitions so that BPF scheduler implementations can use them - * through the generated vmlinux.h. - */ - WRITE_ONCE(v, SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | SCX_KICK_PREEMPT | - SCX_TG_ONLINE); - - scx_idle_init_masks(); - - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); - int n = cpu_to_node(cpu); - - /* local_dsq's sch will be set during scx_root_enable() */ - BUG_ON(init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL)); - - INIT_LIST_HEAD(&rq->scx.runnable_list); - INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals); - - BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick, GFP_KERNEL, n)); - BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick_if_idle, GFP_KERNEL, n)); - BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_preempt, GFP_KERNEL, n)); - BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_wait, GFP_KERNEL, n)); - BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_sync, GFP_KERNEL, n)); - raw_spin_lock_init(&rq->scx.deferred_reenq_lock); - INIT_LIST_HEAD(&rq->scx.deferred_reenq_locals); - INIT_LIST_HEAD(&rq->scx.deferred_reenq_users); - rq->scx.deferred_irq_work = IRQ_WORK_INIT_HARD(deferred_irq_workfn); - rq->scx.kick_cpus_irq_work = IRQ_WORK_INIT_HARD(kick_cpus_irq_workfn); - - if (cpu_online(cpu)) - cpu_rq(cpu)->scx.flags |= SCX_RQ_ONLINE; - } - - register_sysrq_key('S', &sysrq_sched_ext_reset_op); - register_sysrq_key('D', &sysrq_sched_ext_dump_op); - INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); - -#ifdef CONFIG_EXT_SUB_SCHED - BUG_ON(rhashtable_init(&scx_sched_hash, &scx_sched_hash_params)); -#endif /* CONFIG_EXT_SUB_SCHED */ -} - - -/******************************************************************************** - * Helpers that can be called from the BPF scheduler. - */ -static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags) -{ - bool is_local = dsq_id == SCX_DSQ_LOCAL || - (dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON; - - if (*enq_flags & SCX_ENQ_IMMED) { - if (unlikely(!is_local)) { - scx_error(sch, "SCX_ENQ_IMMED on a non-local DSQ 0x%llx", dsq_id); - return false; - } - } else if ((sch->ops.flags & SCX_OPS_ALWAYS_ENQ_IMMED) && is_local) { - *enq_flags |= SCX_ENQ_IMMED; - } - - return true; -} - -static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p, - u64 dsq_id, u64 *enq_flags) -{ - lockdep_assert_irqs_disabled(); - - if (unlikely(!p)) { - scx_error(sch, "called with NULL task"); - return false; - } - - if (unlikely(*enq_flags & __SCX_ENQ_INTERNAL_MASK)) { - scx_error(sch, "invalid enq_flags 0x%llx", *enq_flags); - return false; - } - - /* see SCX_EV_INSERT_NOT_OWNED definition */ - if (unlikely(!scx_task_on_sched(sch, p))) { - __scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1); - return false; - } - - if (!scx_vet_enq_flags(sch, dsq_id, enq_flags)) - return false; - - return true; -} - -static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p, - u64 dsq_id, u64 enq_flags) -{ - struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; - struct task_struct *ddsp_task; - - ddsp_task = __this_cpu_read(direct_dispatch_task); - if (ddsp_task) { - mark_direct_dispatch(sch, ddsp_task, p, dsq_id, enq_flags); - return; - } - - if (unlikely(dspc->cursor >= sch->dsp_max_batch)) { - scx_error(sch, "dispatch buffer overflow"); - return; - } - - dspc->buf[dspc->cursor++] = (struct scx_dsp_buf_ent){ - .task = p, - .qseq = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK, - .dsq_id = dsq_id, - .enq_flags = enq_flags, - }; -} - -__bpf_kfunc_start_defs(); - -/** - * scx_bpf_dsq_insert - Insert a task into the FIFO queue of a DSQ - * @p: task_struct to insert - * @dsq_id: DSQ to insert into - * @slice: duration @p can run for in nsecs, 0 to keep the current value - * @enq_flags: SCX_ENQ_* - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Insert @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe to - * call this function spuriously. Can be called from ops.enqueue(), - * ops.select_cpu(), and ops.dispatch(). - * - * When called from ops.select_cpu() or ops.enqueue(), it's for direct dispatch - * and @p must match the task being enqueued. - * - * When called from ops.select_cpu(), @enq_flags and @dsp_id are stored, and @p - * will be directly inserted into the corresponding dispatch queue after - * ops.select_cpu() returns. If @p is inserted into SCX_DSQ_LOCAL, it will be - * inserted into the local DSQ of the CPU returned by ops.select_cpu(). - * @enq_flags are OR'd with the enqueue flags on the enqueue path before the - * task is inserted. - * - * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id - * and this function can be called upto ops.dispatch_max_batch times to insert - * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the - * remaining slots. scx_bpf_dsq_move_to_local() flushes the batch and resets the - * counter. - * - * This function doesn't have any locking restrictions and may be called under - * BPF locks (in the future when BPF introduces more flexible locking). - * - * @p is allowed to run for @slice. The scheduling path is triggered on slice - * exhaustion. If zero, the current residual slice is maintained. If - * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with - * scx_bpf_kick_cpu() to trigger scheduling. - * - * Returns %true on successful insertion, %false on failure. On the root - * scheduler, %false return triggers scheduler abort and the caller doesn't need - * to check the return value. - */ -__bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id, - u64 slice, u64 enq_flags, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return false; - - if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) - return false; - - if (slice) - p->scx.slice = slice; - else - p->scx.slice = p->scx.slice ?: 1; - - scx_dsq_insert_commit(sch, p, dsq_id, enq_flags); - - return true; -} - -/* - * COMPAT: Will be removed in v6.23 along with the ___v2 suffix. - */ -__bpf_kfunc void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, - u64 slice, u64 enq_flags, - const struct bpf_prog_aux *aux) -{ - scx_bpf_dsq_insert___v2(p, dsq_id, slice, enq_flags, aux); -} - -static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p, - u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) -{ - if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) - return false; - - if (slice) - p->scx.slice = slice; - else - p->scx.slice = p->scx.slice ?: 1; - - p->scx.dsq_vtime = vtime; - - scx_dsq_insert_commit(sch, p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); - - return true; -} - -struct scx_bpf_dsq_insert_vtime_args { - /* @p can't be packed together as KF_RCU is not transitive */ - u64 dsq_id; - u64 slice; - u64 vtime; - u64 enq_flags; -}; - -/** - * __scx_bpf_dsq_insert_vtime - Arg-wrapped vtime DSQ insertion - * @p: task_struct to insert - * @args: struct containing the rest of the arguments - * @args->dsq_id: DSQ to insert into - * @args->slice: duration @p can run for in nsecs, 0 to keep the current value - * @args->vtime: @p's ordering inside the vtime-sorted queue of the target DSQ - * @args->enq_flags: SCX_ENQ_* - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Wrapper kfunc that takes arguments via struct to work around BPF's 5 argument - * limit. BPF programs should use scx_bpf_dsq_insert_vtime() which is provided - * as an inline wrapper in common.bpf.h. - * - * Insert @p into the vtime priority queue of the DSQ identified by - * @args->dsq_id. Tasks queued into the priority queue are ordered by - * @args->vtime. All other aspects are identical to scx_bpf_dsq_insert(). - * - * @args->vtime ordering is according to time_before64() which considers - * wrapping. A numerically larger vtime may indicate an earlier position in the - * ordering and vice-versa. - * - * A DSQ can only be used as a FIFO or priority queue at any given time and this - * function must not be called on a DSQ which already has one or more FIFO tasks - * queued and vice-versa. Also, the built-in DSQs (SCX_DSQ_LOCAL and - * SCX_DSQ_GLOBAL) cannot be used as priority queues. - * - * Returns %true on successful insertion, %false on failure. On the root - * scheduler, %false return triggers scheduler abort and the caller doesn't need - * to check the return value. - */ -__bpf_kfunc bool -__scx_bpf_dsq_insert_vtime(struct task_struct *p, - struct scx_bpf_dsq_insert_vtime_args *args, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return false; - - return scx_dsq_insert_vtime(sch, p, args->dsq_id, args->slice, - args->vtime, args->enq_flags); -} - -/* - * COMPAT: Will be removed in v6.23. - */ -__bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, - u64 slice, u64 vtime, u64 enq_flags) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = rcu_dereference(scx_root); - if (unlikely(!sch)) - return; - -#ifdef CONFIG_EXT_SUB_SCHED - /* - * Disallow if any sub-scheds are attached. There is no way to tell - * which scheduler called us, just error out @p's scheduler. - */ - if (unlikely(!list_empty(&sch->children))) { - scx_error(scx_task_sched(p), "__scx_bpf_dsq_insert_vtime() must be used"); - return; - } -#endif - - scx_dsq_insert_vtime(sch, p, dsq_id, slice, vtime, enq_flags); -} - -__bpf_kfunc_end_defs(); - -BTF_KFUNCS_START(scx_kfunc_ids_enqueue_dispatch) -BTF_ID_FLAGS(func, scx_bpf_dsq_insert, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_dsq_insert___v2, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, __scx_bpf_dsq_insert_vtime, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_dsq_insert_vtime, KF_RCU) -BTF_KFUNCS_END(scx_kfunc_ids_enqueue_dispatch) - -static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { - .owner = THIS_MODULE, - .set = &scx_kfunc_ids_enqueue_dispatch, - .filter = scx_kfunc_context_filter, -}; - -static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, - struct task_struct *p, u64 dsq_id, u64 enq_flags) -{ - struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq; - struct scx_sched *sch; - struct rq *this_rq, *src_rq, *locked_rq; - bool dispatched = false; - bool in_balance; - unsigned long flags; - - /* - * The verifier considers an iterator slot initialized on any - * KF_ITER_NEW return, so a BPF program may legally reach here after - * bpf_iter_scx_dsq_new() failed and left @kit->dsq NULL. - */ - if (unlikely(!src_dsq)) - return false; - - sch = src_dsq->sched; - - if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags)) - return false; - - /* - * If the BPF scheduler keeps calling this function repeatedly, it can - * cause similar live-lock conditions as consume_dispatch_q(). - */ - if (unlikely(READ_ONCE(sch->aborting))) - return false; - - if (unlikely(!scx_task_on_sched(sch, p))) { - scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler", - p->comm, p->pid); - return false; - } - - /* - * Can be called from either ops.dispatch() locking this_rq() or any - * context where no rq lock is held. If latter, lock @p's task_rq which - * we'll likely need anyway. - */ - src_rq = task_rq(p); - - local_irq_save(flags); - this_rq = this_rq(); - in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE; - - if (in_balance) { - if (this_rq != src_rq) { - raw_spin_rq_unlock(this_rq); - raw_spin_rq_lock(src_rq); - } - } else { - raw_spin_rq_lock(src_rq); - } - - locked_rq = src_rq; - raw_spin_lock(&src_dsq->lock); - - /* did someone else get to it while we dropped the locks? */ - if (nldsq_cursor_lost_task(&kit->cursor, src_rq, src_dsq, p)) { - raw_spin_unlock(&src_dsq->lock); - goto out; - } - - /* @p is still on $src_dsq and stable, determine the destination */ - dst_dsq = find_dsq_for_dispatch(sch, this_rq, dsq_id, task_cpu(p)); - - /* - * Apply vtime and slice updates before moving so that the new time is - * visible before inserting into $dst_dsq. @p is still on $src_dsq but - * this is safe as we're locking it. - */ - if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME) - p->scx.dsq_vtime = kit->vtime; - if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE) - p->scx.slice = kit->slice; - - /* execute move */ - locked_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); - dispatched = true; -out: - if (in_balance) { - if (this_rq != locked_rq) { - raw_spin_rq_unlock(locked_rq); - raw_spin_rq_lock(this_rq); - } - } else { - raw_spin_rq_unlock_irqrestore(locked_rq, flags); - } - - kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE | - __SCX_DSQ_ITER_HAS_VTIME); - return dispatched; -} - -__bpf_kfunc_start_defs(); - -/** - * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Can only be called from ops.dispatch(). - */ -__bpf_kfunc u32 scx_bpf_dispatch_nr_slots(const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return 0; - - return sch->dsp_max_batch - __this_cpu_read(sch->pcpu->dsp_ctx.cursor); -} - -/** - * scx_bpf_dispatch_cancel - Cancel the latest dispatch - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Cancel the latest dispatch. Can be called multiple times to cancel further - * dispatches. Can only be called from ops.dispatch(). - */ -__bpf_kfunc void scx_bpf_dispatch_cancel(const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - struct scx_dsp_ctx *dspc; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return; - - dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; - - if (dspc->cursor > 0) - dspc->cursor--; - else - scx_error(sch, "dispatch buffer underflow"); -} - -/** - * scx_bpf_dsq_move_to_local - move a task from a DSQ to the current CPU's local DSQ - * @dsq_id: DSQ to move task from. Must be a user-created DSQ - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * @enq_flags: %SCX_ENQ_* - * - * Move a task from the non-local DSQ identified by @dsq_id to the current CPU's - * local DSQ for execution with @enq_flags applied. Can only be called from - * ops.dispatch(). - * - * Built-in DSQs (%SCX_DSQ_GLOBAL and %SCX_DSQ_LOCAL*) are not supported as - * sources. Local DSQs support reenqueueing (a task can be picked up for - * execution, dequeued for property changes, or reenqueued), but the BPF - * scheduler cannot directly iterate or move tasks from them. %SCX_DSQ_GLOBAL - * is similar but also doesn't support reenqueueing, as it maps to multiple - * per-node DSQs making the scope difficult to define; this may change in the - * future. - * - * This function flushes the in-flight dispatches from scx_bpf_dsq_insert() - * before trying to move from the specified DSQ. It may also grab rq locks and - * thus can't be called under any BPF locks. - * - * Returns %true if a task has been moved, %false if there isn't any task to - * move. - */ -__bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags, - const struct bpf_prog_aux *aux) -{ - struct scx_dispatch_q *dsq; - struct scx_sched *sch; - struct scx_dsp_ctx *dspc; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return false; - - if (!scx_vet_enq_flags(sch, SCX_DSQ_LOCAL, &enq_flags)) - return false; - - dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; - - flush_dispatch_buf(sch, dspc->rq); - - dsq = find_user_dsq(sch, dsq_id); - if (unlikely(!dsq)) { - scx_error(sch, "invalid DSQ ID 0x%016llx", dsq_id); - return false; - } - - if (consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) { - /* - * A successfully consumed task can be dequeued before it starts - * running while the CPU is trying to migrate other dispatched - * tasks. Bump nr_tasks to tell balance_one() to retry on empty - * local DSQ. - */ - dspc->nr_tasks++; - return true; - } else { - return false; - } -} - -/* - * COMPAT: ___v2 was introduced in v7.1. Remove this and ___v2 tag in the future. - */ -__bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id, const struct bpf_prog_aux *aux) -{ - return scx_bpf_dsq_move_to_local___v2(dsq_id, 0, aux); -} - -/** - * scx_bpf_dsq_move_set_slice - Override slice when moving between DSQs - * @it__iter: DSQ iterator in progress - * @slice: duration the moved task can run for in nsecs - * - * Override the slice of the next task that will be moved from @it__iter using - * scx_bpf_dsq_move[_vtime](). If this function is not called, the previous - * slice duration is kept. - */ -__bpf_kfunc void scx_bpf_dsq_move_set_slice(struct bpf_iter_scx_dsq *it__iter, - u64 slice) -{ - struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; - - kit->slice = slice; - kit->cursor.flags |= __SCX_DSQ_ITER_HAS_SLICE; -} - -/** - * scx_bpf_dsq_move_set_vtime - Override vtime when moving between DSQs - * @it__iter: DSQ iterator in progress - * @vtime: task's ordering inside the vtime-sorted queue of the target DSQ - * - * Override the vtime of the next task that will be moved from @it__iter using - * scx_bpf_dsq_move_vtime(). If this function is not called, the previous slice - * vtime is kept. If scx_bpf_dsq_move() is used to dispatch the next task, the - * override is ignored and cleared. - */ -__bpf_kfunc void scx_bpf_dsq_move_set_vtime(struct bpf_iter_scx_dsq *it__iter, - u64 vtime) -{ - struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; - - kit->vtime = vtime; - kit->cursor.flags |= __SCX_DSQ_ITER_HAS_VTIME; -} - -/** - * scx_bpf_dsq_move - Move a task from DSQ iteration to a DSQ - * @it__iter: DSQ iterator in progress - * @p: task to transfer - * @dsq_id: DSQ to move @p to - * @enq_flags: SCX_ENQ_* - * - * Transfer @p which is on the DSQ currently iterated by @it__iter to the DSQ - * specified by @dsq_id. All DSQs - local DSQs, global DSQ and user DSQs - can - * be the destination. - * - * For the transfer to be successful, @p must still be on the DSQ and have been - * queued before the DSQ iteration started. This function doesn't care whether - * @p was obtained from the DSQ iteration. @p just has to be on the DSQ and have - * been queued before the iteration started. - * - * @p's slice is kept by default. Use scx_bpf_dsq_move_set_slice() to update. - * - * Can be called from ops.dispatch() or any BPF context which doesn't hold a rq - * lock (e.g. BPF timers or SYSCALL programs). - * - * Returns %true if @p has been consumed, %false if @p had already been - * consumed, dequeued, or, for sub-scheds, @dsq_id points to a disallowed local - * DSQ. - */ -__bpf_kfunc bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter, - struct task_struct *p, u64 dsq_id, - u64 enq_flags) -{ - return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, - p, dsq_id, enq_flags); -} - -/** - * scx_bpf_dsq_move_vtime - Move a task from DSQ iteration to a PRIQ DSQ - * @it__iter: DSQ iterator in progress - * @p: task to transfer - * @dsq_id: DSQ to move @p to - * @enq_flags: SCX_ENQ_* - * - * Transfer @p which is on the DSQ currently iterated by @it__iter to the - * priority queue of the DSQ specified by @dsq_id. The destination must be a - * user DSQ as only user DSQs support priority queue. - * - * @p's slice and vtime are kept by default. Use scx_bpf_dsq_move_set_slice() - * and scx_bpf_dsq_move_set_vtime() to update. - * - * All other aspects are identical to scx_bpf_dsq_move(). See - * scx_bpf_dsq_insert_vtime() for more information on @vtime. - */ -__bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter, - struct task_struct *p, u64 dsq_id, - u64 enq_flags) -{ - return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, - p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); -} - -#ifdef CONFIG_EXT_SUB_SCHED -/** - * scx_bpf_sub_dispatch - Trigger dispatching on a child scheduler - * @cgroup_id: cgroup ID of the child scheduler to dispatch - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Allows a parent scheduler to trigger dispatching on one of its direct - * child schedulers. The child scheduler runs its dispatch operation to - * move tasks from dispatch queues to the local runqueue. - * - * Returns: true on success, false if cgroup_id is invalid, not a direct - * child, or caller lacks dispatch permission. - */ -__bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux) -{ - struct rq *this_rq = this_rq(); - struct scx_sched *parent, *child; - - guard(rcu)(); - parent = scx_prog_sched(aux); - if (unlikely(!parent)) - return false; - - child = scx_find_sub_sched(cgroup_id); - - if (unlikely(!child)) - return false; - - if (unlikely(scx_parent(child) != parent)) { - scx_error(parent, "trying to dispatch a distant sub-sched on cgroup %llu", - cgroup_id); - return false; - } - - return scx_dispatch_sched(child, this_rq, this_rq->scx.sub_dispatch_prev, - true); -} -#endif /* CONFIG_EXT_SUB_SCHED */ - -__bpf_kfunc_end_defs(); - -BTF_KFUNCS_START(scx_kfunc_ids_dispatch) -BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_dispatch_cancel, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local___v2, KF_IMPLICIT_ARGS) -/* scx_bpf_dsq_move*() also in scx_kfunc_ids_unlocked: callable from unlocked contexts */ -BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) -#ifdef CONFIG_EXT_SUB_SCHED -BTF_ID_FLAGS(func, scx_bpf_sub_dispatch, KF_IMPLICIT_ARGS) -#endif -BTF_KFUNCS_END(scx_kfunc_ids_dispatch) - -static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { - .owner = THIS_MODULE, - .set = &scx_kfunc_ids_dispatch, - .filter = scx_kfunc_context_filter, -}; - -__bpf_kfunc_start_defs(); - -/** - * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Iterate over all of the tasks currently enqueued on the local DSQ of the - * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of - * processed tasks. Can only be called from ops.cpu_release(). - */ -__bpf_kfunc u32 scx_bpf_reenqueue_local(const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - struct rq *rq; - - guard(rcu)(); - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return 0; - - rq = cpu_rq(smp_processor_id()); - lockdep_assert_rq_held(rq); - - return reenq_local(sch, rq, SCX_REENQ_ANY); -} - -__bpf_kfunc_end_defs(); - -BTF_KFUNCS_START(scx_kfunc_ids_cpu_release) -BTF_ID_FLAGS(func, scx_bpf_reenqueue_local, KF_IMPLICIT_ARGS) -BTF_KFUNCS_END(scx_kfunc_ids_cpu_release) - -static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { - .owner = THIS_MODULE, - .set = &scx_kfunc_ids_cpu_release, - .filter = scx_kfunc_context_filter, -}; - -__bpf_kfunc_start_defs(); - -/** - * scx_bpf_create_dsq - Create a custom DSQ - * @dsq_id: DSQ to create - * @node: NUMA node to allocate from - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Create a custom DSQ identified by @dsq_id. Can be called from any sleepable - * scx callback, and any BPF_PROG_TYPE_SYSCALL prog. - */ -__bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node, const struct bpf_prog_aux *aux) -{ - struct scx_dispatch_q *dsq; - struct scx_sched *sch; - s32 ret; - - if (unlikely(node >= (int)nr_node_ids || - (node < 0 && node != NUMA_NO_NODE))) - return -EINVAL; - - if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN)) - return -EINVAL; - - dsq = kmalloc_node(sizeof(*dsq), GFP_KERNEL, node); - if (!dsq) - return -ENOMEM; - - /* - * init_dsq() must be called in GFP_KERNEL context. Init it with NULL - * @sch and update afterwards. - */ - ret = init_dsq(dsq, dsq_id, NULL); - if (ret) { - kfree(dsq); - return ret; - } - - rcu_read_lock(); - - sch = scx_prog_sched(aux); - if (sch) { - dsq->sched = sch; - ret = rhashtable_lookup_insert_fast(&sch->dsq_hash, &dsq->hash_node, - dsq_hash_params); - } else { - ret = -ENODEV; - } - - rcu_read_unlock(); - if (ret) { - exit_dsq(dsq); - kfree(dsq); - } - return ret; -} - -__bpf_kfunc_end_defs(); - -BTF_KFUNCS_START(scx_kfunc_ids_unlocked) -BTF_ID_FLAGS(func, scx_bpf_create_dsq, KF_IMPLICIT_ARGS | KF_SLEEPABLE) -/* also in scx_kfunc_ids_dispatch: also callable from ops.dispatch() */ -BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) -/* also in scx_kfunc_ids_select_cpu: also callable from ops.select_cpu()/ops.enqueue() */ -BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU) -BTF_KFUNCS_END(scx_kfunc_ids_unlocked) - -static const struct btf_kfunc_id_set scx_kfunc_set_unlocked = { - .owner = THIS_MODULE, - .set = &scx_kfunc_ids_unlocked, - .filter = scx_kfunc_context_filter, -}; - -__bpf_kfunc_start_defs(); - -/** - * scx_bpf_task_set_slice - Set task's time slice - * @p: task of interest - * @slice: time slice to set in nsecs - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Set @p's time slice to @slice. Returns %true on success, %false if the - * calling scheduler doesn't have authority over @p. - */ -__bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - sch = scx_prog_sched(aux); - if (unlikely(!sch || !scx_task_on_sched(sch, p))) - return false; - - p->scx.slice = slice; - return true; -} - -/** - * scx_bpf_task_set_dsq_vtime - Set task's virtual time for DSQ ordering - * @p: task of interest - * @vtime: virtual time to set - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Set @p's virtual time to @vtime. Returns %true on success, %false if the - * calling scheduler doesn't have authority over @p. - */ -__bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - sch = scx_prog_sched(aux); - if (unlikely(!sch || !scx_task_on_sched(sch, p))) - return false; - - p->scx.dsq_vtime = vtime; - return true; -} - -static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) -{ - struct rq *this_rq; - unsigned long irq_flags; - - local_irq_save(irq_flags); - - this_rq = this_rq(); - - /* - * While bypassing for PM ops, IRQ handling may not be online which can - * lead to irq_work_queue() malfunction such as infinite busy wait for - * IRQ status update. Suppress kicking. - */ - if (scx_bypassing(sch, cpu_of(this_rq))) - goto out; - - /* - * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting - * rq locks. We can probably be smarter and avoid bouncing if called - * from ops which don't hold a rq lock. - */ - if (flags & SCX_KICK_IDLE) { - struct rq *target_rq = cpu_rq(cpu); - - if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_WAIT))) - scx_error(sch, "PREEMPT/WAIT cannot be used with SCX_KICK_IDLE"); - - if (raw_spin_rq_trylock(target_rq)) { - if (can_skip_idle_kick(target_rq)) { - raw_spin_rq_unlock(target_rq); - goto out; - } - raw_spin_rq_unlock(target_rq); - } - cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick_if_idle); - } else { - cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick); - - if (flags & SCX_KICK_PREEMPT) - cpumask_set_cpu(cpu, this_rq->scx.cpus_to_preempt); - if (flags & SCX_KICK_WAIT) - cpumask_set_cpu(cpu, this_rq->scx.cpus_to_wait); - } - - irq_work_queue(&this_rq->scx.kick_cpus_irq_work); -out: - local_irq_restore(irq_flags); -} - -/** - * scx_bpf_kick_cpu - Trigger reschedule on a CPU - * @cpu: cpu to kick - * @flags: %SCX_KICK_* flags - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or - * trigger rescheduling on a busy CPU. This can be called from any online - * scx_ops operation and the actual kicking is performed asynchronously through - * an irq work. - */ -__bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - sch = scx_prog_sched(aux); - if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) - scx_kick_cpu(sch, cpu, flags); -} - -/** - * scx_bpf_kick_cid - Trigger reschedule on the CPU mapped to @cid - * @cid: cid to kick - * @flags: %SCX_KICK_* flags - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * cid-addressed equivalent of scx_bpf_kick_cpu(). Return 0 on success, - * -errno otherwise. - */ -__bpf_kfunc s32 scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - s32 cpu; - - guard(rcu)(); - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -ENODEV; - cpu = scx_cid_to_cpu(sch, cid); - if (cpu < 0) - return cpu; - scx_kick_cpu(sch, cpu, flags); - return 0; -} - -/** - * scx_bpf_dsq_nr_queued - Return the number of queued tasks - * @dsq_id: id of the DSQ - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Return the number of tasks in the DSQ matching @dsq_id. If not found, - * -%ENOENT is returned. - */ -__bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - struct scx_dispatch_q *dsq; - s32 ret; - - preempt_disable(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) { - ret = -ENODEV; - goto out; - } - - if (dsq_id == SCX_DSQ_LOCAL) { - ret = READ_ONCE(this_rq()->scx.local_dsq.nr); - goto out; - } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { - s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK); - - if (scx_cpu_valid(sch, cpu, NULL)) { - ret = READ_ONCE(cpu_rq(cpu)->scx.local_dsq.nr); - goto out; - } - } else { - dsq = find_user_dsq(sch, dsq_id); - if (dsq) { - ret = READ_ONCE(dsq->nr); - goto out; - } - } - ret = -ENOENT; -out: - preempt_enable(); - return ret; -} - -/** - * scx_bpf_destroy_dsq - Destroy a custom DSQ - * @dsq_id: DSQ to destroy - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with - * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is - * empty and no further tasks are dispatched to it. Ignored if called on a DSQ - * which doesn't exist. Can be called from any online scx_ops operations. - */ -__bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - sch = scx_prog_sched(aux); - if (sch) - destroy_dsq(sch, dsq_id); -} - -/** - * bpf_iter_scx_dsq_new - Create a DSQ iterator - * @it: iterator to initialize - * @dsq_id: DSQ to iterate - * @flags: %SCX_DSQ_ITER_* - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Initialize BPF iterator @it which can be used with bpf_for_each() to walk - * tasks in the DSQ specified by @dsq_id. Iteration using @it only includes - * tasks which are already queued when this function is invoked. - */ -__bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, - u64 flags, const struct bpf_prog_aux *aux) -{ - struct bpf_iter_scx_dsq_kern *kit = (void *)it; - struct scx_sched *sch; - - BUILD_BUG_ON(sizeof(struct bpf_iter_scx_dsq_kern) > - sizeof(struct bpf_iter_scx_dsq)); - BUILD_BUG_ON(__alignof__(struct bpf_iter_scx_dsq_kern) != - __alignof__(struct bpf_iter_scx_dsq)); - BUILD_BUG_ON(__SCX_DSQ_ITER_ALL_FLAGS & - ((1U << __SCX_DSQ_LNODE_PRIV_SHIFT) - 1)); - - /* - * next() and destroy() will be called regardless of the return value. - * Always clear $kit->dsq. - */ - kit->dsq = NULL; - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -ENODEV; - - if (flags & ~__SCX_DSQ_ITER_USER_FLAGS) - return -EINVAL; - - kit->dsq = find_user_dsq(sch, dsq_id); - if (!kit->dsq) - return -ENOENT; - - kit->cursor = INIT_DSQ_LIST_CURSOR(kit->cursor, kit->dsq, flags); - - return 0; -} - -/** - * bpf_iter_scx_dsq_next - Progress a DSQ iterator - * @it: iterator to progress - * - * Return the next task. See bpf_iter_scx_dsq_new(). - */ -__bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) -{ - struct bpf_iter_scx_dsq_kern *kit = (void *)it; - - if (!kit->dsq) - return NULL; - - guard(raw_spinlock_irqsave)(&kit->dsq->lock); - - return nldsq_cursor_next_task(&kit->cursor, kit->dsq); -} - -/** - * bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator - * @it: iterator to destroy - * - * Undo scx_iter_scx_dsq_new(). - */ -__bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) -{ - struct bpf_iter_scx_dsq_kern *kit = (void *)it; - - if (!kit->dsq) - return; - - if (!list_empty(&kit->cursor.node)) { - unsigned long flags; - - raw_spin_lock_irqsave(&kit->dsq->lock, flags); - list_del_init(&kit->cursor.node); - raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); - } - kit->dsq = NULL; -} - -/** - * scx_bpf_dsq_peek - Lockless peek at the first element. - * @dsq_id: DSQ to examine. - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Read the first element in the DSQ. This is semantically equivalent to using - * the DSQ iterator, but is lockfree. Of course, like any lockless operation, - * this provides only a point-in-time snapshot, and the contents may change - * by the time any subsequent locking operation reads the queue. - * - * Returns the pointer, or NULL indicates an empty queue OR internal error. - */ -__bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - struct scx_dispatch_q *dsq; - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return NULL; - - if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN)) { - scx_error(sch, "peek disallowed on builtin DSQ 0x%llx", dsq_id); - return NULL; - } - - dsq = find_user_dsq(sch, dsq_id); - if (unlikely(!dsq)) { - scx_error(sch, "peek on non-existent DSQ 0x%llx", dsq_id); - return NULL; - } - - return rcu_dereference(dsq->first_task); -} - -/** - * scx_bpf_dsq_reenq - Re-enqueue tasks on a DSQ - * @dsq_id: DSQ to re-enqueue - * @reenq_flags: %SCX_RENQ_* - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Iterate over all of the tasks currently enqueued on the DSQ identified by - * @dsq_id, and re-enqueue them in the BPF scheduler. The following DSQs are - * supported: - * - * - Local DSQs (%SCX_DSQ_LOCAL or %SCX_DSQ_LOCAL_ON | $cpu) - * - User DSQs - * - * Re-enqueues are performed asynchronously. Can be called from anywhere. - */ -__bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - struct scx_dispatch_q *dsq; - - guard(preempt)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return; - - if (unlikely(reenq_flags & ~__SCX_REENQ_USER_MASK)) { - scx_error(sch, "invalid SCX_REENQ flags 0x%llx", reenq_flags); - return; - } - - /* not specifying any filter bits is the same as %SCX_REENQ_ANY */ - if (!(reenq_flags & __SCX_REENQ_FILTER_MASK)) - reenq_flags |= SCX_REENQ_ANY; - - dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, smp_processor_id()); - schedule_dsq_reenq(sch, dsq, reenq_flags, scx_locked_rq()); -} - -/** - * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Iterate over all of the tasks currently enqueued on the local DSQ of the - * caller's CPU, and re-enqueue them in the BPF scheduler. Can be called from - * anywhere. - * - * This is now a special case of scx_bpf_dsq_reenq() and may be removed in the - * future. - */ -__bpf_kfunc void scx_bpf_reenqueue_local___v2(const struct bpf_prog_aux *aux) -{ - scx_bpf_dsq_reenq(SCX_DSQ_LOCAL, 0, aux); -} - -__bpf_kfunc_end_defs(); - -__printf(5, 0) -static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf, - size_t line_size, char *fmt, unsigned long long *data, - u32 data__sz) -{ - struct bpf_bprintf_data bprintf_data = { .get_bin_args = true }; - s32 ret; - - if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || - (data__sz && !data)) { - scx_error(sch, "invalid data=%p and data__sz=%u", (void *)data, data__sz); - return -EINVAL; - } - - ret = copy_from_kernel_nofault(data_buf, data, data__sz); - if (ret < 0) { - scx_error(sch, "failed to read data fields (%d)", ret); - return ret; - } - - ret = bpf_bprintf_prepare(fmt, UINT_MAX, data_buf, data__sz / 8, - &bprintf_data); - if (ret < 0) { - scx_error(sch, "format preparation failed (%d)", ret); - return ret; - } - - ret = bstr_printf(line_buf, line_size, fmt, - bprintf_data.bin_args); - bpf_bprintf_cleanup(&bprintf_data); - if (ret < 0) { - scx_error(sch, "(\"%s\", %p, %u) failed to format", fmt, data, data__sz); - return ret; - } - - return ret; -} - -__printf(3, 0) -static s32 bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf, - char *fmt, unsigned long long *data, u32 data__sz) -{ - return __bstr_format(sch, buf->data, buf->line, sizeof(buf->line), - fmt, data, data__sz); -} - -__bpf_kfunc_start_defs(); - -/** - * scx_bpf_exit_bstr - Gracefully exit the BPF scheduler. - * @exit_code: Exit value to pass to user space via struct scx_exit_info. - * @fmt: error message format string - * @data: format string parameters packaged using ___bpf_fill() macro - * @data__sz: @data len, must end in '__sz' for the verifier - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Indicate that the BPF scheduler wants to exit gracefully, and initiate ops - * disabling. - */ -__printf(2, 0) -__bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt, - unsigned long long *data, u32 data__sz, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - unsigned long flags; - - raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); - sch = scx_prog_sched(aux); - if (likely(sch) && - bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) - scx_exit(sch, SCX_EXIT_UNREG_BPF, exit_code, "%s", scx_exit_bstr_buf.line); - raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); -} - -/** - * scx_bpf_error_bstr - Indicate fatal error - * @fmt: error message format string - * @data: format string parameters packaged using ___bpf_fill() macro - * @data__sz: @data len, must end in '__sz' for the verifier - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Indicate that the BPF scheduler encountered a fatal error and initiate ops - * disabling. - */ -__printf(1, 0) -__bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data, - u32 data__sz, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - unsigned long flags; - - raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); - sch = scx_prog_sched(aux); - if (likely(sch) && - bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) - scx_exit(sch, SCX_EXIT_ERROR_BPF, 0, "%s", scx_exit_bstr_buf.line); - raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); -} - -/** - * scx_bpf_dump_bstr - Generate extra debug dump specific to the BPF scheduler - * @fmt: format string - * @data: format string parameters packaged using ___bpf_fill() macro - * @data__sz: @data len, must end in '__sz' for the verifier - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * To be called through scx_bpf_dump() helper from ops.dump(), dump_cpu() and - * dump_task() to generate extra debug dump specific to the BPF scheduler. - * - * The extra dump may be multiple lines. A single line may be split over - * multiple calls. The last line is automatically terminated. - */ -__printf(1, 0) -__bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, - u32 data__sz, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - struct scx_dump_data *dd = &scx_dump_data; - struct scx_bstr_buf *buf = &dd->buf; - s32 ret; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return; - - if (raw_smp_processor_id() != dd->cpu) { - scx_error(sch, "scx_bpf_dump() must only be called from ops.dump() and friends"); - return; - } - - /* append the formatted string to the line buf */ - ret = __bstr_format(sch, buf->data, buf->line + dd->cursor, - sizeof(buf->line) - dd->cursor, fmt, data, data__sz); - if (ret < 0) { - dump_line(dd->s, "%s[!] (\"%s\", %p, %u) failed to format (%d)", - dd->prefix, fmt, data, data__sz, ret); - return; - } - - dd->cursor += ret; - dd->cursor = min_t(s32, dd->cursor, sizeof(buf->line)); - - if (!dd->cursor) - return; - - /* - * If the line buf overflowed or ends in a newline, flush it into the - * dump. This is to allow the caller to generate a single line over - * multiple calls. As ops_dump_flush() can also handle multiple lines in - * the line buf, the only case which can lead to an unexpected - * truncation is when the caller keeps generating newlines in the middle - * instead of the end consecutively. Don't do that. - */ - if (dd->cursor >= sizeof(buf->line) || buf->line[dd->cursor - 1] == '\n') - ops_dump_flush(); -} - -/** - * scx_bpf_cpuperf_cap - Query the maximum relative capacity of a CPU - * @cpu: CPU of interest - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Return the maximum relative capacity of @cpu in relation to the most - * performant CPU in the system. The return value is in the range [1, - * %SCX_CPUPERF_ONE]. See scx_bpf_cpuperf_cur(). - */ -__bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) - return arch_scale_cpu_capacity(cpu); - else - return SCX_CPUPERF_ONE; -} - -/** - * scx_bpf_cidperf_cap - Query the maximum relative capacity of the CPU at @cid - * @cid: cid of the CPU to query - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * cid-addressed equivalent of scx_bpf_cpuperf_cap(). - */ -__bpf_kfunc u32 scx_bpf_cidperf_cap(s32 cid, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - s32 cpu; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return SCX_CPUPERF_ONE; - cpu = scx_cid_to_cpu(sch, cid); - if (cpu < 0) - return SCX_CPUPERF_ONE; - return arch_scale_cpu_capacity(cpu); -} - -/** - * scx_bpf_cpuperf_cur - Query the current relative performance of a CPU - * @cpu: CPU of interest - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Return the current relative performance of @cpu in relation to its maximum. - * The return value is in the range [1, %SCX_CPUPERF_ONE]. - * - * The current performance level of a CPU in relation to the maximum performance - * available in the system can be calculated as follows: - * - * scx_bpf_cpuperf_cap() * scx_bpf_cpuperf_cur() / %SCX_CPUPERF_ONE - * - * The result is in the range [1, %SCX_CPUPERF_ONE]. - */ -__bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) - return arch_scale_freq_capacity(cpu); - else - return SCX_CPUPERF_ONE; -} - -/** - * scx_bpf_cidperf_cur - Query the current performance of the CPU at @cid - * @cid: cid of the CPU to query - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * cid-addressed equivalent of scx_bpf_cpuperf_cur(). - */ -__bpf_kfunc u32 scx_bpf_cidperf_cur(s32 cid, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - s32 cpu; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return SCX_CPUPERF_ONE; - cpu = scx_cid_to_cpu(sch, cid); - if (cpu < 0) - return SCX_CPUPERF_ONE; - return arch_scale_freq_capacity(cpu); -} - -/** - * scx_bpf_cpuperf_set - Set the relative performance target of a CPU - * @cpu: CPU of interest - * @perf: target performance level [0, %SCX_CPUPERF_ONE] - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Set the target performance level of @cpu to @perf. @perf is in linear - * relative scale between 0 and %SCX_CPUPERF_ONE. This determines how the - * schedutil cpufreq governor chooses the target frequency. - * - * The actual performance level chosen, CPU grouping, and the overhead and - * latency of the operations are dependent on the hardware and cpufreq driver in - * use. Consult hardware and cpufreq documentation for more information. The - * current performance level can be monitored using scx_bpf_cpuperf_cur(). - */ -__bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return; - - if (unlikely(perf > SCX_CPUPERF_ONE)) { - scx_error(sch, "Invalid cpuperf target %u for CPU %d", perf, cpu); - return; - } - - if (scx_cpu_valid(sch, cpu, NULL)) { - struct rq *rq = cpu_rq(cpu), *locked_rq = scx_locked_rq(); - struct rq_flags rf; - - /* - * When called with an rq lock held, restrict the operation - * to the corresponding CPU to prevent ABBA deadlocks. - */ - if (locked_rq && rq != locked_rq) { - scx_error(sch, "Invalid target CPU %d", cpu); - return; - } - - /* - * If no rq lock is held, allow to operate on any CPU by - * acquiring the corresponding rq lock. - */ - if (!locked_rq) { - rq_lock_irqsave(rq, &rf); - update_rq_clock(rq); - } - - rq->scx.cpuperf_target = perf; - cpufreq_update_util(rq, 0); - - if (!locked_rq) - rq_unlock_irqrestore(rq, &rf); - } -} - -/** - * scx_bpf_cidperf_set - Set the performance target of the CPU at @cid - * @cid: cid of the CPU to target - * @perf: target performance level [0, %SCX_CPUPERF_ONE] - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * cid-addressed equivalent of scx_bpf_cpuperf_set(). - */ -__bpf_kfunc void scx_bpf_cidperf_set(s32 cid, u32 perf, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - s32 cpu; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return; - cpu = scx_cid_to_cpu(sch, cid); - if (cpu < 0) - return; - scx_bpf_cpuperf_set(cpu, perf, aux); -} - -/** - * scx_bpf_nr_node_ids - Return the number of possible node IDs - * - * All valid node IDs in the system are smaller than the returned value. - */ -__bpf_kfunc u32 scx_bpf_nr_node_ids(void) -{ - return nr_node_ids; -} - -/** - * scx_bpf_nr_cpu_ids - Return the number of possible CPU IDs - * - * All valid CPU IDs in the system are smaller than the returned value. - */ -__bpf_kfunc u32 scx_bpf_nr_cpu_ids(void) -{ - return nr_cpu_ids; -} - -/** - * scx_bpf_nr_cids - Return the size of the cid space - * - * Equals num_possible_cpus(). All valid cids are in [0, return value). - */ -__bpf_kfunc u32 scx_bpf_nr_cids(void) -{ - return num_possible_cpus(); -} - -/** - * scx_bpf_nr_online_cids - Return current count of online CPUs in cid space - * - * Return num_online_cpus(). The standard model restarts the scheduler on - * hotplug, which lets schedulers treat [0, nr_online_cids) as the online - * range. Schedulers that prefer to handle hotplug without a restart should - * install a custom mapping via scx_bpf_cid_override() and track onlining - * through the ops.cid_online / ops.cid_offline callbacks. - */ -__bpf_kfunc u32 scx_bpf_nr_online_cids(void) -{ - return num_online_cpus(); -} - -/** - * scx_bpf_this_cid - Return the cid of the CPU this program is running on - * - * cid-addressed equivalent of bpf_get_smp_processor_id() for scx programs. - * The current cpu is trivially valid, so this is just a table lookup. Return - * -EINVAL if called from a non-SCX program before any scheduler has ever - * been enabled (the cid table is still unallocated at that point). - */ -__bpf_kfunc s32 scx_bpf_this_cid(void) -{ - s16 *tbl = READ_ONCE(scx_cpu_to_cid_tbl); - - if (!tbl) - return -EINVAL; - return tbl[raw_smp_processor_id()]; -} - -/** - * scx_bpf_get_possible_cpumask - Get a referenced kptr to cpu_possible_mask - */ -__bpf_kfunc const struct cpumask *scx_bpf_get_possible_cpumask(void) -{ - return cpu_possible_mask; -} - -/** - * scx_bpf_get_online_cpumask - Get a referenced kptr to cpu_online_mask - */ -__bpf_kfunc const struct cpumask *scx_bpf_get_online_cpumask(void) -{ - return cpu_online_mask; -} - -/** - * scx_bpf_put_cpumask - Release a possible/online cpumask - * @cpumask: cpumask to release - */ -__bpf_kfunc void scx_bpf_put_cpumask(const struct cpumask *cpumask) -{ - /* - * Empty function body because we aren't actually acquiring or releasing - * a reference to a global cpumask, which is read-only in the caller and - * is never released. The acquire / release semantics here are just used - * to make the cpumask is a trusted pointer in the caller. - */ -} - -/** - * scx_bpf_task_running - Is task currently running? - * @p: task of interest - */ -__bpf_kfunc bool scx_bpf_task_running(const struct task_struct *p) -{ - return task_rq(p)->curr == p; -} - -/** - * scx_bpf_task_cpu - CPU a task is currently associated with - * @p: task of interest - */ -__bpf_kfunc s32 scx_bpf_task_cpu(const struct task_struct *p) -{ - return task_cpu(p); -} - -/** - * scx_bpf_task_cid - cid a task is currently associated with - * @p: task of interest - * - * cid-addressed equivalent of scx_bpf_task_cpu(). task_cpu(p) is always a - * valid cpu, so this is just a table lookup. Return -EINVAL if called from - * a non-SCX program before any scheduler has ever been enabled. - */ -__bpf_kfunc s32 scx_bpf_task_cid(const struct task_struct *p) -{ - s16 *tbl = READ_ONCE(scx_cpu_to_cid_tbl); - - if (!tbl) - return -EINVAL; - return tbl[task_cpu(p)]; -} - -/** - * scx_bpf_cpu_rq - Fetch the rq of a CPU - * @cpu: CPU of the rq - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - */ -__bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return NULL; - - if (!scx_cpu_valid(sch, cpu, NULL)) - return NULL; - - if (!sch->warned_deprecated_rq) { - printk_deferred(KERN_WARNING "sched_ext: %s() is deprecated; " - "use scx_bpf_locked_rq() when holding rq lock " - "or scx_bpf_cpu_curr() to read remote curr safely.\n", __func__); - sch->warned_deprecated_rq = true; - } - - return cpu_rq(cpu); -} - -/** - * scx_bpf_locked_rq - Return the rq currently locked by SCX - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Returns the rq if a rq lock is currently held by SCX. - * Otherwise emits an error and returns NULL. - */ -__bpf_kfunc struct rq *scx_bpf_locked_rq(const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - struct rq *rq; - - guard(preempt)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return NULL; - - rq = scx_locked_rq(); - if (!rq) { - scx_error(sch, "accessing rq without holding rq lock"); - return NULL; - } - - return rq; -} - -/** - * scx_bpf_cpu_curr - Return remote CPU's curr task - * @cpu: CPU of interest - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Callers must hold RCU read lock (KF_RCU). - */ -__bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return NULL; - - if (!scx_cpu_valid(sch, cpu, NULL)) - return NULL; - - return rcu_dereference(cpu_rq(cpu)->curr); -} - -/** - * scx_bpf_cid_curr - Return the curr task on the CPU at @cid - * @cid: cid of interest - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * cid-addressed equivalent of scx_bpf_cpu_curr(). Callers must hold RCU - * read lock (KF_RCU). - */ -__bpf_kfunc struct task_struct *scx_bpf_cid_curr(s32 cid, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - s32 cpu; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return NULL; - cpu = scx_cid_to_cpu(sch, cid); - if (cpu < 0) - return NULL; - return rcu_dereference(cpu_rq(cpu)->curr); -} - -/** - * scx_bpf_tid_to_task - Look up a task by its scx tid - * @tid: task ID previously read from p->scx.tid - * - * Returns the task with the given tid, or NULL if no such task exists. The - * returned pointer is valid until the end of the current RCU read section - * (KF_RCU_PROTECTED). Requires SCX_OPS_TID_TO_TASK to be set on the root - * scheduler; otherwise an error is raised and NULL returned. - */ -__bpf_kfunc struct task_struct *scx_bpf_tid_to_task(u64 tid) -{ - struct sched_ext_entity *scx; - - if (!scx_tid_to_task_enabled()) { - struct scx_sched *sch = rcu_dereference(scx_root); - - if (sch) - scx_error(sch, "scx_bpf_tid_to_task() called without SCX_OPS_TID_TO_TASK"); - return NULL; - } - - scx = rhashtable_lookup(&scx_tid_hash, &tid, scx_tid_hash_params); - if (!scx) - return NULL; - - return container_of(scx, struct task_struct, scx); -} - -/** - * scx_bpf_now - Returns a high-performance monotonically non-decreasing - * clock for the current CPU. The clock returned is in nanoseconds. - * - * It provides the following properties: - * - * 1) High performance: Many BPF schedulers call bpf_ktime_get_ns() frequently - * to account for execution time and track tasks' runtime properties. - * Unfortunately, in some hardware platforms, bpf_ktime_get_ns() -- which - * eventually reads a hardware timestamp counter -- is neither performant nor - * scalable. scx_bpf_now() aims to provide a high-performance clock by - * using the rq clock in the scheduler core whenever possible. - * - * 2) High enough resolution for the BPF scheduler use cases: In most BPF - * scheduler use cases, the required clock resolution is lower than the most - * accurate hardware clock (e.g., rdtsc in x86). scx_bpf_now() basically - * uses the rq clock in the scheduler core whenever it is valid. It considers - * that the rq clock is valid from the time the rq clock is updated - * (update_rq_clock) until the rq is unlocked (rq_unpin_lock). - * - * 3) Monotonically non-decreasing clock for the same CPU: scx_bpf_now() - * guarantees the clock never goes backward when comparing them in the same - * CPU. On the other hand, when comparing clocks in different CPUs, there - * is no such guarantee -- the clock can go backward. It provides a - * monotonically *non-decreasing* clock so that it would provide the same - * clock values in two different scx_bpf_now() calls in the same CPU - * during the same period of when the rq clock is valid. - */ -__bpf_kfunc u64 scx_bpf_now(void) -{ - struct rq *rq; - u64 clock; - - preempt_disable(); - - rq = this_rq(); - if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) { - /* - * If the rq clock is valid, use the cached rq clock. - * - * Note that scx_bpf_now() is re-entrant between a process - * context and an interrupt context (e.g., timer interrupt). - * However, we don't need to consider the race between them - * because such race is not observable from a caller. - */ - clock = READ_ONCE(rq->scx.clock); - } else { - /* - * Otherwise, return a fresh rq clock. - * - * The rq clock is updated outside of the rq lock. - * In this case, keep the updated rq clock invalid so the next - * kfunc call outside the rq lock gets a fresh rq clock. - */ - clock = sched_clock_cpu(cpu_of(rq)); - } - - preempt_enable(); - - return clock; -} - -static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *events) -{ - struct scx_event_stats *e_cpu; - int cpu; - - /* Aggregate per-CPU event counters into @events. */ - memset(events, 0, sizeof(*events)); - for_each_possible_cpu(cpu) { - e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats; - scx_agg_event(events, e_cpu, SCX_EV_SELECT_CPU_FALLBACK); - scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); - scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_KEEP_LAST); - scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_EXITING); - scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); - scx_agg_event(events, e_cpu, SCX_EV_REENQ_IMMED); - scx_agg_event(events, e_cpu, SCX_EV_REENQ_LOCAL_REPEAT); - scx_agg_event(events, e_cpu, SCX_EV_REFILL_SLICE_DFL); - scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DURATION); - scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DISPATCH); - scx_agg_event(events, e_cpu, SCX_EV_BYPASS_ACTIVATE); - scx_agg_event(events, e_cpu, SCX_EV_INSERT_NOT_OWNED); - scx_agg_event(events, e_cpu, SCX_EV_SUB_BYPASS_DISPATCH); - } -} - -/* - * scx_bpf_events - Get a system-wide event counter to - * @events: output buffer from a BPF program - * @events__sz: @events len, must end in '__sz'' for the verifier - */ -__bpf_kfunc void scx_bpf_events(struct scx_event_stats *events, - size_t events__sz) -{ - struct scx_sched *sch; - struct scx_event_stats e_sys; - - rcu_read_lock(); - sch = rcu_dereference(scx_root); - if (sch) - scx_read_events(sch, &e_sys); - else - memset(&e_sys, 0, sizeof(e_sys)); - rcu_read_unlock(); - - /* - * We cannot entirely trust a BPF-provided size since a BPF program - * might be compiled against a different vmlinux.h, of which - * scx_event_stats would be larger (a newer vmlinux.h) or smaller - * (an older vmlinux.h). Hence, we use the smaller size to avoid - * memory corruption. - */ - events__sz = min(events__sz, sizeof(*events)); - memcpy(events, &e_sys, events__sz); -} - -#ifdef CONFIG_CGROUP_SCHED -/** - * scx_bpf_task_cgroup - Return the sched cgroup of a task - * @p: task of interest - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with - * from the scheduler's POV. SCX operations should use this function to - * determine @p's current cgroup as, unlike following @p->cgroups, - * @p->sched_task_group is stable for the duration of the SCX op. See - * SCX_CALL_OP_TASK() for details. - */ -__bpf_kfunc struct cgroup *scx_bpf_task_cgroup(struct task_struct *p, - const struct bpf_prog_aux *aux) -{ - struct task_group *tg = p->sched_task_group; - struct cgroup *cgrp = &cgrp_dfl_root.cgrp; - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - goto out; - - if (!scx_kf_arg_task_ok(sch, p)) - goto out; - - cgrp = tg_cgrp(tg); - -out: - cgroup_get(cgrp); - return cgrp; -} -#endif /* CONFIG_CGROUP_SCHED */ - -__bpf_kfunc_end_defs(); - -BTF_KFUNCS_START(scx_kfunc_ids_any) -BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS | KF_RCU); -BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS | KF_RCU); -BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_kick_cid, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_destroy_dsq, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_dsq_peek, KF_IMPLICIT_ARGS | KF_RCU_PROTECTED | KF_RET_NULL) -BTF_ID_FLAGS(func, scx_bpf_dsq_reenq, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_reenqueue_local___v2, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_iter_scx_dsq_new, KF_IMPLICIT_ARGS | KF_ITER_NEW | KF_RCU_PROTECTED) -BTF_ID_FLAGS(func, bpf_iter_scx_dsq_next, KF_ITER_NEXT | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_iter_scx_dsq_destroy, KF_ITER_DESTROY) -BTF_ID_FLAGS(func, scx_bpf_exit_bstr, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_dump_bstr, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cidperf_cap, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cidperf_cur, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cidperf_set, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_nr_node_ids) -BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids) -BTF_ID_FLAGS(func, scx_bpf_nr_cids) -BTF_ID_FLAGS(func, scx_bpf_nr_online_cids) -BTF_ID_FLAGS(func, scx_bpf_this_cid) -BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE) -BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_task_cid, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_IMPLICIT_ARGS | KF_RET_NULL) -BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) -BTF_ID_FLAGS(func, scx_bpf_cid_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) -BTF_ID_FLAGS(func, scx_bpf_tid_to_task, KF_RET_NULL | KF_RCU_PROTECTED) -BTF_ID_FLAGS(func, scx_bpf_now) -BTF_ID_FLAGS(func, scx_bpf_events) -#ifdef CONFIG_CGROUP_SCHED -BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_IMPLICIT_ARGS | KF_RCU | KF_ACQUIRE) -#endif -BTF_KFUNCS_END(scx_kfunc_ids_any) - -static const struct btf_kfunc_id_set scx_kfunc_set_any = { - .owner = THIS_MODULE, - .set = &scx_kfunc_ids_any, - .filter = scx_kfunc_context_filter, -}; - -/* - * cpu-form kfuncs that are forbidden from cid-form schedulers - * (bpf_sched_ext_ops_cid). Programs targeting the cid struct_ops type must - * use the cid-form alternative (cid/cmask kfuncs). - * - * Membership overlaps with scx_kfunc_ids_{any,idle,select_cpu}; the filter - * tests this set independently and rejects matches before the per-op - * allow-list check runs. - * - * pahole/resolve_btfids scans every BTF_ID_FLAGS() at build time and - * intersects flags across duplicate entries, so each entry must carry the - * same flags as the kfunc's primary declaration; otherwise the flags get - * dropped globally. - */ -BTF_KFUNCS_START(scx_kfunc_ids_cpu_only) -BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) -BTF_ID_FLAGS(func, scx_bpf_cpu_node, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE) -BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_IMPLICIT_ARGS | KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_IMPLICIT_ARGS | KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) -BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) -BTF_KFUNCS_END(scx_kfunc_ids_cpu_only) - -/* - * Per-op kfunc allow flags. Each bit corresponds to a context-sensitive kfunc - * group; an op may permit zero or more groups, with the union expressed in - * scx_kf_allow_flags[]. The verifier-time filter (scx_kfunc_context_filter()) - * consults this table to decide whether a context-sensitive kfunc is callable - * from a given SCX op. - */ -enum scx_kf_allow_flags { - SCX_KF_ALLOW_UNLOCKED = 1 << 0, - SCX_KF_ALLOW_INIT = 1 << 1, - SCX_KF_ALLOW_CPU_RELEASE = 1 << 2, - SCX_KF_ALLOW_DISPATCH = 1 << 3, - SCX_KF_ALLOW_ENQUEUE = 1 << 4, - SCX_KF_ALLOW_SELECT_CPU = 1 << 5, -}; - -/* - * Map each SCX op to the union of kfunc groups it permits, indexed by - * SCX_OP_IDX(op). Ops not listed only permit kfuncs that are not - * context-sensitive. - */ -static const u32 scx_kf_allow_flags[] = { - [SCX_OP_IDX(select_cpu)] = SCX_KF_ALLOW_SELECT_CPU | SCX_KF_ALLOW_ENQUEUE, - [SCX_OP_IDX(enqueue)] = SCX_KF_ALLOW_SELECT_CPU | SCX_KF_ALLOW_ENQUEUE, - [SCX_OP_IDX(dispatch)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH, - [SCX_OP_IDX(cpu_release)] = SCX_KF_ALLOW_CPU_RELEASE, - [SCX_OP_IDX(init_task)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(dump)] = SCX_KF_ALLOW_UNLOCKED, -#ifdef CONFIG_EXT_GROUP_SCHED - [SCX_OP_IDX(cgroup_init)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(cgroup_exit)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(cgroup_prep_move)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(cgroup_cancel_move)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(cgroup_set_weight)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(cgroup_set_bandwidth)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(cgroup_set_idle)] = SCX_KF_ALLOW_UNLOCKED, -#endif /* CONFIG_EXT_GROUP_SCHED */ - [SCX_OP_IDX(sub_attach)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(sub_detach)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(cpu_online)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(cpu_offline)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(init)] = SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT, - [SCX_OP_IDX(exit)] = SCX_KF_ALLOW_UNLOCKED, -}; - -/* - * Verifier-time filter for SCX kfuncs. Registered via the .filter field on - * each per-group btf_kfunc_id_set. The BPF core invokes this for every kfunc - * call in the registered hook (BPF_PROG_TYPE_STRUCT_OPS or - * BPF_PROG_TYPE_SYSCALL), regardless of which set originally introduced the - * kfunc - so the filter must short-circuit on kfuncs it doesn't govern by - * falling through to "allow" when none of the SCX sets contain the kfunc. - */ -int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) -{ - bool in_unlocked = btf_id_set8_contains(&scx_kfunc_ids_unlocked, kfunc_id); - bool in_init = btf_id_set8_contains(&scx_kfunc_ids_init, kfunc_id); - bool in_select_cpu = btf_id_set8_contains(&scx_kfunc_ids_select_cpu, kfunc_id); - bool in_enqueue = btf_id_set8_contains(&scx_kfunc_ids_enqueue_dispatch, kfunc_id); - bool in_dispatch = btf_id_set8_contains(&scx_kfunc_ids_dispatch, kfunc_id); - bool in_cpu_release = btf_id_set8_contains(&scx_kfunc_ids_cpu_release, kfunc_id); - bool in_idle = btf_id_set8_contains(&scx_kfunc_ids_idle, kfunc_id); - bool in_any = btf_id_set8_contains(&scx_kfunc_ids_any, kfunc_id); - bool in_cpu_only = btf_id_set8_contains(&scx_kfunc_ids_cpu_only, kfunc_id); - u32 moff, flags; - - /* Not an SCX kfunc - allow. */ - if (!(in_unlocked || in_init || in_select_cpu || in_enqueue || in_dispatch || - in_cpu_release || in_idle || in_any)) - return 0; - - /* SYSCALL progs (e.g. BPF test_run()) may call unlocked and select_cpu kfuncs. */ - if (prog->type == BPF_PROG_TYPE_SYSCALL) - return (in_unlocked || in_select_cpu || in_idle || in_any) ? 0 : -EACCES; - - if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) - return (in_any || in_idle) ? 0 : -EACCES; - - /* - * add_subprog_and_kfunc() collects all kfunc calls, including dead code - * guarded by bpf_ksym_exists(), before check_attach_btf_id() sets - * prog->aux->st_ops. Allow all kfuncs when st_ops is not yet set; - * do_check_main() re-runs the filter with st_ops set and enforces the - * actual restrictions. - */ - if (!prog->aux->st_ops) - return 0; - - /* - * Non-SCX struct_ops: SCX kfuncs are not permitted. - * - * Both bpf_sched_ext_ops (cpu-form) and bpf_sched_ext_ops_cid - * (cid-form) are valid SCX struct_ops. Member offsets match between - * the two (verified by BUILD_BUG_ON in scx_init()), so the shared - * scx_kf_allow_flags[] table indexed by SCX_MOFF_IDX(moff) applies to - * both. - */ - if (prog->aux->st_ops != &bpf_sched_ext_ops && - prog->aux->st_ops != &bpf_sched_ext_ops_cid) - return -EACCES; - - /* - * cid-form schedulers must use cid/cmask kfuncs. cid and cpu are both - * small s32s and trivially confused, so cpu-only kfuncs are rejected at - * load time. The reverse (cpu-form calling cid-form kfuncs) is - * intentionally permissive to ease gradual cpumask -> cid migration. - */ - if (prog->aux->st_ops == &bpf_sched_ext_ops_cid && in_cpu_only) - return -EACCES; - - /* SCX struct_ops: check the per-op allow list. */ - if (in_any || in_idle) - return 0; - - moff = prog->aux->attach_st_ops_member_off; - flags = scx_kf_allow_flags[SCX_MOFF_IDX(moff)]; - - if ((flags & SCX_KF_ALLOW_UNLOCKED) && in_unlocked) - return 0; - if ((flags & SCX_KF_ALLOW_INIT) && in_init) - return 0; - if ((flags & SCX_KF_ALLOW_CPU_RELEASE) && in_cpu_release) - return 0; - if ((flags & SCX_KF_ALLOW_DISPATCH) && in_dispatch) - return 0; - if ((flags & SCX_KF_ALLOW_ENQUEUE) && in_enqueue) - return 0; - if ((flags & SCX_KF_ALLOW_SELECT_CPU) && in_select_cpu) - return 0; - - return -EACCES; -} - -static int __init scx_init(void) -{ - int ret; - - /* - * sched_ext_ops_cid mirrors sched_ext_ops up to and including @priv. - * Both bpf_scx_init_member() and bpf_scx_check_member() use offsets - * from struct sched_ext_ops; sched_ext_ops_cid relies on those offsets - * matching for the shared fields. Catch any drift at boot. - */ -#define CID_OFFSET_MATCH(cpu_field, cid_field) \ - BUILD_BUG_ON(offsetof(struct sched_ext_ops, cpu_field) != \ - offsetof(struct sched_ext_ops_cid, cid_field)) - /* data fields used by bpf_scx_init_member() */ - CID_OFFSET_MATCH(dispatch_max_batch, dispatch_max_batch); - CID_OFFSET_MATCH(flags, flags); - CID_OFFSET_MATCH(name, name); - CID_OFFSET_MATCH(timeout_ms, timeout_ms); - CID_OFFSET_MATCH(exit_dump_len, exit_dump_len); - CID_OFFSET_MATCH(hotplug_seq, hotplug_seq); - CID_OFFSET_MATCH(sub_cgroup_id, sub_cgroup_id); - /* shared callbacks: the union view requires byte-for-byte offset match */ - CID_OFFSET_MATCH(enqueue, enqueue); - CID_OFFSET_MATCH(dequeue, dequeue); - CID_OFFSET_MATCH(dispatch, dispatch); - CID_OFFSET_MATCH(tick, tick); - CID_OFFSET_MATCH(runnable, runnable); - CID_OFFSET_MATCH(running, running); - CID_OFFSET_MATCH(stopping, stopping); - CID_OFFSET_MATCH(quiescent, quiescent); - CID_OFFSET_MATCH(yield, yield); - CID_OFFSET_MATCH(core_sched_before, core_sched_before); - CID_OFFSET_MATCH(set_weight, set_weight); - CID_OFFSET_MATCH(update_idle, update_idle); - CID_OFFSET_MATCH(init_task, init_task); - CID_OFFSET_MATCH(exit_task, exit_task); - CID_OFFSET_MATCH(enable, enable); - CID_OFFSET_MATCH(disable, disable); - CID_OFFSET_MATCH(dump, dump); - CID_OFFSET_MATCH(dump_task, dump_task); - CID_OFFSET_MATCH(sub_attach, sub_attach); - CID_OFFSET_MATCH(sub_detach, sub_detach); - CID_OFFSET_MATCH(init, init); - CID_OFFSET_MATCH(exit, exit); -#ifdef CONFIG_EXT_GROUP_SCHED - CID_OFFSET_MATCH(cgroup_init, cgroup_init); - CID_OFFSET_MATCH(cgroup_exit, cgroup_exit); - CID_OFFSET_MATCH(cgroup_prep_move, cgroup_prep_move); - CID_OFFSET_MATCH(cgroup_move, cgroup_move); - CID_OFFSET_MATCH(cgroup_cancel_move, cgroup_cancel_move); - CID_OFFSET_MATCH(cgroup_set_weight, cgroup_set_weight); - CID_OFFSET_MATCH(cgroup_set_bandwidth, cgroup_set_bandwidth); - CID_OFFSET_MATCH(cgroup_set_idle, cgroup_set_idle); -#endif - /* renamed callbacks must occupy the same slot as their cpu-form sibling */ - CID_OFFSET_MATCH(select_cpu, select_cid); - CID_OFFSET_MATCH(set_cpumask, set_cmask); - CID_OFFSET_MATCH(cpu_online, cid_online); - CID_OFFSET_MATCH(cpu_offline, cid_offline); - CID_OFFSET_MATCH(dump_cpu, dump_cid); - /* @priv tail must align since both share the same data block */ - CID_OFFSET_MATCH(priv, priv); - /* - * cid-form must end exactly at @priv - validate_ops() skips - * cpu_acquire/cpu_release for cid-form because reading those fields - * past the BPF allocation would be UB. - */ - BUILD_BUG_ON(offsetof(struct sched_ext_ops_cid, __end) != - offsetofend(struct sched_ext_ops, priv)); -#undef CID_OFFSET_MATCH - - /* - * kfunc registration can't be done from init_sched_ext_class() as - * register_btf_kfunc_id_set() needs most of the system to be up. - * - * Some kfuncs are context-sensitive and can only be called from - * specific SCX ops. They are grouped into per-context BTF sets, each - * registered with scx_kfunc_context_filter as its .filter callback. The - * BPF core dedups identical filter pointers per hook - * (btf_populate_kfunc_set()), so the filter is invoked exactly once per - * kfunc lookup; it consults scx_kf_allow_flags[] to enforce per-op - * restrictions at verify time. - */ - if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, - &scx_kfunc_set_enqueue_dispatch)) || - (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, - &scx_kfunc_set_dispatch)) || - (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, - &scx_kfunc_set_cpu_release)) || - (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, - &scx_kfunc_set_unlocked)) || - (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, - &scx_kfunc_set_unlocked)) || - (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, - &scx_kfunc_set_any)) || - (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, - &scx_kfunc_set_any)) || - (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, - &scx_kfunc_set_any))) { - pr_err("sched_ext: Failed to register kfunc sets (%d)\n", ret); - return ret; - } - - ret = scx_idle_init(); - if (ret) { - pr_err("sched_ext: Failed to initialize idle tracking (%d)\n", ret); - return ret; - } - - ret = scx_cid_kfunc_init(); - if (ret) { - pr_err("sched_ext: Failed to register cid kfuncs (%d)\n", ret); - return ret; - } - - ret = register_bpf_struct_ops(&bpf_sched_ext_ops, sched_ext_ops); - if (ret) { - pr_err("sched_ext: Failed to register struct_ops (%d)\n", ret); - return ret; - } - - ret = register_bpf_struct_ops(&bpf_sched_ext_ops_cid, sched_ext_ops_cid); - if (ret) { - pr_err("sched_ext: Failed to register cid struct_ops (%d)\n", ret); - return ret; - } - - ret = register_pm_notifier(&scx_pm_notifier); - if (ret) { - pr_err("sched_ext: Failed to register PM notifier (%d)\n", ret); - return ret; - } - - scx_kset = kset_create_and_add("sched_ext", &scx_uevent_ops, kernel_kobj); - if (!scx_kset) { - pr_err("sched_ext: Failed to create /sys/kernel/sched_ext\n"); - return -ENOMEM; - } - - ret = sysfs_create_group(&scx_kset->kobj, &scx_global_attr_group); - if (ret < 0) { - pr_err("sched_ext: Failed to add global attributes\n"); - return ret; - } - - return 0; -} -__initcall(scx_init); diff --git a/kernel/sched/ext.h b/kernel/sched/ext.h deleted file mode 100644 index 0b7fc46aee08..000000000000 --- a/kernel/sched/ext.h +++ /dev/null @@ -1,95 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst - * - * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2022 Tejun Heo - * Copyright (c) 2022 David Vernet - */ -#ifdef CONFIG_SCHED_CLASS_EXT - -void scx_tick(struct rq *rq); -void init_scx_entity(struct sched_ext_entity *scx); -void scx_pre_fork(struct task_struct *p); -int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs); -void scx_post_fork(struct task_struct *p); -void scx_cancel_fork(struct task_struct *p); -bool scx_can_stop_tick(struct rq *rq); -void scx_rq_activate(struct rq *rq); -void scx_rq_deactivate(struct rq *rq); -int scx_check_setscheduler(struct task_struct *p, int policy); -bool task_should_scx(int policy); -bool scx_allow_ttwu_queue(const struct task_struct *p); -void init_sched_ext_class(void); - -static inline u32 scx_cpuperf_target(s32 cpu) -{ - if (scx_enabled()) - return cpu_rq(cpu)->scx.cpuperf_target; - else - return 0; -} - -static inline bool task_on_scx(const struct task_struct *p) -{ - return scx_enabled() && p->sched_class == &ext_sched_class; -} - -#ifdef CONFIG_SCHED_CORE -bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, - bool in_fi); -#endif - -#else /* CONFIG_SCHED_CLASS_EXT */ - -static inline void scx_tick(struct rq *rq) {} -static inline void scx_pre_fork(struct task_struct *p) {} -static inline int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs) { return 0; } -static inline void scx_post_fork(struct task_struct *p) {} -static inline void scx_cancel_fork(struct task_struct *p) {} -static inline u32 scx_cpuperf_target(s32 cpu) { return 0; } -static inline bool scx_can_stop_tick(struct rq *rq) { return true; } -static inline void scx_rq_activate(struct rq *rq) {} -static inline void scx_rq_deactivate(struct rq *rq) {} -static inline int scx_check_setscheduler(struct task_struct *p, int policy) { return 0; } -static inline bool task_on_scx(const struct task_struct *p) { return false; } -static inline bool scx_allow_ttwu_queue(const struct task_struct *p) { return true; } -static inline void init_sched_ext_class(void) {} - -#endif /* CONFIG_SCHED_CLASS_EXT */ - -#ifdef CONFIG_SCHED_CLASS_EXT -void __scx_update_idle(struct rq *rq, bool idle, bool do_notify); - -static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) -{ - if (scx_enabled()) - __scx_update_idle(rq, idle, do_notify); -} -#else -static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) {} -#endif - -#ifdef CONFIG_CGROUP_SCHED -#ifdef CONFIG_EXT_GROUP_SCHED -void scx_tg_init(struct task_group *tg); -int scx_tg_online(struct task_group *tg); -void scx_tg_offline(struct task_group *tg); -int scx_cgroup_can_attach(struct cgroup_taskset *tset); -void scx_cgroup_move_task(struct task_struct *p); -void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); -void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); -void scx_group_set_idle(struct task_group *tg, bool idle); -void scx_group_set_bandwidth(struct task_group *tg, u64 period_us, u64 quota_us, u64 burst_us); -#else /* CONFIG_EXT_GROUP_SCHED */ -static inline void scx_tg_init(struct task_group *tg) {} -static inline int scx_tg_online(struct task_group *tg) { return 0; } -static inline void scx_tg_offline(struct task_group *tg) {} -static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } -static inline void scx_cgroup_move_task(struct task_struct *p) {} -static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} -static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} -static inline void scx_group_set_idle(struct task_group *tg, bool idle) {} -static inline void scx_group_set_bandwidth(struct task_group *tg, u64 period_us, u64 quota_us, u64 burst_us) {} -#endif /* CONFIG_EXT_GROUP_SCHED */ -#endif /* CONFIG_CGROUP_SCHED */ diff --git a/kernel/sched/ext/arena.c b/kernel/sched/ext/arena.c new file mode 100644 index 000000000000..493c2424f842 --- /dev/null +++ b/kernel/sched/ext/arena.c @@ -0,0 +1,131 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * scx_arena_pool: kernel-side sub-allocator over BPF-arena pages. + * + * Each chunk added to @sch->arena_pool comes from one + * bpf_arena_alloc_pages_sleepable() call and is registered at the + * kernel-side mapping address. Callers translate to the BPF-arena form + * themselves if needed. + * + * Allocations grow the pool on demand. Underlying arena pages are released + * when the arena map itself is torn down. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ + +enum scx_arena_consts { + SCX_ARENA_MIN_ORDER = 3, /* 8-byte minimum sub-allocation */ + SCX_ARENA_GROW_PAGES = 4, /* per growth */ +}; + +s32 scx_arena_pool_init(struct scx_sched *sch) +{ + if (!sch->arena_map) + return 0; + + sch->arena_pool = gen_pool_create(SCX_ARENA_MIN_ORDER, NUMA_NO_NODE); + if (!sch->arena_pool) + return -ENOMEM; + return 0; +} + +static void scx_arena_clear_chunk(struct gen_pool *pool, struct gen_pool_chunk *chunk, + void *data) +{ + int order = pool->min_alloc_order; + size_t chunk_sz = chunk->end_addr - chunk->start_addr + 1; + unsigned long end_bit = chunk_sz >> order; + unsigned long b, e; + + for_each_set_bitrange(b, e, chunk->bits, end_bit) + gen_pool_free(pool, chunk->start_addr + (b << order), + (e - b) << order); +} + +/* + * Tear down the pool. Outstanding gen_pool allocations are freed via + * scx_arena_clear_chunk() so gen_pool_destroy() doesn't BUG. The underlying + * arena pages are released when the arena map itself is torn down. + */ +void scx_arena_pool_destroy(struct scx_sched *sch) +{ + if (!sch->arena_pool) + return; + gen_pool_for_each_chunk(sch->arena_pool, scx_arena_clear_chunk, NULL); + gen_pool_destroy(sch->arena_pool); + sch->arena_pool = NULL; +} + +/* + * Grow the pool by @page_cnt pages. bpf_arena_alloc_pages_sleepable() and + * gen_pool_add() (which calls vzalloc(GFP_KERNEL)) require a sleepable + * context. + */ +static int scx_arena_grow(struct scx_sched *sch, u32 page_cnt) +{ + u64 kern_vm_start; + u32 uaddr32; + void *p; + int ret; + + if (!sch->arena_map || !sch->arena_pool) + return -EINVAL; + + p = bpf_arena_alloc_pages_sleepable(sch->arena_map, NULL, + page_cnt, NUMA_NO_NODE, 0); + if (!p) + return -ENOMEM; + + uaddr32 = (u32)(unsigned long)p; + /* arena.o, which defines these, is built only on MMU && 64BIT */ +#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) + kern_vm_start = bpf_arena_map_kern_vm_start(sch->arena_map); +#else + kern_vm_start = 0; +#endif + + ret = gen_pool_add(sch->arena_pool, kern_vm_start + uaddr32, + page_cnt * PAGE_SIZE, NUMA_NO_NODE); + if (ret) { + bpf_arena_free_pages_non_sleepable(sch->arena_map, p, page_cnt); + return ret; + } + return 0; +} + +/* + * Allocate @size bytes from the arena pool. Returns kernel VA on success, NULL + * on failure. May grow the pool via scx_arena_grow() which sleeps. Caller must + * be in a GFP_KERNEL context. + */ +void *scx_arena_alloc(struct scx_sched *sch, size_t size) +{ + unsigned long kern_va; + u32 page_cnt; + + might_sleep(); + + if (!sch->arena_pool) + return NULL; + + while (true) { + kern_va = gen_pool_alloc(sch->arena_pool, size); + if (kern_va) + break; + page_cnt = max_t(u32, SCX_ARENA_GROW_PAGES, + (size + PAGE_SIZE - 1) >> PAGE_SHIFT); + if (scx_arena_grow(sch, page_cnt)) + return NULL; + } + + return (void *)kern_va; +} + +void scx_arena_free(struct scx_sched *sch, void *kern_va, size_t size) +{ + if (sch->arena_pool && kern_va) + gen_pool_free(sch->arena_pool, (unsigned long)kern_va, size); +} diff --git a/kernel/sched/ext/arena.h b/kernel/sched/ext/arena.h new file mode 100644 index 000000000000..4f3610160102 --- /dev/null +++ b/kernel/sched/ext/arena.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Copyright (c) 2025 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2025 Tejun Heo + */ +#ifndef _KERNEL_SCHED_EXT_ARENA_H +#define _KERNEL_SCHED_EXT_ARENA_H + +struct scx_sched; + +s32 scx_arena_pool_init(struct scx_sched *sch); +void scx_arena_pool_destroy(struct scx_sched *sch); +void *scx_arena_alloc(struct scx_sched *sch, size_t size); +void scx_arena_free(struct scx_sched *sch, void *kern_va, size_t size); + +#endif /* _KERNEL_SCHED_EXT_ARENA_H */ diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c new file mode 100644 index 000000000000..aeaea88f34c5 --- /dev/null +++ b/kernel/sched/ext/cid.c @@ -0,0 +1,707 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ +#include + +/* + * cid tables. + * + * Pointers are published once on first enable and never revoked. The default + * mapping is populated before ops.init() runs; scx_bpf_cid_override() commits + * before it returns. As long as the BPF scheduler only uses the tables from + * those points onward, it sees a consistent view. + */ +s16 *scx_cid_to_cpu_tbl; +s16 *scx_cpu_to_cid_tbl; +struct scx_cid_topo *scx_cid_topo; + +#define SCX_CID_TOPO_NEG (struct scx_cid_topo) { \ + .core_cid = -1, .core_idx = -1, .llc_cid = -1, .llc_idx = -1, \ + .node_cid = -1, .node_idx = -1, \ +} + +/* + * Return @cpu's LLC shared_cpu_map. If cacheinfo isn't populated (offline or + * !present), record @cpu in @fallbacks and return its node mask instead - the + * worst that can happen is that the cpu's LLC becomes coarser than reality. + */ +static const struct cpumask *cpu_llc_mask(int cpu, struct cpumask *fallbacks) +{ + struct cpu_cacheinfo *ci = get_cpu_cacheinfo(cpu); + + if (!ci || !ci->info_list || !ci->num_leaves) { + cpumask_set_cpu(cpu, fallbacks); + return cpumask_of_node(cpu_to_node(cpu)); + } + return &ci->info_list[ci->num_leaves - 1].shared_cpu_map; +} + +/* Allocate the cid tables once on first enable; never freed. */ +static s32 scx_cid_arrays_alloc(void) +{ + u32 npossible = num_possible_cpus(); + s16 *cid_to_cpu, *cpu_to_cid; + struct scx_cid_topo *cid_topo; + + if (scx_cid_to_cpu_tbl) + return 0; + + cid_to_cpu = kzalloc_objs(*scx_cid_to_cpu_tbl, npossible, GFP_KERNEL); + cpu_to_cid = kzalloc_objs(*scx_cpu_to_cid_tbl, nr_cpu_ids, GFP_KERNEL); + cid_topo = kmalloc_objs(*scx_cid_topo, npossible, GFP_KERNEL); + + if (!cid_to_cpu || !cpu_to_cid || !cid_topo) { + kfree(cid_to_cpu); + kfree(cpu_to_cid); + kfree(cid_topo); + return -ENOMEM; + } + + WRITE_ONCE(scx_cid_to_cpu_tbl, cid_to_cpu); + WRITE_ONCE(scx_cpu_to_cid_tbl, cpu_to_cid); + WRITE_ONCE(scx_cid_topo, cid_topo); + return 0; +} + +/** + * scx_cid_init - build the cid mapping + * @sch: the scx_sched being initialized; used as the scx_error() target + * + * See "Topological CPU IDs" in cid.h for the model. Walk online cpus by + * intersection at each level (parent_scratch & this_level_mask), which keeps + * containment correct by construction and naturally splits a physical LLC + * straddling two NUMA nodes into two LLC units. The caller must hold + * cpus_read_lock. + */ +s32 scx_cid_init(struct scx_sched *sch) +{ + cpumask_var_t to_walk __free(free_cpumask_var) = CPUMASK_VAR_NULL; + cpumask_var_t node_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; + cpumask_var_t llc_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; + cpumask_var_t core_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; + cpumask_var_t llc_fallback __free(free_cpumask_var) = CPUMASK_VAR_NULL; + cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL; + u32 next_cid = 0; + s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0; + s32 cpu, ret; + + /* CMASK_MAX_WORDS in cid.bpf.h covers NR_CPUS up to 8192 */ + BUILD_BUG_ON(NR_CPUS > 8192); + + lockdep_assert_cpus_held(); + + ret = scx_cid_arrays_alloc(); + if (ret) + return ret; + + if (!zalloc_cpumask_var(&to_walk, GFP_KERNEL) || + !zalloc_cpumask_var(&node_scratch, GFP_KERNEL) || + !zalloc_cpumask_var(&llc_scratch, GFP_KERNEL) || + !zalloc_cpumask_var(&core_scratch, GFP_KERNEL) || + !zalloc_cpumask_var(&llc_fallback, GFP_KERNEL) || + !zalloc_cpumask_var(&online_no_topo, GFP_KERNEL)) + return -ENOMEM; + + /* -1 sentinels for sparse-possible cpu id holes (0 is a valid cid) */ + for (cpu = 0; cpu < nr_cpu_ids; cpu++) + scx_cpu_to_cid_tbl[cpu] = -1; + + cpumask_copy(to_walk, cpu_online_mask); + + while (!cpumask_empty(to_walk)) { + s32 next_cpu = cpumask_first(to_walk); + s32 nid = cpu_to_node(next_cpu); + s32 node_cid = next_cid; + s32 node_idx; + + /* + * No NUMA info: skip and let the tail loop assign a no-topo + * cid. cpumask_of_node(-1) is undefined. + */ + if (nid < 0) { + cpumask_clear_cpu(next_cpu, to_walk); + continue; + } + + node_idx = next_node_idx++; + + /* node_scratch = to_walk & this node */ + cpumask_and(node_scratch, to_walk, cpumask_of_node(nid)); + if (WARN_ON_ONCE(!cpumask_test_cpu(next_cpu, node_scratch))) + return -EINVAL; + + while (!cpumask_empty(node_scratch)) { + s32 ncpu = cpumask_first(node_scratch); + const struct cpumask *llc_mask = cpu_llc_mask(ncpu, llc_fallback); + s32 llc_cid = next_cid; + s32 llc_idx = next_llc_idx++; + + /* llc_scratch = node_scratch & this llc */ + cpumask_and(llc_scratch, node_scratch, llc_mask); + if (WARN_ON_ONCE(!cpumask_test_cpu(ncpu, llc_scratch))) + return -EINVAL; + + while (!cpumask_empty(llc_scratch)) { + s32 lcpu = cpumask_first(llc_scratch); + const struct cpumask *sib = topology_sibling_cpumask(lcpu); + s32 core_cid = next_cid; + s32 core_idx = next_core_idx++; + s32 ccpu; + + /* core_scratch = llc_scratch & this core */ + cpumask_and(core_scratch, llc_scratch, sib); + if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch))) + return -EINVAL; + + for_each_cpu(ccpu, core_scratch) { + s32 cid = next_cid++; + + scx_cid_to_cpu_tbl[cid] = ccpu; + scx_cpu_to_cid_tbl[ccpu] = cid; + scx_cid_topo[cid] = (struct scx_cid_topo){ + .core_cid = core_cid, + .core_idx = core_idx, + .llc_cid = llc_cid, + .llc_idx = llc_idx, + .node_cid = node_cid, + .node_idx = node_idx, + }; + + cpumask_clear_cpu(ccpu, llc_scratch); + cpumask_clear_cpu(ccpu, node_scratch); + cpumask_clear_cpu(ccpu, to_walk); + } + } + } + } + + /* + * No-topo section: any possible cpu without a cid - normally just the + * not-online ones. Collect any currently-online cpus that land here in + * @online_no_topo so we can warn about them at the end. + */ + for_each_cpu(cpu, cpu_possible_mask) { + s32 cid; + + if (__scx_cpu_to_cid(cpu) != -1) + continue; + if (cpu_online(cpu)) + cpumask_set_cpu(cpu, online_no_topo); + + cid = next_cid++; + scx_cid_to_cpu_tbl[cid] = cpu; + scx_cpu_to_cid_tbl[cpu] = cid; + scx_cid_topo[cid] = SCX_CID_TOPO_NEG; + } + + if (!cpumask_empty(llc_fallback)) + pr_warn("scx_cid: cpus without cacheinfo, using node mask as llc: %*pbl\n", + cpumask_pr_args(llc_fallback)); + if (!cpumask_empty(online_no_topo)) + pr_warn("scx_cid: online cpus with no usable topology: %*pbl\n", + cpumask_pr_args(online_no_topo)); + + return 0; +} + +/** + * scx_cmask_clear - Zero every bit in @m's active range + * @m: cmask to clear + * + * Storage past the active range is left as is. + */ +void scx_cmask_clear(struct scx_cmask *m) +{ + u32 nr_words; + + if (!m->nr_cids) + return; + nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1; + memset(m->bits, 0, nr_words * sizeof(u64)); +} + +/** + * scx_cmask_fill - Set every bit in @m's active range + * @m: cmask to fill + * + * Counterpart to scx_cmask_clear(). Storage past the active range is left as is. + */ +void scx_cmask_fill(struct scx_cmask *m) +{ + u32 nr_words, head_bits, tail_bits; + + if (!m->nr_cids) + return; + nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1; + memset(m->bits, 0xff, nr_words * sizeof(u64)); + + /* clear word-0 bits below base */ + head_bits = m->base & 63; + if (head_bits) + m->bits[0] &= ~((1ULL << head_bits) - 1); + + /* clear last-word bits at or past base + nr_cids */ + tail_bits = (m->base + m->nr_cids) & 63; + if (tail_bits) + m->bits[nr_words - 1] &= (1ULL << tail_bits) - 1; +} + +/** + * scx_cpumask_to_cmask - Translate a kernel cpumask into a cmask + * @src: source cpumask + * @dst: cmask to write + * + * Clear @dst's active range and set the bit for each cid whose cpu is in + * @src and lies within that range. Out-of-range cids are silently ignored. + */ +void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst) +{ + s32 cpu; + + scx_cmask_clear(dst); + for_each_cpu(cpu, src) { + s32 cid = __scx_cpu_to_cid(cpu); + + if (cid >= 0) + __scx_cmask_set(cid, dst); + } +} + +__bpf_kfunc_start_defs(); + +/** + * scx_bpf_cid_override - Install an explicit cpu->cid mapping + * @cpu_to_cid: array of nr_cpu_ids s32 entries (cid for each cpu) + * @cpu_to_cid__sz: must be nr_cpu_ids * sizeof(s32) bytes + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * May only be called from ops.init() of the root scheduler. Replace the + * topology-probed cid mapping with the caller-provided one. Each possible cpu + * must map to a unique cid in [0, num_possible_cpus()). Topo info is cleared. + * On invalid input, trigger scx_error() to abort the scheduler. + */ +__bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz, + const struct bpf_prog_aux *aux) +{ + cpumask_var_t seen __free(free_cpumask_var) = CPUMASK_VAR_NULL; + struct scx_sched *sch; + bool alloced; + s32 cpu, cid; + + /* GFP_KERNEL alloc must happen before the rcu read section */ + alloced = zalloc_cpumask_var(&seen, GFP_KERNEL); + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return; + + if (!alloced) { + scx_error(sch, "scx_bpf_cid_override: failed to allocate cpumask"); + return; + } + + if (scx_parent(sch)) { + scx_error(sch, "scx_bpf_cid_override() only allowed from root sched"); + return; + } + + if (cpu_to_cid__sz != nr_cpu_ids * sizeof(s32)) { + scx_error(sch, "scx_bpf_cid_override: expected %zu bytes, got %u", + nr_cpu_ids * sizeof(s32), cpu_to_cid__sz); + return; + } + + for_each_possible_cpu(cpu) { + s32 c = cpu_to_cid[cpu]; + + if (!cid_valid(sch, c)) + return; + if (cpumask_test_and_set_cpu(c, seen)) { + scx_error(sch, "cid %d assigned to multiple cpus", c); + return; + } + scx_cpu_to_cid_tbl[cpu] = c; + scx_cid_to_cpu_tbl[c] = cpu; + } + + /* Invalidate stale topo info - the override carries no topology. */ + for (cid = 0; cid < num_possible_cpus(); cid++) + scx_cid_topo[cid] = SCX_CID_TOPO_NEG; +} + +/** + * scx_bpf_cid_to_cpu - Return the raw CPU id for @cid + * @cid: cid to look up + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Return the raw CPU id for @cid. Trigger scx_error() and return -EINVAL if + * @cid is invalid. The cid<->cpu mapping is static for the lifetime of the + * loaded scheduler, so the BPF side can cache the result to avoid repeated + * kfunc invocations. + */ +__bpf_kfunc s32 scx_bpf_cid_to_cpu(s32 cid, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -EINVAL; + return scx_cid_to_cpu(sch, cid); +} + +/** + * scx_bpf_cpu_to_cid - Return the cid for @cpu + * @cpu: cpu to look up + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Return the cid for @cpu. Trigger scx_error() and return -EINVAL if @cpu is + * invalid. The cid<->cpu mapping is static for the lifetime of the loaded + * scheduler, so the BPF side can cache the result to avoid repeated kfunc + * invocations. + */ +__bpf_kfunc s32 scx_bpf_cpu_to_cid(s32 cpu, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -EINVAL; + return scx_cpu_to_cid(sch, cpu); +} + +/* + * Set ops on cmasks. cmask_walk_op2() shares one walk across mutating + * (and/or/copy/andnot) and predicate (subset/intersects) two-cmask forms; + * cmask_walk_op1() does the same shape over a single cmask range. Every public + * entry passes a compile-time-constant @op; cmask_walk_op{1,2}() and + * cmask_word_op{1,2}() are __always_inline so the inner switch collapses to the + * selected op and cmask_op2_is_pred() folds the predicate early-exit out of + * mutating ops. + * + * Two-cmask ops only touch @dst bits inside the intersection of the two ranges; + * bits outside stay untouched. In particular, scx_cmask_copy() does NOT zero + * @dst bits that lie outside @src's range. + * + * The _RACY variants are otherwise identical to their non-racy counterpart but + * read @src word-by-word via data_race(). Memory ordering with concurrent + * writers is the caller's responsibility. + */ +enum cmask_op2 { + /* mutating */ + CMASK_OP2_AND, + CMASK_OP2_OR, + CMASK_OP2_OR_RACY, + CMASK_OP2_COPY, + CMASK_OP2_COPY_RACY, + CMASK_OP2_ANDNOT, + /* predicates - short-circuit when the per-word result is true */ + CMASK_OP2_SUBSET, + CMASK_OP2_INTERSECTS, +}; + +static __always_inline bool cmask_op2_is_pred(const enum cmask_op2 op) +{ + return op == CMASK_OP2_SUBSET || op == CMASK_OP2_INTERSECTS; +} + +static __always_inline bool cmask_word_op2(u64 *av, const u64 *bp, u64 mask, + const enum cmask_op2 op) +{ + switch (op) { + case CMASK_OP2_AND: + *av &= ~mask | *bp; + return false; + case CMASK_OP2_OR: + *av |= *bp & mask; + return false; + case CMASK_OP2_OR_RACY: + *av |= data_race(*bp) & mask; + return false; + case CMASK_OP2_COPY: + *av = (*av & ~mask) | (*bp & mask); + return false; + case CMASK_OP2_COPY_RACY: + *av = (*av & ~mask) | (data_race(*bp) & mask); + return false; + case CMASK_OP2_ANDNOT: + *av &= ~(*bp & mask); + return false; + case CMASK_OP2_SUBSET: + /* stop on the first bit in @sub not set in @super */ + return (*bp & ~*av) & mask; + case CMASK_OP2_INTERSECTS: + return (*av & *bp) & mask; + } + unreachable(); +} + +/* + * Walk the intersection of [@a_base, @a_base + @a_nr_cids) with [@b_base, + * @b_base + @b_nr_cids) word by word, applying @op. Mutating ops walk all words + * and return false; predicates return true on the first word whose per-word + * test is true. Empty intersection returns false (matches "no bits to consider" + * for both mutate and predicate). + * + * Base/nr_cids are taken as parameters so callers with snapshotted bounds can + * drive the walk with values independent of the cmask's header. + */ +static __always_inline bool cmask_walk_op2(u64 *a_bits, u32 a_base, u32 a_nr_cids, + const u64 *b_bits, u32 b_base, u32 b_nr_cids, + const enum cmask_op2 op) +{ + u32 lo = max(a_base, b_base); + u32 hi = min(a_base + a_nr_cids, b_base + b_nr_cids); + u32 a_word_off = a_base / 64; + u32 b_word_off = b_base / 64; + u32 lo_word = lo / 64; + u32 hi_word = (hi - 1) / 64; + u64 head_mask = GENMASK_U64(63, lo & 63); + u64 tail_mask = GENMASK_U64((hi - 1) & 63, 0); + u32 w; + + if (lo >= hi) + return false; + + if (lo_word == hi_word) + return cmask_word_op2(&a_bits[lo_word - a_word_off], + &b_bits[lo_word - b_word_off], + head_mask & tail_mask, op); + + if (cmask_word_op2(&a_bits[lo_word - a_word_off], + &b_bits[lo_word - b_word_off], head_mask, op) && + cmask_op2_is_pred(op)) + return true; + + for (w = lo_word + 1; w < hi_word; w++) + if (cmask_word_op2(&a_bits[w - a_word_off], + &b_bits[w - b_word_off], ~0ULL, op) && + cmask_op2_is_pred(op)) + return true; + + return cmask_word_op2(&a_bits[hi_word - a_word_off], + &b_bits[hi_word - b_word_off], tail_mask, op); +} + +enum cmask_op1 { + CMASK_OP1_ANY_SET, +}; + +static __always_inline bool cmask_word_op1(const u64 *ap, u64 mask, + const enum cmask_op1 op) +{ + switch (op) { + case CMASK_OP1_ANY_SET: + return *ap & mask; + } + unreachable(); +} + +/* + * Walk [@a_base, @a_base + @a_nr_cids) of @a_bits word by word, applying @op. + * Returns true on the first word whose per-word test is true; returns false if + * no word matches or the range is empty. All current op1s short-circuit on + * per-word true; if a non-predicate op1 lands here, add a cmask_op1_is_pred() + * guard analogous to cmask_op2_is_pred(). + */ +static __always_inline bool cmask_walk_op1(const u64 *a_bits, u32 a_base, + u32 a_nr_cids, + const enum cmask_op1 op) +{ + u32 lo = a_base; + u32 hi = a_base + a_nr_cids; + u32 a_word_off = a_base / 64; + u32 lo_word = lo / 64; + u32 hi_word = (hi - 1) / 64; + u64 head_mask = GENMASK_U64(63, lo & 63); + u64 tail_mask = GENMASK_U64((hi - 1) & 63, 0); + u32 w; + + if (lo >= hi) + return false; + + if (lo_word == hi_word) + return cmask_word_op1(&a_bits[lo_word - a_word_off], + head_mask & tail_mask, op); + + if (cmask_word_op1(&a_bits[lo_word - a_word_off], head_mask, op)) + return true; + for (w = lo_word + 1; w < hi_word; w++) + if (cmask_word_op1(&a_bits[w - a_word_off], ~0ULL, op)) + return true; + return cmask_word_op1(&a_bits[hi_word - a_word_off], tail_mask, op); +} + +void scx_cmask_and(struct scx_cmask *dst, const struct scx_cmask *src) +{ + cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_AND); +} + +void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src) +{ + cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_OR); +} + +/** + * scx_cmask_or_racy - OR @src into @dst, reading @src without locking + * + * @src is read word-by-word through data_race(). Same per-bit independence + * rationale as scx_cmask_copy_racy(). Memory ordering with writers is the + * caller's responsibility. + */ +void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src) +{ + cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_OR_RACY); +} + +void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src) +{ + cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_COPY); +} + +/** + * scx_cmask_copy_racy - Snapshot @src into @dst without locking + * + * @src is read word-by-word through data_race(). Head/tail masking matches + * scx_cmask_copy(). Each bit in a cmask is independent, so partial updates + * just leave some bits fresher than others. Memory ordering with writers is + * the caller's responsibility. + */ +void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src) +{ + cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_COPY_RACY); +} + +void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src) +{ + cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_ANDNOT); +} + +/* + * Return true if @cm has any bit set in [@lo, @hi). Caller must ensure + * [@lo, @hi) is contained in @cm's range. + */ +static bool cmask_any_set_in_range(const struct scx_cmask *cm, u32 lo, u32 hi) +{ + if (lo >= hi) + return false; + return cmask_walk_op1(&cm->bits[lo / 64 - cm->base / 64], lo, hi - lo, + CMASK_OP1_ANY_SET); +} + +/** + * scx_cmask_subset - test whether @sub is a subset of @super + * @sub: cmask to test + * @super: cmask to test against + * + * Return true iff every set bit of @sub is also set in @super. + */ +bool scx_cmask_subset(const struct scx_cmask *sub, const struct scx_cmask *super) +{ + u32 super_end = super->base + super->nr_cids; + u32 sub_end = sub->base + sub->nr_cids; + + /* + * Set bits in @sub outside @super's range can't be in @super, so any + * such bit means not a subset. The walk below only visits words + * common to both ranges, so these need a separate scan. + */ + if (sub->base < super->base && + cmask_any_set_in_range(sub, sub->base, min(super->base, sub_end))) + return false; + if (sub_end > super_end && + cmask_any_set_in_range(sub, max(sub->base, super_end), sub_end)) + return false; + + return !cmask_walk_op2((u64 *)super->bits, super->base, super->nr_cids, + sub->bits, sub->base, sub->nr_cids, CMASK_OP2_SUBSET); +} + +bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b) +{ + return cmask_walk_op2((u64 *)a->bits, a->base, a->nr_cids, + b->bits, b->base, b->nr_cids, CMASK_OP2_INTERSECTS); +} + +/** + * scx_cmask_empty - Test whether @m has no bits set + * @m: cmask to test + * + * Return true iff @m's active range has no bits set. + */ +bool scx_cmask_empty(const struct scx_cmask *m) +{ + return !cmask_any_set_in_range(m, m->base, m->base + m->nr_cids); +} + +/** + * scx_bpf_cid_topo - Copy out per-cid topology info + * @cid: cid to look up + * @out__uninit: where to copy the topology info; fully written by this call + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Fill @out__uninit with the topology info for @cid. Trigger scx_error() if + * @cid is out of range. If @cid is valid but in the no-topo section, all fields + * are set to -1. + */ +__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out__uninit, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch) || !cid_valid(sch, cid)) { + *out__uninit = SCX_CID_TOPO_NEG; + return; + } + + *out__uninit = READ_ONCE(scx_cid_topo)[cid]; +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(scx_kfunc_ids_init) +BTF_ID_FLAGS(func, scx_bpf_cid_override, KF_IMPLICIT_ARGS | KF_SLEEPABLE) +BTF_KFUNCS_END(scx_kfunc_ids_init) + +static const struct btf_kfunc_id_set scx_kfunc_set_init = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_init, + .filter = scx_kfunc_context_filter, +}; + +BTF_KFUNCS_START(scx_kfunc_ids_cid) +BTF_ID_FLAGS(func, scx_bpf_cid_to_cpu, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpu_to_cid, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cid_topo, KF_IMPLICIT_ARGS) +BTF_KFUNCS_END(scx_kfunc_ids_cid) + +static const struct btf_kfunc_id_set scx_kfunc_set_cid = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_cid, +}; + +int scx_cid_kfunc_init(void) +{ + return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_cid) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_cid) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_cid); +} diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h new file mode 100644 index 000000000000..6e657fd147b0 --- /dev/null +++ b/kernel/sched/ext/cid.h @@ -0,0 +1,271 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Topological CPU IDs (cids) + * -------------------------- + * + * Raw cpu numbers are clumsy for sharding work and communication across + * topology units, especially from BPF: the space can be sparse, numerical + * closeness doesn't imply topological closeness (x86 hyperthreading often puts + * SMT siblings far apart), and a range of cpu ids doesn't mean anything. + * Sub-scheds make this acute - cpu allocation, revocation and other state are + * constantly communicated across sub-scheds, and passing whole cpumasks scales + * poorly with cpu count. cpumasks are also awkward in BPF: a variable-length + * kernel type sized for the maximum NR_CPUS (4k), with verbose helper sequences + * for every op. + * + * cids give every cpu a dense, topology-ordered id. CPUs sharing a core, LLC or + * NUMA node get contiguous cid ranges, so a topology unit becomes a (start, + * length) slice of cid space. Communication can pass a slice instead of a + * cpumask, and BPF code can process, for example, a u64 word's worth of cids at + * a time. + * + * The mapping is built once at root scheduler enable time by walking the + * topology of online cpus only. Going by online cpus is out of necessity: + * depending on the arch, topology info isn't reliably available for offline + * cpus. The expected usage model is restarting the scheduler on hotplug events + * so the mapping is rebuilt against the new online set. A scheduler that wants + * to handle hotplug without a restart can provide its own cid and shard mapping + * through the override interface. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ +#ifndef _KERNEL_SCHED_EXT_CID_H +#define _KERNEL_SCHED_EXT_CID_H + +struct scx_sched; + +/* + * Cid space (total is always num_possible_cpus()) is laid out with + * topology-annotated cids first, then no-topo cids at the tail. The + * topology-annotated block covers the cpus that were online when scx_cid_init() + * ran and remains valid even after those cpus go offline. The tail block covers + * possible-but-not-online cpus and carries all-(-1) topo info (see + * scx_cid_topo); callers detect it via the -1 sentinels. + * + * See the comment above the table definitions in cid.c for the + * memory-ordering and visibility contract. + */ +extern s16 *scx_cid_to_cpu_tbl; +extern s16 *scx_cpu_to_cid_tbl; +extern struct scx_cid_topo *scx_cid_topo; +extern struct btf_id_set8 scx_kfunc_ids_init; + +void scx_cmask_clear(struct scx_cmask *m); +void scx_cmask_fill(struct scx_cmask *m); +void scx_cmask_and(struct scx_cmask *dst, const struct scx_cmask *src); +void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src); +void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src); +void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src); +void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src); +void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src); +bool scx_cmask_subset(const struct scx_cmask *sub, const struct scx_cmask *super); +bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b); +bool scx_cmask_empty(const struct scx_cmask *m); +s32 scx_cid_init(struct scx_sched *sch); +int scx_cid_kfunc_init(void); +void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst); + +/** + * cid_valid - Verify a cid value, to be used on ops input args + * @sch: scx_sched to abort on error + * @cid: cid which came from a BPF ops + * + * Return true if @cid is in [0, num_possible_cpus()). On failure, trigger + * scx_error() and return false. + */ +static inline bool cid_valid(struct scx_sched *sch, s32 cid) +{ + if (likely(cid >= 0 && cid < num_possible_cpus())) + return true; + scx_error(sch, "invalid cid %d", cid); + return false; +} + +/** + * __scx_cid_to_cpu - Unchecked cid->cpu table lookup + * @cid: cid to look up. Must be in [0, num_possible_cpus()). + * + * Intended for callsites that have already validated @cid and that hold a + * non-NULL @sch from scx_prog_sched() - a live sched implies the table has + * been allocated, so no NULL check is needed here. + */ +static inline s32 __scx_cid_to_cpu(s32 cid) +{ + /* READ_ONCE pairs with WRITE_ONCE in scx_cid_arrays_alloc() */ + return READ_ONCE(scx_cid_to_cpu_tbl)[cid]; +} + +/** + * __scx_cpu_to_cid - Unchecked cpu->cid table lookup + * @cpu: cpu to look up. Must be a valid possible cpu id. + * + * Same usage constraints as __scx_cid_to_cpu(). + */ +static inline s32 __scx_cpu_to_cid(s32 cpu) +{ + return READ_ONCE(scx_cpu_to_cid_tbl)[cpu]; +} + +/** + * scx_cid_to_cpu - Translate @cid to its cpu + * @sch: scx_sched for error reporting + * @cid: cid to look up + * + * Return the cpu for @cid or a negative errno on failure. Invalid cid triggers + * scx_error() on @sch. The cid arrays are allocated on first scheduler enable + * and never freed, so the returned cpu is stable for the lifetime of the loaded + * scheduler. + */ +static inline s32 scx_cid_to_cpu(struct scx_sched *sch, s32 cid) +{ + if (!cid_valid(sch, cid)) + return -EINVAL; + return __scx_cid_to_cpu(cid); +} + +/** + * scx_cpu_to_cid - Translate @cpu to its cid + * @sch: scx_sched for error reporting + * @cpu: cpu to look up + * + * Return the cid for @cpu or a negative errno on failure. Invalid cpu triggers + * scx_error() on @sch. Same lifetime guarantee as scx_cid_to_cpu(). + */ +static inline s32 scx_cpu_to_cid(struct scx_sched *sch, s32 cpu) +{ + if (!scx_cpu_valid(sch, cpu, NULL)) + return -EINVAL; + return __scx_cpu_to_cid(cpu); +} + +/** + * scx_is_cid_type - Test whether the active scheduler hierarchy is cid-form + */ +static inline bool scx_is_cid_type(void) +{ + return static_branch_unlikely(&__scx_is_cid_type); +} + +static inline bool __scx_cmask_contains(u32 cid, const struct scx_cmask *m) +{ + return likely(cid >= m->base && cid < m->base + m->nr_cids); +} + +/* Word in bits[] covering @cid. @cid must satisfy __scx_cmask_contains(). */ +static inline u64 *__scx_cmask_word(u32 cid, const struct scx_cmask *m) +{ + return (u64 *)&m->bits[cid / 64 - m->base / 64]; +} + +/** + * __scx_cmask_init - Initialize @m with explicit storage capacity + * @m: cmask to initialize + * @base: first cid of the active range + * @nr_cids: number of cids in the active range + * @alloc_cids: storage capacity in cids, at least @nr_cids + * + * Use when storage is sized larger than the initial active range. All of + * bits[] is zeroed. + */ +static inline void __scx_cmask_init(struct scx_cmask *m, u32 base, u32 nr_cids, + u32 alloc_cids) +{ + if (WARN_ON_ONCE(alloc_cids < nr_cids)) + nr_cids = alloc_cids; + + m->base = base; + m->nr_cids = nr_cids; + m->alloc_words = SCX_CMASK_NR_WORDS(alloc_cids); + memset(m->bits, 0, m->alloc_words * sizeof(u64)); +} + +/** + * scx_cmask_init - Initialize @m on tight storage + * @m: cmask to initialize + * @base: first cid of the active range + * @nr_cids: number of cids in the active range + * + * All of bits[] is zeroed. + */ +static inline void scx_cmask_init(struct scx_cmask *m, u32 base, u32 nr_cids) +{ + __scx_cmask_init(m, base, nr_cids, nr_cids); +} + +/** + * scx_cmask_reframe - Reshape @m's active range without resizing storage + * @m: cmask to reframe + * @base: new active range base + * @nr_cids: new active range length, must fit within @m->alloc_words + * + * Body bits within the new range become garbage - only the head and tail + * words are zeroed to keep the padding invariant. + */ +static inline void scx_cmask_reframe(struct scx_cmask *m, u32 base, u32 nr_cids) +{ + if (WARN_ON_ONCE(SCX_CMASK_NR_WORDS(nr_cids) > m->alloc_words)) + return; + + if (nr_cids) { + u32 last_word = ((base & 63) + nr_cids - 1) / 64; + + m->bits[0] = 0; + m->bits[last_word] = 0; + } + + m->base = base; + m->nr_cids = nr_cids; +} + +static inline void __scx_cmask_set(u32 cid, struct scx_cmask *m) +{ + if (!__scx_cmask_contains(cid, m)) + return; + *__scx_cmask_word(cid, m) |= BIT_U64(cid & 63); +} + +/** + * scx_cmask_test - test whether @cid is set in @m + * @cid: cid to test + * @m: cmask to test + * + * Return %false if @cid is outside @m's active range. Otherwise return the + * bit's value. Read via READ_ONCE so callers can race set/clear writers. + */ +static inline bool scx_cmask_test(u32 cid, const struct scx_cmask *m) +{ + if (!__scx_cmask_contains(cid, m)) + return false; + return READ_ONCE(*__scx_cmask_word(cid, m)) & BIT_U64(cid & 63); +} + +/* + * Words of bits[] the active range spans, 0 if empty. Tighter than the storage + * SCX_CMASK_NR_WORDS() sizes for the worst-case base alignment. + */ +static inline u32 scx_cmask_nr_used_words(const struct scx_cmask *m) +{ + if (!m->nr_cids) + return 0; + return ((m->base & 63) + m->nr_cids - 1) / 64 + 1; +} + +/** + * scx_cmask_for_each_cid - iterate set cids in @m + * @cid: s32 loop var that receives each set cid in turn + * @m: cmask to iterate + * + * Visits set bits within @m's active range in ascending order. Scans only the + * words the active range spans, where head and tail padding is kept zero, so + * no per-cid range check is needed. + */ +#define scx_cmask_for_each_cid(cid, m) \ + for (u64 __bs = (m)->base & ~63u, __wi = 0, \ + __nw = scx_cmask_nr_used_words(m); \ + __wi < __nw; __wi++) \ + for (u64 __w = READ_ONCE((m)->bits[__wi]); \ + __w && ((cid) = __bs + __wi * 64 + __ffs64(__w), true); \ + __w &= __w - 1) + +#endif /* _KERNEL_SCHED_EXT_CID_H */ diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c new file mode 100644 index 000000000000..00fe6cc6d7e2 --- /dev/null +++ b/kernel/sched/ext/ext.c @@ -0,0 +1,10882 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + */ + +static DEFINE_RAW_SPINLOCK(scx_sched_lock); + +/* + * NOTE: sched_ext is in the process of growing multiple scheduler support and + * scx_root usage is in a transitional state. Naked dereferences are safe if the + * caller is one of the tasks attached to SCX and explicit RCU dereference is + * necessary otherwise. Naked scx_root dereferences trigger sparse warnings but + * are used as temporary markers to indicate that the dereferences need to be + * updated to point to the associated scheduler instances rather than scx_root. + */ +struct scx_sched __rcu *scx_root; + +/* + * All scheds, writers must hold both scx_enable_mutex and scx_sched_lock. + * Readers can hold either or rcu_read_lock(). + */ +static LIST_HEAD(scx_sched_all); + +#ifdef CONFIG_EXT_SUB_SCHED +static const struct rhashtable_params scx_sched_hash_params = { + .key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id), + .key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id), + .head_offset = offsetof(struct scx_sched, hash_node), + .insecure_elasticity = true, /* inserted under scx_sched_lock */ +}; + +static struct rhashtable scx_sched_hash; +#endif + +/* see SCX_OPS_TID_TO_TASK */ +static const struct rhashtable_params scx_tid_hash_params = { + .key_len = sizeof_field(struct sched_ext_entity, tid), + .key_offset = offsetof(struct sched_ext_entity, tid), + .head_offset = offsetof(struct sched_ext_entity, tid_hash_node), + .insecure_elasticity = true, /* inserted/removed under scx_tasks_lock */ +}; +static struct rhashtable scx_tid_hash; + +/* + * During exit, a task may schedule after losing its PIDs. When disabling the + * BPF scheduler, we need to be able to iterate tasks in every state to + * guarantee system safety. Maintain a dedicated task list which contains every + * task between its fork and eventual free. + */ +static DEFINE_RAW_SPINLOCK(scx_tasks_lock); +static LIST_HEAD(scx_tasks); + +/* ops enable/disable */ +static DEFINE_MUTEX(scx_enable_mutex); +DEFINE_STATIC_KEY_FALSE(__scx_enabled); +DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); +static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED); +static DEFINE_RAW_SPINLOCK(scx_bypass_lock); +static bool scx_init_task_enabled; +static bool scx_switching_all; +DEFINE_STATIC_KEY_FALSE(__scx_switched_all); +static DEFINE_STATIC_KEY_FALSE(__scx_tid_to_task_enabled); + +/* + * True once SCX_OPS_TID_TO_TASK has been negotiated with the root scheduler + * and the tid->task table is live. Wraps the static key so callers don't + * take the address, and hints "likely enabled" for the common case where + * the feature is in use. + */ +static inline bool scx_tid_to_task_enabled(void) +{ + return static_branch_likely(&__scx_tid_to_task_enabled); +} + +static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0); +static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0); + +/* Global cursor for the per-CPU tid allocator. Starts at 1; tid 0 is reserved. */ +static atomic64_t scx_tid_cursor = ATOMIC64_INIT(1); + +#ifdef CONFIG_EXT_SUB_SCHED +/* + * The sub sched being enabled. Used by scx_disable_and_exit_task() to exit + * tasks for the sub-sched being enabled. Use a global variable instead of a + * per-task field as all enables are serialized. + */ +static struct scx_sched *scx_enabling_sub_sched; +#else +#define scx_enabling_sub_sched (struct scx_sched *)NULL +#endif /* CONFIG_EXT_SUB_SCHED */ + +/* + * A monotonically increasing sequence number that is incremented every time a + * scheduler is enabled. This can be used to check if any custom sched_ext + * scheduler has ever been used in the system. + */ +static atomic_long_t scx_enable_seq = ATOMIC_LONG_INIT(0); + +/* + * Watchdog interval. All scx_sched's share a single watchdog timer and the + * interval is half of the shortest sch->watchdog_timeout. + */ +static unsigned long scx_watchdog_interval; + +/* + * The last time the delayed work was run. This delayed work relies on + * ksoftirqd being able to run to service timer interrupts, so it's possible + * that this work itself could get wedged. To account for this, we check that + * it's not stalled in the timer tick, and trigger an error if it is. + */ +static unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; + +static struct delayed_work scx_watchdog_work; + +/* + * For %SCX_KICK_WAIT: Each CPU has a pointer to an array of kick_sync sequence + * numbers. The arrays are allocated with kvzalloc() as size can exceed percpu + * allocator limits on large machines. O(nr_cpu_ids^2) allocation, allocated + * lazily when enabling and freed when disabling to avoid waste when sched_ext + * isn't active. + */ +struct scx_kick_syncs { + struct rcu_head rcu; + unsigned long syncs[]; +}; + +static DEFINE_PER_CPU(struct scx_kick_syncs __rcu *, scx_kick_syncs); + +/* + * Per-CPU buffered allocator state for p->scx.tid. Each CPU pulls a chunk of + * SCX_TID_CHUNK ids from scx_tid_cursor and hands them out locally without + * further synchronization. See scx_alloc_tid(). + */ +struct scx_tid_alloc { + u64 next; + u64 end; +}; +static DEFINE_PER_CPU(struct scx_tid_alloc, scx_tid_alloc); + +/* + * Direct dispatch marker. + * + * Non-NULL values are used for direct dispatch from enqueue path. A valid + * pointer points to the task currently being enqueued. An ERR_PTR value is used + * to indicate that direct dispatch has already happened. + */ +static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); + +static const struct rhashtable_params dsq_hash_params = { + .key_len = sizeof_field(struct scx_dispatch_q, id), + .key_offset = offsetof(struct scx_dispatch_q, id), + .head_offset = offsetof(struct scx_dispatch_q, hash_node), +}; + +static LLIST_HEAD(dsqs_to_free); + +/* string formatting from BPF */ +struct scx_bstr_buf { + u64 data[MAX_BPRINTF_VARARGS]; + char line[SCX_EXIT_MSG_LEN]; +}; + +static DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock); +static struct scx_bstr_buf scx_exit_bstr_buf; + +/* ops debug dump */ +static DEFINE_RAW_SPINLOCK(scx_dump_lock); + +struct scx_dump_data { + s32 cpu; + bool first; + s32 cursor; + struct seq_buf *s; + const char *prefix; + struct scx_bstr_buf buf; +}; + +static struct scx_dump_data scx_dump_data = { + .cpu = -1, +}; + +/* /sys/kernel/sched_ext interface */ +static struct kset *scx_kset; + +/* + * Parameters that can be adjusted through /sys/module/sched_ext/parameters. + * There usually is no reason to modify these as normal scheduler operation + * shouldn't be affected by them. The knobs are primarily for debugging. + */ +static unsigned int scx_slice_bypass_us = SCX_SLICE_BYPASS / NSEC_PER_USEC; +static unsigned int scx_bypass_lb_intv_us = SCX_BYPASS_LB_DFL_INTV_US; + +static int set_slice_us(const char *val, const struct kernel_param *kp) +{ + return param_set_uint_minmax(val, kp, 100, 100 * USEC_PER_MSEC); +} + +static const struct kernel_param_ops slice_us_param_ops = { + .set = set_slice_us, + .get = param_get_uint, +}; + +static int set_bypass_lb_intv_us(const char *val, const struct kernel_param *kp) +{ + return param_set_uint_minmax(val, kp, 0, 10 * USEC_PER_SEC); +} + +static const struct kernel_param_ops bypass_lb_intv_us_param_ops = { + .set = set_bypass_lb_intv_us, + .get = param_get_uint, +}; + +#undef MODULE_PARAM_PREFIX +#define MODULE_PARAM_PREFIX "sched_ext." + +module_param_cb(slice_bypass_us, &slice_us_param_ops, &scx_slice_bypass_us, 0600); +MODULE_PARM_DESC(slice_bypass_us, "bypass slice in microseconds, applied on [un]load (100us to 100ms)"); +module_param_cb(bypass_lb_intv_us, &bypass_lb_intv_us_param_ops, &scx_bypass_lb_intv_us, 0600); +MODULE_PARM_DESC(bypass_lb_intv_us, "bypass load balance interval in microseconds (0 (disable) to 10s)"); + +#undef MODULE_PARAM_PREFIX + +#define CREATE_TRACE_POINTS +#include + +static void run_deferred(struct rq *rq); +static bool task_dead_and_done(struct task_struct *p); +static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); +static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind); + +__printf(5, 6) bool __scx_exit(struct scx_sched *sch, + enum scx_exit_kind kind, s64 exit_code, + s32 exit_cpu, const char *fmt, ...) +{ + va_list args; + bool ret; + + va_start(args, fmt); + ret = scx_vexit(sch, kind, exit_code, exit_cpu, fmt, args); + va_end(args); + + return ret; +} + +#define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op) + +static long jiffies_delta_msecs(unsigned long at, unsigned long now) +{ + if (time_after(at, now)) + return jiffies_to_msecs(at - now); + else + return -(long)jiffies_to_msecs(now - at); +} + +static bool u32_before(u32 a, u32 b) +{ + return (s32)(a - b) < 0; +} + +#ifdef CONFIG_EXT_SUB_SCHED +/** + * scx_parent - Find the parent sched + * @sch: sched to find the parent of + * + * Returns the parent scheduler or %NULL if @sch is root. + */ +static struct scx_sched *scx_parent(struct scx_sched *sch) +{ + if (sch->level) + return sch->ancestors[sch->level - 1]; + else + return NULL; +} + +/** + * scx_next_descendant_pre - find the next descendant for pre-order walk + * @pos: the current position (%NULL to initiate traversal) + * @root: sched whose descendants to walk + * + * To be used by scx_for_each_descendant_pre(). Find the next descendant to + * visit for pre-order traversal of @root's descendants. @root is included in + * the iteration and the first node to be visited. + */ +static struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, + struct scx_sched *root) +{ + struct scx_sched *next; + + lockdep_assert(lockdep_is_held(&scx_enable_mutex) || + lockdep_is_held(&scx_sched_lock)); + + /* if first iteration, visit @root */ + if (!pos) + return root; + + /* visit the first child if exists */ + next = list_first_entry_or_null(&pos->children, struct scx_sched, sibling); + if (next) + return next; + + /* no child, visit my or the closest ancestor's next sibling */ + while (pos != root) { + if (!list_is_last(&pos->sibling, &scx_parent(pos)->children)) + return list_next_entry(pos, sibling); + pos = scx_parent(pos); + } + + return NULL; +} + +static struct scx_sched *scx_find_sub_sched(u64 cgroup_id) +{ + return rhashtable_lookup(&scx_sched_hash, &cgroup_id, + scx_sched_hash_params); +} + +static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) +{ + rcu_assign_pointer(p->scx.sched, sch); +} +#else /* CONFIG_EXT_SUB_SCHED */ +static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; } +static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } +static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} +#endif /* CONFIG_EXT_SUB_SCHED */ + +/** + * scx_is_descendant - Test whether sched is a descendant + * @sch: sched to test + * @ancestor: ancestor sched to test against + * + * Test whether @sch is a descendant of @ancestor. + */ +static bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor) +{ + if (sch->level < ancestor->level) + return false; + return sch->ancestors[ancestor->level] == ancestor; +} + +/** + * scx_for_each_descendant_pre - pre-order walk of a sched's descendants + * @pos: iteration cursor + * @root: sched to walk the descendants of + * + * Walk @root's descendants. @root is included in the iteration and the first + * node to be visited. Must be called with either scx_enable_mutex or + * scx_sched_lock held. + */ +#define scx_for_each_descendant_pre(pos, root) \ + for ((pos) = scx_next_descendant_pre(NULL, (root)); (pos); \ + (pos) = scx_next_descendant_pre((pos), (root))) + +static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch, s32 cpu) +{ + return &sch->pnode[cpu_to_node(cpu)]->global_dsq; +} + +static struct scx_dispatch_q *find_user_dsq(struct scx_sched *sch, u64 dsq_id) +{ + return rhashtable_lookup(&sch->dsq_hash, &dsq_id, dsq_hash_params); +} + +static const struct sched_class *scx_setscheduler_class(struct task_struct *p) +{ + if (p->sched_class == &stop_sched_class) + return &stop_sched_class; + + return __setscheduler_class(p->policy, p->prio); +} + +static struct scx_dispatch_q *bypass_dsq(struct scx_sched *sch, s32 cpu) +{ + return &per_cpu_ptr(sch->pcpu, cpu)->bypass_dsq; +} + +static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu) +{ +#ifdef CONFIG_EXT_SUB_SCHED + /* + * If @sch is a sub-sched which is bypassing, its tasks should go into + * the bypass DSQs of the nearest ancestor which is not bypassing. The + * not-bypassing ancestor is responsible for scheduling all tasks from + * bypassing sub-trees. If all ancestors including root are bypassing, + * all tasks should go to the root's bypass DSQs. + * + * Whenever a sched starts bypassing, all runnable tasks in its subtree + * are re-enqueued after scx_bypassing() is turned on, guaranteeing that + * all tasks are transferred to the right DSQs. + */ + while (scx_parent(sch) && scx_bypassing(sch, cpu)) + sch = scx_parent(sch); +#endif /* CONFIG_EXT_SUB_SCHED */ + + return bypass_dsq(sch, cpu); +} + +/** + * bypass_dsp_enabled - Check if bypass dispatch path is enabled + * @sch: scheduler to check + * + * When a descendant scheduler enters bypass mode, bypassed tasks are scheduled + * by the nearest non-bypassing ancestor, or the root scheduler if all ancestors + * are bypassing. In the former case, the ancestor is not itself bypassing but + * its bypass DSQs will be populated with bypassed tasks from descendants. Thus, + * the ancestor's bypass dispatch path must be active even though its own + * bypass_depth remains zero. + * + * This function checks bypass_dsp_enable_depth which is managed separately from + * bypass_depth to enable this decoupling. See enable_bypass_dsp() and + * disable_bypass_dsp(). + */ +static bool bypass_dsp_enabled(struct scx_sched *sch) +{ + return unlikely(atomic_read(&sch->bypass_dsp_enable_depth)); +} + +/** + * rq_is_open - Is the rq available for immediate execution of an SCX task? + * @rq: rq to test + * @enq_flags: optional %SCX_ENQ_* of the task being enqueued + * + * Returns %true if @rq is currently open for executing an SCX task. After a + * %false return, @rq is guaranteed to invoke SCX dispatch path at least once + * before going to idle and not inserting a task into @rq's local DSQ after a + * %false return doesn't cause @rq to stall. + */ +static bool rq_is_open(struct rq *rq, u64 enq_flags) +{ + lockdep_assert_rq_held(rq); + + /* + * A higher-priority class task is either running or in the process of + * waking up on @rq. + */ + if (sched_class_above(rq->next_class, &ext_sched_class)) + return false; + + /* + * @rq is either in transition to or in idle and there is no + * higher-priority class task waking up on it. + */ + if (sched_class_above(&ext_sched_class, rq->next_class)) + return true; + + /* + * @rq is either picking, in transition to, or running an SCX task. + */ + + /* + * If we're in the dispatch path holding rq lock, $curr may or may not + * be ready depending on whether the on-going dispatch decides to extend + * $curr's slice. We say yes here and resolve it at the end of dispatch. + * See balance_one(). + */ + if (rq->scx.flags & SCX_RQ_IN_BALANCE) + return true; + + /* + * %SCX_ENQ_PREEMPT clears $curr's slice if on SCX and kicks dispatch, + * so allow it to avoid spuriously triggering reenq on a combined + * PREEMPT|IMMED insertion. + */ + if (enq_flags & SCX_ENQ_PREEMPT) + return true; + + /* + * @rq is either in transition to or running an SCX task and can't go + * idle without another SCX dispatch cycle. + */ + return false; +} + +/* + * Track the rq currently locked. + * + * This allows kfuncs to safely operate on rq from any scx ops callback, + * knowing which rq is already locked. + */ +DEFINE_PER_CPU(struct rq *, scx_locked_rq_state); + +static inline void update_locked_rq(struct rq *rq) +{ + /* + * Check whether @rq is actually locked. This can help expose bugs + * or incorrect assumptions about the context in which a kfunc or + * callback is executed. + */ + if (rq) + lockdep_assert_rq_held(rq); + __this_cpu_write(scx_locked_rq_state, rq); +} + +/* + * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not + * clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit. + */ +#define SCX_CALL_OP(sch, op, locked_rq, args...) \ +do { \ + struct rq *__prev_locked_rq; \ + \ + if (locked_rq) { \ + __prev_locked_rq = scx_locked_rq(); \ + update_locked_rq(locked_rq); \ + } \ + (sch)->ops.op(args); \ + if (locked_rq) \ + update_locked_rq(__prev_locked_rq); \ +} while (0) + +/* + * Flipped on enable per sch->is_cid_type. Declared in internal.h so + * subsystem inlines can read it. + */ +DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type); + +/* + * scx_cpu_arg() wraps a cpu arg being handed to an SCX op. For cid-form + * schedulers it resolves to the matching cid; for cpu-form it passes @cpu + * through. scx_cpu_ret() is the inverse for a cpu/cid returned from an op + * (currently only ops.select_cpu); it validates the BPF-supplied cid and + * triggers scx_error() on @sch if invalid. + */ +static s32 scx_cpu_arg(s32 cpu) +{ + if (scx_is_cid_type()) + return __scx_cpu_to_cid(cpu); + return cpu; +} + +static s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid) +{ + if (cpu_or_cid < 0 || !scx_is_cid_type()) + return cpu_or_cid; + return scx_cid_to_cpu(sch, cpu_or_cid); +} + +#define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \ +({ \ + struct rq *__prev_locked_rq; \ + __typeof__((sch)->ops.op(args)) __ret; \ + \ + if (locked_rq) { \ + __prev_locked_rq = scx_locked_rq(); \ + update_locked_rq(locked_rq); \ + } \ + __ret = (sch)->ops.op(args); \ + if (locked_rq) \ + update_locked_rq(__prev_locked_rq); \ + __ret; \ +}) + +/* + * SCX_CALL_OP_TASK*() invokes an SCX op that takes one or two task arguments + * and records them in current->scx.kf_tasks[] for the duration of the call. A + * kfunc invoked from inside such an op can then use + * scx_kf_arg_task_ok() to verify that its task argument is one of + * those subject tasks. + * + * Every SCX_CALL_OP_TASK*() call site invokes its op with @p's rq lock held - + * either via the @locked_rq argument here, or (for ops.select_cpu()) via @p's + * pi_lock held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu. + * So if kf_tasks[] is set, @p's scheduler-protected fields are stable. + * + * kf_tasks[] can not stack, so task-based SCX ops must not nest. The + * WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants + * while a previous one is still in progress. + */ +#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \ +do { \ + WARN_ON_ONCE(current->scx.kf_tasks[0]); \ + current->scx.kf_tasks[0] = task; \ + SCX_CALL_OP((sch), op, locked_rq, task, ##args); \ + current->scx.kf_tasks[0] = NULL; \ +} while (0) + +#define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \ +({ \ + __typeof__((sch)->ops.op(task, ##args)) __ret; \ + WARN_ON_ONCE(current->scx.kf_tasks[0]); \ + current->scx.kf_tasks[0] = task; \ + __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \ + current->scx.kf_tasks[0] = NULL; \ + __ret; \ +}) + +#define SCX_CALL_OP_2TASKS_RET(sch, op, locked_rq, task0, task1, args...) \ +({ \ + __typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \ + WARN_ON_ONCE(current->scx.kf_tasks[0]); \ + current->scx.kf_tasks[0] = task0; \ + current->scx.kf_tasks[1] = task1; \ + __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task0, task1, ##args); \ + current->scx.kf_tasks[0] = NULL; \ + current->scx.kf_tasks[1] = NULL; \ + __ret; \ +}) + +/** + * scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task + * @sch: scx_sched being invoked + * @rq: rq to update as the currently-locked rq, or NULL + * @task: task whose affinity is changing + * @cpumask: new cpumask + * + * For cid-form schedulers, translate @cpumask to a cmask via the per-cpu + * scratch in cid.c and dispatch through the ops_cid union view. Caller + * must hold @rq's rq lock so this_cpu_ptr is stable across the call. + */ +static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, + struct task_struct *task, + const struct cpumask *cpumask) +{ + WARN_ON_ONCE(current->scx.kf_tasks[0]); + current->scx.kf_tasks[0] = task; + if (rq) + update_locked_rq(rq); + + if (scx_is_cid_type()) { + struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); + /* + * Build the per-CPU arena cmask and hand BPF its arena address. + * Caller holds the rq lock with IRQs disabled, which makes us + * the sole user of the scratch area. + */ + scx_cpumask_to_cmask(cpumask, kern_va); + sch->ops_cid.set_cmask(task, scx_kaddr_to_arena(sch, kern_va)); + } else { + sch->ops.set_cpumask(task, cpumask); + } + + if (rq) + update_locked_rq(NULL); + current->scx.kf_tasks[0] = NULL; +} + +/* see SCX_CALL_OP_TASK() */ +static __always_inline bool scx_kf_arg_task_ok(struct scx_sched *sch, + struct task_struct *p) +{ + if (unlikely((p != current->scx.kf_tasks[0] && + p != current->scx.kf_tasks[1]))) { + scx_error(sch, "called on a task not being operated on"); + return false; + } + + return true; +} + +enum scx_dsq_iter_flags { + /* iterate in the reverse dispatch order */ + SCX_DSQ_ITER_REV = 1U << 16, + + __SCX_DSQ_ITER_HAS_SLICE = 1U << 30, + __SCX_DSQ_ITER_HAS_VTIME = 1U << 31, + + __SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV, + __SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS | + __SCX_DSQ_ITER_HAS_SLICE | + __SCX_DSQ_ITER_HAS_VTIME, +}; + +/** + * nldsq_next_task - Iterate to the next task in a non-local DSQ + * @dsq: non-local dsq being iterated + * @cur: current position, %NULL to start iteration + * @rev: walk backwards + * + * Returns %NULL when iteration is finished. + */ +static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq, + struct task_struct *cur, bool rev) +{ + struct list_head *list_node; + struct scx_dsq_list_node *dsq_lnode; + + lockdep_assert_held(&dsq->lock); + + if (cur) + list_node = &cur->scx.dsq_list.node; + else + list_node = &dsq->list; + + /* find the next task, need to skip BPF iteration cursors */ + do { + if (rev) + list_node = list_node->prev; + else + list_node = list_node->next; + + if (list_node == &dsq->list) + return NULL; + + dsq_lnode = container_of(list_node, struct scx_dsq_list_node, + node); + } while (dsq_lnode->flags & SCX_DSQ_LNODE_ITER_CURSOR); + + return container_of(dsq_lnode, struct task_struct, scx.dsq_list); +} + +#define nldsq_for_each_task(p, dsq) \ + for ((p) = nldsq_next_task((dsq), NULL, false); (p); \ + (p) = nldsq_next_task((dsq), (p), false)) + +/** + * nldsq_cursor_next_task - Iterate to the next task given a cursor in a non-local DSQ + * @cursor: scx_dsq_list_node initialized with INIT_DSQ_LIST_CURSOR() + * @dsq: non-local dsq being iterated + * + * Find the next task in a cursor based iteration. The caller must have + * initialized @cursor using INIT_DSQ_LIST_CURSOR() and can release the DSQ lock + * between the iteration steps. + * + * Only tasks which were queued before @cursor was initialized are visible. This + * bounds the iteration and guarantees that vtime never jumps in the other + * direction while iterating. + */ +static struct task_struct *nldsq_cursor_next_task(struct scx_dsq_list_node *cursor, + struct scx_dispatch_q *dsq) +{ + bool rev = cursor->flags & SCX_DSQ_ITER_REV; + struct task_struct *p; + + lockdep_assert_held(&dsq->lock); + BUG_ON(!(cursor->flags & SCX_DSQ_LNODE_ITER_CURSOR)); + + if (list_empty(&cursor->node)) + p = NULL; + else + p = container_of(cursor, struct task_struct, scx.dsq_list); + + /* skip cursors and tasks that were queued after @cursor init */ + do { + p = nldsq_next_task(dsq, p, rev); + } while (p && unlikely(u32_before(cursor->priv, p->scx.dsq_seq))); + + if (p) { + if (rev) + list_move_tail(&cursor->node, &p->scx.dsq_list.node); + else + list_move(&cursor->node, &p->scx.dsq_list.node); + } else { + list_del_init(&cursor->node); + } + + return p; +} + +/** + * nldsq_cursor_lost_task - Test whether someone else took the task since iteration + * @cursor: scx_dsq_list_node initialized with INIT_DSQ_LIST_CURSOR() + * @rq: rq @p was on + * @dsq: dsq @p was on + * @p: target task + * + * @p is a task returned by nldsq_cursor_next_task(). The locks may have been + * dropped and re-acquired inbetween. Verify that no one else took or is in the + * process of taking @p from @dsq. + * + * On %false return, the caller can assume full ownership of @p. + */ +static bool nldsq_cursor_lost_task(struct scx_dsq_list_node *cursor, + struct rq *rq, struct scx_dispatch_q *dsq, + struct task_struct *p) +{ + lockdep_assert_rq_held(rq); + lockdep_assert_held(&dsq->lock); + + /* + * @p could have already left $src_dsq, got re-enqueud, or be in the + * process of being consumed by someone else. + */ + if (unlikely(p->scx.dsq != dsq || + u32_before(cursor->priv, p->scx.dsq_seq) || + p->scx.holding_cpu >= 0)) + return true; + + /* if @p has stayed on @dsq, its rq couldn't have changed */ + if (WARN_ON_ONCE(rq != task_rq(p))) + return true; + + return false; +} + +/* + * BPF DSQ iterator. Tasks in a non-local DSQ can be iterated in [reverse] + * dispatch order. BPF-visible iterator is opaque and larger to allow future + * changes without breaking backward compatibility. Can be used with + * bpf_for_each(). See bpf_iter_scx_dsq_*(). + */ +struct bpf_iter_scx_dsq_kern { + struct scx_dsq_list_node cursor; + struct scx_dispatch_q *dsq; + u64 slice; + u64 vtime; +} __attribute__((aligned(8))); + +struct bpf_iter_scx_dsq { + u64 __opaque[6]; +} __attribute__((aligned(8))); + + +static u32 scx_get_task_state(const struct task_struct *p) +{ + return p->scx.flags & SCX_TASK_STATE_MASK; +} + +static void scx_set_task_state(struct task_struct *p, u32 state) +{ + u32 prev_state = scx_get_task_state(p); + bool warn = false; + + switch (state) { + case SCX_TASK_NONE: + warn = prev_state == SCX_TASK_DEAD; + break; + case SCX_TASK_INIT_BEGIN: + warn = prev_state != SCX_TASK_NONE; + break; + case SCX_TASK_INIT: + warn = prev_state != SCX_TASK_INIT_BEGIN; + p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; + break; + case SCX_TASK_READY: + warn = !(prev_state == SCX_TASK_INIT || + prev_state == SCX_TASK_ENABLED); + break; + case SCX_TASK_ENABLED: + warn = prev_state != SCX_TASK_READY; + break; + case SCX_TASK_DEAD: + warn = !(prev_state == SCX_TASK_NONE || + prev_state == SCX_TASK_INIT_BEGIN); + break; + default: + WARN_ONCE(1, "sched_ext: Invalid task state %d -> %d for %s[%d]", + prev_state, state, p->comm, p->pid); + return; + } + + WARN_ONCE(warn, "sched_ext: Invalid task state transition 0x%x -> 0x%x for %s[%d]", + prev_state, state, p->comm, p->pid); + + p->scx.flags &= ~SCX_TASK_STATE_MASK; + p->scx.flags |= state; +} + +/* + * SCX task iterator. + */ +struct scx_task_iter { + struct sched_ext_entity cursor; + struct task_struct *locked_task; + struct rq *rq; + struct rq_flags rf; + u32 cnt; + bool list_locked; +#ifdef CONFIG_EXT_SUB_SCHED + struct cgroup *cgrp; + struct cgroup_subsys_state *css_pos; + struct css_task_iter css_iter; +#endif +}; + +/** + * scx_task_iter_start - Lock scx_tasks_lock and start a task iteration + * @iter: iterator to init + * @cgrp: Optional root of cgroup subhierarchy to iterate + * + * Initialize @iter. Once initialized, @iter must eventually be stopped with + * scx_task_iter_stop(). + * + * If @cgrp is %NULL, scx_tasks is used for iteration and this function returns + * with scx_tasks_lock held and @iter->cursor inserted into scx_tasks. + * + * If @cgrp is not %NULL, @cgrp and its descendants' tasks are walked using + * @iter->css_iter. The caller must be holding cgroup_lock() to prevent cgroup + * task migrations. + * + * The two modes of iterations are largely independent and it's likely that + * scx_tasks can be removed in favor of always using cgroup iteration if + * CONFIG_SCHED_CLASS_EXT depends on CONFIG_CGROUPS. + * + * scx_tasks_lock and the rq lock may be released using scx_task_iter_unlock() + * between this and the first next() call or between any two next() calls. If + * the locks are released between two next() calls, the caller is responsible + * for ensuring that the task being iterated remains accessible either through + * RCU read lock or obtaining a reference count. + * + * All tasks which existed when the iteration started are guaranteed to be + * visited as long as they are not dead. + */ +static void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp) +{ + memset(iter, 0, sizeof(*iter)); + +#ifdef CONFIG_EXT_SUB_SCHED + if (cgrp) { + lockdep_assert_held(&cgroup_mutex); + iter->cgrp = cgrp; + iter->css_pos = css_next_descendant_pre(NULL, &iter->cgrp->self); + css_task_iter_start(iter->css_pos, CSS_TASK_ITER_WITH_DEAD, + &iter->css_iter); + return; + } +#endif + raw_spin_lock_irq(&scx_tasks_lock); + + iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; + list_add(&iter->cursor.tasks_node, &scx_tasks); + iter->list_locked = true; +} + +static void __scx_task_iter_rq_unlock(struct scx_task_iter *iter) +{ + if (iter->locked_task) { + __balance_callbacks(iter->rq, &iter->rf); + task_rq_unlock(iter->rq, iter->locked_task, &iter->rf); + iter->locked_task = NULL; + } +} + +/** + * scx_task_iter_unlock - Unlock rq and scx_tasks_lock held by a task iterator + * @iter: iterator to unlock + * + * If @iter is in the middle of a locked iteration, it may be locking the rq of + * the task currently being visited in addition to scx_tasks_lock. Unlock both. + * This function can be safely called anytime during an iteration. The next + * iterator operation will automatically restore the necessary locking. + */ +static void scx_task_iter_unlock(struct scx_task_iter *iter) +{ + __scx_task_iter_rq_unlock(iter); + if (iter->list_locked) { + iter->list_locked = false; + raw_spin_unlock_irq(&scx_tasks_lock); + } +} + +static void __scx_task_iter_maybe_relock(struct scx_task_iter *iter) +{ + if (!iter->list_locked) { + raw_spin_lock_irq(&scx_tasks_lock); + iter->list_locked = true; + } +} + +/** + * scx_task_iter_relock - Re-acquire scx_tasks_lock and, optionally, @p's rq + * @iter: iterator to relock + * @p: task whose rq to lock, or %NULL for scx_tasks_lock only + * + * Counterpart to scx_task_iter_unlock(). Locking @p's rq is optional. Once + * re-acquired, both locks are managed by the iterator from here on. + */ +static void scx_task_iter_relock(struct scx_task_iter *iter, + struct task_struct *p) +{ + __scx_task_iter_maybe_relock(iter); + if (p) { + iter->rq = task_rq_lock(p, &iter->rf); + iter->locked_task = p; + } +} + +/** + * scx_task_iter_stop - Stop a task iteration and unlock scx_tasks_lock + * @iter: iterator to exit + * + * Exit a previously initialized @iter. Must be called with scx_tasks_lock held + * which is released on return. If the iterator holds a task's rq lock, that rq + * lock is also released. See scx_task_iter_start() for details. + */ +static void scx_task_iter_stop(struct scx_task_iter *iter) +{ +#ifdef CONFIG_EXT_SUB_SCHED + if (iter->cgrp) { + if (iter->css_pos) + css_task_iter_end(&iter->css_iter); + __scx_task_iter_rq_unlock(iter); + return; + } +#endif + __scx_task_iter_maybe_relock(iter); + list_del_init(&iter->cursor.tasks_node); + scx_task_iter_unlock(iter); +} + +/** + * scx_task_iter_next - Next task + * @iter: iterator to walk + * + * Visit the next task. See scx_task_iter_start() for details. Locks are dropped + * and re-acquired every %SCX_TASK_ITER_BATCH iterations to avoid causing stalls + * by holding scx_tasks_lock for too long. + */ +static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) +{ + struct list_head *cursor = &iter->cursor.tasks_node; + struct sched_ext_entity *pos; + + if (!(++iter->cnt % SCX_TASK_ITER_BATCH)) { + scx_task_iter_unlock(iter); + cond_resched(); + } + +#ifdef CONFIG_EXT_SUB_SCHED + if (iter->cgrp) { + while (iter->css_pos) { + struct task_struct *p; + + p = css_task_iter_next(&iter->css_iter); + if (p) + return p; + + css_task_iter_end(&iter->css_iter); + iter->css_pos = css_next_descendant_pre(iter->css_pos, + &iter->cgrp->self); + if (iter->css_pos) + css_task_iter_start(iter->css_pos, CSS_TASK_ITER_WITH_DEAD, + &iter->css_iter); + } + return NULL; + } +#endif + __scx_task_iter_maybe_relock(iter); + + list_for_each_entry(pos, cursor, tasks_node) { + if (&pos->tasks_node == &scx_tasks) + return NULL; + if (!(pos->flags & SCX_TASK_CURSOR)) { + list_move(cursor, &pos->tasks_node); + return container_of(pos, struct task_struct, scx); + } + } + + /* can't happen, should always terminate at scx_tasks above */ + BUG(); +} + +/** + * scx_task_iter_next_locked - Next non-idle task with its rq locked + * @iter: iterator to walk + * + * Visit the non-idle task with its rq lock held. Allows callers to specify + * whether they would like to filter out dead tasks. See scx_task_iter_start() + * for details. + */ +static struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) +{ + struct task_struct *p; + + __scx_task_iter_rq_unlock(iter); + + while ((p = scx_task_iter_next(iter))) { + /* + * scx_task_iter is used to prepare and move tasks into SCX + * while loading the BPF scheduler and vice-versa while + * unloading. The init_tasks ("swappers") should be excluded + * from the iteration because: + * + * - It's unsafe to use __setschduler_prio() on an init_task to + * determine the sched_class to use as it won't preserve its + * idle_sched_class. + * + * - ops.init/exit_task() can easily be confused if called with + * init_tasks as they, e.g., share PID 0. + * + * As init_tasks are never scheduled through SCX, they can be + * skipped safely. Note that is_idle_task() which tests %PF_IDLE + * doesn't work here: + * + * - %PF_IDLE may not be set for an init_task whose CPU hasn't + * yet been onlined. + * + * - %PF_IDLE can be set on tasks that are not init_tasks. See + * play_idle_precise() used by CONFIG_IDLE_INJECT. + * + * Test for idle_sched_class as only init_tasks are on it. + */ + if (p->sched_class == &idle_sched_class) + continue; + + iter->rq = task_rq_lock(p, &iter->rf); + iter->locked_task = p; + + /* + * cgroup_task_dead() removes the dead tasks from cset->tasks + * after sched_ext_dead() and cgroup iteration may see tasks + * which already finished sched_ext_dead(). %SCX_TASK_DEAD is + * set by sched_ext_dead() under @p's rq lock. Test it to + * avoid visiting tasks which are already dead from SCX POV. + */ + if (scx_get_task_state(p) == SCX_TASK_DEAD) { + __scx_task_iter_rq_unlock(iter); + continue; + } + + return p; + } + return NULL; +} + +/** + * scx_add_event - Increase an event counter for 'name' by 'cnt' + * @sch: scx_sched to account events for + * @name: an event name defined in struct scx_event_stats + * @cnt: the number of the event occurred + * + * This can be used when preemption is not disabled. + */ +#define scx_add_event(sch, name, cnt) do { \ + this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ + trace_sched_ext_event(#name, (cnt)); \ +} while(0) + +/** + * __scx_add_event - Increase an event counter for 'name' by 'cnt' + * @sch: scx_sched to account events for + * @name: an event name defined in struct scx_event_stats + * @cnt: the number of the event occurred + * + * This should be used only when preemption is disabled. + */ +#define __scx_add_event(sch, name, cnt) do { \ + __this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ + trace_sched_ext_event(#name, cnt); \ +} while(0) + +/** + * scx_agg_event - Aggregate an event counter 'kind' from 'src_e' to 'dst_e' + * @dst_e: destination event stats + * @src_e: source event stats + * @kind: a kind of event to be aggregated + */ +#define scx_agg_event(dst_e, src_e, kind) do { \ + (dst_e)->kind += READ_ONCE((src_e)->kind); \ +} while(0) + +/** + * scx_dump_event - Dump an event 'kind' in 'events' to 's' + * @s: output seq_buf + * @events: event stats + * @kind: a kind of event to dump + */ +#define scx_dump_event(s, events, kind) do { \ + dump_line(&(s), "%40s: %16lld", #kind, (events)->kind); \ +} while (0) + + +static void scx_read_events(struct scx_sched *sch, + struct scx_event_stats *events); + +static enum scx_enable_state scx_enable_state(void) +{ + return atomic_read(&scx_enable_state_var); +} + +static enum scx_enable_state scx_set_enable_state(enum scx_enable_state to) +{ + return atomic_xchg(&scx_enable_state_var, to); +} + +static bool scx_tryset_enable_state(enum scx_enable_state to, + enum scx_enable_state from) +{ + int from_v = from; + + return atomic_try_cmpxchg(&scx_enable_state_var, &from_v, to); +} + +/** + * wait_ops_state - Busy-wait the specified ops state to end + * @p: target task + * @opss: state to wait the end of + * + * Busy-wait for @p to transition out of @opss. This can only be used when the + * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also + * has load_acquire semantics to ensure that the caller can see the updates made + * in the enqueueing and dispatching paths. + */ +static void wait_ops_state(struct task_struct *p, unsigned long opss) +{ + do { + cpu_relax(); + } while (atomic_long_read_acquire(&p->scx.ops_state) == opss); +} + +static inline bool __cpu_valid(s32 cpu) +{ + return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); +} + +/** + * scx_cpu_valid - Verify a cpu number, to be used on ops input args + * @sch: scx_sched to abort on error + * @cpu: cpu number which came from a BPF ops + * @where: extra information reported on error + * + * @cpu is a cpu number which came from the BPF scheduler and can be any value. + * Verify that it is in range and one of the possible cpus. If invalid, trigger + * an ops error. + */ +bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where) +{ + if (__cpu_valid(cpu)) { + return true; + } else { + scx_error(sch, "invalid CPU %d%s%s", cpu, where ? " " : "", where ?: ""); + return false; + } +} + +/** + * ops_sanitize_err - Sanitize a -errno value + * @sch: scx_sched to error out on error + * @ops_name: operation to blame on failure + * @err: -errno value to sanitize + * + * Verify @err is a valid -errno. If not, trigger scx_error() and return + * -%EPROTO. This is necessary because returning a rogue -errno up the chain can + * cause misbehaviors. For an example, a large negative return from + * ops.init_task() triggers an oops when passed up the call chain because the + * value fails IS_ERR() test after being encoded with ERR_PTR() and then is + * handled as a pointer. + */ +static int ops_sanitize_err(struct scx_sched *sch, const char *ops_name, s32 err) +{ + if (err < 0 && err >= -MAX_ERRNO) + return err; + + scx_error(sch, "ops.%s() returned an invalid errno %d", ops_name, err); + return -EPROTO; +} + +static void deferred_bal_cb_workfn(struct rq *rq) +{ + run_deferred(rq); +} + +static void deferred_irq_workfn(struct irq_work *irq_work) +{ + struct rq *rq = container_of(irq_work, struct rq, scx.deferred_irq_work); + + raw_spin_rq_lock(rq); + run_deferred(rq); + raw_spin_rq_unlock(rq); +} + +/** + * schedule_deferred - Schedule execution of deferred actions on an rq + * @rq: target rq + * + * Schedule execution of deferred actions on @rq. Deferred actions are executed + * with @rq locked but unpinned, and thus can unlock @rq to e.g. migrate tasks + * to other rqs. + */ +static void schedule_deferred(struct rq *rq) +{ + /* + * This is the fallback when schedule_deferred_locked() can't use + * the cheaper balance callback or wakeup hook paths (the target + * CPU is not in balance or wakeup). Currently, this is primarily + * hit by reenqueue operations targeting a remote CPU. + * + * Queue on the target CPU. The deferred work can run from any CPU + * correctly - the _locked() path already processes remote rqs from + * the calling CPU - but targeting the owning CPU allows IPI delivery + * without waiting for the calling CPU to re-enable IRQs and is + * cheaper as the reenqueue runs locally. + */ + irq_work_queue_on(&rq->scx.deferred_irq_work, cpu_of(rq)); +} + +/** + * schedule_deferred_locked - Schedule execution of deferred actions on an rq + * @rq: target rq + * + * Schedule execution of deferred actions on @rq. Equivalent to + * schedule_deferred() but requires @rq to be locked and can be more efficient. + */ +static void schedule_deferred_locked(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + + /* + * If in the middle of waking up a task, task_woken_scx() will be called + * afterwards which will then run the deferred actions, no need to + * schedule anything. + */ + if (rq->scx.flags & SCX_RQ_IN_WAKEUP) + return; + + /* Don't do anything if there already is a deferred operation. */ + if (rq->scx.flags & SCX_RQ_BAL_CB_PENDING) + return; + + /* + * If in balance, the balance callbacks will be called before rq lock is + * released. Schedule one. + * + * + * We can't directly insert the callback into the + * rq's list: The call can drop its lock and make the pending balance + * callback visible to unrelated code paths that call rq_pin_lock(). + * + * Just let balance_one() know that it must do it itself. + */ + if (rq->scx.flags & SCX_RQ_IN_BALANCE) { + rq->scx.flags |= SCX_RQ_BAL_CB_PENDING; + return; + } + + /* + * No scheduler hooks available. Use the generic irq_work path. The + * above WAKEUP and BALANCE paths should cover most of the cases and the + * time to IRQ re-enable shouldn't be long. + */ + schedule_deferred(rq); +} + +static void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, + u64 reenq_flags, struct rq *locked_rq) +{ + struct rq *rq; + + /* + * Allowing reenqueues doesn't make sense while bypassing. This also + * blocks from new reenqueues to be scheduled on dead scheds. + */ + if (unlikely(READ_ONCE(sch->bypass_depth))) + return; + + if (dsq->id == SCX_DSQ_LOCAL) { + rq = container_of(dsq, struct rq, scx.local_dsq); + + struct scx_sched_pcpu *sch_pcpu = per_cpu_ptr(sch->pcpu, cpu_of(rq)); + struct scx_deferred_reenq_local *drl = &sch_pcpu->deferred_reenq_local; + + /* + * Pairs with smp_mb() in process_deferred_reenq_locals() and + * guarantees that there is a reenq_local() afterwards. + */ + smp_mb(); + + if (list_empty(&drl->node) || + (READ_ONCE(drl->flags) & reenq_flags) != reenq_flags) { + + guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); + + if (list_empty(&drl->node)) + list_move_tail(&drl->node, &rq->scx.deferred_reenq_locals); + WRITE_ONCE(drl->flags, drl->flags | reenq_flags); + } + } else if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) { + rq = this_rq(); + + struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu, cpu_of(rq)); + struct scx_deferred_reenq_user *dru = &dsq_pcpu->deferred_reenq_user; + + /* + * Pairs with smp_mb() in process_deferred_reenq_users() and + * guarantees that there is a reenq_user() afterwards. + */ + smp_mb(); + + if (list_empty(&dru->node) || + (READ_ONCE(dru->flags) & reenq_flags) != reenq_flags) { + + guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); + + if (list_empty(&dru->node)) + list_move_tail(&dru->node, &rq->scx.deferred_reenq_users); + WRITE_ONCE(dru->flags, dru->flags | reenq_flags); + } + } else { + scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id); + return; + } + + if (rq == locked_rq) + schedule_deferred_locked(rq); + else + schedule_deferred(rq); +} + +static void schedule_reenq_local(struct rq *rq, u64 reenq_flags) +{ + struct scx_sched *root = rcu_dereference_sched(scx_root); + + if (WARN_ON_ONCE(!root)) + return; + + schedule_dsq_reenq(root, &rq->scx.local_dsq, reenq_flags, rq); +} + +/** + * touch_core_sched - Update timestamp used for core-sched task ordering + * @rq: rq to read clock from, must be locked + * @p: task to update the timestamp for + * + * Update @p->scx.core_sched_at timestamp. This is used by scx_prio_less() to + * implement global or local-DSQ FIFO ordering for core-sched. Should be called + * when a task becomes runnable and its turn on the CPU ends (e.g. slice + * exhaustion). + */ +static void touch_core_sched(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_rq_held(rq); + +#ifdef CONFIG_SCHED_CORE + /* + * It's okay to update the timestamp spuriously. Use + * sched_core_disabled() which is cheaper than enabled(). + * + * As this is used to determine ordering between tasks of sibling CPUs, + * it may be better to use per-core dispatch sequence instead. + */ + if (!sched_core_disabled()) + p->scx.core_sched_at = sched_clock_cpu(cpu_of(rq)); +#endif +} + +/** + * touch_core_sched_dispatch - Update core-sched timestamp on dispatch + * @rq: rq to read clock from, must be locked + * @p: task being dispatched + * + * If the BPF scheduler implements custom core-sched ordering via + * ops.core_sched_before(), @p->scx.core_sched_at is used to implement FIFO + * ordering within each local DSQ. This function is called from dispatch paths + * and updates @p->scx.core_sched_at if custom core-sched ordering is in effect. + */ +static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_rq_held(rq); + +#ifdef CONFIG_SCHED_CORE + if (unlikely(SCX_HAS_OP(scx_root, core_sched_before))) + touch_core_sched(rq, p); +#endif +} + +static void update_curr_scx(struct rq *rq) +{ + struct task_struct *curr = rq->curr; + s64 delta_exec; + + delta_exec = update_curr_common(rq); + if (unlikely(delta_exec <= 0)) + return; + + if (curr->scx.slice != SCX_SLICE_INF) { + curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec); + if (!curr->scx.slice) + touch_core_sched(rq, curr); + } + + dl_server_update(&rq->ext_server, delta_exec); +} + +static bool scx_dsq_priq_less(struct rb_node *node_a, + const struct rb_node *node_b) +{ + const struct task_struct *a = + container_of(node_a, struct task_struct, scx.dsq_priq); + const struct task_struct *b = + container_of(node_b, struct task_struct, scx.dsq_priq); + + return time_before64(a->scx.dsq_vtime, b->scx.dsq_vtime); +} + +static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags) +{ + /* scx_bpf_dsq_nr_queued() reads ->nr without locking, use WRITE_ONCE() */ + WRITE_ONCE(dsq->nr, dsq->nr + 1); + + /* + * Once @p reaches a local DSQ, it can only leave it by being dispatched + * to the CPU or dequeued. In both cases, the only way @p can go back to + * the BPF sched is through enqueueing. If being inserted into a local + * DSQ with IMMED, persist the state until the next enqueueing event in + * do_enqueue_task() so that we can maintain IMMED protection through + * e.g. SAVE/RESTORE cycles and slice extensions. + */ + if (enq_flags & SCX_ENQ_IMMED) { + if (unlikely(dsq->id != SCX_DSQ_LOCAL)) { + WARN_ON_ONCE(!(enq_flags & SCX_ENQ_GDSQ_FALLBACK)); + return; + } + p->scx.flags |= SCX_TASK_IMMED; + } + + if (p->scx.flags & SCX_TASK_IMMED) { + struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); + + if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) + return; + + rq->scx.nr_immed++; + + /* + * If @rq already had other tasks or the current task is not + * done yet, @p can't go on the CPU immediately. Re-enqueue. + */ + if (unlikely(dsq->nr > 1 || !rq_is_open(rq, enq_flags))) + schedule_reenq_local(rq, 0); + } +} + +static void dsq_dec_nr(struct scx_dispatch_q *dsq, struct task_struct *p) +{ + /* see dsq_inc_nr() */ + WRITE_ONCE(dsq->nr, dsq->nr - 1); + + if (p->scx.flags & SCX_TASK_IMMED) { + struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); + + if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL) || + WARN_ON_ONCE(rq->scx.nr_immed <= 0)) + return; + + rq->scx.nr_immed--; + } +} + +static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) +{ + p->scx.slice = READ_ONCE(sch->slice_dfl); + __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); +} + +/* + * Return true if @p is moving due to an internal SCX migration, false + * otherwise. + */ +static inline bool task_scx_migrating(struct task_struct *p) +{ + /* + * We only need to check sticky_cpu: it is set to the destination + * CPU in move_remote_task_to_local_dsq() before deactivate_task() + * and cleared when the task is enqueued on the destination, so it + * is only non-negative during an internal SCX migration. + */ + return p->scx.sticky_cpu >= 0; +} + +/* + * Call ops.dequeue() if the task is in BPF custody and not migrating. + * Clears %SCX_TASK_IN_CUSTODY when the callback is invoked. + */ +static void call_task_dequeue(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 deq_flags) +{ + if (!(p->scx.flags & SCX_TASK_IN_CUSTODY) || task_scx_migrating(p)) + return; + + if (SCX_HAS_OP(sch, dequeue)) + SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags); + + p->scx.flags &= ~SCX_TASK_IN_CUSTODY; +} + +static void local_dsq_post_enq(struct scx_sched *sch, struct scx_dispatch_q *dsq, + struct task_struct *p, u64 enq_flags) +{ + struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); + + call_task_dequeue(sch, rq, p, 0); + + /* + * Note that @rq's lock may be dropped between this enqueue and @p + * actually getting on CPU. This gives higher-class tasks (e.g. RT) + * an opportunity to wake up on @rq and prevent @p from running. + * Here are some concrete examples: + * + * Example 1: + * + * We dispatch two tasks from a single ops.dispatch(): + * - First, a local task to this CPU's local DSQ; + * - Second, a local/remote task to a remote CPU's local DSQ. + * We must drop the local rq lock in order to finish the second + * dispatch. In that time, an RT task can wake up on the local rq. + * + * Example 2: + * + * We dispatch a local/remote task to a remote CPU's local DSQ. + * We must drop the remote rq lock before the dispatched task can run, + * which gives an RT task an opportunity to wake up on the remote rq. + * + * Both examples work the same if we replace dispatching with moving + * the tasks from a user-created DSQ. + * + * We must detect these wakeups so that we can re-enqueue IMMED tasks + * from @rq's local DSQ. scx_wakeup_preempt() serves exactly this + * purpose, but for it to be invoked, we must ensure that we bump + * @rq->next_class to &ext_sched_class if it's currently idle. + * + * wakeup_preempt() does the bumping, and since we only invoke it if + * @rq->next_class is below &ext_sched_class, it will also + * resched_curr(rq). + */ + if (sched_class_above(p->sched_class, rq->next_class)) + wakeup_preempt(rq, p, 0); + + /* + * If @rq is in balance, the CPU is already vacant and looking for the + * next task to run. No need to preempt or trigger resched after moving + * @p into its local DSQ. + * Note that the wakeup_preempt() above may have already triggered + * a resched if @rq->next_class was idle. It's harmless, since + * need_resched is cleared immediately after task pick. + */ + if (rq->scx.flags & SCX_RQ_IN_BALANCE) + return; + + if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && + rq->curr->sched_class == &ext_sched_class) { + rq->curr->scx.slice = 0; + resched_curr(rq); + } +} + +static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq, + struct scx_dispatch_q *dsq, struct task_struct *p, + u64 enq_flags) +{ + bool is_local = dsq->id == SCX_DSQ_LOCAL; + + WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); + WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || + !RB_EMPTY_NODE(&p->scx.dsq_priq)); + + if (!is_local) { + raw_spin_lock_nested(&dsq->lock, + (enq_flags & SCX_ENQ_NESTED) ? SINGLE_DEPTH_NESTING : 0); + + if (unlikely(dsq->id == SCX_DSQ_INVALID)) { + scx_error(sch, "attempting to dispatch to a destroyed dsq"); + /* fall back to the global dsq */ + raw_spin_unlock(&dsq->lock); + dsq = find_global_dsq(sch, task_cpu(p)); + raw_spin_lock(&dsq->lock); + } + } + + if (unlikely((dsq->id & SCX_DSQ_FLAG_BUILTIN) && + (enq_flags & SCX_ENQ_DSQ_PRIQ))) { + /* + * SCX_DSQ_LOCAL and SCX_DSQ_GLOBAL DSQs always consume from + * their FIFO queues. To avoid confusion and accidentally + * starving vtime-dispatched tasks by FIFO-dispatched tasks, we + * disallow any internal DSQ from doing vtime ordering of + * tasks. + */ + scx_error(sch, "cannot use vtime ordering for built-in DSQs"); + enq_flags &= ~SCX_ENQ_DSQ_PRIQ; + } + + if (enq_flags & SCX_ENQ_DSQ_PRIQ) { + struct rb_node *rbp; + + /* + * A PRIQ DSQ shouldn't be using FIFO enqueueing. As tasks are + * linked to both the rbtree and list on PRIQs, this can only be + * tested easily when adding the first task. + */ + if (unlikely(RB_EMPTY_ROOT(&dsq->priq) && + nldsq_next_task(dsq, NULL, false))) + scx_error(sch, "DSQ ID 0x%016llx already had FIFO-enqueued tasks", + dsq->id); + + p->scx.dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; + rb_add(&p->scx.dsq_priq, &dsq->priq, scx_dsq_priq_less); + + /* + * Find the previous task and insert after it on the list so + * that @dsq->list is vtime ordered. + */ + rbp = rb_prev(&p->scx.dsq_priq); + if (rbp) { + struct task_struct *prev = + container_of(rbp, struct task_struct, + scx.dsq_priq); + list_add(&p->scx.dsq_list.node, &prev->scx.dsq_list.node); + /* first task unchanged - no update needed */ + } else { + list_add(&p->scx.dsq_list.node, &dsq->list); + /* not builtin and new task is at head - use fastpath */ + rcu_assign_pointer(dsq->first_task, p); + } + } else { + /* a FIFO DSQ shouldn't be using PRIQ enqueuing */ + if (unlikely(!RB_EMPTY_ROOT(&dsq->priq))) + scx_error(sch, "DSQ ID 0x%016llx already had PRIQ-enqueued tasks", + dsq->id); + + if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) { + list_add(&p->scx.dsq_list.node, &dsq->list); + /* new task inserted at head - use fastpath */ + if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) + rcu_assign_pointer(dsq->first_task, p); + } else { + /* + * dsq->list can contain parked BPF iterator cursors, so + * list_empty() here isn't a reliable proxy for "no real + * task in the DSQ". Test dsq->first_task directly. + */ + list_add_tail(&p->scx.dsq_list.node, &dsq->list); + if (!dsq->first_task && !(dsq->id & SCX_DSQ_FLAG_BUILTIN)) + rcu_assign_pointer(dsq->first_task, p); + } + } + + /* seq records the order tasks are queued, used by BPF DSQ iterator */ + WRITE_ONCE(dsq->seq, dsq->seq + 1); + p->scx.dsq_seq = dsq->seq; + + dsq_inc_nr(dsq, p, enq_flags); + p->scx.dsq = dsq; + + /* + * Update custody and call ops.dequeue() before clearing ops_state: + * once ops_state is cleared, waiters in ops_dequeue() can proceed + * and dequeue_task_scx() will RMW p->scx.flags. If we clear + * ops_state first, both sides would modify p->scx.flags + * concurrently in a non-atomic way. + */ + if (is_local) { + local_dsq_post_enq(sch, dsq, p, enq_flags); + } else { + /* + * Task on global/bypass DSQ: leave custody, task on + * non-terminal DSQ: enter custody. + */ + if (dsq->id == SCX_DSQ_GLOBAL || dsq->id == SCX_DSQ_BYPASS) + call_task_dequeue(sch, rq, p, 0); + else + p->scx.flags |= SCX_TASK_IN_CUSTODY; + + raw_spin_unlock(&dsq->lock); + } + + /* + * We're transitioning out of QUEUEING or DISPATCHING. store_release to + * match waiters' load_acquire. + */ + if (enq_flags & SCX_ENQ_CLEAR_OPSS) + atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); +} + +static void task_unlink_from_dsq(struct task_struct *p, + struct scx_dispatch_q *dsq) +{ + WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node)); + + if (p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { + rb_erase(&p->scx.dsq_priq, &dsq->priq); + RB_CLEAR_NODE(&p->scx.dsq_priq); + p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; + } + + list_del_init(&p->scx.dsq_list.node); + dsq_dec_nr(dsq, p); + + if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN) && dsq->first_task == p) { + struct task_struct *first_task; + + first_task = nldsq_next_task(dsq, NULL, false); + rcu_assign_pointer(dsq->first_task, first_task); + } +} + +static void dispatch_dequeue(struct rq *rq, struct task_struct *p) +{ + struct scx_dispatch_q *dsq = p->scx.dsq; + bool is_local = dsq == &rq->scx.local_dsq; + + lockdep_assert_rq_held(rq); + + if (!dsq) { + /* + * If !dsq && on-list, @p is on @rq's ddsp_deferred_locals. + * Unlinking is all that's needed to cancel. + */ + if (unlikely(!list_empty(&p->scx.dsq_list.node))) + list_del_init(&p->scx.dsq_list.node); + + /* + * When dispatching directly from the BPF scheduler to a local + * DSQ, the task isn't associated with any DSQ but + * @p->scx.holding_cpu may be set under the protection of + * %SCX_OPSS_DISPATCHING. + */ + if (p->scx.holding_cpu >= 0) + p->scx.holding_cpu = -1; + + return; + } + + if (!is_local) + raw_spin_lock(&dsq->lock); + + /* + * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx.dsq_* can't + * change underneath us. + */ + if (p->scx.holding_cpu < 0) { + /* @p must still be on @dsq, dequeue */ + task_unlink_from_dsq(p, dsq); + } else { + /* + * We're racing against dispatch_to_local_dsq() which already + * removed @p from @dsq and set @p->scx.holding_cpu. Clear the + * holding_cpu which tells dispatch_to_local_dsq() that it lost + * the race. + */ + WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node)); + p->scx.holding_cpu = -1; + } + p->scx.dsq = NULL; + + if (!is_local) + raw_spin_unlock(&dsq->lock); +} + +/* + * Abbreviated version of dispatch_dequeue() that can be used when both @p's rq + * and dsq are locked. + */ +static void dispatch_dequeue_locked(struct task_struct *p, + struct scx_dispatch_q *dsq) +{ + lockdep_assert_rq_held(task_rq(p)); + lockdep_assert_held(&dsq->lock); + + task_unlink_from_dsq(p, dsq); + p->scx.dsq = NULL; +} + +static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch, + struct rq *rq, u64 dsq_id, + s32 tcpu) +{ + struct scx_dispatch_q *dsq; + + if (dsq_id == SCX_DSQ_LOCAL) + return &rq->scx.local_dsq; + + if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { + s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK); + + if (!scx_cpu_valid(sch, cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict")) + return find_global_dsq(sch, tcpu); + + return &cpu_rq(cpu)->scx.local_dsq; + } + + if (dsq_id == SCX_DSQ_GLOBAL) + dsq = find_global_dsq(sch, tcpu); + else + dsq = find_user_dsq(sch, dsq_id); + + if (unlikely(!dsq)) { + scx_error(sch, "non-existent DSQ 0x%llx", dsq_id); + return find_global_dsq(sch, tcpu); + } + + return dsq; +} + +static void mark_direct_dispatch(struct scx_sched *sch, + struct task_struct *ddsp_task, + struct task_struct *p, u64 dsq_id, + u64 enq_flags) +{ + /* + * Mark that dispatch already happened from ops.select_cpu() or + * ops.enqueue() by spoiling direct_dispatch_task with a non-NULL value + * which can never match a valid task pointer. + */ + __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); + + /* @p must match the task on the enqueue path */ + if (unlikely(p != ddsp_task)) { + if (IS_ERR(ddsp_task)) + scx_error(sch, "%s[%d] already direct-dispatched", + p->comm, p->pid); + else + scx_error(sch, "scheduling for %s[%d] but trying to direct-dispatch %s[%d]", + ddsp_task->comm, ddsp_task->pid, + p->comm, p->pid); + return; + } + + WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID); + WARN_ON_ONCE(p->scx.ddsp_enq_flags); + + p->scx.ddsp_dsq_id = dsq_id; + p->scx.ddsp_enq_flags = enq_flags; +} + +/* + * Clear @p direct dispatch state when leaving the scheduler. + * + * Direct dispatch state must be cleared in the following cases: + * - direct_dispatch(): cleared on the synchronous enqueue path, deferred + * dispatch keeps the state until consumed + * - process_ddsp_deferred_locals(): cleared after consuming deferred state, + * - do_enqueue_task(): cleared on enqueue fallbacks where the dispatch + * verdict is ignored (local/global/bypass) + * - dequeue_task_scx(): cleared after dispatch_dequeue(), covering deferred + * cancellation and holding_cpu races + * - scx_disable_task(): cleared for queued wakeup tasks, which are excluded by + * the scx_bypass() loop, so that stale state is not reused by a subsequent + * scheduler instance + */ +static inline void clear_direct_dispatch(struct task_struct *p) +{ + p->scx.ddsp_dsq_id = SCX_DSQ_INVALID; + p->scx.ddsp_enq_flags = 0; +} + +static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, + u64 enq_flags) +{ + struct rq *rq = task_rq(p); + struct scx_dispatch_q *dsq = + find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p)); + u64 ddsp_enq_flags; + + touch_core_sched_dispatch(rq, p); + + p->scx.ddsp_enq_flags |= enq_flags; + + /* + * We are in the enqueue path with @rq locked and pinned, and thus can't + * double lock a remote rq and enqueue to its local DSQ. For + * DSQ_LOCAL_ON verdicts targeting the local DSQ of a remote CPU, defer + * the enqueue so that it's executed when @rq can be unlocked. + */ + if (dsq->id == SCX_DSQ_LOCAL && dsq != &rq->scx.local_dsq) { + unsigned long opss; + + opss = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_STATE_MASK; + + switch (opss & SCX_OPSS_STATE_MASK) { + case SCX_OPSS_NONE: + break; + case SCX_OPSS_QUEUEING: + /* + * As @p was never passed to the BPF side, _release is + * not strictly necessary. Still do it for consistency. + */ + atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); + break; + default: + WARN_ONCE(true, "sched_ext: %s[%d] has invalid ops state 0x%lx in direct_dispatch()", + p->comm, p->pid, opss); + atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); + break; + } + + WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); + list_add_tail(&p->scx.dsq_list.node, + &rq->scx.ddsp_deferred_locals); + schedule_deferred_locked(rq); + return; + } + + ddsp_enq_flags = p->scx.ddsp_enq_flags; + clear_direct_dispatch(p); + + dispatch_enqueue(sch, rq, dsq, p, ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); +} + +static bool scx_rq_online(struct rq *rq) +{ + /* + * Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE indicates + * the online state as seen from the BPF scheduler. cpu_active() test + * guarantees that, if this function returns %true, %SCX_RQ_ONLINE will + * stay set until the current scheduling operation is complete even if + * we aren't locking @rq. + */ + return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq))); +} + +static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, + int sticky_cpu) +{ + struct scx_sched *sch = scx_task_sched(p); + struct task_struct **ddsp_taskp; + struct scx_dispatch_q *dsq; + unsigned long qseq; + + WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_QUEUED)); + + /* internal movements - rq migration / RESTORE */ + if (sticky_cpu == cpu_of(rq)) + goto local_norefill; + + /* + * Clear persistent TASK_IMMED for fresh enqueues, see dsq_inc_nr(). + * Note that exiting and migration-disabled tasks that skip + * ops.enqueue() below will lose IMMED protection unless + * %SCX_OPS_ENQ_EXITING / %SCX_OPS_ENQ_MIGRATION_DISABLED are set. + */ + p->scx.flags &= ~SCX_TASK_IMMED; + + /* + * If !scx_rq_online(), we already told the BPF scheduler that the CPU + * is offline and are just running the hotplug path. Don't bother the + * BPF scheduler. + */ + if (!scx_rq_online(rq)) + goto local; + + if (scx_bypassing(sch, cpu_of(rq))) { + __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); + goto bypass; + } + + if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) + goto direct; + + /* see %SCX_OPS_ENQ_EXITING */ + if (!(sch->ops.flags & SCX_OPS_ENQ_EXITING) && + unlikely(p->flags & PF_EXITING)) { + __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1); + goto local; + } + + /* see %SCX_OPS_ENQ_MIGRATION_DISABLED */ + if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) && + is_migration_disabled(p)) { + __scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1); + goto local; + } + + if (unlikely(!SCX_HAS_OP(sch, enqueue))) + goto global; + + /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ + qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; + + WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); + atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq); + + ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); + WARN_ON_ONCE(*ddsp_taskp); + *ddsp_taskp = p; + + SCX_CALL_OP_TASK(sch, enqueue, rq, p, enq_flags); + + *ddsp_taskp = NULL; + if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) + goto direct; + + /* + * Task is now in BPF scheduler's custody. Set %SCX_TASK_IN_CUSTODY + * so ops.dequeue() is called when it leaves custody. + */ + p->scx.flags |= SCX_TASK_IN_CUSTODY; + + /* + * If not directly dispatched, QUEUEING isn't clear yet and dispatch or + * dequeue may be waiting. The store_release matches their load_acquire. + */ + atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq); + return; + +direct: + direct_dispatch(sch, p, enq_flags); + return; +local_norefill: + dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, enq_flags); + return; +local: + dsq = &rq->scx.local_dsq; + goto enqueue; +global: + dsq = find_global_dsq(sch, task_cpu(p)); + goto enqueue; +bypass: + dsq = bypass_enq_target_dsq(sch, task_cpu(p)); + goto enqueue; + +enqueue: + /* + * For task-ordering, slice refill must be treated as implying the end + * of the current slice. Otherwise, the longer @p stays on the CPU, the + * higher priority it becomes from scx_prio_less()'s POV. + */ + touch_core_sched(rq, p); + refill_task_slice_dfl(sch, p); + clear_direct_dispatch(p); + dispatch_enqueue(sch, rq, dsq, p, enq_flags); +} + +static bool task_runnable(const struct task_struct *p) +{ + return !list_empty(&p->scx.runnable_node); +} + +static void set_task_runnable(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_rq_held(rq); + + if (p->scx.flags & SCX_TASK_RESET_RUNNABLE_AT) { + p->scx.runnable_at = jiffies; + p->scx.flags &= ~SCX_TASK_RESET_RUNNABLE_AT; + } + + /* + * list_add_tail() must be used. scx_bypass() depends on tasks being + * appended to the runnable_list. + */ + list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list); +} + +static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at) +{ + list_del_init(&p->scx.runnable_node); + if (reset_runnable_at) + p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; +} + +static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_flags) +{ + struct scx_sched *sch = scx_task_sched(p); + int sticky_cpu = p->scx.sticky_cpu; + u64 enq_flags = core_enq_flags | rq->scx.extra_enq_flags; + + if (enq_flags & ENQUEUE_WAKEUP) + rq->scx.flags |= SCX_RQ_IN_WAKEUP; + + /* + * Restoring a running task will be immediately followed by + * set_next_task_scx() which expects the task to not be on the BPF + * scheduler as tasks can only start running through local DSQs. Force + * direct-dispatch into the local DSQ by setting the sticky_cpu. + */ + if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) + sticky_cpu = cpu_of(rq); + + if (p->scx.flags & SCX_TASK_QUEUED) { + WARN_ON_ONCE(!task_runnable(p)); + goto out; + } + + set_task_runnable(rq, p); + p->scx.flags |= SCX_TASK_QUEUED; + rq->scx.nr_running++; + add_nr_running(rq, 1); + + if (SCX_HAS_OP(sch, runnable) && !task_on_rq_migrating(p)) + SCX_CALL_OP_TASK(sch, runnable, rq, p, enq_flags); + + if (enq_flags & SCX_ENQ_WAKEUP) + touch_core_sched(rq, p); + + /* Start dl_server if this is the first task being enqueued */ + if (rq->scx.nr_running == 1) + dl_server_start(&rq->ext_server); + + do_enqueue_task(rq, p, enq_flags, sticky_cpu); + + if (sticky_cpu >= 0) + p->scx.sticky_cpu = -1; +out: + rq->scx.flags &= ~SCX_RQ_IN_WAKEUP; + + if ((enq_flags & SCX_ENQ_CPU_SELECTED) && + unlikely(cpu_of(rq) != p->scx.selected_cpu)) + __scx_add_event(sch, SCX_EV_SELECT_CPU_FALLBACK, 1); +} + +static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags) +{ + struct scx_sched *sch = scx_task_sched(p); + unsigned long opss; + + /* dequeue is always temporary, don't reset runnable_at */ + clr_task_runnable(p, false); + +retry: + /* acquire ensures that we see the preceding updates on QUEUED */ + opss = atomic_long_read_acquire(&p->scx.ops_state); + + switch (opss & SCX_OPSS_STATE_MASK) { + case SCX_OPSS_NONE: + break; + case SCX_OPSS_QUEUEING: + /* + * QUEUEING is started and finished while holding @p's rq lock. + * As we're holding the rq lock now, we shouldn't see QUEUEING. + */ + BUG(); + case SCX_OPSS_QUEUED: + /* + * A queued task must always be in BPF scheduler's custody. If + * SCX_TASK_IN_CUSTODY is clear, finish_dispatch() on another + * CPU has already passed call_task_dequeue() (which clears the + * flag), but has not yet written SCX_OPSS_NONE. That final + * store does not require this rq's lock, so retrying with + * cpu_relax() is bounded: we will observe NONE (or DISPATCHING, + * handled by the fallthrough) on a subsequent iteration. + */ + if (unlikely(!(READ_ONCE(p->scx.flags) & SCX_TASK_IN_CUSTODY))) { + cpu_relax(); + goto retry; + } + + if (atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, + SCX_OPSS_NONE)) + break; + fallthrough; + case SCX_OPSS_DISPATCHING: + /* + * If @p is being dispatched from the BPF scheduler to a DSQ, + * wait for the transfer to complete so that @p doesn't get + * added to its DSQ after dequeueing is complete. + * + * As we're waiting on DISPATCHING with the rq locked, the + * dispatching side shouldn't try to lock the rq while + * DISPATCHING is set. See dispatch_to_local_dsq(). + * + * DISPATCHING shouldn't have qseq set and control can reach + * here with NONE @opss from the above QUEUED case block. + * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. + */ + wait_ops_state(p, SCX_OPSS_DISPATCHING); + BUG_ON(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); + break; + } + + /* + * Call ops.dequeue() if the task is still in BPF custody. + * + * The code that clears ops_state to %SCX_OPSS_NONE does not always + * clear %SCX_TASK_IN_CUSTODY: in dispatch_to_local_dsq(), when + * we're moving a task that was in %SCX_OPSS_DISPATCHING to a + * remote CPU's local DSQ, we only set ops_state to %SCX_OPSS_NONE + * so that a concurrent dequeue can proceed, but we clear + * %SCX_TASK_IN_CUSTODY only when we later enqueue or move the + * task. So we can see NONE + IN_CUSTODY here and we must handle + * it. Similarly, after waiting on %SCX_OPSS_DISPATCHING we see + * NONE but the task may still have %SCX_TASK_IN_CUSTODY set until + * it is enqueued on the destination. + */ + call_task_dequeue(sch, rq, p, deq_flags); +} + +static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_flags) +{ + struct scx_sched *sch = scx_task_sched(p); + u64 deq_flags = core_deq_flags; + + /* + * Set %SCX_DEQ_SCHED_CHANGE when the dequeue is due to a property + * change (not sleep or core-sched pick). + */ + if (!(deq_flags & (DEQUEUE_SLEEP | SCX_DEQ_CORE_SCHED_EXEC))) + deq_flags |= SCX_DEQ_SCHED_CHANGE; + + if (!(p->scx.flags & SCX_TASK_QUEUED)) { + WARN_ON_ONCE(task_runnable(p)); + return true; + } + + ops_dequeue(rq, p, deq_flags); + + /* + * A currently running task which is going off @rq first gets dequeued + * and then stops running. As we want running <-> stopping transitions + * to be contained within runnable <-> quiescent transitions, trigger + * ->stopping() early here instead of in put_prev_task_scx(). + * + * @p may go through multiple stopping <-> running transitions between + * here and put_prev_task_scx() if task attribute changes occur while + * balance_one() leaves @rq unlocked. However, they don't contain any + * information meaningful to the BPF scheduler and can be suppressed by + * skipping the callbacks if the task is !QUEUED. + */ + if (SCX_HAS_OP(sch, stopping) && task_current(rq, p)) { + update_curr_scx(rq); + SCX_CALL_OP_TASK(sch, stopping, rq, p, false); + } + + if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p)) + SCX_CALL_OP_TASK(sch, quiescent, rq, p, deq_flags); + + if (deq_flags & SCX_DEQ_SLEEP) + p->scx.flags |= SCX_TASK_DEQD_FOR_SLEEP; + else + p->scx.flags &= ~SCX_TASK_DEQD_FOR_SLEEP; + + p->scx.flags &= ~SCX_TASK_QUEUED; + rq->scx.nr_running--; + sub_nr_running(rq, 1); + + dispatch_dequeue(rq, p); + clear_direct_dispatch(p); + return true; +} + +static void yield_task_scx(struct rq *rq) +{ + struct task_struct *p = rq->donor; + struct scx_sched *sch = scx_task_sched(p); + + if (SCX_HAS_OP(sch, yield)) + SCX_CALL_OP_2TASKS_RET(sch, yield, rq, p, NULL); + else + p->scx.slice = 0; +} + +static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) +{ + struct task_struct *from = rq->donor; + struct scx_sched *sch = scx_task_sched(from); + + if (SCX_HAS_OP(sch, yield) && sch == scx_task_sched(to)) + return SCX_CALL_OP_2TASKS_RET(sch, yield, rq, from, to); + else + return false; +} + +static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_flags) +{ + /* + * Preemption between SCX tasks is implemented by resetting the victim + * task's slice to 0 and triggering reschedule on the target CPU. + * Nothing to do. + */ + if (p->sched_class == &ext_sched_class) + return; + + /* + * Getting preempted by a higher-priority class. Reenqueue IMMED tasks. + * This captures all preemption cases including: + * + * - A SCX task is currently running. + * + * - @rq is waking from idle due to a SCX task waking to it. + * + * - A higher-priority wakes up while SCX dispatch is in progress. + */ + if (rq->scx.nr_immed) + schedule_reenq_local(rq, 0); +} + +static void move_local_task_to_local_dsq(struct scx_sched *sch, + struct task_struct *p, u64 enq_flags, + struct scx_dispatch_q *src_dsq, + struct rq *dst_rq) +{ + struct scx_dispatch_q *dst_dsq = &dst_rq->scx.local_dsq; + + /* @dsq is locked and @p is on @dst_rq */ + lockdep_assert_held(&src_dsq->lock); + lockdep_assert_rq_held(dst_rq); + + WARN_ON_ONCE(p->scx.holding_cpu >= 0); + + if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) + list_add(&p->scx.dsq_list.node, &dst_dsq->list); + else + list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list); + + dsq_inc_nr(dst_dsq, p, enq_flags); + p->scx.dsq = dst_dsq; + + local_dsq_post_enq(sch, dst_dsq, p, enq_flags); +} + +/** + * move_remote_task_to_local_dsq - Move a task from a foreign rq to a local DSQ + * @p: task to move + * @enq_flags: %SCX_ENQ_* + * @src_rq: rq to move the task from, locked on entry, released on return + * @dst_rq: rq to move the task into, locked on return + * + * Move @p which is currently on @src_rq to @dst_rq's local DSQ. + */ +static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, + struct rq *src_rq, struct rq *dst_rq) +{ + lockdep_assert_rq_held(src_rq); + + /* + * Set sticky_cpu before deactivate_task() to properly mark the + * beginning of an SCX-internal migration. + */ + p->scx.sticky_cpu = cpu_of(dst_rq); + deactivate_task(src_rq, p, 0); + set_task_cpu(p, cpu_of(dst_rq)); + + raw_spin_rq_unlock(src_rq); + raw_spin_rq_lock(dst_rq); + + /* + * We want to pass scx-specific enq_flags but activate_task() will + * truncate the upper 32 bit. As we own @rq, we can pass them through + * @rq->scx.extra_enq_flags instead. + */ + WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)); + WARN_ON_ONCE(dst_rq->scx.extra_enq_flags); + dst_rq->scx.extra_enq_flags = enq_flags; + activate_task(dst_rq, p, 0); + dst_rq->scx.extra_enq_flags = 0; +} + +/* + * Similar to kernel/sched/core.c::is_cpu_allowed(). However, there are two + * differences: + * + * - is_cpu_allowed() asks "Can this task run on this CPU?" while + * task_can_run_on_remote_rq() asks "Can the BPF scheduler migrate the task to + * this CPU?". + * + * While migration is disabled, is_cpu_allowed() has to say "yes" as the task + * must be allowed to finish on the CPU that it's currently on regardless of + * the CPU state. However, task_can_run_on_remote_rq() must say "no" as the + * BPF scheduler shouldn't attempt to migrate a task which has migration + * disabled. + * + * - The BPF scheduler is bypassed while the rq is offline and we can always say + * no to the BPF scheduler initiated migrations while offline. + * + * The caller must ensure that @p and @rq are on different CPUs. + */ +static bool task_can_run_on_remote_rq(struct scx_sched *sch, + struct task_struct *p, struct rq *rq, + bool enforce) +{ + s32 cpu = cpu_of(rq); + + WARN_ON_ONCE(task_cpu(p) == cpu); + + /* + * If @p has migration disabled, @p->cpus_ptr is updated to contain only + * the pinned CPU in migrate_disable_switch() while @p is being switched + * out. However, put_prev_task_scx() is called before @p->cpus_ptr is + * updated and thus another CPU may see @p on a DSQ inbetween leading to + * @p passing the below task_allowed_on_cpu() check while migration is + * disabled. + * + * Test the migration disabled state first as the race window is narrow + * and the BPF scheduler failing to check migration disabled state can + * easily be masked if task_allowed_on_cpu() is done first. + */ + if (unlikely(is_migration_disabled(p))) { + if (enforce) + scx_error(sch, "SCX_DSQ_LOCAL[_ON] cannot move migration disabled %s[%d] from CPU %d to %d", + p->comm, p->pid, task_cpu(p), cpu); + return false; + } + + /* + * We don't require the BPF scheduler to avoid dispatching to offline + * CPUs mostly for convenience but also because CPUs can go offline + * between scx_bpf_dsq_insert() calls and here. Trigger error iff the + * picked CPU is outside the allowed mask. + */ + if (!task_allowed_on_cpu(p, cpu)) { + if (enforce) + scx_error(sch, "SCX_DSQ_LOCAL[_ON] target CPU %d not allowed for %s[%d]", + cpu, p->comm, p->pid); + return false; + } + + if (!scx_rq_online(rq)) { + if (enforce) + __scx_add_event(sch, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE, 1); + return false; + } + + return true; +} + +/** + * unlink_dsq_and_lock_src_rq() - Unlink task from its DSQ and lock its task_rq + * @p: target task + * @dsq: locked DSQ @p is currently on + * @src_rq: rq @p is currently on, stable with @dsq locked + * + * Called with @dsq locked but no rq's locked. We want to move @p to a different + * DSQ, including any local DSQ, but are not locking @src_rq. Locking @src_rq is + * required when transferring into a local DSQ. Even when transferring into a + * non-local DSQ, it's better to use the same mechanism to protect against + * dequeues and maintain the invariant that @p->scx.dsq can only change while + * @src_rq is locked, which e.g. scx_dump_task() depends on. + * + * We want to grab @src_rq but that can deadlock if we try while locking @dsq, + * so we want to unlink @p from @dsq, drop its lock and then lock @src_rq. As + * this may race with dequeue, which can't drop the rq lock or fail, do a little + * dancing from our side. + * + * @p->scx.holding_cpu is set to this CPU before @dsq is unlocked. If @p gets + * dequeued after we unlock @dsq but before locking @src_rq, the holding_cpu + * would be cleared to -1. While other cpus may have updated it to different + * values afterwards, as this operation can't be preempted or recurse, the + * holding_cpu can never become this CPU again before we're done. Thus, we can + * tell whether we lost to dequeue by testing whether the holding_cpu still + * points to this CPU. See dispatch_dequeue() for the counterpart. + * + * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is + * still valid. %false if lost to dequeue. + */ +static bool unlink_dsq_and_lock_src_rq(struct task_struct *p, + struct scx_dispatch_q *dsq, + struct rq *src_rq) +{ + s32 cpu = raw_smp_processor_id(); + + lockdep_assert_held(&dsq->lock); + + WARN_ON_ONCE(p->scx.holding_cpu >= 0); + task_unlink_from_dsq(p, dsq); + p->scx.holding_cpu = cpu; + + raw_spin_unlock(&dsq->lock); + raw_spin_rq_lock(src_rq); + + /* task_rq couldn't have changed if we're still the holding cpu */ + return likely(p->scx.holding_cpu == cpu) && + !WARN_ON_ONCE(src_rq != task_rq(p)); +} + +static bool consume_remote_task(struct rq *this_rq, + struct task_struct *p, u64 enq_flags, + struct scx_dispatch_q *dsq, struct rq *src_rq) +{ + raw_spin_rq_unlock(this_rq); + + if (unlink_dsq_and_lock_src_rq(p, dsq, src_rq)) { + move_remote_task_to_local_dsq(p, enq_flags, src_rq, this_rq); + return true; + } else { + raw_spin_rq_unlock(src_rq); + raw_spin_rq_lock(this_rq); + return false; + } +} + +/** + * move_task_between_dsqs() - Move a task from one DSQ to another + * @sch: scx_sched being operated on + * @p: target task + * @enq_flags: %SCX_ENQ_* + * @src_dsq: DSQ @p is currently on, must not be a local DSQ + * @dst_dsq: DSQ @p is being moved to, can be any DSQ + * + * Must be called with @p's task_rq and @src_dsq locked. If @dst_dsq is a local + * DSQ and @p is on a different CPU, @p will be migrated and thus its task_rq + * will change. As @p's task_rq is locked, this function doesn't need to use the + * holding_cpu mechanism. + * + * On return, @src_dsq is unlocked and only @p's new task_rq, which is the + * return value, is locked. + */ +static struct rq *move_task_between_dsqs(struct scx_sched *sch, + struct task_struct *p, u64 enq_flags, + struct scx_dispatch_q *src_dsq, + struct scx_dispatch_q *dst_dsq) +{ + struct rq *src_rq = task_rq(p), *dst_rq; + + BUG_ON(src_dsq->id == SCX_DSQ_LOCAL); + lockdep_assert_held(&src_dsq->lock); + lockdep_assert_rq_held(src_rq); + + if (dst_dsq->id == SCX_DSQ_LOCAL) { + dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); + if (src_rq != dst_rq && + unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { + dst_dsq = find_global_dsq(sch, task_cpu(p)); + dst_rq = src_rq; + enq_flags |= SCX_ENQ_GDSQ_FALLBACK; + } + } else { + /* no need to migrate if destination is a non-local DSQ */ + dst_rq = src_rq; + } + + /* + * Move @p into $dst_dsq. If $dst_dsq is the local DSQ of a different + * CPU, @p will be migrated. + */ + if (dst_dsq->id == SCX_DSQ_LOCAL) { + /* @p is going from a non-local DSQ to a local DSQ */ + if (src_rq == dst_rq) { + task_unlink_from_dsq(p, src_dsq); + move_local_task_to_local_dsq(sch, p, enq_flags, + src_dsq, dst_rq); + raw_spin_unlock(&src_dsq->lock); + } else { + raw_spin_unlock(&src_dsq->lock); + move_remote_task_to_local_dsq(p, enq_flags, + src_rq, dst_rq); + } + } else { + /* + * @p is going from a non-local DSQ to a non-local DSQ. As + * $src_dsq is already locked, do an abbreviated dequeue. + */ + dispatch_dequeue_locked(p, src_dsq); + raw_spin_unlock(&src_dsq->lock); + + dispatch_enqueue(sch, dst_rq, dst_dsq, p, enq_flags); + } + + return dst_rq; +} + +static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq, + struct scx_dispatch_q *dsq, u64 enq_flags) +{ + struct task_struct *p; +retry: + /* + * The caller can't expect to successfully consume a task if the task's + * addition to @dsq isn't guaranteed to be visible somehow. Test + * @dsq->list without locking and skip if it seems empty. + */ + if (list_empty(&dsq->list)) + return false; + + raw_spin_lock(&dsq->lock); + + nldsq_for_each_task(p, dsq) { + struct rq *task_rq = task_rq(p); + + /* + * This loop can lead to multiple lockup scenarios, e.g. the BPF + * scheduler can put an enormous number of affinitized tasks into + * a contended DSQ, or the outer retry loop can repeatedly race + * against scx_bypass() dequeueing tasks from @dsq trying to put + * the system into the bypass mode. This can easily live-lock the + * machine. If aborting, exit from all non-bypass DSQs. + */ + if (unlikely(READ_ONCE(sch->aborting)) && dsq->id != SCX_DSQ_BYPASS) + break; + + if (rq == task_rq) { + task_unlink_from_dsq(p, dsq); + move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq); + raw_spin_unlock(&dsq->lock); + return true; + } + + if (task_can_run_on_remote_rq(sch, p, rq, false)) { + if (likely(consume_remote_task(rq, p, enq_flags, dsq, task_rq))) + return true; + goto retry; + } + } + + raw_spin_unlock(&dsq->lock); + return false; +} + +static bool consume_global_dsq(struct scx_sched *sch, struct rq *rq) +{ + int node = cpu_to_node(cpu_of(rq)); + + return consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq, 0); +} + +/** + * dispatch_to_local_dsq - Dispatch a task to a local dsq + * @sch: scx_sched being operated on + * @rq: current rq which is locked + * @dst_dsq: destination DSQ + * @p: task to dispatch + * @enq_flags: %SCX_ENQ_* + * + * We're holding @rq lock and want to dispatch @p to @dst_dsq which is a local + * DSQ. This function performs all the synchronization dancing needed because + * local DSQs are protected with rq locks. + * + * The caller must have exclusive ownership of @p (e.g. through + * %SCX_OPSS_DISPATCHING). + */ +static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, + struct scx_dispatch_q *dst_dsq, + struct task_struct *p, u64 enq_flags) +{ + struct rq *src_rq = task_rq(p); + struct rq *dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); + struct rq *locked_rq = rq; + + /* + * We're synchronized against dequeue through DISPATCHING. As @p can't + * be dequeued, its task_rq and cpus_allowed are stable too. + * + * If dispatching to @rq that @p is already on, no lock dancing needed. + */ + if (rq == src_rq && rq == dst_rq) { + dispatch_enqueue(sch, rq, dst_dsq, p, + enq_flags | SCX_ENQ_CLEAR_OPSS); + return; + } + + if (src_rq != dst_rq && + unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { + dispatch_enqueue(sch, rq, find_global_dsq(sch, task_cpu(p)), p, + enq_flags | SCX_ENQ_CLEAR_OPSS | SCX_ENQ_GDSQ_FALLBACK); + return; + } + + /* + * @p is on a possibly remote @src_rq which we need to lock to move the + * task. If dequeue is in progress, it'd be locking @src_rq and waiting + * on DISPATCHING, so we can't grab @src_rq lock while holding + * DISPATCHING. + * + * As DISPATCHING guarantees that @p is wholly ours, we can pretend that + * we're moving from a DSQ and use the same mechanism - mark the task + * under transfer with holding_cpu, release DISPATCHING and then follow + * the same protocol. See unlink_dsq_and_lock_src_rq(). + */ + p->scx.holding_cpu = raw_smp_processor_id(); + + /* store_release ensures that dequeue sees the above */ + atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); + + /* switch to @src_rq lock */ + if (locked_rq != src_rq) { + raw_spin_rq_unlock(locked_rq); + locked_rq = src_rq; + raw_spin_rq_lock(src_rq); + } + + /* task_rq couldn't have changed if we're still the holding cpu */ + if (likely(p->scx.holding_cpu == raw_smp_processor_id()) && + !WARN_ON_ONCE(src_rq != task_rq(p))) { + /* + * If @p is staying on the same rq, there's no need to go + * through the full deactivate/activate cycle. Optimize by + * abbreviating move_remote_task_to_local_dsq(). + */ + if (src_rq == dst_rq) { + p->scx.holding_cpu = -1; + dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, + enq_flags); + } else { + move_remote_task_to_local_dsq(p, enq_flags, + src_rq, dst_rq); + /* task has been moved to dst_rq, which is now locked */ + locked_rq = dst_rq; + } + + /* if the destination CPU is idle, wake it up */ + if (sched_class_above(p->sched_class, dst_rq->curr->sched_class)) + resched_curr(dst_rq); + } + + /* switch back to @rq lock */ + if (locked_rq != rq) { + raw_spin_rq_unlock(locked_rq); + raw_spin_rq_lock(rq); + } +} + +/** + * finish_dispatch - Asynchronously finish dispatching a task + * @rq: current rq which is locked + * @p: task to finish dispatching + * @qseq_at_dispatch: qseq when @p started getting dispatched + * @dsq_id: destination DSQ ID + * @enq_flags: %SCX_ENQ_* + * + * Dispatching to local DSQs may need to wait for queueing to complete or + * require rq lock dancing. As we don't wanna do either while inside + * ops.dispatch() to avoid locking order inversion, we split dispatching into + * two parts. scx_bpf_dsq_insert() which is called by ops.dispatch() records the + * task and its qseq. Once ops.dispatch() returns, this function is called to + * finish up. + * + * There is no guarantee that @p is still valid for dispatching or even that it + * was valid in the first place. Make sure that the task is still owned by the + * BPF scheduler and claim the ownership before dispatching. + */ +static void finish_dispatch(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, + unsigned long qseq_at_dispatch, + u64 dsq_id, u64 enq_flags) +{ + struct scx_dispatch_q *dsq; + unsigned long opss; + + touch_core_sched_dispatch(rq, p); +retry: + /* + * No need for _acquire here. @p is accessed only after a successful + * try_cmpxchg to DISPATCHING. + */ + opss = atomic_long_read(&p->scx.ops_state); + + switch (opss & SCX_OPSS_STATE_MASK) { + case SCX_OPSS_DISPATCHING: + case SCX_OPSS_NONE: + /* someone else already got to it */ + return; + case SCX_OPSS_QUEUED: + /* + * If qseq doesn't match, @p has gone through at least one + * dispatch/dequeue and re-enqueue cycle between + * scx_bpf_dsq_insert() and here and we have no claim on it. + */ + if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) + return; + + /* see SCX_EV_INSERT_NOT_OWNED definition */ + if (unlikely(!scx_task_on_sched(sch, p))) { + __scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1); + return; + } + + /* + * While we know @p is accessible, we don't yet have a claim on + * it - the BPF scheduler is allowed to dispatch tasks + * spuriously and there can be a racing dequeue attempt. Let's + * claim @p by atomically transitioning it from QUEUED to + * DISPATCHING. + */ + if (likely(atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, + SCX_OPSS_DISPATCHING))) + break; + goto retry; + case SCX_OPSS_QUEUEING: + /* + * do_enqueue_task() is in the process of transferring the task + * to the BPF scheduler while holding @p's rq lock. As we aren't + * holding any kernel or BPF resource that the enqueue path may + * depend upon, it's safe to wait. + */ + wait_ops_state(p, opss); + goto retry; + } + + BUG_ON(!(p->scx.flags & SCX_TASK_QUEUED)); + + dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, task_cpu(p)); + + if (dsq->id == SCX_DSQ_LOCAL) + dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); + else + dispatch_enqueue(sch, rq, dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); +} + +static void flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) +{ + struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; + u32 u; + + for (u = 0; u < dspc->cursor; u++) { + struct scx_dsp_buf_ent *ent = &dspc->buf[u]; + + finish_dispatch(sch, rq, ent->task, ent->qseq, ent->dsq_id, + ent->enq_flags); + } + + dspc->nr_tasks += dspc->cursor; + dspc->cursor = 0; +} + +static inline void maybe_queue_balance_callback(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + + if (!(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) + return; + + queue_balance_callback(rq, &rq->scx.deferred_bal_cb, + deferred_bal_cb_workfn); + + rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; +} + +/* + * One user of this function is scx_bpf_dispatch() which can be called + * recursively as sub-sched dispatches nest. Always inline to reduce stack usage + * from the call frame. + */ +static __always_inline bool +scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, + struct task_struct *prev, bool nested) +{ + struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; + int nr_loops = SCX_DSP_MAX_LOOPS; + s32 cpu = cpu_of(rq); + bool prev_on_sch = (prev->sched_class == &ext_sched_class) && + scx_task_on_sched(sch, prev); + + if (consume_global_dsq(sch, rq)) + return true; + + if (bypass_dsp_enabled(sch)) { + /* if @sch is bypassing, only the bypass DSQs are active */ + if (scx_bypassing(sch, cpu)) + return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0); + +#ifdef CONFIG_EXT_SUB_SCHED + /* + * If @sch isn't bypassing but its children are, @sch is + * responsible for making forward progress for both its own + * tasks that aren't bypassing and the bypassing descendants' + * tasks. The following implements a simple built-in behavior - + * let each CPU try to run the bypass DSQ every Nth time. + * + * Later, if necessary, we can add an ops flag to suppress the + * auto-consumption and a kfunc to consume the bypass DSQ and, + * so that the BPF scheduler can fully control scheduling of + * bypassed tasks. + */ + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + + if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && + consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0)) { + __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); + return true; + } +#endif /* CONFIG_EXT_SUB_SCHED */ + } + + if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq)) + return false; + + dspc->rq = rq; + + /* + * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, + * the local DSQ might still end up empty after a successful + * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() + * produced some tasks, retry. The BPF scheduler may depend on this + * looping behavior to simplify its implementation. + */ + do { + dspc->nr_tasks = 0; + + if (nested) { + SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), + prev_on_sch ? prev : NULL); + } else { + /* stash @prev so that nested invocations can access it */ + rq->scx.sub_dispatch_prev = prev; + SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), + prev_on_sch ? prev : NULL); + rq->scx.sub_dispatch_prev = NULL; + } + + flush_dispatch_buf(sch, rq); + + if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) { + rq->scx.flags |= SCX_RQ_BAL_KEEP; + return true; + } + if (rq->scx.local_dsq.nr) + return true; + if (consume_global_dsq(sch, rq)) + return true; + + /* + * ops.dispatch() can trap us in this loop by repeatedly + * dispatching ineligible tasks. Break out once in a while to + * allow the watchdog to run. As IRQ can't be enabled in + * balance(), we want to complete this scheduling cycle and then + * start a new one. IOW, we want to call resched_curr() on the + * next, most likely idle, task, not the current one. Use + * __scx_bpf_kick_cpu() for deferred kicking. + */ + if (unlikely(!--nr_loops)) { + scx_kick_cpu(sch, cpu, 0); + break; + } + } while (dspc->nr_tasks); + + /* + * Prevent the CPU from going idle while bypassed descendants have tasks + * queued. Without this fallback, bypassed tasks could stall if the host + * scheduler's ops.dispatch() doesn't yield any tasks. + */ + if (bypass_dsp_enabled(sch)) + return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0); + + return false; +} + +static int balance_one(struct rq *rq, struct task_struct *prev) +{ + struct scx_sched *sch = scx_root; + s32 cpu = cpu_of(rq); + + lockdep_assert_rq_held(rq); + rq->scx.flags |= SCX_RQ_IN_BALANCE; + rq->scx.flags &= ~SCX_RQ_BAL_KEEP; + + if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) && + unlikely(rq->scx.cpu_released)) { + /* + * If the previous sched_class for the current CPU was not SCX, + * notify the BPF scheduler that it again has control of the + * core. This callback complements ->cpu_release(), which is + * emitted in switch_class(). + */ + if (sch->ops.cpu_acquire) + SCX_CALL_OP(sch, cpu_acquire, rq, cpu, NULL); + rq->scx.cpu_released = false; + } + + if (prev->sched_class == &ext_sched_class) { + update_curr_scx(rq); + + /* + * If @prev is runnable & has slice left, it has priority and + * fetching more just increases latency for the fetched tasks. + * Tell pick_task_scx() to keep running @prev. If the BPF + * scheduler wants to handle this explicitly, it should + * implement ->cpu_release(). + * + * See scx_disable_workfn() for the explanation on the bypassing + * test. + */ + if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice && + !scx_bypassing(sch, cpu)) { + rq->scx.flags |= SCX_RQ_BAL_KEEP; + goto has_tasks; + } + } + + /* if there already are tasks to run, nothing to do */ + if (rq->scx.local_dsq.nr) + goto has_tasks; + + if (scx_dispatch_sched(sch, rq, prev, false)) + goto has_tasks; + + /* + * Didn't find another task to run. Keep running @prev unless + * %SCX_OPS_ENQ_LAST is in effect. + */ + if ((prev->scx.flags & SCX_TASK_QUEUED) && + (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu))) { + rq->scx.flags |= SCX_RQ_BAL_KEEP; + __scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1); + goto has_tasks; + } + rq->scx.flags &= ~SCX_RQ_IN_BALANCE; + return false; + +has_tasks: + /* + * @rq may have extra IMMED tasks without reenq scheduled: + * + * - rq_is_open() can't reliably tell when and how slice is going to be + * modified for $curr and allows IMMED tasks to be queued while + * dispatch is in progress. + * + * - A non-IMMED HEAD task can get queued in front of an IMMED task + * between the IMMED queueing and the subsequent scheduling event. + */ + if (unlikely(rq->scx.local_dsq.nr > 1 && rq->scx.nr_immed)) + schedule_reenq_local(rq, 0); + + rq->scx.flags &= ~SCX_RQ_IN_BALANCE; + return true; +} + +static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) +{ + struct scx_sched *sch = scx_task_sched(p); + + if (p->scx.flags & SCX_TASK_QUEUED) { + /* + * Core-sched might decide to execute @p before it is + * dispatched. Call ops_dequeue() to notify the BPF scheduler. + */ + ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC); + dispatch_dequeue(rq, p); + } + + p->se.exec_start = rq_clock_task(rq); + + /* see dequeue_task_scx() on why we skip when !QUEUED */ + if (SCX_HAS_OP(sch, running) && (p->scx.flags & SCX_TASK_QUEUED)) + SCX_CALL_OP_TASK(sch, running, rq, p); + + clr_task_runnable(p, true); + + /* + * @p is getting newly scheduled or got kicked after someone updated its + * slice. Refresh whether tick can be stopped. See scx_can_stop_tick(). + */ + if ((p->scx.slice == SCX_SLICE_INF) != + (bool)(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) { + if (p->scx.slice == SCX_SLICE_INF) + rq->scx.flags |= SCX_RQ_CAN_STOP_TICK; + else + rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK; + + sched_update_tick_dependency(rq); + + /* + * For now, let's refresh the load_avgs just when transitioning + * in and out of nohz. In the future, we might want to add a + * mechanism which calls the following periodically on + * tick-stopped CPUs. + */ + update_other_load_avgs(rq); + } +} + +static enum scx_cpu_preempt_reason +preempt_reason_from_class(const struct sched_class *class) +{ + if (class == &stop_sched_class) + return SCX_CPU_PREEMPT_STOP; + if (class == &dl_sched_class) + return SCX_CPU_PREEMPT_DL; + if (class == &rt_sched_class) + return SCX_CPU_PREEMPT_RT; + return SCX_CPU_PREEMPT_UNKNOWN; +} + +static void switch_class(struct rq *rq, struct task_struct *next) +{ + struct scx_sched *sch = scx_root; + const struct sched_class *next_class = next->sched_class; + + if (!(sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT)) + return; + + /* + * The callback is conceptually meant to convey that the CPU is no + * longer under the control of SCX. Therefore, don't invoke the callback + * if the next class is below SCX (in which case the BPF scheduler has + * actively decided not to schedule any tasks on the CPU). + */ + if (sched_class_above(&ext_sched_class, next_class)) + return; + + /* + * At this point we know that SCX was preempted by a higher priority + * sched_class, so invoke the ->cpu_release() callback if we have not + * done so already. We only send the callback once between SCX being + * preempted, and it regaining control of the CPU. + * + * ->cpu_release() complements ->cpu_acquire(), which is emitted the + * next time that balance_one() is invoked. + */ + if (!rq->scx.cpu_released) { + if (sch->ops.cpu_release) { + struct scx_cpu_release_args args = { + .reason = preempt_reason_from_class(next_class), + .task = next, + }; + + SCX_CALL_OP(sch, cpu_release, rq, cpu_of(rq), &args); + } + rq->scx.cpu_released = true; + } +} + +static void put_prev_task_scx(struct rq *rq, struct task_struct *p, + struct task_struct *next) +{ + struct scx_sched *sch = scx_task_sched(p); + + /* see kick_sync_wait_bal_cb() */ + smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); + + update_curr_scx(rq); + + /* see dequeue_task_scx() on why we skip when !QUEUED */ + if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED)) + SCX_CALL_OP_TASK(sch, stopping, rq, p, true); + + if (p->scx.flags & SCX_TASK_QUEUED) { + set_task_runnable(rq, p); + + /* + * If @p has slice left and is being put, @p is getting + * preempted by a higher priority scheduler class or core-sched + * forcing a different task. Leave it at the head of the local + * DSQ unless it was an IMMED task. IMMED tasks should not + * linger on a busy CPU, reenqueue them to the BPF scheduler. + */ + if (p->scx.slice && !scx_bypassing(sch, cpu_of(rq))) { + if (p->scx.flags & SCX_TASK_IMMED) { + p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; + do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + } else { + dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, SCX_ENQ_HEAD); + } + goto switch_class; + } + + /* + * If @p is runnable but we're about to enter a lower + * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell + * ops.enqueue() that @p is the only one available for this cpu, + * which should trigger an explicit follow-up scheduling event. + */ + if (next && sched_class_above(&ext_sched_class, next->sched_class)) { + WARN_ON_ONCE(!(sch->ops.flags & SCX_OPS_ENQ_LAST)); + do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); + } else { + do_enqueue_task(rq, p, 0, -1); + } + } + +switch_class: + if (next && next->sched_class != &ext_sched_class) + switch_class(rq, next); +} + +static void kick_sync_wait_bal_cb(struct rq *rq) +{ + struct scx_kick_syncs __rcu *ks = __this_cpu_read(scx_kick_syncs); + unsigned long *ksyncs = rcu_dereference_sched(ks)->syncs; + bool waited; + s32 cpu; + + /* + * Drop rq lock and enable IRQs while waiting. IRQs must be enabled + * — a target CPU may be waiting for us to process an IPI (e.g. TLB + * flush) while we wait for its kick_sync to advance. + * + * Also, keep advancing our own kick_sync so that new kick_sync waits + * targeting us, which can start after we drop the lock, cannot form + * cyclic dependencies. + */ +retry: + waited = false; + for_each_cpu(cpu, rq->scx.cpus_to_sync) { + /* + * smp_load_acquire() pairs with smp_store_release() on + * kick_sync updates on the target CPUs. + */ + if (cpu == cpu_of(rq) || + smp_load_acquire(&cpu_rq(cpu)->scx.kick_sync) != ksyncs[cpu]) { + cpumask_clear_cpu(cpu, rq->scx.cpus_to_sync); + continue; + } + + raw_spin_rq_unlock_irq(rq); + while (READ_ONCE(cpu_rq(cpu)->scx.kick_sync) == ksyncs[cpu]) { + smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); + cpu_relax(); + } + raw_spin_rq_lock_irq(rq); + waited = true; + } + + if (waited) + goto retry; +} + +static struct task_struct *first_local_task(struct rq *rq) +{ + return list_first_entry_or_null(&rq->scx.local_dsq.list, + struct task_struct, scx.dsq_list.node); +} + +static struct task_struct * +do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) +{ + struct task_struct *prev = rq->curr; + bool keep_prev; + struct task_struct *p; + + /* see kick_sync_wait_bal_cb() */ + smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); + + rq_modified_begin(rq, &ext_sched_class); + + rq_unpin_lock(rq, rf); + balance_one(rq, prev); + rq_repin_lock(rq, rf); + maybe_queue_balance_callback(rq); + + /* + * Defer to a balance callback which can drop rq lock and enable + * IRQs. Waiting directly in the pick path would deadlock against + * CPUs sending us IPIs (e.g. TLB flushes) while we wait for them. + */ + if (unlikely(rq->scx.kick_sync_pending)) { + rq->scx.kick_sync_pending = false; + queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, + kick_sync_wait_bal_cb); + } + + /* + * If any higher-priority sched class enqueued a runnable task on + * this rq during balance_one(), abort and return RETRY_TASK, so + * that the scheduler loop can restart. + * + * If @force_scx is true, always try to pick a SCHED_EXT task, + * regardless of any higher-priority sched classes activity. + */ + if (!force_scx && rq_modified_above(rq, &ext_sched_class)) + return RETRY_TASK; + + keep_prev = rq->scx.flags & SCX_RQ_BAL_KEEP; + if (unlikely(keep_prev && + prev->sched_class != &ext_sched_class)) { + WARN_ON_ONCE(scx_enable_state() == SCX_ENABLED); + keep_prev = false; + } + + /* + * If balance_one() is telling us to keep running @prev, replenish slice + * if necessary and keep running @prev. Otherwise, pop the first one + * from the local DSQ. + */ + if (keep_prev) { + p = prev; + if (!p->scx.slice) + refill_task_slice_dfl(scx_task_sched(p), p); + } else { + p = first_local_task(rq); + if (!p) + return NULL; + + if (unlikely(!p->scx.slice)) { + struct scx_sched *sch = scx_task_sched(p); + + if (!scx_bypassing(sch, cpu_of(rq)) && + !sch->warned_zero_slice) { + printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in %s()\n", + p->comm, p->pid, __func__); + sch->warned_zero_slice = true; + } + refill_task_slice_dfl(sch, p); + } + } + + return p; +} + +static struct task_struct *pick_task_scx(struct rq *rq, struct rq_flags *rf) +{ + return do_pick_task_scx(rq, rf, false); +} + +/* + * Select the next task to run from the ext scheduling class. + * + * Use do_pick_task_scx() directly with @force_scx enabled, since the + * dl_server must always select a sched_ext task. + */ +static struct task_struct * +ext_server_pick_task(struct sched_dl_entity *dl_se, struct rq_flags *rf) +{ + if (!scx_enabled()) + return NULL; + + return do_pick_task_scx(dl_se->rq, rf, true); +} + +/* + * Initialize the ext server deadline entity. + */ +void ext_server_init(struct rq *rq) +{ + struct sched_dl_entity *dl_se = &rq->ext_server; + + init_dl_entity(dl_se); + + dl_server_init(dl_se, rq, ext_server_pick_task); +} + +#ifdef CONFIG_SCHED_CORE +/** + * scx_prio_less - Task ordering for core-sched + * @a: task A + * @b: task B + * @in_fi: in forced idle state + * + * Core-sched is implemented as an additional scheduling layer on top of the + * usual sched_class'es and needs to find out the expected task ordering. For + * SCX, core-sched calls this function to interrogate the task ordering. + * + * Unless overridden by ops.core_sched_before(), @p->scx.core_sched_at is used + * to implement the default task ordering. The older the timestamp, the higher + * priority the task - the global FIFO ordering matching the default scheduling + * behavior. + * + * When ops.core_sched_before() is enabled, @p->scx.core_sched_at is used to + * implement FIFO ordering within each local DSQ. See pick_task_scx(). + */ +bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, + bool in_fi) +{ + struct scx_sched *sch_a = scx_task_sched(a); + struct scx_sched *sch_b = scx_task_sched(b); + + /* + * The const qualifiers are dropped from task_struct pointers when + * calling ops.core_sched_before(). Accesses are controlled by the + * verifier. + */ + if (sch_a == sch_b && SCX_HAS_OP(sch_a, core_sched_before) && + !scx_bypassing(sch_a, task_cpu(a))) + return SCX_CALL_OP_2TASKS_RET(sch_a, core_sched_before, + task_rq(a), + (struct task_struct *)a, + (struct task_struct *)b); + else + return time_after64(a->scx.core_sched_at, b->scx.core_sched_at); +} +#endif /* CONFIG_SCHED_CORE */ + +static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) +{ + struct scx_sched *sch = scx_task_sched(p); + bool bypassing; + + /* + * sched_exec() calls with %WF_EXEC when @p is about to exec(2) as it + * can be a good migration opportunity with low cache and memory + * footprint. Returning a CPU different than @prev_cpu triggers + * immediate rq migration. However, for SCX, as the current rq + * association doesn't dictate where the task is going to run, this + * doesn't fit well. If necessary, we can later add a dedicated method + * which can decide to preempt self to force it through the regular + * scheduling path. + */ + if (unlikely(wake_flags & WF_EXEC)) + return prev_cpu; + + bypassing = scx_bypassing(sch, task_cpu(p)); + if (likely(SCX_HAS_OP(sch, select_cpu)) && !bypassing) { + s32 cpu; + struct task_struct **ddsp_taskp; + + ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); + WARN_ON_ONCE(*ddsp_taskp); + *ddsp_taskp = p; + + this_rq()->scx.in_select_cpu = true; + cpu = SCX_CALL_OP_TASK_RET(sch, select_cpu, NULL, p, + scx_cpu_arg(prev_cpu), wake_flags); + cpu = scx_cpu_ret(sch, cpu); + this_rq()->scx.in_select_cpu = false; + p->scx.selected_cpu = cpu; + *ddsp_taskp = NULL; + if (scx_cpu_valid(sch, cpu, "from ops.select_cpu()")) + return cpu; + else + return prev_cpu; + } else { + s32 cpu; + + cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, NULL, 0); + if (cpu >= 0) { + refill_task_slice_dfl(sch, p); + p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL; + } else { + cpu = prev_cpu; + } + p->scx.selected_cpu = cpu; + + if (bypassing) + __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); + return cpu; + } +} + +static void task_woken_scx(struct rq *rq, struct task_struct *p) +{ + run_deferred(rq); +} + +static void set_cpus_allowed_scx(struct task_struct *p, + struct affinity_context *ac) +{ + struct scx_sched *sch = scx_task_sched(p); + + set_cpus_allowed_common(p, ac); + + if (task_dead_and_done(p)) + return; + + /* + * The effective cpumask is stored in @p->cpus_ptr which may temporarily + * differ from the configured one in @p->cpus_mask. Always tell the bpf + * scheduler the effective one. + * + * Fine-grained memory write control is enforced by BPF making the const + * designation pointless. Cast it away when calling the operation. + */ + if (SCX_HAS_OP(sch, set_cpumask)) + scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr); +} + +static void handle_hotplug(struct rq *rq, bool online) +{ + struct scx_sched *sch = scx_root; + s32 cpu = cpu_of(rq); + + atomic_long_inc(&scx_hotplug_seq); + + /* + * scx_root updates are protected by cpus_read_lock() and will stay + * stable here. Note that we can't depend on scx_enabled() test as the + * hotplug ops need to be enabled before __scx_enabled is set. + */ + if (unlikely(!sch)) + return; + + if (scx_enabled()) + scx_idle_update_selcpu_topology(&sch->ops); + + if (online && SCX_HAS_OP(sch, cpu_online)) + SCX_CALL_OP(sch, cpu_online, NULL, scx_cpu_arg(cpu)); + else if (!online && SCX_HAS_OP(sch, cpu_offline)) + SCX_CALL_OP(sch, cpu_offline, NULL, scx_cpu_arg(cpu)); + else + scx_exit(sch, SCX_EXIT_UNREG_KERN, + SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, + "cpu %d going %s, exiting scheduler", cpu, + online ? "online" : "offline"); +} + +void scx_rq_activate(struct rq *rq) +{ + handle_hotplug(rq, true); +} + +void scx_rq_deactivate(struct rq *rq) +{ + handle_hotplug(rq, false); +} + +static void rq_online_scx(struct rq *rq) +{ + rq->scx.flags |= SCX_RQ_ONLINE; +} + +static void rq_offline_scx(struct rq *rq) +{ + rq->scx.flags &= ~SCX_RQ_ONLINE; +} + +static bool check_rq_for_timeouts(struct rq *rq) +{ + struct scx_sched *sch; + struct task_struct *p; + struct rq_flags rf; + bool timed_out = false; + + rq_lock_irqsave(rq, &rf); + sch = rcu_dereference_bh(scx_root); + if (unlikely(!sch)) + goto out_unlock; + + list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) { + struct scx_sched *sch = scx_task_sched(p); + unsigned long last_runnable = p->scx.runnable_at; + + if (unlikely(time_after(jiffies, + last_runnable + READ_ONCE(sch->watchdog_timeout)))) { + u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); + + __scx_exit(sch, SCX_EXIT_ERROR_STALL, 0, cpu_of(rq), + "%s[%d] failed to run for %u.%03us", + p->comm, p->pid, dur_ms / 1000, + dur_ms % 1000); + timed_out = true; + break; + } + } +out_unlock: + rq_unlock_irqrestore(rq, &rf); + return timed_out; +} + +static void scx_watchdog_workfn(struct work_struct *work) +{ + unsigned long intv; + int cpu; + + WRITE_ONCE(scx_watchdog_timestamp, jiffies); + + for_each_online_cpu(cpu) { + if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) + break; + + cond_resched(); + } + + intv = READ_ONCE(scx_watchdog_interval); + if (intv < ULONG_MAX) + queue_delayed_work(system_dfl_wq, to_delayed_work(work), intv); +} + +void scx_tick(struct rq *rq) +{ + struct scx_sched *root; + unsigned long last_check; + + if (!scx_enabled()) + return; + + root = rcu_dereference_bh(scx_root); + if (unlikely(!root)) + return; + + last_check = READ_ONCE(scx_watchdog_timestamp); + if (unlikely(time_after(jiffies, + last_check + READ_ONCE(root->watchdog_timeout)))) { + u32 dur_ms = jiffies_to_msecs(jiffies - last_check); + + scx_exit(root, SCX_EXIT_ERROR_STALL, 0, + "watchdog failed to check in for %u.%03us", + dur_ms / 1000, dur_ms % 1000); + } + + update_other_load_avgs(rq); +} + +static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) +{ + struct scx_sched *sch = scx_task_sched(curr); + + update_curr_scx(rq); + + /* + * While disabling, always resched and refresh core-sched timestamp as + * we can't trust the slice management or ops.core_sched_before(). + */ + if (scx_bypassing(sch, cpu_of(rq))) { + curr->scx.slice = 0; + touch_core_sched(rq, curr); + } else if (SCX_HAS_OP(sch, tick)) { + SCX_CALL_OP_TASK(sch, tick, rq, curr); + } + + if (!curr->scx.slice) + resched_curr(rq); +} + +#ifdef CONFIG_EXT_GROUP_SCHED +static struct cgroup *tg_cgrp(struct task_group *tg) +{ + /* + * If CGROUP_SCHED is disabled, @tg is NULL. If @tg is an autogroup, + * @tg->css.cgroup is NULL. In both cases, @tg can be treated as the + * root cgroup. + */ + if (tg && tg->css.cgroup) + return tg->css.cgroup; + else + return &cgrp_dfl_root.cgrp; +} + +#define SCX_INIT_TASK_ARGS_CGROUP(tg) .cgroup = tg_cgrp(tg), + +#else /* CONFIG_EXT_GROUP_SCHED */ + +#define SCX_INIT_TASK_ARGS_CGROUP(tg) + +#endif /* CONFIG_EXT_GROUP_SCHED */ + +static int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork) +{ + int ret; + + p->scx.disallow = false; + + if (SCX_HAS_OP(sch, init_task)) { + struct scx_init_task_args args = { + SCX_INIT_TASK_ARGS_CGROUP(task_group(p)) + .fork = fork, + }; + + ret = SCX_CALL_OP_RET(sch, init_task, NULL, p, &args); + if (unlikely(ret)) { + ret = ops_sanitize_err(sch, "init_task", ret); + return ret; + } + } + + if (p->scx.disallow) { + if (unlikely(scx_parent(sch))) { + scx_error(sch, "non-root ops.init_task() set task->scx.disallow for %s[%d]", + p->comm, p->pid); + } else if (unlikely(fork)) { + scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] during fork", + p->comm, p->pid); + } else { + struct rq *rq; + struct rq_flags rf; + + rq = task_rq_lock(p, &rf); + + /* + * We're in the load path and @p->policy will be applied + * right after. Reverting @p->policy here and rejecting + * %SCHED_EXT transitions from scx_check_setscheduler() + * guarantees that if ops.init_task() sets @p->disallow, + * @p can never be in SCX. + */ + if (p->policy == SCHED_EXT) { + p->policy = SCHED_NORMAL; + atomic_long_inc(&scx_nr_rejected); + } + + task_rq_unlock(rq, p, &rf); + } + } + + return 0; +} + +static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p) +{ + struct rq *rq = task_rq(p); + u32 weight; + + lockdep_assert_rq_held(rq); + + /* + * Verify the task is not in BPF scheduler's custody. If flag + * transitions are consistent, the flag should always be clear + * here. + */ + WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY); + + /* + * Set the weight before calling ops.enable() so that the scheduler + * doesn't see a stale value if they inspect the task struct. + */ + if (task_has_idle_policy(p)) + weight = WEIGHT_IDLEPRIO; + else + weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; + + p->scx.weight = sched_weight_to_cgroup(weight); + + if (SCX_HAS_OP(sch, enable)) + SCX_CALL_OP_TASK(sch, enable, rq, p); + + if (SCX_HAS_OP(sch, set_weight)) + SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); +} + +static void scx_enable_task(struct scx_sched *sch, struct task_struct *p) +{ + __scx_enable_task(sch, p); + scx_set_task_state(p, SCX_TASK_ENABLED); +} + +static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) +{ + struct rq *rq = task_rq(p); + + lockdep_assert_rq_held(rq); + WARN_ON_ONCE(scx_get_task_state(p) != SCX_TASK_ENABLED); + + clear_direct_dispatch(p); + + if (SCX_HAS_OP(sch, disable)) + SCX_CALL_OP_TASK(sch, disable, rq, p); + scx_set_task_state(p, SCX_TASK_READY); + + /* + * Verify the task is not in BPF scheduler's custody. If flag + * transitions are consistent, the flag should always be clear + * here. + */ + WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY); +} + +static void __scx_disable_and_exit_task(struct scx_sched *sch, + struct task_struct *p) +{ + struct scx_exit_task_args args = { + .cancelled = false, + }; + + lockdep_assert_held(&p->pi_lock); + lockdep_assert_rq_held(task_rq(p)); + + switch (scx_get_task_state(p)) { + case SCX_TASK_NONE: + return; + case SCX_TASK_INIT: + args.cancelled = true; + break; + case SCX_TASK_READY: + break; + case SCX_TASK_ENABLED: + scx_disable_task(sch, p); + break; + default: + WARN_ON_ONCE(true); + return; + } + + if (SCX_HAS_OP(sch, exit_task)) + SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args); +} + +/* + * Undo a completed __scx_init_task(sch, p, false) when scx_enable_task() never + * ran. The task state has not been transitioned, so this mirrors the + * SCX_TASK_INIT branch in __scx_disable_and_exit_task(). + */ +static void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p) +{ + struct scx_exit_task_args args = { .cancelled = true }; + + lockdep_assert_held(&p->pi_lock); + lockdep_assert_rq_held(task_rq(p)); + + if (SCX_HAS_OP(sch, exit_task)) + SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args); +} + +static void scx_disable_and_exit_task(struct scx_sched *sch, + struct task_struct *p) +{ + __scx_disable_and_exit_task(sch, p); + + /* + * If set, @p exited between __scx_init_task() and scx_enable_task() in + * scx_sub_enable() and is initialized for both the associated sched and + * its parent. Exit for the child too - scx_enable_task() never ran for + * it, so undo only init_task. The flag is only set on the sub-enable + * path, so it's always clear when @p arrives here in %SCX_TASK_NONE. + */ + if (p->scx.flags & SCX_TASK_SUB_INIT) { + if (!WARN_ON_ONCE(!scx_enabling_sub_sched)) + scx_sub_init_cancel_task(scx_enabling_sub_sched, p); + p->scx.flags &= ~SCX_TASK_SUB_INIT; + } + + scx_set_task_sched(p, NULL); + scx_set_task_state(p, SCX_TASK_NONE); +} + +void init_scx_entity(struct sched_ext_entity *scx) +{ + memset(scx, 0, sizeof(*scx)); + INIT_LIST_HEAD(&scx->dsq_list.node); + RB_CLEAR_NODE(&scx->dsq_priq); + scx->sticky_cpu = -1; + scx->holding_cpu = -1; + INIT_LIST_HEAD(&scx->runnable_node); + scx->runnable_at = jiffies; + scx->ddsp_dsq_id = SCX_DSQ_INVALID; + scx->slice = SCX_SLICE_DFL; +} + +/* See scx_tid_alloc / scx_tid_cursor. */ +static u64 scx_alloc_tid(void) +{ + struct scx_tid_alloc *ta; + + guard(preempt)(); + ta = this_cpu_ptr(&scx_tid_alloc); + + if (unlikely(ta->next >= ta->end)) { + ta->next = atomic64_fetch_add(SCX_TID_CHUNK, &scx_tid_cursor); + ta->end = ta->next + SCX_TID_CHUNK; + } + return ta->next++; +} + +static void scx_tid_hash_insert(struct task_struct *p) +{ + int ret; + + lockdep_assert_held(&scx_tasks_lock); + + ret = rhashtable_lookup_insert_fast(&scx_tid_hash, + &p->scx.tid_hash_node, + scx_tid_hash_params); + WARN_ON_ONCE(ret); +} + +void scx_pre_fork(struct task_struct *p) +{ + /* + * BPF scheduler enable/disable paths want to be able to iterate and + * update all tasks which can become complex when racing forks. As + * enable/disable are very cold paths, let's use a percpu_rwsem to + * exclude forks. + */ + percpu_down_read(&scx_fork_rwsem); +} + +int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs) +{ + s32 ret; + + percpu_rwsem_assert_held(&scx_fork_rwsem); + + p->scx.tid = scx_alloc_tid(); + + if (scx_init_task_enabled) { +#ifdef CONFIG_EXT_SUB_SCHED + struct scx_sched *sch = kargs->cset->dfl_cgrp->scx_sched; +#else + struct scx_sched *sch = scx_root; +#endif + scx_set_task_state(p, SCX_TASK_INIT_BEGIN); + ret = __scx_init_task(sch, p, true); + if (unlikely(ret)) { + scx_set_task_state(p, SCX_TASK_NONE); + return ret; + } + scx_set_task_state(p, SCX_TASK_INIT); + scx_set_task_sched(p, sch); + } + + return 0; +} + +void scx_post_fork(struct task_struct *p) +{ + if (scx_init_task_enabled) { + scx_set_task_state(p, SCX_TASK_READY); + + /* + * Enable the task immediately if it's running on sched_ext. + * Otherwise, it'll be enabled in switching_to_scx() if and + * when it's ever configured to run with a SCHED_EXT policy. + */ + if (p->sched_class == &ext_sched_class) { + struct rq_flags rf; + struct rq *rq; + + rq = task_rq_lock(p, &rf); + scx_enable_task(scx_task_sched(p), p); + task_rq_unlock(rq, p, &rf); + } + } + + scoped_guard(raw_spinlock_irq, &scx_tasks_lock) { + list_add_tail(&p->scx.tasks_node, &scx_tasks); + if (scx_tid_to_task_enabled()) + scx_tid_hash_insert(p); + } + + percpu_up_read(&scx_fork_rwsem); +} + +void scx_cancel_fork(struct task_struct *p) +{ + if (scx_enabled()) { + struct rq *rq; + struct rq_flags rf; + + rq = task_rq_lock(p, &rf); + WARN_ON_ONCE(scx_get_task_state(p) >= SCX_TASK_READY); + scx_disable_and_exit_task(scx_task_sched(p), p); + task_rq_unlock(rq, p, &rf); + } + + percpu_up_read(&scx_fork_rwsem); +} + +/** + * task_dead_and_done - Is a task dead and done running? + * @p: target task + * + * Once sched_ext_dead() removes the dead task from scx_tasks and exits it, the + * task no longer exists from SCX's POV. However, certain sched_class ops may be + * invoked on these dead tasks leading to failures - e.g. sched_setscheduler() + * may try to switch a task which finished sched_ext_dead() back into SCX + * triggering invalid SCX task state transitions and worse. + * + * Once a task has finished the final switch, sched_ext_dead() is the only thing + * that needs to happen on the task. Use this test to short-circuit sched_class + * operations which may be called on dead tasks. + */ +static bool task_dead_and_done(struct task_struct *p) +{ + struct rq *rq = task_rq(p); + + lockdep_assert_rq_held(rq); + + /* + * In do_task_dead(), a dying task sets %TASK_DEAD with preemption + * disabled and __schedule(). If @p has %TASK_DEAD set and off CPU, @p + * won't ever run again. + */ + return unlikely(READ_ONCE(p->__state) == TASK_DEAD) && + !task_on_cpu(rq, p); +} + +void sched_ext_dead(struct task_struct *p) +{ + /* + * By the time control reaches here, @p has %TASK_DEAD set, switched out + * for the last time and then dropped the rq lock - task_dead_and_done() + * should be returning %true nullifying the straggling sched_class ops. + * Remove from scx_tasks and exit @p. + */ + scoped_guard(raw_spinlock_irqsave, &scx_tasks_lock) { + list_del_init(&p->scx.tasks_node); + if (scx_tid_to_task_enabled()) + rhashtable_remove_fast(&scx_tid_hash, + &p->scx.tid_hash_node, + scx_tid_hash_params); + } + + /* + * @p is off scx_tasks and wholly ours. scx_root_enable()'s READY -> + * ENABLED transitions can't race us. Disable ops for @p. + * + * %SCX_TASK_DEAD synchronizes against cgroup task iteration - see + * scx_task_iter_next_locked(). NONE tasks need no marking: cgroup + * iteration is only used from sub-sched paths, which require root + * enabled. Root enable transitions every live task to at least READY. + * + * %INIT_BEGIN means ops.init_task() is running for @p. Don't call + * into ops; transition to %DEAD so the post-init recheck unwinds + * via scx_sub_init_cancel_task(). + */ + if (scx_get_task_state(p) != SCX_TASK_NONE) { + struct rq_flags rf; + struct rq *rq; + + rq = task_rq_lock(p, &rf); + if (scx_get_task_state(p) != SCX_TASK_INIT_BEGIN) + scx_disable_and_exit_task(scx_task_sched(p), p); + scx_set_task_state(p, SCX_TASK_DEAD); + task_rq_unlock(rq, p, &rf); + } +} + +static void reweight_task_scx(struct rq *rq, struct task_struct *p, + const struct load_weight *lw) +{ + struct scx_sched *sch = scx_task_sched(p); + + lockdep_assert_rq_held(task_rq(p)); + + if (task_dead_and_done(p)) + return; + + p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight)); + if (SCX_HAS_OP(sch, set_weight)) + SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); +} + +static void prio_changed_scx(struct rq *rq, struct task_struct *p, u64 oldprio) +{ +} + +static void switching_to_scx(struct rq *rq, struct task_struct *p) +{ + struct scx_sched *sch = scx_task_sched(p); + + if (task_dead_and_done(p)) + return; + + scx_enable_task(sch, p); + + /* + * set_cpus_allowed_scx() is not called while @p is associated with a + * different scheduler class. Keep the BPF scheduler up-to-date. + */ + if (SCX_HAS_OP(sch, set_cpumask)) + scx_call_op_set_cpumask(sch, rq, p, (struct cpumask *)p->cpus_ptr); +} + +static void switched_from_scx(struct rq *rq, struct task_struct *p) +{ + if (task_dead_and_done(p)) + return; + + /* + * %NONE means SCX is no longer tracking @p at the task level (e.g. + * scx_fail_parent() handed @p back to the parent at NONE pending the + * parent's own teardown). There is nothing to disable; calling + * scx_disable_task() would WARN on the non-%ENABLED state and trigger a + * NONE -> READY validation failure. + */ + if (scx_get_task_state(p) == SCX_TASK_NONE) + return; + + scx_disable_task(scx_task_sched(p), p); +} + +static void switched_to_scx(struct rq *rq, struct task_struct *p) {} + +int scx_check_setscheduler(struct task_struct *p, int policy) +{ + lockdep_assert_rq_held(task_rq(p)); + + /* if disallow, reject transitioning into SCX */ + if (scx_enabled() && READ_ONCE(p->scx.disallow) && + p->policy != policy && policy == SCHED_EXT) + return -EACCES; + + return 0; +} + +static void process_ddsp_deferred_locals(struct rq *rq) +{ + struct task_struct *p; + + lockdep_assert_rq_held(rq); + + /* + * Now that @rq can be unlocked, execute the deferred enqueueing of + * tasks directly dispatched to the local DSQs of other CPUs. See + * direct_dispatch(). Keep popping from the head instead of using + * list_for_each_entry_safe() as dispatch_local_dsq() may unlock @rq + * temporarily. + */ + while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals, + struct task_struct, scx.dsq_list.node))) { + struct scx_sched *sch = scx_task_sched(p); + struct scx_dispatch_q *dsq; + u64 dsq_id = p->scx.ddsp_dsq_id; + u64 enq_flags = p->scx.ddsp_enq_flags; + + list_del_init(&p->scx.dsq_list.node); + clear_direct_dispatch(p); + + dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p)); + if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) + dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); + } +} + +/* + * Determine whether @p should be reenqueued from a local DSQ. + * + * @reenq_flags is mutable and accumulates state across the DSQ walk: + * + * - %SCX_REENQ_TSR_NOT_FIRST: Set after the first task is visited. "First" + * tracks position in the DSQ list, not among IMMED tasks. A non-IMMED task at + * the head consumes the first slot. + * + * - %SCX_REENQ_TSR_RQ_OPEN: Set by reenq_local() before the walk if + * rq_is_open() is true. + * + * An IMMED task is kept (returns %false) only if it's the first task in the DSQ + * AND the current task is done — i.e. it will execute immediately. All other + * IMMED tasks are reenqueued. This means if a non-IMMED task sits at the head, + * every IMMED task behind it gets reenqueued. + * + * Reenqueued tasks go through ops.enqueue() with %SCX_ENQ_REENQ | + * %SCX_TASK_REENQ_IMMED. If the BPF scheduler dispatches back to the same local + * DSQ with %SCX_ENQ_IMMED while the CPU is still unavailable, this triggers + * another reenq cycle. Repetitions are bounded by %SCX_REENQ_LOCAL_MAX_REPEAT + * in process_deferred_reenq_locals(). + */ +static bool local_task_should_reenq(struct task_struct *p, u64 *reenq_flags, u32 *reason) +{ + bool first; + + first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST); + *reenq_flags |= SCX_REENQ_TSR_NOT_FIRST; + + *reason = SCX_TASK_REENQ_KFUNC; + + if ((p->scx.flags & SCX_TASK_IMMED) && + (!first || !(*reenq_flags & SCX_REENQ_TSR_RQ_OPEN))) { + __scx_add_event(scx_task_sched(p), SCX_EV_REENQ_IMMED, 1); + *reason = SCX_TASK_REENQ_IMMED; + return true; + } + + return *reenq_flags & SCX_REENQ_ANY; +} + +static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) +{ + LIST_HEAD(tasks); + u32 nr_enqueued = 0; + struct task_struct *p, *n; + + lockdep_assert_rq_held(rq); + + if (WARN_ON_ONCE(reenq_flags & __SCX_REENQ_TSR_MASK)) + reenq_flags &= ~__SCX_REENQ_TSR_MASK; + if (rq_is_open(rq, 0)) + reenq_flags |= SCX_REENQ_TSR_RQ_OPEN; + + /* + * The BPF scheduler may choose to dispatch tasks back to + * @rq->scx.local_dsq. Move all candidate tasks off to a private list + * first to avoid processing the same tasks repeatedly. + */ + list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list, + scx.dsq_list.node) { + struct scx_sched *task_sch = scx_task_sched(p); + u32 reason; + + /* + * If @p is being migrated, @p's current CPU may not agree with + * its allowed CPUs and the migration_cpu_stop is about to + * deactivate and re-activate @p anyway. Skip re-enqueueing. + * + * While racing sched property changes may also dequeue and + * re-enqueue a migrating task while its current CPU and allowed + * CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to + * the current local DSQ for running tasks and thus are not + * visible to the BPF scheduler. + */ + if (p->migration_pending) + continue; + + if (!scx_is_descendant(task_sch, sch)) + continue; + + if (!local_task_should_reenq(p, &reenq_flags, &reason)) + continue; + + dispatch_dequeue(rq, p); + + if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + p->scx.flags |= reason; + + list_add_tail(&p->scx.dsq_list.node, &tasks); + } + + list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { + list_del_init(&p->scx.dsq_list.node); + + do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); + + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + nr_enqueued++; + } + + return nr_enqueued; +} + +static void process_deferred_reenq_locals(struct rq *rq) +{ + u64 seq = ++rq->scx.deferred_reenq_locals_seq; + + lockdep_assert_rq_held(rq); + + while (true) { + struct scx_sched *sch; + u64 reenq_flags; + bool skip = false; + + scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { + struct scx_deferred_reenq_local *drl = + list_first_entry_or_null(&rq->scx.deferred_reenq_locals, + struct scx_deferred_reenq_local, + node); + struct scx_sched_pcpu *sch_pcpu; + + if (!drl) + return; + + sch_pcpu = container_of(drl, struct scx_sched_pcpu, + deferred_reenq_local); + sch = sch_pcpu->sch; + + reenq_flags = drl->flags; + WRITE_ONCE(drl->flags, 0); + list_del_init(&drl->node); + + if (likely(drl->seq != seq)) { + drl->seq = seq; + drl->cnt = 0; + } else { + if (unlikely(++drl->cnt > SCX_REENQ_LOCAL_MAX_REPEAT)) { + scx_error(sch, "SCX_ENQ_REENQ on SCX_DSQ_LOCAL repeated %u times", + drl->cnt); + skip = true; + } + + __scx_add_event(sch, SCX_EV_REENQ_LOCAL_REPEAT, 1); + } + } + + if (!skip) { + /* see schedule_dsq_reenq() */ + smp_mb(); + + reenq_local(sch, rq, reenq_flags); + } + } +} + +static bool user_task_should_reenq(struct task_struct *p, u64 reenq_flags, u32 *reason) +{ + *reason = SCX_TASK_REENQ_KFUNC; + return reenq_flags & SCX_REENQ_ANY; +} + +static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flags) +{ + struct rq *locked_rq = rq; + struct scx_sched *sch = dsq->sched; + struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, dsq, 0); + struct task_struct *p; + s32 nr_enqueued = 0; + + lockdep_assert_rq_held(rq); + + raw_spin_lock(&dsq->lock); + + while (likely(!READ_ONCE(sch->bypass_depth))) { + struct rq *task_rq; + u32 reason; + + p = nldsq_cursor_next_task(&cursor, dsq); + if (!p) + break; + + if (!user_task_should_reenq(p, reenq_flags, &reason)) + continue; + + task_rq = task_rq(p); + + if (locked_rq != task_rq) { + if (locked_rq) + raw_spin_rq_unlock(locked_rq); + if (unlikely(!raw_spin_rq_trylock(task_rq))) { + raw_spin_unlock(&dsq->lock); + raw_spin_rq_lock(task_rq); + raw_spin_lock(&dsq->lock); + } + locked_rq = task_rq; + + /* did we lose @p while switching locks? */ + if (nldsq_cursor_lost_task(&cursor, task_rq, dsq, p)) + continue; + } + + /* @p is on @dsq, its rq and @dsq are locked */ + dispatch_dequeue_locked(p, dsq); + raw_spin_unlock(&dsq->lock); + + if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + p->scx.flags |= reason; + + do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1); + + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + + if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) { + raw_spin_rq_unlock(locked_rq); + locked_rq = NULL; + cpu_relax(); + } + + raw_spin_lock(&dsq->lock); + } + + list_del_init(&cursor.node); + raw_spin_unlock(&dsq->lock); + + if (locked_rq != rq) { + if (locked_rq) + raw_spin_rq_unlock(locked_rq); + raw_spin_rq_lock(rq); + } +} + +static void process_deferred_reenq_users(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + + while (true) { + struct scx_dispatch_q *dsq; + u64 reenq_flags; + + scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { + struct scx_deferred_reenq_user *dru = + list_first_entry_or_null(&rq->scx.deferred_reenq_users, + struct scx_deferred_reenq_user, + node); + struct scx_dsq_pcpu *dsq_pcpu; + + if (!dru) + return; + + dsq_pcpu = container_of(dru, struct scx_dsq_pcpu, + deferred_reenq_user); + dsq = dsq_pcpu->dsq; + reenq_flags = dru->flags; + WRITE_ONCE(dru->flags, 0); + list_del_init(&dru->node); + } + + /* see schedule_dsq_reenq() */ + smp_mb(); + + BUG_ON(dsq->id & SCX_DSQ_FLAG_BUILTIN); + reenq_user(rq, dsq, reenq_flags); + } +} + +static void run_deferred(struct rq *rq) +{ + process_ddsp_deferred_locals(rq); + + if (!list_empty(&rq->scx.deferred_reenq_locals)) + process_deferred_reenq_locals(rq); + + if (!list_empty(&rq->scx.deferred_reenq_users)) + process_deferred_reenq_users(rq); +} + +#ifdef CONFIG_NO_HZ_FULL +bool scx_can_stop_tick(struct rq *rq) +{ + struct task_struct *p = rq->curr; + struct scx_sched *sch = scx_task_sched(p); + + if (p->sched_class != &ext_sched_class) + return true; + + if (scx_bypassing(sch, cpu_of(rq))) + return false; + + /* + * @rq can dispatch from different DSQs, so we can't tell whether it + * needs the tick or not by looking at nr_running. Allow stopping ticks + * iff the BPF scheduler indicated so. See set_next_task_scx(). + */ + return rq->scx.flags & SCX_RQ_CAN_STOP_TICK; +} +#endif + +#ifdef CONFIG_EXT_GROUP_SCHED + +DEFINE_STATIC_PERCPU_RWSEM(scx_cgroup_ops_rwsem); +static bool scx_cgroup_enabled; + +void scx_tg_init(struct task_group *tg) +{ + tg->scx.weight = CGROUP_WEIGHT_DFL; + tg->scx.bw_period_us = default_bw_period_us(); + tg->scx.bw_quota_us = RUNTIME_INF; + tg->scx.idle = false; +} + +int scx_tg_online(struct task_group *tg) +{ + struct scx_sched *sch = scx_root; + int ret = 0; + + WARN_ON_ONCE(tg->scx.flags & (SCX_TG_ONLINE | SCX_TG_INITED)); + + if (scx_cgroup_enabled) { + if (SCX_HAS_OP(sch, cgroup_init)) { + struct scx_cgroup_init_args args = + { .weight = tg->scx.weight, + .bw_period_us = tg->scx.bw_period_us, + .bw_quota_us = tg->scx.bw_quota_us, + .bw_burst_us = tg->scx.bw_burst_us }; + + ret = SCX_CALL_OP_RET(sch, cgroup_init, + NULL, tg->css.cgroup, &args); + if (ret) + ret = ops_sanitize_err(sch, "cgroup_init", ret); + } + if (ret == 0) + tg->scx.flags |= SCX_TG_ONLINE | SCX_TG_INITED; + } else { + tg->scx.flags |= SCX_TG_ONLINE; + } + + return ret; +} + +void scx_tg_offline(struct task_group *tg) +{ + struct scx_sched *sch = scx_root; + + WARN_ON_ONCE(!(tg->scx.flags & SCX_TG_ONLINE)); + + if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_exit) && + (tg->scx.flags & SCX_TG_INITED)) + SCX_CALL_OP(sch, cgroup_exit, NULL, tg->css.cgroup); + tg->scx.flags &= ~(SCX_TG_ONLINE | SCX_TG_INITED); +} + +int scx_cgroup_can_attach(struct cgroup_taskset *tset) +{ + struct scx_sched *sch = scx_root; + struct cgroup_subsys_state *css; + struct task_struct *p; + int ret; + + if (!scx_cgroup_enabled) + return 0; + + cgroup_taskset_for_each(p, css, tset) { + struct cgroup *from = tg_cgrp(task_group(p)); + struct cgroup *to = tg_cgrp(css_tg(css)); + + WARN_ON_ONCE(p->scx.cgrp_moving_from); + + /* + * sched_move_task() omits identity migrations. Let's match the + * behavior so that ops.cgroup_prep_move() and ops.cgroup_move() + * always match one-to-one. + */ + if (from == to) + continue; + + if (SCX_HAS_OP(sch, cgroup_prep_move)) { + ret = SCX_CALL_OP_RET(sch, cgroup_prep_move, NULL, + p, from, css->cgroup); + if (ret) + goto err; + } + + p->scx.cgrp_moving_from = from; + } + + return 0; + +err: + cgroup_taskset_for_each(p, css, tset) { + if (SCX_HAS_OP(sch, cgroup_cancel_move) && + p->scx.cgrp_moving_from) + SCX_CALL_OP(sch, cgroup_cancel_move, NULL, + p, p->scx.cgrp_moving_from, css->cgroup); + p->scx.cgrp_moving_from = NULL; + } + + return ops_sanitize_err(sch, "cgroup_prep_move", ret); +} + +void scx_cgroup_move_task(struct task_struct *p) +{ + struct scx_sched *sch = scx_root; + + if (!scx_cgroup_enabled) + return; + + /* + * scx_cgroup_can_attach() sets cgrp_moving_from only when the task's + * cgroup changes. Migration keys off css rather than cgroup identity, + * so it can hand an unchanged-cgroup task here with cgrp_moving_from + * NULL. Nothing to report to the BPF scheduler then, so skip it and + * keep prep_move and move paired. + */ + if (SCX_HAS_OP(sch, cgroup_move) && p->scx.cgrp_moving_from) + SCX_CALL_OP_TASK(sch, cgroup_move, task_rq(p), + p, p->scx.cgrp_moving_from, + tg_cgrp(task_group(p))); + p->scx.cgrp_moving_from = NULL; +} + +void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) +{ + struct scx_sched *sch = scx_root; + struct cgroup_subsys_state *css; + struct task_struct *p; + + if (!scx_cgroup_enabled) + return; + + cgroup_taskset_for_each(p, css, tset) { + if (SCX_HAS_OP(sch, cgroup_cancel_move) && + p->scx.cgrp_moving_from) + SCX_CALL_OP(sch, cgroup_cancel_move, NULL, + p, p->scx.cgrp_moving_from, css->cgroup); + p->scx.cgrp_moving_from = NULL; + } +} + +void scx_group_set_weight(struct task_group *tg, unsigned long weight) +{ + struct scx_sched *sch; + + percpu_down_read(&scx_cgroup_ops_rwsem); + sch = scx_root; + + if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_weight) && + tg->scx.weight != weight) + SCX_CALL_OP(sch, cgroup_set_weight, NULL, tg_cgrp(tg), weight); + + tg->scx.weight = weight; + + percpu_up_read(&scx_cgroup_ops_rwsem); +} + +void scx_group_set_idle(struct task_group *tg, bool idle) +{ + struct scx_sched *sch; + + percpu_down_read(&scx_cgroup_ops_rwsem); + sch = scx_root; + + if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_idle)) + SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle); + + /* Update the task group's idle state */ + tg->scx.idle = idle; + + percpu_up_read(&scx_cgroup_ops_rwsem); +} + +void scx_group_set_bandwidth(struct task_group *tg, + u64 period_us, u64 quota_us, u64 burst_us) +{ + struct scx_sched *sch; + + percpu_down_read(&scx_cgroup_ops_rwsem); + sch = scx_root; + + if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_bandwidth) && + (tg->scx.bw_period_us != period_us || + tg->scx.bw_quota_us != quota_us || + tg->scx.bw_burst_us != burst_us)) + SCX_CALL_OP(sch, cgroup_set_bandwidth, NULL, + tg_cgrp(tg), period_us, quota_us, burst_us); + + tg->scx.bw_period_us = period_us; + tg->scx.bw_quota_us = quota_us; + tg->scx.bw_burst_us = burst_us; + + percpu_up_read(&scx_cgroup_ops_rwsem); +} +#endif /* CONFIG_EXT_GROUP_SCHED */ + +#if defined(CONFIG_EXT_GROUP_SCHED) || defined(CONFIG_EXT_SUB_SCHED) +static struct cgroup *root_cgroup(void) +{ + return &cgrp_dfl_root.cgrp; +} + +static void scx_cgroup_lock(void) +{ +#ifdef CONFIG_EXT_GROUP_SCHED + percpu_down_write(&scx_cgroup_ops_rwsem); +#endif + cgroup_lock(); +} + +static void scx_cgroup_unlock(void) +{ + cgroup_unlock(); +#ifdef CONFIG_EXT_GROUP_SCHED + percpu_up_write(&scx_cgroup_ops_rwsem); +#endif +} +#else /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */ +static inline struct cgroup *root_cgroup(void) { return NULL; } +static inline void scx_cgroup_lock(void) {} +static inline void scx_cgroup_unlock(void) {} +#endif /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */ + +#ifdef CONFIG_EXT_SUB_SCHED +static struct cgroup *sch_cgroup(struct scx_sched *sch) +{ + return sch->cgrp; +} + +/* for each descendant of @cgrp including self, set ->scx_sched to @sch */ +static void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) +{ + struct cgroup *pos; + struct cgroup_subsys_state *css; + + cgroup_for_each_live_descendant_pre(pos, css, cgrp) + rcu_assign_pointer(pos->scx_sched, sch); +} +#else /* CONFIG_EXT_SUB_SCHED */ +static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; } +static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {} +#endif /* CONFIG_EXT_SUB_SCHED */ + +/* + * Omitted operations: + * + * - migrate_task_rq: Unnecessary as task to cpu mapping is transient. + * + * - task_fork/dead: We need fork/dead notifications for all tasks regardless of + * their current sched_class. Call them directly from sched core instead. + */ +DEFINE_SCHED_CLASS(ext) = { + .enqueue_task = enqueue_task_scx, + .dequeue_task = dequeue_task_scx, + .yield_task = yield_task_scx, + .yield_to_task = yield_to_task_scx, + + .wakeup_preempt = wakeup_preempt_scx, + + .pick_task = pick_task_scx, + + .put_prev_task = put_prev_task_scx, + .set_next_task = set_next_task_scx, + + .select_task_rq = select_task_rq_scx, + .task_woken = task_woken_scx, + .set_cpus_allowed = set_cpus_allowed_scx, + + .rq_online = rq_online_scx, + .rq_offline = rq_offline_scx, + + .task_tick = task_tick_scx, + + .switching_to = switching_to_scx, + .switched_from = switched_from_scx, + .switched_to = switched_to_scx, + .reweight_task = reweight_task_scx, + .prio_changed = prio_changed_scx, + + .update_curr = update_curr_scx, + +#ifdef CONFIG_UCLAMP_TASK + .uclamp_enabled = 1, +#endif +}; + +static s32 init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, + struct scx_sched *sch) +{ + s32 cpu; + + memset(dsq, 0, sizeof(*dsq)); + + raw_spin_lock_init(&dsq->lock); + INIT_LIST_HEAD(&dsq->list); + dsq->id = dsq_id; + dsq->sched = sch; + + dsq->pcpu = alloc_percpu(struct scx_dsq_pcpu); + if (!dsq->pcpu) + return -ENOMEM; + + for_each_possible_cpu(cpu) { + struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); + + pcpu->dsq = dsq; + INIT_LIST_HEAD(&pcpu->deferred_reenq_user.node); + } + + return 0; +} + +static void exit_dsq(struct scx_dispatch_q *dsq) +{ + s32 cpu; + + for_each_possible_cpu(cpu) { + struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); + struct scx_deferred_reenq_user *dru = &pcpu->deferred_reenq_user; + struct rq *rq = cpu_rq(cpu); + + /* + * There must have been a RCU grace period since the last + * insertion and @dsq should be off the deferred list by now. + */ + if (WARN_ON_ONCE(!list_empty(&dru->node))) { + guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); + list_del_init(&dru->node); + } + } + + free_percpu(dsq->pcpu); +} + +static void free_dsq_rcufn(struct rcu_head *rcu) +{ + struct scx_dispatch_q *dsq = container_of(rcu, struct scx_dispatch_q, rcu); + + exit_dsq(dsq); + kfree(dsq); +} + +static void free_dsq_irq_workfn(struct irq_work *irq_work) +{ + struct llist_node *to_free = llist_del_all(&dsqs_to_free); + struct scx_dispatch_q *dsq, *tmp_dsq; + + llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) + call_rcu(&dsq->rcu, free_dsq_rcufn); +} + +static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); + +static void destroy_dsq(struct scx_sched *sch, u64 dsq_id) +{ + struct scx_dispatch_q *dsq; + unsigned long flags; + + rcu_read_lock(); + + dsq = find_user_dsq(sch, dsq_id); + if (!dsq) + goto out_unlock_rcu; + + raw_spin_lock_irqsave(&dsq->lock, flags); + + if (dsq->nr) { + scx_error(sch, "attempting to destroy in-use dsq 0x%016llx (nr=%u)", + dsq->id, dsq->nr); + goto out_unlock_dsq; + } + + if (rhashtable_remove_fast(&sch->dsq_hash, &dsq->hash_node, + dsq_hash_params)) + goto out_unlock_dsq; + + /* + * Mark dead by invalidating ->id to prevent dispatch_enqueue() from + * queueing more tasks. As this function can be called from anywhere, + * freeing is bounced through an irq work to avoid nesting RCU + * operations inside scheduler locks. + */ + dsq->id = SCX_DSQ_INVALID; + if (llist_add(&dsq->free_node, &dsqs_to_free)) + irq_work_queue(&free_dsq_irq_work); + +out_unlock_dsq: + raw_spin_unlock_irqrestore(&dsq->lock, flags); +out_unlock_rcu: + rcu_read_unlock(); +} + +#ifdef CONFIG_EXT_GROUP_SCHED +static void scx_cgroup_exit(struct scx_sched *sch) +{ + struct cgroup_subsys_state *css; + + scx_cgroup_enabled = false; + + /* + * scx_tg_on/offline() are excluded through cgroup_lock(). If we walk + * cgroups and exit all the inited ones, all online cgroups are exited. + */ + css_for_each_descendant_post(css, &root_task_group.css) { + struct task_group *tg = css_tg(css); + + if (!(tg->scx.flags & SCX_TG_INITED)) + continue; + tg->scx.flags &= ~SCX_TG_INITED; + + if (!sch->ops.cgroup_exit) + continue; + + SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); + } +} + +static int scx_cgroup_init(struct scx_sched *sch) +{ + struct cgroup_subsys_state *css; + int ret; + + /* + * scx_tg_on/offline() are excluded through cgroup_lock(). If we walk + * cgroups and init, all online cgroups are initialized. + */ + css_for_each_descendant_pre(css, &root_task_group.css) { + struct task_group *tg = css_tg(css); + struct scx_cgroup_init_args args = { + .weight = tg->scx.weight, + .bw_period_us = tg->scx.bw_period_us, + .bw_quota_us = tg->scx.bw_quota_us, + .bw_burst_us = tg->scx.bw_burst_us, + }; + + if ((tg->scx.flags & + (SCX_TG_ONLINE | SCX_TG_INITED)) != SCX_TG_ONLINE) + continue; + + if (!sch->ops.cgroup_init) { + tg->scx.flags |= SCX_TG_INITED; + continue; + } + + ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, + css->cgroup, &args); + if (ret) { + scx_error(sch, "ops.cgroup_init() failed (%d)", ret); + return ret; + } + tg->scx.flags |= SCX_TG_INITED; + } + + WARN_ON_ONCE(scx_cgroup_enabled); + scx_cgroup_enabled = true; + + return 0; +} + +#else +static void scx_cgroup_exit(struct scx_sched *sch) {} +static int scx_cgroup_init(struct scx_sched *sch) { return 0; } +#endif + + +/******************************************************************************** + * Sysfs interface and ops enable/disable. + */ + +#define SCX_ATTR(_name) \ + static struct kobj_attribute scx_attr_##_name = { \ + .attr = { .name = __stringify(_name), .mode = 0444 }, \ + .show = scx_attr_##_name##_show, \ + } + +static ssize_t scx_attr_state_show(struct kobject *kobj, + struct kobj_attribute *ka, char *buf) +{ + return sysfs_emit(buf, "%s\n", scx_enable_state_str[scx_enable_state()]); +} +SCX_ATTR(state); + +static ssize_t scx_attr_switch_all_show(struct kobject *kobj, + struct kobj_attribute *ka, char *buf) +{ + return sysfs_emit(buf, "%d\n", READ_ONCE(scx_switching_all)); +} +SCX_ATTR(switch_all); + +static ssize_t scx_attr_nr_rejected_show(struct kobject *kobj, + struct kobj_attribute *ka, char *buf) +{ + return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_nr_rejected)); +} +SCX_ATTR(nr_rejected); + +static ssize_t scx_attr_hotplug_seq_show(struct kobject *kobj, + struct kobj_attribute *ka, char *buf) +{ + return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_hotplug_seq)); +} +SCX_ATTR(hotplug_seq); + +static ssize_t scx_attr_enable_seq_show(struct kobject *kobj, + struct kobj_attribute *ka, char *buf) +{ + return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_enable_seq)); +} +SCX_ATTR(enable_seq); + +static struct attribute *scx_global_attrs[] = { + &scx_attr_state.attr, + &scx_attr_switch_all.attr, + &scx_attr_nr_rejected.attr, + &scx_attr_hotplug_seq.attr, + &scx_attr_enable_seq.attr, + NULL, +}; + +static const struct attribute_group scx_global_attr_group = { + .attrs = scx_global_attrs, +}; + +static void free_pnode(struct scx_sched_pnode *pnode); +static void free_exit_info(struct scx_exit_info *ei); + +static s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) +{ + size_t size = struct_size_t(struct scx_cmask, bits, + SCX_CMASK_NR_WORDS(num_possible_cpus())); + int cpu; + + if (!sch->is_cid_type || !sch->arena_pool) + return 0; + + sch->set_cmask_scratch = alloc_percpu(struct scx_cmask *); + if (!sch->set_cmask_scratch) + return -ENOMEM; + + for_each_possible_cpu(cpu) { + struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu); + + *slot = scx_arena_alloc(sch, size); + if (!*slot) + return -ENOMEM; + scx_cmask_init(*slot, 0, num_possible_cpus()); + } + return 0; +} + +static void scx_set_cmask_scratch_free(struct scx_sched *sch) +{ + size_t size = struct_size_t(struct scx_cmask, bits, + SCX_CMASK_NR_WORDS(num_possible_cpus())); + int cpu; + + if (!sch->set_cmask_scratch) + return; + + for_each_possible_cpu(cpu) { + struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu); + + scx_arena_free(sch, *slot, size); + } + free_percpu(sch->set_cmask_scratch); + sch->set_cmask_scratch = NULL; +} + +static void scx_sched_free_rcu_work(struct work_struct *work) +{ + struct rcu_work *rcu_work = to_rcu_work(work); + struct scx_sched *sch = container_of(rcu_work, struct scx_sched, rcu_work); + struct rhashtable_iter rht_iter; + struct scx_dispatch_q *dsq; + int cpu, node; + + irq_work_sync(&sch->disable_irq_work); + kthread_destroy_worker(sch->helper); + timer_shutdown_sync(&sch->bypass_lb_timer); + free_cpumask_var(sch->bypass_lb_donee_cpumask); + free_cpumask_var(sch->bypass_lb_resched_cpumask); + +#ifdef CONFIG_EXT_SUB_SCHED + kfree(sch->cgrp_path); + if (sch_cgroup(sch)) + cgroup_put(sch_cgroup(sch)); + if (sch->sub_kset) + kobject_put(&sch->sub_kset->kobj); +#endif /* CONFIG_EXT_SUB_SCHED */ + + for_each_possible_cpu(cpu) { + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + + /* + * $sch would have entered bypass mode before the RCU grace + * period. As that blocks new deferrals, all + * deferred_reenq_local_node's must be off-list by now. + */ + WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node)); + + exit_dsq(bypass_dsq(sch, cpu)); + } + + free_percpu(sch->pcpu); + + for_each_node_state(node, N_POSSIBLE) + free_pnode(sch->pnode[node]); + kfree(sch->pnode); + + rhashtable_walk_enter(&sch->dsq_hash, &rht_iter); + do { + rhashtable_walk_start(&rht_iter); + + while (!IS_ERR_OR_NULL((dsq = rhashtable_walk_next(&rht_iter)))) + destroy_dsq(sch, dsq->id); + + rhashtable_walk_stop(&rht_iter); + } while (dsq == ERR_PTR(-EAGAIN)); + rhashtable_walk_exit(&rht_iter); + + rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); + free_exit_info(sch->exit_info); + scx_set_cmask_scratch_free(sch); + scx_arena_pool_destroy(sch); + if (sch->arena_map) + bpf_map_put(sch->arena_map); + kfree(sch); +} + +static void scx_kobj_release(struct kobject *kobj) +{ + struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); + + INIT_RCU_WORK(&sch->rcu_work, scx_sched_free_rcu_work); + queue_rcu_work(system_dfl_wq, &sch->rcu_work); +} + +static ssize_t scx_attr_ops_show(struct kobject *kobj, + struct kobj_attribute *ka, char *buf) +{ + struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); + + return sysfs_emit(buf, "%s\n", sch->ops.name); +} +SCX_ATTR(ops); + +#define scx_attr_event_show(buf, at, events, kind) ({ \ + sysfs_emit_at(buf, at, "%s %llu\n", #kind, (events)->kind); \ +}) + +static ssize_t scx_attr_events_show(struct kobject *kobj, + struct kobj_attribute *ka, char *buf) +{ + struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); + struct scx_event_stats events; + int at = 0; + + scx_read_events(sch, &events); + at += scx_attr_event_show(buf, at, &events, SCX_EV_SELECT_CPU_FALLBACK); + at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); + at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_KEEP_LAST); + at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_EXITING); + at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); + at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_IMMED); + at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_LOCAL_REPEAT); + at += scx_attr_event_show(buf, at, &events, SCX_EV_REFILL_SLICE_DFL); + at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DURATION); + at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DISPATCH); + at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_ACTIVATE); + at += scx_attr_event_show(buf, at, &events, SCX_EV_INSERT_NOT_OWNED); + at += scx_attr_event_show(buf, at, &events, SCX_EV_SUB_BYPASS_DISPATCH); + return at; +} +SCX_ATTR(events); + +static struct attribute *scx_sched_attrs[] = { + &scx_attr_ops.attr, + &scx_attr_events.attr, + NULL, +}; +ATTRIBUTE_GROUPS(scx_sched); + +static const struct kobj_type scx_ktype = { + .release = scx_kobj_release, + .sysfs_ops = &kobj_sysfs_ops, + .default_groups = scx_sched_groups, +}; + +static int scx_uevent(const struct kobject *kobj, struct kobj_uevent_env *env) +{ + const struct scx_sched *sch; + + /* + * scx_uevent() can be reached by both scx_sched kobjects (scx_ktype) + * and sub-scheduler kset kobjects (kset_ktype) through the parent + * chain walk. Filter out the latter to avoid invalid casts. + */ + if (kobj->ktype != &scx_ktype) + return 0; + + sch = container_of(kobj, struct scx_sched, kobj); + + return add_uevent_var(env, "SCXOPS=%s", sch->ops.name); +} + +static const struct kset_uevent_ops scx_uevent_ops = { + .uevent = scx_uevent, +}; + +/* + * Used by sched_fork() and __setscheduler_prio() to pick the matching + * sched_class. dl/rt are already handled. + */ +bool task_should_scx(int policy) +{ + /* if disabled, nothing should be on it */ + if (!scx_enabled()) + return false; + + /* scx is taking over all SCHED_OTHER and SCHED_EXT tasks */ + if (READ_ONCE(scx_switching_all)) + return true; + + /* + * scx is tearing down - keep new SCHED_EXT tasks out. + * + * Must come after scx_switching_all test, which serves as a proxy + * for __scx_switched_all. While __scx_switched_all is set, we must + * return true via the branch above: a fork routed to fair would + * stall because next_active_class() skips fair. + * + * This can develop into a deadlock - scx holds scx_enable_mutex across + * kthread_create() in scx_alloc_and_add_sched(); if the new kthread is + * the stalled task, the disable path can never grab the mutex to clear + * scx_switching_all. + */ + if (unlikely(scx_enable_state() == SCX_DISABLING)) + return false; + + return policy == SCHED_EXT; +} + +bool scx_allow_ttwu_queue(const struct task_struct *p) +{ + struct scx_sched *sch; + + if (!scx_enabled()) + return true; + + sch = scx_task_sched(p); + if (unlikely(!sch)) + return true; + + if (sch->ops.flags & SCX_OPS_ALLOW_QUEUED_WAKEUP) + return true; + + if (unlikely(p->sched_class != &ext_sched_class)) + return true; + + return false; +} + +/** + * handle_lockup - sched_ext common lockup handler + * @fmt: format string + * + * Called on system stall or lockup condition and initiates abort of sched_ext + * if enabled, which may resolve the reported lockup. + * + * Returns %true if sched_ext is enabled and abort was initiated, which may + * resolve the lockup. %false if sched_ext is not enabled or abort was already + * initiated by someone else. + */ +static __printf(1, 2) bool handle_lockup(const char *fmt, ...) +{ + struct scx_sched *sch; + va_list args; + bool ret; + + guard(rcu)(); + + sch = rcu_dereference(scx_root); + if (unlikely(!sch)) + return false; + + switch (scx_enable_state()) { + case SCX_ENABLING: + case SCX_ENABLED: + va_start(args, fmt); + ret = scx_verror(sch, fmt, args); + va_end(args); + return ret; + default: + return false; + } +} + +/** + * scx_rcu_cpu_stall - sched_ext RCU CPU stall handler + * + * While there are various reasons why RCU CPU stalls can occur on a system + * that may not be caused by the current BPF scheduler, try kicking out the + * current scheduler in an attempt to recover the system to a good state before + * issuing panics. + * + * Returns %true if sched_ext is enabled and abort was initiated, which may + * resolve the reported RCU stall. %false if sched_ext is not enabled or someone + * else already initiated abort. + */ +bool scx_rcu_cpu_stall(void) +{ + return handle_lockup("RCU CPU stall detected!"); +} + +/** + * scx_softlockup - sched_ext softlockup handler + * @dur_s: number of seconds of CPU stuck due to soft lockup + * + * On some multi-socket setups (e.g. 2x Intel 8480c), the BPF scheduler can + * live-lock the system by making many CPUs target the same DSQ to the point + * where soft-lockup detection triggers. This function is called from + * soft-lockup watchdog when the triggering point is close and tries to unjam + * the system and aborting the BPF scheduler. + */ +void scx_softlockup(u32 dur_s) +{ + if (!handle_lockup("soft lockup - CPU %d stuck for %us", smp_processor_id(), dur_s)) + return; + + printk_deferred(KERN_ERR "sched_ext: Soft lockup - CPU %d stuck for %us, disabling BPF scheduler\n", + smp_processor_id(), dur_s); +} + +/* + * scx_hardlockup() runs from NMI and eventually calls scx_claim_exit(), + * which takes scx_sched_lock. scx_sched_lock isn't NMI-safe and grabbing + * it from NMI context can lead to deadlocks. Defer via irq_work; the + * disable path runs off irq_work anyway. + */ +static atomic_t scx_hardlockup_cpu = ATOMIC_INIT(-1); + +static void scx_hardlockup_irq_workfn(struct irq_work *work) +{ + int cpu = atomic_xchg(&scx_hardlockup_cpu, -1); + + if (cpu >= 0 && handle_lockup("hard lockup - CPU %d", cpu)) + printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n", + cpu); +} + +static DEFINE_IRQ_WORK(scx_hardlockup_irq_work, scx_hardlockup_irq_workfn); + +/** + * scx_hardlockup - sched_ext hardlockup handler + * + * A poorly behaving BPF scheduler can trigger hard lockup by e.g. putting + * numerous affinitized tasks in a single queue and directing all CPUs at it. + * Try kicking out the current scheduler in an attempt to recover the system to + * a good state before taking more drastic actions. + * + * Queues an irq_work; the handle_lockup() call happens in IRQ context (see + * scx_hardlockup_irq_workfn). + * + * Returns %true if sched_ext is enabled and the work was queued, %false + * otherwise. + */ +bool scx_hardlockup(int cpu) +{ + if (!rcu_access_pointer(scx_root)) + return false; + + atomic_cmpxchg(&scx_hardlockup_cpu, -1, cpu); + irq_work_queue(&scx_hardlockup_irq_work); + return true; +} + +static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, + struct cpumask *donee_mask, struct cpumask *resched_mask, + u32 nr_donor_target, u32 nr_donee_target) +{ + struct rq *donor_rq = cpu_rq(donor); + struct scx_dispatch_q *donor_dsq = bypass_dsq(sch, donor); + struct task_struct *p, *n; + struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, donor_dsq, 0); + s32 delta = READ_ONCE(donor_dsq->nr) - nr_donor_target; + u32 nr_balanced = 0, min_delta_us; + + /* + * All we want to guarantee is reasonable forward progress. No reason to + * fine tune. Assuming every task on @donor_dsq runs their full slice, + * consider offloading iff the total queued duration is over the + * threshold. + */ + min_delta_us = READ_ONCE(scx_bypass_lb_intv_us) / SCX_BYPASS_LB_MIN_DELTA_DIV; + if (delta < DIV_ROUND_UP(min_delta_us, READ_ONCE(scx_slice_bypass_us))) + return 0; + + raw_spin_rq_lock_irq(donor_rq); + raw_spin_lock(&donor_dsq->lock); + list_add(&cursor.node, &donor_dsq->list); +resume: + n = container_of(&cursor, struct task_struct, scx.dsq_list); + n = nldsq_next_task(donor_dsq, n, false); + + while ((p = n)) { + struct scx_dispatch_q *donee_dsq; + int donee; + + n = nldsq_next_task(donor_dsq, n, false); + + if (donor_dsq->nr <= nr_donor_target) + break; + + if (cpumask_empty(donee_mask)) + break; + + /* + * If an earlier pass placed @p on @donor_dsq from a different + * CPU and the donee hasn't consumed it yet, @p is still on the + * previous CPU and task_rq(@p) != @donor_rq. @p can't be moved + * without its rq locked. Skip. + */ + if (task_rq(p) != donor_rq) + continue; + + donee = cpumask_any_and_distribute(donee_mask, p->cpus_ptr); + if (donee >= nr_cpu_ids) + continue; + + donee_dsq = bypass_dsq(sch, donee); + + /* + * $p's rq is not locked but $p's DSQ lock protects its + * scheduling properties making this test safe. + */ + if (!task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false)) + continue; + + /* + * Moving $p from one non-local DSQ to another. The source rq + * and DSQ are already locked. Do an abbreviated dequeue and + * then perform enqueue without unlocking $donor_dsq. + * + * We don't want to drop and reacquire the lock on each + * iteration as @donor_dsq can be very long and potentially + * highly contended. Donee DSQs are less likely to be contended. + * The nested locking is safe as only this LB moves tasks + * between bypass DSQs. + */ + dispatch_dequeue_locked(p, donor_dsq); + dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, SCX_ENQ_NESTED); + + /* + * $donee might have been idle and need to be woken up. No need + * to be clever. Kick every CPU that receives tasks. + */ + cpumask_set_cpu(donee, resched_mask); + + if (READ_ONCE(donee_dsq->nr) >= nr_donee_target) + cpumask_clear_cpu(donee, donee_mask); + + nr_balanced++; + if (!(nr_balanced % SCX_BYPASS_LB_BATCH) && n) { + list_move_tail(&cursor.node, &n->scx.dsq_list.node); + raw_spin_unlock(&donor_dsq->lock); + raw_spin_rq_unlock_irq(donor_rq); + cpu_relax(); + raw_spin_rq_lock_irq(donor_rq); + raw_spin_lock(&donor_dsq->lock); + goto resume; + } + } + + list_del_init(&cursor.node); + raw_spin_unlock(&donor_dsq->lock); + raw_spin_rq_unlock_irq(donor_rq); + + return nr_balanced; +} + +static void bypass_lb_node(struct scx_sched *sch, int node) +{ + const struct cpumask *node_mask = cpumask_of_node(node); + struct cpumask *donee_mask = sch->bypass_lb_donee_cpumask; + struct cpumask *resched_mask = sch->bypass_lb_resched_cpumask; + u32 nr_tasks = 0, nr_cpus = 0, nr_balanced = 0; + u32 nr_target, nr_donor_target; + u32 before_min = U32_MAX, before_max = 0; + u32 after_min = U32_MAX, after_max = 0; + int cpu; + + /* count the target tasks and CPUs */ + for_each_cpu_and(cpu, cpu_online_mask, node_mask) { + u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr); + + nr_tasks += nr; + nr_cpus++; + + before_min = min(nr, before_min); + before_max = max(nr, before_max); + } + + if (!nr_cpus) + return; + + /* + * We don't want CPUs to have more than $nr_donor_target tasks and + * balancing to fill donee CPUs upto $nr_target. Once targets are + * calculated, find the donee CPUs. + */ + nr_target = DIV_ROUND_UP(nr_tasks, nr_cpus); + nr_donor_target = DIV_ROUND_UP(nr_target * SCX_BYPASS_LB_DONOR_PCT, 100); + + cpumask_clear(donee_mask); + for_each_cpu_and(cpu, cpu_online_mask, node_mask) { + if (READ_ONCE(bypass_dsq(sch, cpu)->nr) < nr_target) + cpumask_set_cpu(cpu, donee_mask); + } + + /* iterate !donee CPUs and see if they should be offloaded */ + cpumask_clear(resched_mask); + for_each_cpu_and(cpu, cpu_online_mask, node_mask) { + if (cpumask_empty(donee_mask)) + break; + if (cpumask_test_cpu(cpu, donee_mask)) + continue; + if (READ_ONCE(bypass_dsq(sch, cpu)->nr) <= nr_donor_target) + continue; + + nr_balanced += bypass_lb_cpu(sch, cpu, donee_mask, resched_mask, + nr_donor_target, nr_target); + } + + for_each_cpu(cpu, resched_mask) + resched_cpu(cpu); + + for_each_cpu_and(cpu, cpu_online_mask, node_mask) { + u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr); + + after_min = min(nr, after_min); + after_max = max(nr, after_max); + + } + + trace_sched_ext_bypass_lb(node, nr_cpus, nr_tasks, nr_balanced, + before_min, before_max, after_min, after_max); +} + +/* + * In bypass mode, all tasks are put on the per-CPU bypass DSQs. If the machine + * is over-saturated and the BPF scheduler skewed tasks into few CPUs, some + * bypass DSQs can be overloaded. If there are enough tasks to saturate other + * lightly loaded CPUs, such imbalance can lead to very high execution latency + * on the overloaded CPUs and thus to hung tasks and RCU stalls. To avoid such + * outcomes, a simple load balancing mechanism is implemented by the following + * timer which runs periodically while bypass mode is in effect. + */ +static void scx_bypass_lb_timerfn(struct timer_list *timer) +{ + struct scx_sched *sch = container_of(timer, struct scx_sched, bypass_lb_timer); + int node; + u32 intv_us; + + if (!bypass_dsp_enabled(sch)) + return; + + for_each_node_with_cpus(node) + bypass_lb_node(sch, node); + + intv_us = READ_ONCE(scx_bypass_lb_intv_us); + if (intv_us) + mod_timer(timer, jiffies + usecs_to_jiffies(intv_us)); +} + +static bool inc_bypass_depth(struct scx_sched *sch) +{ + lockdep_assert_held(&scx_bypass_lock); + + WARN_ON_ONCE(sch->bypass_depth < 0); + WRITE_ONCE(sch->bypass_depth, sch->bypass_depth + 1); + if (sch->bypass_depth != 1) + return false; + + WRITE_ONCE(sch->slice_dfl, READ_ONCE(scx_slice_bypass_us) * NSEC_PER_USEC); + sch->bypass_timestamp = ktime_get_ns(); + scx_add_event(sch, SCX_EV_BYPASS_ACTIVATE, 1); + return true; +} + +static bool dec_bypass_depth(struct scx_sched *sch) +{ + lockdep_assert_held(&scx_bypass_lock); + + WARN_ON_ONCE(sch->bypass_depth < 1); + WRITE_ONCE(sch->bypass_depth, sch->bypass_depth - 1); + if (sch->bypass_depth != 0) + return false; + + WRITE_ONCE(sch->slice_dfl, SCX_SLICE_DFL); + scx_add_event(sch, SCX_EV_BYPASS_DURATION, + ktime_get_ns() - sch->bypass_timestamp); + return true; +} + +static void enable_bypass_dsp(struct scx_sched *sch) +{ + struct scx_sched *host = scx_parent(sch) ?: sch; + u32 intv_us = READ_ONCE(scx_bypass_lb_intv_us); + s32 ret; + + /* + * @sch->bypass_depth transitioning from 0 to 1 triggers enabling. + * Shouldn't stagger. + */ + if (WARN_ON_ONCE(test_and_set_bit(0, &sch->bypass_dsp_claim))) + return; + + /* + * When a sub-sched bypasses, its tasks are queued on the bypass DSQs of + * the nearest non-bypassing ancestor or root. As enable_bypass_dsp() is + * called iff @sch is not already bypassed due to an ancestor bypassing, + * we can assume that the parent is not bypassing and thus will be the + * host of the bypass DSQs. + * + * While the situation may change in the future, the following + * guarantees that the nearest non-bypassing ancestor or root has bypass + * dispatch enabled while a descendant is bypassing, which is all that's + * required. + * + * bypass_dsp_enabled() test is used to determine whether to enter the + * bypass dispatch handling path from both bypassing and hosting scheds. + * Bump enable depth on both @sch and bypass dispatch host. + */ + ret = atomic_inc_return(&sch->bypass_dsp_enable_depth); + WARN_ON_ONCE(ret <= 0); + + if (host != sch) { + ret = atomic_inc_return(&host->bypass_dsp_enable_depth); + WARN_ON_ONCE(ret <= 0); + } + + /* + * The LB timer will stop running if bypass dispatch is disabled. Start + * after enabling bypass dispatch. + */ + if (intv_us && !timer_pending(&host->bypass_lb_timer)) + mod_timer(&host->bypass_lb_timer, + jiffies + usecs_to_jiffies(intv_us)); +} + +/* may be called without holding scx_bypass_lock */ +static void disable_bypass_dsp(struct scx_sched *sch) +{ + s32 ret; + + if (!test_and_clear_bit(0, &sch->bypass_dsp_claim)) + return; + + ret = atomic_dec_return(&sch->bypass_dsp_enable_depth); + WARN_ON_ONCE(ret < 0); + + if (scx_parent(sch)) { + ret = atomic_dec_return(&scx_parent(sch)->bypass_dsp_enable_depth); + WARN_ON_ONCE(ret < 0); + } +} + +/** + * scx_bypass - [Un]bypass scx_ops and guarantee forward progress + * @sch: sched to bypass + * @bypass: true for bypass, false for unbypass + * + * Bypassing guarantees that all runnable tasks make forward progress without + * trusting the BPF scheduler. We can't grab any mutexes or rwsems as they might + * be held by tasks that the BPF scheduler is forgetting to run, which + * unfortunately also excludes toggling the static branches. + * + * Let's work around by overriding a couple ops and modifying behaviors based on + * the DISABLING state and then cycling the queued tasks through dequeue/enqueue + * to force global FIFO scheduling. + * + * - ops.select_cpu() is ignored and the default select_cpu() is used. + * + * - ops.enqueue() is ignored and tasks are queued in simple global FIFO order. + * %SCX_OPS_ENQ_LAST is also ignored. + * + * - ops.dispatch() is ignored. + * + * - balance_one() does not set %SCX_RQ_BAL_KEEP on non-zero slice as slice + * can't be trusted. Whenever a tick triggers, the running task is rotated to + * the tail of the queue with core_sched_at touched. + * + * - pick_next_task() suppresses zero slice warning. + * + * - scx_kick_cpu() is disabled to avoid irq_work malfunction during PM + * operations. + * + * - scx_prio_less() reverts to the default core_sched_at order. + */ +static void scx_bypass(struct scx_sched *sch, bool bypass) +{ + struct scx_sched *pos; + unsigned long flags; + int cpu; + + raw_spin_lock_irqsave(&scx_bypass_lock, flags); + + if (bypass) { + if (!inc_bypass_depth(sch)) + goto unlock; + + enable_bypass_dsp(sch); + } else { + if (!dec_bypass_depth(sch)) + goto unlock; + } + + /* + * Bypass state is propagated to all descendants - an scx_sched bypasses + * if itself or any of its ancestors are in bypass mode. + */ + raw_spin_lock(&scx_sched_lock); + scx_for_each_descendant_pre(pos, sch) { + if (pos == sch) + continue; + if (bypass) + inc_bypass_depth(pos); + else + dec_bypass_depth(pos); + } + raw_spin_unlock(&scx_sched_lock); + + /* + * No task property is changing. We just need to make sure all currently + * queued tasks are re-queued according to the new scx_bypassing() + * state. As an optimization, walk each rq's runnable_list instead of + * the scx_tasks list. + * + * This function can't trust the scheduler and thus can't use + * cpus_read_lock(). Walk all possible CPUs instead of online. + */ + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + struct task_struct *p, *n; + + raw_spin_rq_lock(rq); + raw_spin_lock(&scx_sched_lock); + + scx_for_each_descendant_pre(pos, sch) { + struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu); + + if (pos->bypass_depth) + pcpu->flags |= SCX_SCHED_PCPU_BYPASSING; + else + pcpu->flags &= ~SCX_SCHED_PCPU_BYPASSING; + } + + raw_spin_unlock(&scx_sched_lock); + + /* + * We need to guarantee that no tasks are on the BPF scheduler + * while bypassing. Either we see enabled or the enable path + * sees scx_bypassing() before moving tasks to SCX. + */ + if (!scx_enabled()) { + raw_spin_rq_unlock(rq); + continue; + } + + /* + * The use of list_for_each_entry_safe_reverse() is required + * because each task is going to be removed from and added back + * to the runnable_list during iteration. Because they're added + * to the tail of the list, safe reverse iteration can still + * visit all nodes. + */ + list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, + scx.runnable_node) { + if (!scx_is_descendant(scx_task_sched(p), sch)) + continue; + + /* cycling deq/enq is enough, see the function comment */ + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + /* nothing */ ; + } + } + + /* resched to restore ticks and idle state */ + if (cpu_online(cpu) || cpu == smp_processor_id()) + resched_curr(rq); + + raw_spin_rq_unlock(rq); + } + + /* disarming must come after moving all tasks out of the bypass DSQs */ + if (!bypass) + disable_bypass_dsp(sch); +unlock: + raw_spin_unlock_irqrestore(&scx_bypass_lock, flags); +} + +static void free_exit_info(struct scx_exit_info *ei) +{ + kvfree(ei->dump); + kfree(ei->msg); + kfree(ei->bt); + kfree(ei); +} + +static struct scx_exit_info *alloc_exit_info(size_t exit_dump_len) +{ + struct scx_exit_info *ei; + + ei = kzalloc_obj(*ei); + if (!ei) + return NULL; + + ei->exit_cpu = -1; + ei->bt = kzalloc_objs(ei->bt[0], SCX_EXIT_BT_LEN); + ei->msg = kzalloc(SCX_EXIT_MSG_LEN, GFP_KERNEL); + ei->dump = kvzalloc(exit_dump_len, GFP_KERNEL); + + if (!ei->bt || !ei->msg || !ei->dump) { + free_exit_info(ei); + return NULL; + } + + return ei; +} + +static const char *scx_exit_reason(enum scx_exit_kind kind) +{ + switch (kind) { + case SCX_EXIT_UNREG: + return "unregistered from user space"; + case SCX_EXIT_UNREG_BPF: + return "unregistered from BPF"; + case SCX_EXIT_UNREG_KERN: + return "unregistered from the main kernel"; + case SCX_EXIT_SYSRQ: + return "disabled by sysrq-S"; + case SCX_EXIT_PARENT: + return "parent exiting"; + case SCX_EXIT_ERROR: + return "runtime error"; + case SCX_EXIT_ERROR_BPF: + return "scx_bpf_error"; + case SCX_EXIT_ERROR_STALL: + return "runnable task stall"; + default: + return ""; + } +} + +static void free_kick_syncs(void) +{ + int cpu; + + for_each_possible_cpu(cpu) { + struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); + struct scx_kick_syncs *to_free; + + to_free = rcu_replace_pointer(*ksyncs, NULL, true); + if (to_free) + kvfree_rcu(to_free, rcu); + } +} + +static void refresh_watchdog(void) +{ + struct scx_sched *sch; + unsigned long intv = ULONG_MAX; + + /* take the shortest timeout and use its half for watchdog interval */ + rcu_read_lock(); + list_for_each_entry_rcu(sch, &scx_sched_all, all) + intv = max(min(intv, sch->watchdog_timeout / 2), 1); + rcu_read_unlock(); + + WRITE_ONCE(scx_watchdog_timestamp, jiffies); + WRITE_ONCE(scx_watchdog_interval, intv); + + if (intv < ULONG_MAX) + mod_delayed_work(system_dfl_wq, &scx_watchdog_work, intv); + else + cancel_delayed_work_sync(&scx_watchdog_work); +} + +static s32 scx_link_sched(struct scx_sched *sch) +{ + const char *err_msg = ""; + s32 ret = 0; + + scoped_guard(raw_spinlock_irq, &scx_sched_lock) { +#ifdef CONFIG_EXT_SUB_SCHED + struct scx_sched *parent = scx_parent(sch); + + if (parent) { + /* + * scx_claim_exit() propagates exit_kind transition to + * its sub-scheds while holding scx_sched_lock - either + * we can see the parent's non-NONE exit_kind or the + * parent can shoot us down. + */ + if (atomic_read(&parent->exit_kind) != SCX_EXIT_NONE) { + err_msg = "parent disabled"; + ret = -ENOENT; + break; + } + + ret = rhashtable_lookup_insert_fast(&scx_sched_hash, + &sch->hash_node, scx_sched_hash_params); + if (ret) { + err_msg = "failed to insert into scx_sched_hash"; + break; + } + + list_add_tail(&sch->sibling, &parent->children); + } +#endif /* CONFIG_EXT_SUB_SCHED */ + + list_add_tail_rcu(&sch->all, &scx_sched_all); + } + + /* + * scx_error() takes scx_sched_lock via scx_claim_exit(), so it must run after + * the guard above is released. + */ + if (ret) { + scx_error(sch, "%s (%d)", err_msg, ret); + return ret; + } + + refresh_watchdog(); + return 0; +} + +static void scx_unlink_sched(struct scx_sched *sch) +{ + scoped_guard(raw_spinlock_irq, &scx_sched_lock) { +#ifdef CONFIG_EXT_SUB_SCHED + if (scx_parent(sch)) { + rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node, + scx_sched_hash_params); + list_del_init(&sch->sibling); + } +#endif /* CONFIG_EXT_SUB_SCHED */ + list_del_rcu(&sch->all); + } + + refresh_watchdog(); +} + +/* + * Called to disable future dumps and wait for in-progress one while disabling + * @sch. Once @sch becomes empty during disable, there's no point in dumping it. + * This prevents calling dump ops on a dead sch. + */ +static void scx_disable_dump(struct scx_sched *sch) +{ + guard(raw_spinlock_irqsave)(&scx_dump_lock); + sch->dump_disabled = true; +} + +static void scx_log_sched_disable(struct scx_sched *sch) +{ + struct scx_exit_info *ei = sch->exit_info; + const char *type = scx_parent(sch) ? "sub-scheduler" : "scheduler"; + + if (ei->kind >= SCX_EXIT_ERROR) { + pr_err("sched_ext: BPF %s \"%s\" disabled (%s)\n", type, + sch->ops.name, ei->reason); + + if (ei->msg[0] != '\0') + pr_err("sched_ext: %s: %s\n", sch->ops.name, ei->msg); +#ifdef CONFIG_STACKTRACE + stack_trace_print(ei->bt, ei->bt_len, 2); +#endif + } else { + pr_info("sched_ext: BPF %s \"%s\" disabled (%s)\n", type, + sch->ops.name, ei->reason); + } +} + +#ifdef CONFIG_EXT_SUB_SCHED +static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq); + +static void drain_descendants(struct scx_sched *sch) +{ + /* + * Child scheds that finished the critical part of disabling will take + * themselves off @sch->children. Wait for it to drain. As propagation + * is recursive, empty @sch->children means that all proper descendant + * scheds reached unlinking stage. + */ + wait_event(scx_unlink_waitq, list_empty(&sch->children)); +} + +static void scx_fail_parent(struct scx_sched *sch, + struct task_struct *failed, s32 fail_code) +{ + struct scx_sched *parent = scx_parent(sch); + struct scx_task_iter sti; + struct task_struct *p; + + scx_error(parent, "ops.init_task() failed (%d) for %s[%d] while disabling a sub-scheduler", + fail_code, failed->comm, failed->pid); + + /* + * Once $parent is bypassed, it's safe to put SCX_TASK_NONE tasks into + * it. This may cause downstream failures on the BPF side but $parent is + * dying anyway. + */ + scx_bypass(parent, true); + + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + if (scx_task_on_sched(parent, p)) + continue; + + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_disable_and_exit_task(sch, p); + scx_set_task_sched(p, parent); + } + } + scx_task_iter_stop(&sti); +} + +static void scx_sub_disable(struct scx_sched *sch) +{ + struct scx_sched *parent = scx_parent(sch); + struct scx_task_iter sti; + struct task_struct *p; + int ret; + + /* + * Guarantee forward progress and wait for descendants to be disabled. + * To limit disruptions, $parent is not bypassed. Tasks are fully + * prepped and then inserted back into $parent. + */ + scx_bypass(sch, true); + drain_descendants(sch); + + /* + * Here, every runnable task is guaranteed to make forward progress and + * we can safely use blocking synchronization constructs. Actually + * disable ops. + */ + mutex_lock(&scx_enable_mutex); + percpu_down_write(&scx_fork_rwsem); + scx_cgroup_lock(); + + set_cgroup_sched(sch_cgroup(sch), parent); + + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + struct rq *rq; + struct rq_flags rf; + + /* filter out duplicate visits */ + if (scx_task_on_sched(parent, p)) + continue; + + /* + * By the time control reaches here, all descendant schedulers + * should already have been disabled. + */ + WARN_ON_ONCE(!scx_task_on_sched(sch, p)); + + /* + * @p is pinned by the iter: css_task_iter_next() takes a + * reference and holds it until the next iter_next() call, so + * @p->usage is guaranteed > 0. + */ + get_task_struct(p); + + scx_task_iter_unlock(&sti); + + /* + * $p is READY or ENABLED on @sch. Initialize for $parent, + * disable and exit from @sch, and then switch over to $parent. + * + * If a task fails to initialize for $parent, the only available + * action is disabling $parent too. While this allows disabling + * of a child sched to cause the parent scheduler to fail, the + * failure can only originate from ops.init_task() of the + * parent. A child can't directly affect the parent through its + * own failures. + */ + ret = __scx_init_task(parent, p, false); + if (ret) { + scx_fail_parent(sch, p, ret); + put_task_struct(p); + break; + } + + rq = task_rq_lock(p, &rf); + + if (scx_get_task_state(p) == SCX_TASK_DEAD) { + /* + * sched_ext_dead() raced us between __scx_init_task() + * and this rq lock and ran exit_task() on @sch (the + * sched @p was on at that point), not on $parent. + * $parent's just-completed init is owed an exit_task() + * and we issue it here. + */ + scx_sub_init_cancel_task(parent, p); + task_rq_unlock(rq, p, &rf); + put_task_struct(p); + continue; + } + + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + /* + * $p is initialized for $parent and still attached to + * @sch. Disable and exit for @sch, switch over to + * $parent, override the state to READY to account for + * $p having already been initialized, and then enable. + */ + scx_disable_and_exit_task(sch, p); + scx_set_task_state(p, SCX_TASK_INIT_BEGIN); + scx_set_task_state(p, SCX_TASK_INIT); + scx_set_task_sched(p, parent); + scx_set_task_state(p, SCX_TASK_READY); + scx_enable_task(parent, p); + } + + task_rq_unlock(rq, p, &rf); + put_task_struct(p); + } + scx_task_iter_stop(&sti); + + scx_disable_dump(sch); + + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); + + /* + * All tasks are moved off of @sch but there may still be on-going + * operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use + * the expedited version as ancestors may be waiting in bypass mode. + * Also, tell the parent that there is no need to keep running bypass + * DSQs for us. + */ + synchronize_rcu_expedited(); + disable_bypass_dsp(sch); + + scx_unlink_sched(sch); + + mutex_unlock(&scx_enable_mutex); + + /* + * @sch is now unlinked from the parent's children list. Notify and call + * ops.sub_detach/exit(). Note that ops.sub_detach/exit() must be called + * after unlinking and releasing all locks. See scx_claim_exit(). + */ + wake_up_all(&scx_unlink_waitq); + + if (parent->ops.sub_detach && sch->sub_attached) { + struct scx_sub_detach_args sub_detach_args = { + .ops = &sch->ops, + .cgroup_path = sch->cgrp_path, + }; + SCX_CALL_OP(parent, sub_detach, NULL, + &sub_detach_args); + } + + scx_log_sched_disable(sch); + + if (sch->ops.exit) + SCX_CALL_OP(sch, exit, NULL, sch->exit_info); + if (sch->sub_kset) + kobject_del(&sch->sub_kset->kobj); + kobject_del(&sch->kobj); +} +#else /* CONFIG_EXT_SUB_SCHED */ +static inline void drain_descendants(struct scx_sched *sch) { } +static inline void scx_sub_disable(struct scx_sched *sch) { } +#endif /* CONFIG_EXT_SUB_SCHED */ + +static void scx_root_disable(struct scx_sched *sch) +{ + struct scx_task_iter sti; + struct task_struct *p; + bool was_switched_all; + int cpu; + + /* guarantee forward progress and wait for descendants to be disabled */ + scx_bypass(sch, true); + drain_descendants(sch); + + switch (scx_set_enable_state(SCX_DISABLING)) { + case SCX_DISABLING: + WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); + break; + case SCX_DISABLED: + pr_warn("sched_ext: ops error detected without ops (%s)\n", + sch->exit_info->msg); + WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING); + goto done; + default: + break; + } + + /* + * Here, every runnable task is guaranteed to make forward progress and + * we can safely use blocking synchronization constructs. Actually + * disable ops. + */ + mutex_lock(&scx_enable_mutex); + + was_switched_all = scx_switched_all(); + + static_branch_disable(&__scx_switched_all); + WRITE_ONCE(scx_switching_all, false); + + /* + * Shut down cgroup support before tasks so that the cgroup attach path + * doesn't race against scx_disable_and_exit_task(). + */ + scx_cgroup_lock(); + scx_cgroup_exit(sch); + scx_cgroup_unlock(); + + /* + * The BPF scheduler is going away. All tasks including %TASK_DEAD ones + * must be switched out and exited synchronously. + */ + percpu_down_write(&scx_fork_rwsem); + + scx_init_task_enabled = false; + + scx_task_iter_start(&sti, NULL); + while ((p = scx_task_iter_next_locked(&sti))) { + unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; + const struct sched_class *old_class = p->sched_class; + const struct sched_class *new_class = scx_setscheduler_class(p); + + update_rq_clock(task_rq(p)); + + if (old_class != new_class) + queue_flags |= DEQUEUE_CLASS; + + scoped_guard (sched_change, p, queue_flags) { + p->sched_class = new_class; + } + + scx_disable_and_exit_task(scx_task_sched(p), p); + } + scx_task_iter_stop(&sti); + + scx_disable_dump(sch); + + scx_cgroup_lock(); + set_cgroup_sched(sch_cgroup(sch), NULL); + scx_cgroup_unlock(); + + percpu_up_write(&scx_fork_rwsem); + + /* + * Invalidate all the rq clocks to prevent getting outdated + * rq clocks from a previous scx scheduler. + * + * Also re-balance the dl_server bandwidth reservations: detach + * ext_server (no more sched_ext tasks) and reinstate fair_server if it + * was previously detached because we were running in full mode. + * + * Unlike the enable path, this runs on a recovery path that cannot + * fail, so we use dl_server_swap_bw() to atomically free ext_server's + * bandwidth and reclaim it for fair_server under the same dl_b lock. + * + * The swap can still fail with -EBUSY if someone bumped ext_server's + * runtime via debugfs between enable and disable; in that narrow case + * both servers end up detached and we just WARN. + */ + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + scx_rq_clock_invalidate(rq); + + scoped_guard(rq_lock_irqsave, rq) { + update_rq_clock(rq); + if (was_switched_all) { + if (WARN_ON_ONCE(dl_server_swap_bw(&rq->ext_server, + &rq->fair_server))) + pr_warn("failed to re-attach fair_server on CPU %d\n", cpu); + } else { + dl_server_detach_bw(&rq->ext_server); + } + } + } + + /* no task is on scx, turn off all the switches and flush in-progress calls */ + static_branch_disable(&__scx_enabled); + static_branch_disable(&__scx_is_cid_type); + if (sch->ops.flags & SCX_OPS_TID_TO_TASK) + static_branch_disable(&__scx_tid_to_task_enabled); + bitmap_zero(sch->has_op, SCX_OPI_END); + scx_idle_disable(); + synchronize_rcu(); + if (sch->ops.flags & SCX_OPS_TID_TO_TASK) + rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL); + + scx_log_sched_disable(sch); + + if (sch->ops.exit) + SCX_CALL_OP(sch, exit, NULL, sch->exit_info); + + scx_unlink_sched(sch); + + /* + * scx_root clearing must be inside cpus_read_lock(). See + * handle_hotplug(). + */ + cpus_read_lock(); + RCU_INIT_POINTER(scx_root, NULL); + cpus_read_unlock(); + + /* + * Delete the kobject from the hierarchy synchronously. Otherwise, sysfs + * could observe an object of the same name still in the hierarchy when + * the next scheduler is loaded. + */ +#ifdef CONFIG_EXT_SUB_SCHED + if (sch->sub_kset) + kobject_del(&sch->sub_kset->kobj); +#endif + kobject_del(&sch->kobj); + + free_kick_syncs(); + + mutex_unlock(&scx_enable_mutex); + + WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING); +done: + scx_bypass(sch, false); +} + +/* + * Claim the exit on @sch. The caller must ensure that the helper kthread work + * is kicked before the current task can be preempted. Once exit_kind is + * claimed, scx_error() can no longer trigger, so if the current task gets + * preempted and the BPF scheduler fails to schedule it back, the helper work + * will never be kicked and the whole system can wedge. + */ +static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind) +{ + int none = SCX_EXIT_NONE; + + lockdep_assert_preemption_disabled(); + + if (WARN_ON_ONCE(kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE)) + kind = SCX_EXIT_ERROR; + + if (!atomic_try_cmpxchg(&sch->exit_kind, &none, kind)) + return false; + + /* + * Some CPUs may be trapped in the dispatch paths. Set the aborting + * flag to break potential live-lock scenarios, ensuring we can + * successfully reach scx_bypass(). + */ + WRITE_ONCE(sch->aborting, true); + + /* + * Propagate exits to descendants immediately. Each has a dedicated + * helper kthread and can run in parallel. While most of disabling is + * serialized, running them in separate threads allows parallelizing + * ops.exit(), which can take arbitrarily long prolonging bypass mode. + * + * To guarantee forward progress, this propagation must be in-line so + * that ->aborting is synchronously asserted for all sub-scheds. The + * propagation is also the interlocking point against sub-sched + * attachment. See scx_link_sched(). + * + * This doesn't cause recursions as propagation only takes place for + * non-propagation exits. + */ + if (kind != SCX_EXIT_PARENT) { + scoped_guard (raw_spinlock_irqsave, &scx_sched_lock) { + struct scx_sched *pos; + scx_for_each_descendant_pre(pos, sch) + scx_disable(pos, SCX_EXIT_PARENT); + } + } + + return true; +} + +static void scx_disable_workfn(struct kthread_work *work) +{ + struct scx_sched *sch = container_of(work, struct scx_sched, disable_work); + struct scx_exit_info *ei = sch->exit_info; + int kind; + + kind = atomic_read(&sch->exit_kind); + while (true) { + if (kind == SCX_EXIT_DONE) /* already disabled? */ + return; + WARN_ON_ONCE(kind == SCX_EXIT_NONE); + if (atomic_try_cmpxchg(&sch->exit_kind, &kind, SCX_EXIT_DONE)) + break; + } + ei->kind = kind; + ei->reason = scx_exit_reason(ei->kind); + + if (scx_parent(sch)) + scx_sub_disable(sch); + else + scx_root_disable(sch); +} + +static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind) +{ + guard(preempt)(); + if (scx_claim_exit(sch, kind)) + irq_work_queue(&sch->disable_irq_work); +} + +/** + * scx_flush_disable_work - flush the disable work and wait for it to finish + * @sch: the scheduler + * + * sch->disable_work might still not queued, causing kthread_flush_work() + * as a noop. Syncing the irq_work first is required to guarantee the + * kthread work has been queued before waiting for it. + */ +static void scx_flush_disable_work(struct scx_sched *sch) +{ + int kind; + + do { + irq_work_sync(&sch->disable_irq_work); + kthread_flush_work(&sch->disable_work); + kind = atomic_read(&sch->exit_kind); + } while (kind != SCX_EXIT_NONE && kind != SCX_EXIT_DONE); +} + +static void dump_newline(struct seq_buf *s) +{ + trace_sched_ext_dump(""); + + /* @s may be zero sized and seq_buf triggers WARN if so */ + if (s->size) + seq_buf_putc(s, '\n'); +} + +static __printf(2, 3) void dump_line(struct seq_buf *s, const char *fmt, ...) +{ + va_list args; + +#ifdef CONFIG_TRACEPOINTS + if (trace_sched_ext_dump_enabled()) { + /* protected by scx_dump_lock */ + static char line_buf[SCX_EXIT_MSG_LEN]; + + va_start(args, fmt); + vscnprintf(line_buf, sizeof(line_buf), fmt, args); + va_end(args); + + trace_call__sched_ext_dump(line_buf); + } +#endif + /* @s may be zero sized and seq_buf triggers WARN if so */ + if (s->size) { + va_start(args, fmt); + seq_buf_vprintf(s, fmt, args); + va_end(args); + + seq_buf_putc(s, '\n'); + } +} + +static void dump_stack_trace(struct seq_buf *s, const char *prefix, + const unsigned long *bt, unsigned int len) +{ + unsigned int i; + + for (i = 0; i < len; i++) + dump_line(s, "%s%pS", prefix, (void *)bt[i]); +} + +static void ops_dump_init(struct seq_buf *s, const char *prefix) +{ + struct scx_dump_data *dd = &scx_dump_data; + + lockdep_assert_irqs_disabled(); + + dd->cpu = smp_processor_id(); /* allow scx_bpf_dump() */ + dd->first = true; + dd->cursor = 0; + dd->s = s; + dd->prefix = prefix; +} + +static void ops_dump_flush(void) +{ + struct scx_dump_data *dd = &scx_dump_data; + char *line = dd->buf.line; + + if (!dd->cursor) + return; + + /* + * There's something to flush and this is the first line. Insert a blank + * line to distinguish ops dump. + */ + if (dd->first) { + dump_newline(dd->s); + dd->first = false; + } + + /* + * There may be multiple lines in $line. Scan and emit each line + * separately. + */ + while (true) { + char *end = line; + char c; + + while (*end != '\n' && *end != '\0') + end++; + + /* + * If $line overflowed, it may not have newline at the end. + * Always emit with a newline. + */ + c = *end; + *end = '\0'; + dump_line(dd->s, "%s%s", dd->prefix, line); + if (c == '\0') + break; + + /* move to the next line */ + end++; + if (*end == '\0') + break; + line = end; + } + + dd->cursor = 0; +} + +static void ops_dump_exit(void) +{ + ops_dump_flush(); + scx_dump_data.cpu = -1; +} + +static void scx_dump_task(struct scx_sched *sch, struct seq_buf *s, struct scx_dump_ctx *dctx, + struct rq *rq, struct task_struct *p, char marker) +{ + static unsigned long bt[SCX_EXIT_BT_LEN]; + struct scx_sched *task_sch = scx_task_sched(p); + const char *own_marker; + char sch_id_buf[32]; + char dsq_id_buf[19] = "(n/a)"; + unsigned long ops_state = atomic_long_read(&p->scx.ops_state); + unsigned int bt_len = 0; + + own_marker = task_sch == sch ? "*" : ""; + + if (task_sch->level == 0) + scnprintf(sch_id_buf, sizeof(sch_id_buf), "root"); + else + scnprintf(sch_id_buf, sizeof(sch_id_buf), "sub%d-%llu", + task_sch->level, task_sch->ops.sub_cgroup_id); + + if (p->scx.dsq) + scnprintf(dsq_id_buf, sizeof(dsq_id_buf), "0x%llx", + (unsigned long long)p->scx.dsq->id); + + dump_newline(s); + dump_line(s, " %c%c %s[%d] %s%s %+ldms", + marker, task_state_to_char(p), p->comm, p->pid, + own_marker, sch_id_buf, + jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies)); + dump_line(s, " scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu", + scx_get_task_state(p) >> SCX_TASK_STATE_SHIFT, + p->scx.flags & ~SCX_TASK_STATE_MASK, + p->scx.dsq_flags, ops_state & SCX_OPSS_STATE_MASK, + ops_state >> SCX_OPSS_QSEQ_SHIFT); + dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s", + p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf); + dump_line(s, " dsq_vtime=%llu slice=%llu weight=%u", + p->scx.dsq_vtime, p->scx.slice, p->scx.weight); + dump_line(s, " cpus=%*pb no_mig=%u", cpumask_pr_args(p->cpus_ptr), + p->migration_disabled); + + if (SCX_HAS_OP(sch, dump_task)) { + ops_dump_init(s, " "); + SCX_CALL_OP(sch, dump_task, rq, dctx, p); + ops_dump_exit(); + } + +#ifdef CONFIG_STACKTRACE + bt_len = stack_trace_save_tsk(p, bt, SCX_EXIT_BT_LEN, 1); +#endif + if (bt_len) { + dump_newline(s); + dump_stack_trace(s, " ", bt, bt_len); + } +} + +static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, + struct scx_dump_ctx *dctx, int cpu, + bool dump_all_tasks) +{ + struct rq *rq = cpu_rq(cpu); + struct rq_flags rf; + struct task_struct *p; + struct seq_buf ns; + size_t avail, used; + char *buf; + bool idle; + + rq_lock_irqsave(rq, &rf); + + idle = list_empty(&rq->scx.runnable_list) && + rq->curr->sched_class == &idle_sched_class; + + if (idle && !SCX_HAS_OP(sch, dump_cpu)) + goto next; + + /* + * We don't yet know whether ops.dump_cpu() will produce output + * and we may want to skip the default CPU dump if it doesn't. + * Use a nested seq_buf to generate the standard dump so that we + * can decide whether to commit later. + */ + avail = seq_buf_get_buf(s, &buf); + seq_buf_init(&ns, buf, avail); + + dump_newline(&ns); + dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu", + cpu, rq->scx.nr_running, rq->scx.flags, + rq->scx.cpu_released, rq->scx.ops_qseq, + rq->scx.kick_sync); + dump_line(&ns, " curr=%s[%d] class=%ps", + rq->curr->comm, rq->curr->pid, + rq->curr->sched_class); + if (!cpumask_empty(rq->scx.cpus_to_kick)) + dump_line(&ns, " cpus_to_kick : %*pb", + cpumask_pr_args(rq->scx.cpus_to_kick)); + if (!cpumask_empty(rq->scx.cpus_to_kick_if_idle)) + dump_line(&ns, " idle_to_kick : %*pb", + cpumask_pr_args(rq->scx.cpus_to_kick_if_idle)); + if (!cpumask_empty(rq->scx.cpus_to_preempt)) + dump_line(&ns, " cpus_to_preempt: %*pb", + cpumask_pr_args(rq->scx.cpus_to_preempt)); + if (!cpumask_empty(rq->scx.cpus_to_wait)) + dump_line(&ns, " cpus_to_wait : %*pb", + cpumask_pr_args(rq->scx.cpus_to_wait)); + if (!cpumask_empty(rq->scx.cpus_to_sync)) + dump_line(&ns, " cpus_to_sync : %*pb", + cpumask_pr_args(rq->scx.cpus_to_sync)); + + used = seq_buf_used(&ns); + if (SCX_HAS_OP(sch, dump_cpu)) { + ops_dump_init(&ns, " "); + SCX_CALL_OP(sch, dump_cpu, rq, dctx, scx_cpu_arg(cpu), idle); + ops_dump_exit(); + } + + /* + * If idle && nothing generated by ops.dump_cpu(), there's + * nothing interesting. Skip. + */ + if (idle && used == seq_buf_used(&ns)) + goto next; + + /* + * $s may already have overflowed when $ns was created. If so, + * calling commit on it will trigger BUG. + */ + if (avail) { + seq_buf_commit(s, seq_buf_used(&ns)); + if (seq_buf_has_overflowed(&ns)) + seq_buf_set_overflow(s); + } + + if (rq->curr->sched_class == &ext_sched_class && + (dump_all_tasks || scx_task_on_sched(sch, rq->curr))) + scx_dump_task(sch, s, dctx, rq, rq->curr, '*'); + + list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) + if (dump_all_tasks || scx_task_on_sched(sch, p)) + scx_dump_task(sch, s, dctx, rq, p, ' '); +next: + rq_unlock_irqrestore(rq, &rf); +} + +/* + * Dump scheduler state. If @dump_all_tasks is true, dump all tasks regardless + * of which scheduler they belong to. If false, only dump tasks owned by @sch. + * For SysRq-D dumps, @dump_all_tasks=false since all schedulers are dumped + * separately. For error dumps, @dump_all_tasks=true since only the failing + * scheduler is dumped. + */ +static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, + size_t dump_len, bool dump_all_tasks) +{ + static const char trunc_marker[] = "\n\n~~~~ TRUNCATED ~~~~\n"; + struct scx_dump_ctx dctx = { + .kind = ei->kind, + .exit_code = ei->exit_code, + .reason = ei->reason, + .at_ns = ktime_get_ns(), + .at_jiffies = jiffies, + }; + struct seq_buf s; + struct scx_event_stats events; + int cpu; + + guard(raw_spinlock_irqsave)(&scx_dump_lock); + + if (sch->dump_disabled) + return; + + seq_buf_init(&s, ei->dump, dump_len); + +#ifdef CONFIG_EXT_SUB_SCHED + if (sch->level == 0) + dump_line(&s, "%s: root", sch->ops.name); + else + dump_line(&s, "%s: sub%d-%llu %s", + sch->ops.name, sch->level, sch->ops.sub_cgroup_id, + sch->cgrp_path); +#endif + if (ei->kind == SCX_EXIT_NONE) { + dump_line(&s, "Debug dump triggered by %s", ei->reason); + } else { + if (ei->exit_cpu >= 0) + dump_line(&s, "%s[%d] triggered exit kind %d on CPU %d:", + current->comm, current->pid, ei->kind, + ei->exit_cpu); + else + dump_line(&s, "%s[%d] triggered exit kind %d:", + current->comm, current->pid, ei->kind); + dump_line(&s, " %s (%s)", ei->reason, ei->msg); + dump_newline(&s); + dump_line(&s, "Backtrace:"); + dump_stack_trace(&s, " ", ei->bt, ei->bt_len); + } + + if (SCX_HAS_OP(sch, dump)) { + ops_dump_init(&s, ""); + SCX_CALL_OP(sch, dump, NULL, &dctx); + ops_dump_exit(); + } + + dump_newline(&s); + dump_line(&s, "CPU states"); + dump_line(&s, "----------"); + + /* + * Dump the exit CPU first so it isn't lost to dump truncation, then + * walk the rest in order, skipping the one already dumped. + */ + if (ei->exit_cpu >= 0) + scx_dump_cpu(sch, &s, &dctx, ei->exit_cpu, dump_all_tasks); + for_each_possible_cpu(cpu) { + if (cpu != ei->exit_cpu) + scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); + } + + dump_newline(&s); + dump_line(&s, "Event counters"); + dump_line(&s, "--------------"); + + scx_read_events(sch, &events); + scx_dump_event(s, &events, SCX_EV_SELECT_CPU_FALLBACK); + scx_dump_event(s, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); + scx_dump_event(s, &events, SCX_EV_DISPATCH_KEEP_LAST); + scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_EXITING); + scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); + scx_dump_event(s, &events, SCX_EV_REENQ_IMMED); + scx_dump_event(s, &events, SCX_EV_REENQ_LOCAL_REPEAT); + scx_dump_event(s, &events, SCX_EV_REFILL_SLICE_DFL); + scx_dump_event(s, &events, SCX_EV_BYPASS_DURATION); + scx_dump_event(s, &events, SCX_EV_BYPASS_DISPATCH); + scx_dump_event(s, &events, SCX_EV_BYPASS_ACTIVATE); + scx_dump_event(s, &events, SCX_EV_INSERT_NOT_OWNED); + scx_dump_event(s, &events, SCX_EV_SUB_BYPASS_DISPATCH); + + if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker)) + memcpy(ei->dump + dump_len - sizeof(trunc_marker), + trunc_marker, sizeof(trunc_marker)); +} + +static void scx_disable_irq_workfn(struct irq_work *irq_work) +{ + struct scx_sched *sch = container_of(irq_work, struct scx_sched, disable_irq_work); + struct scx_exit_info *ei = sch->exit_info; + + if (ei->kind >= SCX_EXIT_ERROR) + scx_dump_state(sch, ei, sch->ops.exit_dump_len, true); + + kthread_queue_work(sch->helper, &sch->disable_work); +} + +bool scx_vexit(struct scx_sched *sch, + enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu, + const char *fmt, va_list args) +{ + struct scx_exit_info *ei = sch->exit_info; + + guard(preempt)(); + + if (!scx_claim_exit(sch, kind)) + return false; + + ei->exit_code = exit_code; +#ifdef CONFIG_STACKTRACE + if (kind >= SCX_EXIT_ERROR) + ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1); +#endif + vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args); + + /* + * Set ei->kind and ->reason for scx_dump_state(). They'll be set again + * in scx_disable_workfn(). + */ + ei->kind = kind; + ei->reason = scx_exit_reason(ei->kind); + ei->exit_cpu = exit_cpu; + + irq_work_queue(&sch->disable_irq_work); + return true; +} + +static int alloc_kick_syncs(void) +{ + int cpu; + + /* + * Allocate per-CPU arrays sized by nr_cpu_ids. Use kvzalloc as size + * can exceed percpu allocator limits on large machines. + */ + for_each_possible_cpu(cpu) { + struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); + struct scx_kick_syncs *new_ksyncs; + + WARN_ON_ONCE(rcu_access_pointer(*ksyncs)); + + new_ksyncs = kvzalloc_node(struct_size(new_ksyncs, syncs, nr_cpu_ids), + GFP_KERNEL, cpu_to_node(cpu)); + if (!new_ksyncs) { + free_kick_syncs(); + return -ENOMEM; + } + + rcu_assign_pointer(*ksyncs, new_ksyncs); + } + + return 0; +} + +static void free_pnode(struct scx_sched_pnode *pnode) +{ + if (!pnode) + return; + exit_dsq(&pnode->global_dsq); + kfree(pnode); +} + +static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node) +{ + struct scx_sched_pnode *pnode; + + pnode = kzalloc_node(sizeof(*pnode), GFP_KERNEL, node); + if (!pnode) + return NULL; + + if (init_dsq(&pnode->global_dsq, SCX_DSQ_GLOBAL, sch)) { + kfree(pnode); + return NULL; + } + + return pnode; +} + +/* + * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid + * starvation. During the READY -> ENABLED task switching loop, the calling + * thread's sched_class gets switched from fair to ext. As fair has higher + * priority than ext, the calling thread can be indefinitely starved under + * fair-class saturation, leading to a system hang. + */ +struct scx_enable_cmd { + struct kthread_work work; + union { + struct sched_ext_ops *ops; + struct sched_ext_ops_cid *ops_cid; + }; + bool is_cid_type; + struct bpf_map *arena_map; /* arena ref to transfer to sch */ + int ret; +}; + +/* + * Allocate and initialize a new scx_sched. @cgrp's reference is always + * consumed whether the function succeeds or fails. + */ +static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, + struct cgroup *cgrp, + struct scx_sched *parent) +{ + struct sched_ext_ops *ops = cmd->ops; + struct scx_sched *sch; + s32 level = parent ? parent->level + 1 : 0; + s32 node, cpu, ret, bypass_fail_cpu = nr_cpu_ids; + + sch = kzalloc_flex(*sch, ancestors, level + 1); + if (!sch) { + ret = -ENOMEM; + goto err_put_cgrp; + } + + sch->exit_info = alloc_exit_info(ops->exit_dump_len); + if (!sch->exit_info) { + ret = -ENOMEM; + goto err_free_sch; + } + + ret = rhashtable_init(&sch->dsq_hash, &dsq_hash_params); + if (ret < 0) + goto err_free_ei; + + sch->pnode = kzalloc_objs(sch->pnode[0], nr_node_ids); + if (!sch->pnode) { + ret = -ENOMEM; + goto err_free_hash; + } + + for_each_node_state(node, N_POSSIBLE) { + sch->pnode[node] = alloc_pnode(sch, node); + if (!sch->pnode[node]) { + ret = -ENOMEM; + goto err_free_pnode; + } + } + + sch->dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; + sch->pcpu = __alloc_percpu(struct_size_t(struct scx_sched_pcpu, + dsp_ctx.buf, sch->dsp_max_batch), + __alignof__(struct scx_sched_pcpu)); + if (!sch->pcpu) { + ret = -ENOMEM; + goto err_free_pnode; + } + + for_each_possible_cpu(cpu) { + ret = init_dsq(bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch); + if (ret) { + bypass_fail_cpu = cpu; + goto err_free_pcpu; + } + } + + for_each_possible_cpu(cpu) { + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + + pcpu->sch = sch; + INIT_LIST_HEAD(&pcpu->deferred_reenq_local.node); + } + + sch->helper = kthread_run_worker(0, "sched_ext_helper"); + if (IS_ERR(sch->helper)) { + ret = PTR_ERR(sch->helper); + goto err_free_pcpu; + } + + sched_set_fifo(sch->helper->task); + + if (parent) + memcpy(sch->ancestors, parent->ancestors, + level * sizeof(parent->ancestors[0])); + sch->ancestors[level] = sch; + sch->level = level; + + if (ops->timeout_ms) + sch->watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); + else + sch->watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; + + sch->slice_dfl = SCX_SLICE_DFL; + atomic_set(&sch->exit_kind, SCX_EXIT_NONE); + sch->disable_irq_work = IRQ_WORK_INIT_HARD(scx_disable_irq_workfn); + kthread_init_work(&sch->disable_work, scx_disable_workfn); + timer_setup(&sch->bypass_lb_timer, scx_bypass_lb_timerfn, 0); + + if (!alloc_cpumask_var(&sch->bypass_lb_donee_cpumask, GFP_KERNEL)) { + ret = -ENOMEM; + goto err_stop_helper; + } + if (!alloc_cpumask_var(&sch->bypass_lb_resched_cpumask, GFP_KERNEL)) { + ret = -ENOMEM; + goto err_free_lb_cpumask; + } + /* + * Copy ops through the right union view. For cid-form the source is + * struct sched_ext_ops_cid which lacks the trailing cpu_acquire/ + * cpu_release; those stay zero from kzalloc. + */ + if (cmd->is_cid_type) { + sch->ops_cid = *cmd->ops_cid; + sch->is_cid_type = true; + } else { + sch->ops = *cmd->ops; + } + + rcu_assign_pointer(ops->priv, sch); + + sch->kobj.kset = scx_kset; + INIT_LIST_HEAD(&sch->all); + +#ifdef CONFIG_EXT_SUB_SCHED + char *buf = kzalloc(PATH_MAX, GFP_KERNEL); + if (!buf) { + ret = -ENOMEM; + goto err_free_lb_resched; + } + cgroup_path(cgrp, buf, PATH_MAX); + sch->cgrp_path = kstrdup(buf, GFP_KERNEL); + kfree(buf); + if (!sch->cgrp_path) { + ret = -ENOMEM; + goto err_free_lb_resched; + } + + sch->cgrp = cgrp; + INIT_LIST_HEAD(&sch->children); + INIT_LIST_HEAD(&sch->sibling); + + if (parent) + ret = kobject_init_and_add(&sch->kobj, &scx_ktype, + &parent->sub_kset->kobj, + "sub-%llu", cgroup_id(cgrp)); + else + ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root"); + + if (ret < 0) { + RCU_INIT_POINTER(ops->priv, NULL); + kobject_put(&sch->kobj); + return ERR_PTR(ret); + } + + if (ops->sub_attach) { + sch->sub_kset = kset_create_and_add("sub", NULL, &sch->kobj); + if (!sch->sub_kset) { + RCU_INIT_POINTER(ops->priv, NULL); + kobject_put(&sch->kobj); + return ERR_PTR(-ENOMEM); + } + } +#else /* CONFIG_EXT_SUB_SCHED */ + ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root"); + if (ret < 0) { + RCU_INIT_POINTER(ops->priv, NULL); + kobject_put(&sch->kobj); + return ERR_PTR(ret); + } +#endif /* CONFIG_EXT_SUB_SCHED */ + + /* + * Consume the arena_map ref bpf_scx_reg_cid() took. Defer to here so + * earlier failure paths leave cmd->arena_map set and bpf_scx_reg_cid + * drops the ref. After this point, sch owns the ref and any cleanup + * runs through scx_sched_free_rcu_work() which puts it. + */ + sch->arena_map = cmd->arena_map; + /* BPF arena is only available on MMU && 64BIT */ +#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) + if (sch->arena_map) + sch->arena_kern_base = bpf_arena_map_kern_vm_start(sch->arena_map); +#endif + cmd->arena_map = NULL; + return sch; + +#ifdef CONFIG_EXT_SUB_SCHED +err_free_lb_resched: + RCU_INIT_POINTER(ops->priv, NULL); + free_cpumask_var(sch->bypass_lb_resched_cpumask); +#endif +err_free_lb_cpumask: + free_cpumask_var(sch->bypass_lb_donee_cpumask); +err_stop_helper: + kthread_destroy_worker(sch->helper); +err_free_pcpu: + for_each_possible_cpu(cpu) { + if (cpu == bypass_fail_cpu) + break; + exit_dsq(bypass_dsq(sch, cpu)); + } + free_percpu(sch->pcpu); +err_free_pnode: + for_each_node_state(node, N_POSSIBLE) + free_pnode(sch->pnode[node]); + kfree(sch->pnode); +err_free_hash: + rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); +err_free_ei: + free_exit_info(sch->exit_info); +err_free_sch: + kfree(sch); +err_put_cgrp: +#ifdef CONFIG_EXT_SUB_SCHED + cgroup_put(cgrp); +#endif + return ERR_PTR(ret); +} + +static int check_hotplug_seq(struct scx_sched *sch, + const struct sched_ext_ops *ops) +{ + unsigned long long global_hotplug_seq; + + /* + * If a hotplug event has occurred between when a scheduler was + * initialized, and when we were able to attach, exit and notify user + * space about it. + */ + if (ops->hotplug_seq) { + global_hotplug_seq = atomic_long_read(&scx_hotplug_seq); + if (ops->hotplug_seq != global_hotplug_seq) { + scx_exit(sch, SCX_EXIT_UNREG_KERN, + SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, + "expected hotplug seq %llu did not match actual %llu", + ops->hotplug_seq, global_hotplug_seq); + return -EBUSY; + } + } + + return 0; +} + +static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) +{ + /* + * It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the + * ops.enqueue() callback isn't implemented. + */ + if ((ops->flags & SCX_OPS_ENQ_LAST) && !ops->enqueue) { + scx_error(sch, "SCX_OPS_ENQ_LAST requires ops.enqueue() to be implemented"); + return -EINVAL; + } + + /* + * SCX_OPS_TID_TO_TASK is enabled by the root scheduler. A sub-sched + * may set it to declare a dependency; reject if the root hasn't + * enabled it. + */ + if ((ops->flags & SCX_OPS_TID_TO_TASK) && scx_parent(sch) && + !(scx_root->ops.flags & SCX_OPS_TID_TO_TASK)) { + scx_error(sch, "SCX_OPS_TID_TO_TASK requires root scheduler to enable it"); + return -EINVAL; + } + + /* + * SCX_OPS_BUILTIN_IDLE_PER_NODE requires built-in CPU idle + * selection policy to be enabled. + */ + if ((ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE) && + (ops->update_idle && !(ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE))) { + scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE requires CPU idle selection enabled"); + return -EINVAL; + } + + /* + * cid-form's struct is shorter and doesn't include the cpu_acquire / + * cpu_release tail; reading those fields off a cid-form @ops would + * run past the BPF allocation. Skip for cid-form. + */ + if (!sch->is_cid_type && (ops->cpu_acquire || ops->cpu_release)) + pr_warn("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n"); + + /* + * Sub-scheduler support is tied to the cid-form struct_ops. A sub-sched + * attaches through a cid-form-only interface (sub_attach/sub_detach), + * and a root that accepts sub-scheds must expose cid-form state to + * them. Reject cpu-form schedulers on either side. + */ + if (!sch->is_cid_type) { + if (scx_parent(sch)) { + scx_error(sch, "sub-sched requires cid-form struct_ops"); + return -EINVAL; + } + if (ops->sub_attach || ops->sub_detach) { + scx_error(sch, "sub_attach/sub_detach requires cid-form struct_ops"); + return -EINVAL; + } + } + + return 0; +} + +static void scx_root_enable_workfn(struct kthread_work *work) +{ + struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work); + struct sched_ext_ops *ops = cmd->ops; + struct cgroup *cgrp = root_cgroup(); + struct scx_sched *sch; + struct scx_task_iter sti; + struct task_struct *p; + int i, cpu, ret; + + mutex_lock(&scx_enable_mutex); + + if (scx_enable_state() != SCX_DISABLED) { + ret = -EBUSY; + goto err_unlock; + } + + /* + * @ops->priv binds @ops to its scx_sched instance. It is set here by + * scx_alloc_and_add_sched() and cleared at the tail of bpf_scx_unreg(), + * which runs after scx_root_disable() has dropped scx_enable_mutex. If + * it's still non-NULL here, a previous attachment on @ops has not + * finished tearing down; proceeding would let the in-flight unreg's + * RCU_INIT_POINTER(NULL) clobber the @ops->priv we are about to assign. + */ + if (rcu_access_pointer(ops->priv)) { + ret = -EBUSY; + goto err_unlock; + } + + ret = alloc_kick_syncs(); + if (ret) + goto err_unlock; + + if (ops->flags & SCX_OPS_TID_TO_TASK) { + ret = rhashtable_init(&scx_tid_hash, &scx_tid_hash_params); + if (ret) + goto err_free_ksyncs; + } + +#ifdef CONFIG_EXT_SUB_SCHED + cgroup_get(cgrp); +#endif + sch = scx_alloc_and_add_sched(cmd, cgrp, NULL); + if (IS_ERR(sch)) { + ret = PTR_ERR(sch); + goto err_free_tid_hash; + } + + if (sch->is_cid_type) + static_branch_enable(&__scx_is_cid_type); + + /* + * Transition to ENABLING and clear exit info to arm the disable path. + * Failure triggers full disabling from here on. + */ + WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED); + WARN_ON_ONCE(scx_root); + + atomic_long_set(&scx_nr_rejected, 0); + + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + rq->scx.local_dsq.sched = sch; + rq->scx.cpuperf_target = SCX_CPUPERF_ONE; + } + + /* + * Keep CPUs stable during enable so that the BPF scheduler can track + * online CPUs by watching ->on/offline_cpu() after ->init(). + */ + cpus_read_lock(); + + /* + * Build the cid mapping before publishing scx_root. The cid kfuncs + * dereference the cid arrays unconditionally once scx_prog_sched() + * returns non-NULL; the rcu_assign_pointer() below pairs with their + * rcu_dereference() to make the populated arrays visible. + */ + ret = scx_cid_init(sch); + if (ret) { + cpus_read_unlock(); + goto err_disable; + } + + /* + * Make the scheduler instance visible. Must be inside cpus_read_lock(). + * See handle_hotplug(). + */ + rcu_assign_pointer(scx_root, sch); + + ret = scx_link_sched(sch); + if (ret) { + cpus_read_unlock(); + goto err_disable; + } + + scx_idle_enable(ops); + + if (sch->ops.init) { + ret = SCX_CALL_OP_RET(sch, init, NULL); + if (ret) { + ret = ops_sanitize_err(sch, "init", ret); + cpus_read_unlock(); + scx_error(sch, "ops.init() failed (%d)", ret); + goto err_disable; + } + sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; + } + + ret = scx_arena_pool_init(sch); + if (ret) { + cpus_read_unlock(); + goto err_disable; + } + + ret = scx_set_cmask_scratch_alloc(sch); + if (ret) { + cpus_read_unlock(); + goto err_disable; + } + + for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++) + if (((void (**)(void))ops)[i]) + set_bit(i, sch->has_op); + + ret = check_hotplug_seq(sch, ops); + if (ret) { + cpus_read_unlock(); + goto err_disable; + } + scx_idle_update_selcpu_topology(ops); + + cpus_read_unlock(); + + ret = validate_ops(sch, ops); + if (ret) + goto err_disable; + + /* + * Attach the ext_server bandwidth reservation before anything is + * committed so that we can fail the enable if the root domain cannot + * accommodate it. The matching fair_server detach is deferred to the + * tail of this function, after the switch is fully committed and can no + * longer fail. + * + * On failure, err_disable funnels into scx_root_disable() which + * detaches ext_server, so partially-attached state is cleaned up + * automatically. + */ + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + scoped_guard(rq_lock_irqsave, rq) { + update_rq_clock(rq); + ret = dl_server_attach_bw(&rq->ext_server); + } + if (ret) { + pr_warn("sched_ext: failed to attach ext_server on CPU %d (%d)\n", + cpu, ret); + goto err_disable; + } + } + + /* + * Once __scx_enabled is set, %current can be switched to SCX anytime. + * This can lead to stalls as some BPF schedulers (e.g. userspace + * scheduling) may not function correctly before all tasks are switched. + * Init in bypass mode to guarantee forward progress. + */ + scx_bypass(sch, true); + + for (i = SCX_OPI_NORMAL_BEGIN; i < SCX_OPI_NORMAL_END; i++) + if (((void (**)(void))ops)[i]) + set_bit(i, sch->has_op); + + if (sch->ops.cpu_acquire || sch->ops.cpu_release) + sch->ops.flags |= SCX_OPS_HAS_CPU_PREEMPT; + + /* + * Lock out forks, cgroup on/offlining and moves before opening the + * floodgate so that they don't wander into the operations prematurely. + */ + percpu_down_write(&scx_fork_rwsem); + + WARN_ON_ONCE(scx_init_task_enabled); + scx_init_task_enabled = true; + + /* flip under fork_rwsem; the iter below covers existing tasks */ + if (ops->flags & SCX_OPS_TID_TO_TASK) + static_branch_enable(&__scx_tid_to_task_enabled); + + /* + * Enable ops for every task. Fork is excluded by scx_fork_rwsem + * preventing new tasks from being added. No need to exclude tasks + * leaving as sched_ext_free() can handle both prepped and enabled + * tasks. Prep all tasks first and then enable them with preemption + * disabled. + * + * All cgroups should be initialized before scx_init_task() so that the + * BPF scheduler can reliably track each task's cgroup membership from + * scx_init_task(). Lock out cgroup on/offlining and task migrations + * while tasks are being initialized so that scx_cgroup_can_attach() + * never sees uninitialized tasks. + */ + scx_cgroup_lock(); + set_cgroup_sched(sch_cgroup(sch), sch); + ret = scx_cgroup_init(sch); + if (ret) + goto err_disable_unlock_all; + + scx_task_iter_start(&sti, NULL); + while ((p = scx_task_iter_next_locked(&sti))) { + /* + * @p is in scx_tasks under scx_tasks_lock, and SCX_TASK_DEAD + * tasks are filtered by scx_task_iter_next_locked(). + * sched_ext_dead() removes @p from scx_tasks under the same + * lock before put_task_struct_rcu_user() runs, so @p->usage + * is guaranteed > 0 here. + */ + get_task_struct(p); + + /* + * Set %INIT_BEGIN under the iter's rq lock so that a concurrent + * sched_ext_dead() does not call ops.exit_task() on @p while + * ops.init_task() is running. If sched_ext_dead() runs before + * this store, it has already removed @p from scx_tasks and the + * iter won't visit @p; if it runs after, it observes + * %INIT_BEGIN and transitions to %DEAD without calling ops, + * leaving the post-init recheck below to unwind. + */ + scx_set_task_state(p, SCX_TASK_INIT_BEGIN); + scx_task_iter_unlock(&sti); + + ret = __scx_init_task(sch, p, false); + + scx_task_iter_relock(&sti, p); + + if (unlikely(ret)) { + if (scx_get_task_state(p) != SCX_TASK_DEAD) + scx_set_task_state(p, SCX_TASK_NONE); + scx_task_iter_stop(&sti); + scx_error(sch, "ops.init_task() failed (%d) for %s[%d]", + ret, p->comm, p->pid); + put_task_struct(p); + goto err_disable_unlock_all; + } + + if (scx_get_task_state(p) == SCX_TASK_DEAD) { + /* + * sched_ext_dead() observed %INIT_BEGIN and set %DEAD. + * ops.exit_task() is owed to the sched __scx_init_task() + * ran against; call it now. + */ + scx_sub_init_cancel_task(sch, p); + } else { + scx_set_task_state(p, SCX_TASK_INIT); + scx_set_task_sched(p, sch); + scx_set_task_state(p, SCX_TASK_READY); + } + + /* + * Insert into the tid hash. scx_tasks_lock is held by the iter; + * list_empty() guards against sched_ext_dead() having taken @p + * off the list while init ran unlocked. + */ + if (scx_tid_to_task_enabled() && !list_empty(&p->scx.tasks_node)) + scx_tid_hash_insert(p); + + put_task_struct(p); + } + scx_task_iter_stop(&sti); + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); + + /* + * All tasks are READY. It's safe to turn on scx_enabled() and switch + * all eligible tasks. + */ + WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL)); + static_branch_enable(&__scx_enabled); + + /* + * We're fully committed and can't fail. The task READY -> ENABLED + * transitions here are synchronized against sched_ext_free() through + * scx_tasks_lock. + */ + percpu_down_write(&scx_fork_rwsem); + scx_task_iter_start(&sti, NULL); + while ((p = scx_task_iter_next_locked(&sti))) { + unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE; + const struct sched_class *old_class = p->sched_class; + const struct sched_class *new_class = scx_setscheduler_class(p); + + if (scx_get_task_state(p) != SCX_TASK_READY) + continue; + + if (old_class != new_class) + queue_flags |= DEQUEUE_CLASS; + + scoped_guard (sched_change, p, queue_flags) { + p->scx.slice = READ_ONCE(sch->slice_dfl); + p->sched_class = new_class; + } + } + scx_task_iter_stop(&sti); + percpu_up_write(&scx_fork_rwsem); + + scx_bypass(sch, false); + + if (!scx_tryset_enable_state(SCX_ENABLED, SCX_ENABLING)) { + WARN_ON_ONCE(atomic_read(&sch->exit_kind) == SCX_EXIT_NONE); + goto err_disable; + } + + if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL)) + static_branch_enable(&__scx_switched_all); + + /* + * Detach the fair_server bandwidth reservation now that the switch + * is fully committed. In full mode (!SCX_OPS_SWITCH_PARTIAL) no + * task will ever run in the fair class, so give that bandwidth + * back to the RT class. The matching ext_server attach already + * happened earlier; this only releases bandwidth and cannot fail. + * + * In partial mode keep fair_server attached. + */ + if (scx_switched_all()) { + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + guard(rq_lock_irqsave)(rq); + update_rq_clock(rq); + dl_server_detach_bw(&rq->fair_server); + } + } + + pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n", + sch->ops.name, scx_switched_all() ? "" : " (partial)"); + kobject_uevent(&sch->kobj, KOBJ_ADD); + mutex_unlock(&scx_enable_mutex); + + atomic_long_inc(&scx_enable_seq); + + cmd->ret = 0; + return; + +err_free_tid_hash: + if (ops->flags & SCX_OPS_TID_TO_TASK) + rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL); +err_free_ksyncs: + free_kick_syncs(); +err_unlock: + mutex_unlock(&scx_enable_mutex); + cmd->ret = ret; + return; + +err_disable_unlock_all: + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); + /* we'll soon enter disable path, keep bypass on */ +err_disable: + mutex_unlock(&scx_enable_mutex); + /* + * Returning an error code here would not pass all the error information + * to userspace. Record errno using scx_error() for cases scx_error() + * wasn't already invoked and exit indicating success so that the error + * is notified through ops.exit() with all the details. + * + * Flush scx_disable_work to ensure that error is reported before init + * completion. sch's base reference will be put by bpf_scx_unreg(). + */ + scx_error(sch, "scx_root_enable() failed (%d)", ret); + scx_flush_disable_work(sch); + cmd->ret = 0; +} + +#ifdef CONFIG_EXT_SUB_SCHED +/* verify that a scheduler can be attached to @cgrp and return the parent */ +static struct scx_sched *find_parent_sched(struct cgroup *cgrp) +{ + struct scx_sched *parent = cgrp->scx_sched; + struct scx_sched *pos; + + lockdep_assert_held(&scx_sched_lock); + + /* can't attach twice to the same cgroup */ + if (parent->cgrp == cgrp) + return ERR_PTR(-EBUSY); + + /* does $parent allow sub-scheds? */ + if (!parent->ops.sub_attach) + return ERR_PTR(-EOPNOTSUPP); + + /* can't insert between $parent and its exiting children */ + list_for_each_entry(pos, &parent->children, sibling) + if (cgroup_is_descendant(pos->cgrp, cgrp)) + return ERR_PTR(-EBUSY); + + return parent; +} + +static bool assert_task_ready_or_enabled(struct task_struct *p) +{ + u32 state = scx_get_task_state(p); + + switch (state) { + case SCX_TASK_READY: + case SCX_TASK_ENABLED: + return true; + default: + WARN_ONCE(true, "sched_ext: Invalid task state %d for %s[%d] during enabling sub sched", + state, p->comm, p->pid); + return false; + } +} + +static void scx_sub_enable_workfn(struct kthread_work *work) +{ + struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work); + struct sched_ext_ops *ops = cmd->ops; + struct cgroup *cgrp; + struct scx_sched *parent, *sch; + struct scx_task_iter sti; + struct task_struct *p; + s32 i, ret; + + mutex_lock(&scx_enable_mutex); + + if (!scx_enabled()) { + ret = -ENODEV; + goto out_unlock; + } + + /* See scx_root_enable_workfn() for the @ops->priv check. */ + if (rcu_access_pointer(ops->priv)) { + ret = -EBUSY; + goto out_unlock; + } + + cgrp = cgroup_get_from_id(ops->sub_cgroup_id); + if (IS_ERR(cgrp)) { + ret = PTR_ERR(cgrp); + goto out_unlock; + } + + raw_spin_lock_irq(&scx_sched_lock); + parent = find_parent_sched(cgrp); + if (IS_ERR(parent)) { + raw_spin_unlock_irq(&scx_sched_lock); + ret = PTR_ERR(parent); + goto out_put_cgrp; + } + kobject_get(&parent->kobj); + raw_spin_unlock_irq(&scx_sched_lock); + + /* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */ + sch = scx_alloc_and_add_sched(cmd, cgrp, parent); + kobject_put(&parent->kobj); + if (IS_ERR(sch)) { + ret = PTR_ERR(sch); + goto out_unlock; + } + + ret = scx_link_sched(sch); + if (ret) + goto err_disable; + + if (sch->level >= SCX_SUB_MAX_DEPTH) { + scx_error(sch, "max nesting depth %d violated", + SCX_SUB_MAX_DEPTH); + goto err_disable; + } + + if (sch->ops.init) { + ret = SCX_CALL_OP_RET(sch, init, NULL); + if (ret) { + ret = ops_sanitize_err(sch, "init", ret); + scx_error(sch, "ops.init() failed (%d)", ret); + goto err_disable; + } + sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; + } + + ret = scx_arena_pool_init(sch); + if (ret) + goto err_disable; + + ret = scx_set_cmask_scratch_alloc(sch); + if (ret) + goto err_disable; + + if (validate_ops(sch, ops)) + goto err_disable; + + struct scx_sub_attach_args sub_attach_args = { + .ops = &sch->ops, + .cgroup_path = sch->cgrp_path, + }; + + ret = SCX_CALL_OP_RET(parent, sub_attach, NULL, + &sub_attach_args); + if (ret) { + ret = ops_sanitize_err(sch, "sub_attach", ret); + scx_error(sch, "parent rejected (%d)", ret); + goto err_disable; + } + sch->sub_attached = true; + + scx_bypass(sch, true); + + for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) + if (((void (**)(void))ops)[i]) + set_bit(i, sch->has_op); + + percpu_down_write(&scx_fork_rwsem); + scx_cgroup_lock(); + + /* + * Set cgroup->scx_sched's and check CSS_ONLINE. Either we see + * !CSS_ONLINE or scx_cgroup_lifetime_notify() sees and shoots us down. + */ + set_cgroup_sched(sch_cgroup(sch), sch); + if (!(cgrp->self.flags & CSS_ONLINE)) { + scx_error(sch, "cgroup is not online"); + goto err_unlock_and_disable; + } + + /* + * Initialize tasks for the new child $sch without exiting them for + * $parent so that the tasks can always be reverted back to $parent + * sched on child init failure. + */ + WARN_ON_ONCE(scx_enabling_sub_sched); + scx_enabling_sub_sched = sch; + + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + struct rq *rq; + struct rq_flags rf; + + /* + * Task iteration may visit the same task twice when racing + * against exiting. Use %SCX_TASK_SUB_INIT to mark tasks which + * finished __scx_init_task() and skip if set. + * + * A task may exit and get freed between __scx_init_task() + * completion and scx_enable_task(). In such cases, + * scx_disable_and_exit_task() must exit the task for both the + * parent and child scheds. + */ + if (p->scx.flags & SCX_TASK_SUB_INIT) + continue; + + /* @p is pinned by the iter; see scx_sub_disable() */ + get_task_struct(p); + + if (!assert_task_ready_or_enabled(p)) { + ret = -EINVAL; + goto abort; + } + + scx_task_iter_unlock(&sti); + + /* + * As $p is still on $parent, it can't be transitioned to INIT. + * Let's worry about task state later. Use __scx_init_task(). + */ + ret = __scx_init_task(sch, p, false); + if (ret) + goto abort; + + rq = task_rq_lock(p, &rf); + + if (scx_get_task_state(p) == SCX_TASK_DEAD) { + /* + * sched_ext_dead() raced us between __scx_init_task() + * and this rq lock and ran exit_task() on $parent (the + * sched @p was on at that point), not on @sch. @sch's + * just-completed init is owed an exit_task() and we + * issue it here. + */ + scx_sub_init_cancel_task(sch, p); + task_rq_unlock(rq, p, &rf); + put_task_struct(p); + continue; + } + + p->scx.flags |= SCX_TASK_SUB_INIT; + task_rq_unlock(rq, p, &rf); + + put_task_struct(p); + } + scx_task_iter_stop(&sti); + + /* + * All tasks are prepped. Disable/exit tasks for $parent and enable for + * the new @sch. + */ + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + /* + * Use clearing of %SCX_TASK_SUB_INIT to detect and skip + * duplicate iterations. + */ + if (!(p->scx.flags & SCX_TASK_SUB_INIT)) + continue; + + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + /* + * $p must be either READY or ENABLED. If ENABLED, + * __scx_disabled_and_exit_task() first disables and + * makes it READY. However, after exiting $p, it will + * leave $p as READY. + */ + assert_task_ready_or_enabled(p); + __scx_disable_and_exit_task(parent, p); + + /* + * $p is now only initialized for @sch and READY, which + * is what we want. Assign it to @sch and enable. + */ + scx_set_task_sched(p, sch); + scx_enable_task(sch, p); + + p->scx.flags &= ~SCX_TASK_SUB_INIT; + } + } + scx_task_iter_stop(&sti); + + scx_enabling_sub_sched = NULL; + + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); + + scx_bypass(sch, false); + + pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name); + kobject_uevent(&sch->kobj, KOBJ_ADD); + ret = 0; + goto out_unlock; + +out_put_cgrp: + cgroup_put(cgrp); +out_unlock: + mutex_unlock(&scx_enable_mutex); + cmd->ret = ret; + return; + +abort: + put_task_struct(p); + scx_task_iter_stop(&sti); + + /* + * Undo __scx_init_task() for tasks we marked. scx_enable_task() never + * ran for @sch on them, so calling scx_disable_task() here would invoke + * ops.disable() without a matching ops.enable(). scx_enabling_sub_sched + * must stay set until SUB_INIT is cleared from every marked task - + * scx_disable_and_exit_task() reads it when a task exits concurrently. + */ + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + if (p->scx.flags & SCX_TASK_SUB_INIT) { + scx_sub_init_cancel_task(sch, p); + p->scx.flags &= ~SCX_TASK_SUB_INIT; + } + } + scx_task_iter_stop(&sti); + scx_enabling_sub_sched = NULL; +err_unlock_and_disable: + /* we'll soon enter disable path, keep bypass on */ + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); +err_disable: + mutex_unlock(&scx_enable_mutex); + scx_flush_disable_work(sch); + cmd->ret = 0; +} + +static s32 scx_cgroup_lifetime_notify(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct cgroup *cgrp = data; + struct cgroup *parent = cgroup_parent(cgrp); + + if (!cgroup_on_dfl(cgrp)) + return NOTIFY_OK; + + switch (action) { + case CGROUP_LIFETIME_ONLINE: + /* inherit ->scx_sched from $parent */ + if (parent) + rcu_assign_pointer(cgrp->scx_sched, parent->scx_sched); + break; + case CGROUP_LIFETIME_OFFLINE: + /* if there is a sched attached, shoot it down */ + if (cgrp->scx_sched && cgrp->scx_sched->cgrp == cgrp) + scx_exit(cgrp->scx_sched, SCX_EXIT_UNREG_KERN, + SCX_ECODE_RSN_CGROUP_OFFLINE, + "cgroup %llu going offline", cgroup_id(cgrp)); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block scx_cgroup_lifetime_nb = { + .notifier_call = scx_cgroup_lifetime_notify, +}; + +static s32 __init scx_cgroup_lifetime_notifier_init(void) +{ + return blocking_notifier_chain_register(&cgroup_lifetime_notifier, + &scx_cgroup_lifetime_nb); +} +core_initcall(scx_cgroup_lifetime_notifier_init); +#endif /* CONFIG_EXT_SUB_SCHED */ + +static s32 scx_enable(struct scx_enable_cmd *cmd, struct bpf_link *link) +{ + static struct kthread_worker *helper; + static DEFINE_MUTEX(helper_mutex); + + if (housekeeping_enabled(HK_TYPE_DOMAIN_BOOT)) { + pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation\n"); + return -EINVAL; + } + + if (!READ_ONCE(helper)) { + mutex_lock(&helper_mutex); + if (!helper) { + struct kthread_worker *w = + kthread_run_worker(0, "scx_enable_helper"); + if (IS_ERR_OR_NULL(w)) { + mutex_unlock(&helper_mutex); + return -ENOMEM; + } + sched_set_fifo(w->task); + WRITE_ONCE(helper, w); + } + mutex_unlock(&helper_mutex); + } + +#ifdef CONFIG_EXT_SUB_SCHED + if (cmd->ops->sub_cgroup_id > 1) + kthread_init_work(&cmd->work, scx_sub_enable_workfn); + else +#endif /* CONFIG_EXT_SUB_SCHED */ + kthread_init_work(&cmd->work, scx_root_enable_workfn); + + kthread_queue_work(READ_ONCE(helper), &cmd->work); + kthread_flush_work(&cmd->work); + return cmd->ret; +} + + +/******************************************************************************** + * bpf_struct_ops plumbing. + */ +#include +#include +#include + +static const struct btf_type *task_struct_type; + +static bool bpf_scx_is_valid_access(int off, int size, + enum bpf_access_type type, + const struct bpf_prog *prog, + struct bpf_insn_access_aux *info) +{ + if (type != BPF_READ) + return false; + if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) + return false; + if (off % size != 0) + return false; + + return btf_ctx_access(off, size, type, prog, info); +} + +static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, + const struct bpf_reg_state *reg, int off, + int size) +{ + const struct btf_type *t; + + t = btf_type_by_id(reg->btf, reg->btf_id); + if (t == task_struct_type) { + /* + * COMPAT: Will be removed in v6.23. + */ + if ((off >= offsetof(struct task_struct, scx.slice) && + off + size <= offsetofend(struct task_struct, scx.slice)) || + (off >= offsetof(struct task_struct, scx.dsq_vtime) && + off + size <= offsetofend(struct task_struct, scx.dsq_vtime))) { + pr_warn("sched_ext: Writing directly to p->scx.slice/dsq_vtime is deprecated, use scx_bpf_task_set_slice/dsq_vtime()"); + return SCALAR_VALUE; + } + + if (off >= offsetof(struct task_struct, scx.disallow) && + off + size <= offsetofend(struct task_struct, scx.disallow)) + return SCALAR_VALUE; + } + + return -EACCES; +} + +static const struct bpf_verifier_ops bpf_scx_verifier_ops = { + .get_func_proto = bpf_base_func_proto, + .is_valid_access = bpf_scx_is_valid_access, + .btf_struct_access = bpf_scx_btf_struct_access, +}; + +static int bpf_scx_init_member(const struct btf_type *t, + const struct btf_member *member, + void *kdata, const void *udata) +{ + const struct sched_ext_ops *uops = udata; + struct sched_ext_ops *ops = kdata; + u32 moff = __btf_member_bit_offset(t, member) / 8; + int ret; + + switch (moff) { + case offsetof(struct sched_ext_ops, dispatch_max_batch): + if (*(u32 *)(udata + moff) > INT_MAX) + return -E2BIG; + ops->dispatch_max_batch = *(u32 *)(udata + moff); + return 1; + case offsetof(struct sched_ext_ops, flags): + if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) + return -EINVAL; + ops->flags = *(u64 *)(udata + moff); + return 1; + case offsetof(struct sched_ext_ops, name): + ret = bpf_obj_name_cpy(ops->name, uops->name, + sizeof(ops->name)); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + return 1; + case offsetof(struct sched_ext_ops, timeout_ms): + if (msecs_to_jiffies(*(u32 *)(udata + moff)) > + SCX_WATCHDOG_MAX_TIMEOUT) + return -E2BIG; + ops->timeout_ms = *(u32 *)(udata + moff); + return 1; + case offsetof(struct sched_ext_ops, exit_dump_len): + ops->exit_dump_len = + *(u32 *)(udata + moff) ?: SCX_EXIT_DUMP_DFL_LEN; + return 1; + case offsetof(struct sched_ext_ops, hotplug_seq): + ops->hotplug_seq = *(u64 *)(udata + moff); + return 1; +#ifdef CONFIG_EXT_SUB_SCHED + case offsetof(struct sched_ext_ops, sub_cgroup_id): + ops->sub_cgroup_id = *(u64 *)(udata + moff); + return 1; +#endif /* CONFIG_EXT_SUB_SCHED */ + } + + return 0; +} + +#ifdef CONFIG_EXT_SUB_SCHED +static void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog) +{ + struct scx_sched *sch; + + guard(rcu)(); + sch = scx_prog_sched(prog->aux); + if (unlikely(!sch)) + return; + + scx_error(sch, "dispatch recursion detected"); +} +#endif /* CONFIG_EXT_SUB_SCHED */ + +static int bpf_scx_check_member(const struct btf_type *t, + const struct btf_member *member, + const struct bpf_prog *prog) +{ + u32 moff = __btf_member_bit_offset(t, member) / 8; + + switch (moff) { + case offsetof(struct sched_ext_ops, init_task): +#ifdef CONFIG_EXT_GROUP_SCHED + case offsetof(struct sched_ext_ops, cgroup_init): + case offsetof(struct sched_ext_ops, cgroup_exit): + case offsetof(struct sched_ext_ops, cgroup_prep_move): +#endif + case offsetof(struct sched_ext_ops, cpu_online): + case offsetof(struct sched_ext_ops, cpu_offline): + case offsetof(struct sched_ext_ops, init): + case offsetof(struct sched_ext_ops, exit): + case offsetof(struct sched_ext_ops, sub_attach): + case offsetof(struct sched_ext_ops, sub_detach): + break; + default: + if (prog->sleepable) + return -EINVAL; + } + +#ifdef CONFIG_EXT_SUB_SCHED + /* + * Enable private stack for operations that can nest along the + * hierarchy. + * + * XXX - Ideally, we should only do this for scheds that allow + * sub-scheds and sub-scheds themselves but I don't know how to access + * struct_ops from here. + */ + switch (moff) { + case offsetof(struct sched_ext_ops, dispatch): + prog->aux->priv_stack_requested = true; + prog->aux->recursion_detected = scx_pstack_recursion_on_dispatch; + } +#endif /* CONFIG_EXT_SUB_SCHED */ + + return 0; +} + +static int bpf_scx_reg(void *kdata, struct bpf_link *link) +{ + struct scx_enable_cmd cmd = { .ops = kdata }; + + return scx_enable(&cmd, link); +} + +struct scx_arena_scan { + struct bpf_map *arena; + int err; +}; + +/* + * The verifier enforces one arena per BPF program, so each struct_ops + * member prog contributes at most one arena via bpf_prog_arena(). + * Require all non-NULL contributions to match. + */ +static int scx_arena_scan_prog(struct bpf_prog *prog, void *data) +{ + struct scx_arena_scan *s = data; + struct bpf_map *arena = NULL; + + /* arena.o, which defines these, is built only on MMU && 64BIT */ +#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) + arena = bpf_prog_arena(prog); +#endif + if (!arena) + return 0; + if (s->arena && s->arena != arena) { + s->err = -EINVAL; + return 1; + } + s->arena = arena; + return 0; +} + +static int bpf_scx_reg_cid(void *kdata, struct bpf_link *link) +{ + struct scx_enable_cmd cmd = { .ops_cid = kdata, .is_cid_type = true }; + struct scx_arena_scan scan = {}; + int ret; + + bpf_struct_ops_for_each_prog(kdata, scx_arena_scan_prog, &scan); + if (scan.err) { + pr_err("sched_ext: cid-form scheduler uses multiple arena maps\n"); + return scan.err; + } + if (!scan.arena) { + pr_err("sched_ext: cid-form scheduler must use a BPF arena map\n"); + return -EINVAL; + } + + bpf_map_inc(scan.arena); + cmd.arena_map = scan.arena; + ret = scx_enable(&cmd, link); + if (cmd.arena_map) /* not consumed by scx_alloc_and_add_sched() */ + bpf_map_put(cmd.arena_map); + return ret; +} + +static void bpf_scx_unreg(void *kdata, struct bpf_link *link) +{ + struct sched_ext_ops *ops = kdata; + struct scx_sched *sch = rcu_dereference_protected(ops->priv, true); + + scx_disable(sch, SCX_EXIT_UNREG); + scx_flush_disable_work(sch); + RCU_INIT_POINTER(ops->priv, NULL); + kobject_put(&sch->kobj); +} + +static int bpf_scx_init(struct btf *btf) +{ + task_struct_type = btf_type_by_id(btf, btf_tracing_ids[BTF_TRACING_TYPE_TASK]); + + return 0; +} + +static int bpf_scx_update(void *kdata, void *old_kdata, struct bpf_link *link) +{ + /* + * sched_ext does not support updating the actively-loaded BPF + * scheduler, as registering a BPF scheduler can always fail if the + * scheduler returns an error code for e.g. ops.init(), ops.init_task(), + * etc. Similarly, we can always race with unregistration happening + * elsewhere, such as with sysrq. + */ + return -EOPNOTSUPP; +} + +static int bpf_scx_validate(void *kdata) +{ + return 0; +} + +static s32 sched_ext_ops__select_cpu(struct task_struct *p, s32 prev_cpu, u64 wake_flags) { return -EINVAL; } +static void sched_ext_ops__enqueue(struct task_struct *p, u64 enq_flags) {} +static void sched_ext_ops__dequeue(struct task_struct *p, u64 enq_flags) {} +static void sched_ext_ops__dispatch(s32 prev_cpu, struct task_struct *prev__nullable) {} +static void sched_ext_ops__tick(struct task_struct *p) {} +static void sched_ext_ops__runnable(struct task_struct *p, u64 enq_flags) {} +static void sched_ext_ops__running(struct task_struct *p) {} +static void sched_ext_ops__stopping(struct task_struct *p, bool runnable) {} +static void sched_ext_ops__quiescent(struct task_struct *p, u64 deq_flags) {} +static bool sched_ext_ops__yield(struct task_struct *from, struct task_struct *to__nullable) { return false; } +static bool sched_ext_ops__core_sched_before(struct task_struct *a, struct task_struct *b) { return false; } +static void sched_ext_ops__set_weight(struct task_struct *p, u32 weight) {} +static void sched_ext_ops__set_cpumask(struct task_struct *p, const struct cpumask *mask) {} +static void sched_ext_ops__update_idle(s32 cpu, bool idle) {} +static void sched_ext_ops__cpu_acquire(s32 cpu, struct scx_cpu_acquire_args *args) {} +static void sched_ext_ops__cpu_release(s32 cpu, struct scx_cpu_release_args *args) {} +static s32 sched_ext_ops__init_task(struct task_struct *p, struct scx_init_task_args *args) { return -EINVAL; } +static void sched_ext_ops__exit_task(struct task_struct *p, struct scx_exit_task_args *args) {} +static void sched_ext_ops__enable(struct task_struct *p) {} +static void sched_ext_ops__disable(struct task_struct *p) {} +#ifdef CONFIG_EXT_GROUP_SCHED +static s32 sched_ext_ops__cgroup_init(struct cgroup *cgrp, struct scx_cgroup_init_args *args) { return -EINVAL; } +static void sched_ext_ops__cgroup_exit(struct cgroup *cgrp) {} +static s32 sched_ext_ops__cgroup_prep_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) { return -EINVAL; } +static void sched_ext_ops__cgroup_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {} +static void sched_ext_ops__cgroup_cancel_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {} +static void sched_ext_ops__cgroup_set_weight(struct cgroup *cgrp, u32 weight) {} +static void sched_ext_ops__cgroup_set_bandwidth(struct cgroup *cgrp, u64 period_us, u64 quota_us, u64 burst_us) {} +static void sched_ext_ops__cgroup_set_idle(struct cgroup *cgrp, bool idle) {} +#endif /* CONFIG_EXT_GROUP_SCHED */ +static s32 sched_ext_ops__sub_attach(struct scx_sub_attach_args *args) { return -EINVAL; } +static void sched_ext_ops__sub_detach(struct scx_sub_detach_args *args) {} +static void sched_ext_ops__cpu_online(s32 cpu) {} +static void sched_ext_ops__cpu_offline(s32 cpu) {} +static s32 sched_ext_ops__init(void) { return -EINVAL; } +static void sched_ext_ops__exit(struct scx_exit_info *info) {} +static void sched_ext_ops__dump(struct scx_dump_ctx *ctx) {} +static void sched_ext_ops__dump_cpu(struct scx_dump_ctx *ctx, s32 cpu, bool idle) {} +static void sched_ext_ops__dump_task(struct scx_dump_ctx *ctx, struct task_struct *p) {} + +static struct sched_ext_ops __bpf_ops_sched_ext_ops = { + .select_cpu = sched_ext_ops__select_cpu, + .enqueue = sched_ext_ops__enqueue, + .dequeue = sched_ext_ops__dequeue, + .dispatch = sched_ext_ops__dispatch, + .tick = sched_ext_ops__tick, + .runnable = sched_ext_ops__runnable, + .running = sched_ext_ops__running, + .stopping = sched_ext_ops__stopping, + .quiescent = sched_ext_ops__quiescent, + .yield = sched_ext_ops__yield, + .core_sched_before = sched_ext_ops__core_sched_before, + .set_weight = sched_ext_ops__set_weight, + .set_cpumask = sched_ext_ops__set_cpumask, + .update_idle = sched_ext_ops__update_idle, + .cpu_acquire = sched_ext_ops__cpu_acquire, + .cpu_release = sched_ext_ops__cpu_release, + .init_task = sched_ext_ops__init_task, + .exit_task = sched_ext_ops__exit_task, + .enable = sched_ext_ops__enable, + .disable = sched_ext_ops__disable, +#ifdef CONFIG_EXT_GROUP_SCHED + .cgroup_init = sched_ext_ops__cgroup_init, + .cgroup_exit = sched_ext_ops__cgroup_exit, + .cgroup_prep_move = sched_ext_ops__cgroup_prep_move, + .cgroup_move = sched_ext_ops__cgroup_move, + .cgroup_cancel_move = sched_ext_ops__cgroup_cancel_move, + .cgroup_set_weight = sched_ext_ops__cgroup_set_weight, + .cgroup_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth, + .cgroup_set_idle = sched_ext_ops__cgroup_set_idle, +#endif + .sub_attach = sched_ext_ops__sub_attach, + .sub_detach = sched_ext_ops__sub_detach, + .cpu_online = sched_ext_ops__cpu_online, + .cpu_offline = sched_ext_ops__cpu_offline, + .init = sched_ext_ops__init, + .exit = sched_ext_ops__exit, + .dump = sched_ext_ops__dump, + .dump_cpu = sched_ext_ops__dump_cpu, + .dump_task = sched_ext_ops__dump_task, +}; + +static struct bpf_struct_ops bpf_sched_ext_ops = { + .verifier_ops = &bpf_scx_verifier_ops, + .reg = bpf_scx_reg, + .unreg = bpf_scx_unreg, + .check_member = bpf_scx_check_member, + .init_member = bpf_scx_init_member, + .init = bpf_scx_init, + .update = bpf_scx_update, + .validate = bpf_scx_validate, + .name = "sched_ext_ops", + .owner = THIS_MODULE, + .cfi_stubs = &__bpf_ops_sched_ext_ops +}; + +/* + * cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types + * identical, only param names differ across structs) are reused; only + * set_cmask needs a fresh stub since the second argument type differs. + */ +static void sched_ext_ops_cid__set_cmask(struct task_struct *p, + const struct scx_cmask *cmask) {} + +static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { + .select_cid = sched_ext_ops__select_cpu, + .enqueue = sched_ext_ops__enqueue, + .dequeue = sched_ext_ops__dequeue, + .dispatch = sched_ext_ops__dispatch, + .tick = sched_ext_ops__tick, + .runnable = sched_ext_ops__runnable, + .running = sched_ext_ops__running, + .stopping = sched_ext_ops__stopping, + .quiescent = sched_ext_ops__quiescent, + .yield = sched_ext_ops__yield, + .core_sched_before = sched_ext_ops__core_sched_before, + .set_weight = sched_ext_ops__set_weight, + .set_cmask = sched_ext_ops_cid__set_cmask, + .update_idle = sched_ext_ops__update_idle, + .init_task = sched_ext_ops__init_task, + .exit_task = sched_ext_ops__exit_task, + .enable = sched_ext_ops__enable, + .disable = sched_ext_ops__disable, +#ifdef CONFIG_EXT_GROUP_SCHED + .cgroup_init = sched_ext_ops__cgroup_init, + .cgroup_exit = sched_ext_ops__cgroup_exit, + .cgroup_prep_move = sched_ext_ops__cgroup_prep_move, + .cgroup_move = sched_ext_ops__cgroup_move, + .cgroup_cancel_move = sched_ext_ops__cgroup_cancel_move, + .cgroup_set_weight = sched_ext_ops__cgroup_set_weight, + .cgroup_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth, + .cgroup_set_idle = sched_ext_ops__cgroup_set_idle, +#endif + .sub_attach = sched_ext_ops__sub_attach, + .sub_detach = sched_ext_ops__sub_detach, + .cid_online = sched_ext_ops__cpu_online, + .cid_offline = sched_ext_ops__cpu_offline, + .init = sched_ext_ops__init, + .exit = sched_ext_ops__exit, + .dump = sched_ext_ops__dump, + .dump_cid = sched_ext_ops__dump_cpu, + .dump_task = sched_ext_ops__dump_task, +}; + +/* + * The cid-form struct_ops shares all bpf_struct_ops hooks with the cpu form. + * init_member, check_member, reg, unreg, etc. process kdata as the byte block + * verified to match by the BUILD_BUG_ON checks in scx_init(). + */ +static struct bpf_struct_ops bpf_sched_ext_ops_cid = { + .verifier_ops = &bpf_scx_verifier_ops, + .reg = bpf_scx_reg_cid, + .unreg = bpf_scx_unreg, + .check_member = bpf_scx_check_member, + .init_member = bpf_scx_init_member, + .init = bpf_scx_init, + .update = bpf_scx_update, + .validate = bpf_scx_validate, + .name = "sched_ext_ops_cid", + .owner = THIS_MODULE, + .cfi_stubs = &__bpf_ops_sched_ext_ops_cid +}; + + +/******************************************************************************** + * System integration and init. + */ + +static void sysrq_handle_sched_ext_reset(u8 key) +{ + struct scx_sched *sch; + + sch = rcu_dereference(scx_root); + if (likely(sch)) + scx_disable(sch, SCX_EXIT_SYSRQ); + else + pr_info("sched_ext: BPF schedulers not loaded\n"); +} + +static const struct sysrq_key_op sysrq_sched_ext_reset_op = { + .handler = sysrq_handle_sched_ext_reset, + .help_msg = "reset-sched-ext(S)", + .action_msg = "Disable sched_ext and revert all tasks to CFS", + .enable_mask = SYSRQ_ENABLE_RTNICE, +}; + +static void sysrq_handle_sched_ext_dump(u8 key) +{ + struct scx_exit_info ei = { + .kind = SCX_EXIT_NONE, + .exit_cpu = -1, + .reason = "SysRq-D", + }; + struct scx_sched *sch; + + list_for_each_entry_rcu(sch, &scx_sched_all, all) + scx_dump_state(sch, &ei, 0, false); +} + +static const struct sysrq_key_op sysrq_sched_ext_dump_op = { + .handler = sysrq_handle_sched_ext_dump, + .help_msg = "dump-sched-ext(D)", + .action_msg = "Trigger sched_ext debug dump", + .enable_mask = SYSRQ_ENABLE_RTNICE, +}; + +static bool can_skip_idle_kick(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + + /* + * We can skip idle kicking if @rq is going to go through at least one + * full SCX scheduling cycle before going idle. Just checking whether + * curr is not idle is insufficient because we could be racing + * balance_one() trying to pull the next task from a remote rq, which + * may fail, and @rq may become idle afterwards. + * + * The race window is small and we don't and can't guarantee that @rq is + * only kicked while idle anyway. Skip only when sure. + */ + return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_BALANCE); +} + +static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *ksyncs) +{ + struct rq *rq = cpu_rq(cpu); + struct scx_rq *this_scx = &this_rq->scx; + const struct sched_class *cur_class; + bool should_wait = false; + unsigned long flags; + + raw_spin_rq_lock_irqsave(rq, flags); + cur_class = rq->curr->sched_class; + + /* + * During CPU hotplug, a CPU may depend on kicking itself to make + * forward progress. Allow kicking self regardless of online state. If + * @cpu is running a higher class task, we have no control over @cpu. + * Skip kicking. + */ + if ((cpu_online(cpu) || cpu == cpu_of(this_rq)) && + !sched_class_above(cur_class, &ext_sched_class)) { + if (cpumask_test_cpu(cpu, this_scx->cpus_to_preempt)) { + if (cur_class == &ext_sched_class) + rq->curr->scx.slice = 0; + cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); + } + + if (cpumask_test_cpu(cpu, this_scx->cpus_to_wait)) { + if (cur_class == &ext_sched_class) { + cpumask_set_cpu(cpu, this_scx->cpus_to_sync); + ksyncs[cpu] = rq->scx.kick_sync; + should_wait = true; + } + cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); + } + + resched_curr(rq); + } else { + cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); + cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); + } + + raw_spin_rq_unlock_irqrestore(rq, flags); + + return should_wait; +} + +static void kick_one_cpu_if_idle(s32 cpu, struct rq *this_rq) +{ + struct rq *rq = cpu_rq(cpu); + unsigned long flags; + + raw_spin_rq_lock_irqsave(rq, flags); + + if (!can_skip_idle_kick(rq) && + (cpu_online(cpu) || cpu == cpu_of(this_rq))) + resched_curr(rq); + + raw_spin_rq_unlock_irqrestore(rq, flags); +} + +static void kick_cpus_irq_workfn(struct irq_work *irq_work) +{ + struct rq *this_rq = this_rq(); + struct scx_rq *this_scx = &this_rq->scx; + struct scx_kick_syncs __rcu *ksyncs_pcpu = __this_cpu_read(scx_kick_syncs); + bool should_wait = false; + unsigned long *ksyncs; + s32 cpu; + + /* can race with free_kick_syncs() during scheduler disable */ + if (unlikely(!ksyncs_pcpu)) + return; + + ksyncs = rcu_dereference_bh(ksyncs_pcpu)->syncs; + + for_each_cpu(cpu, this_scx->cpus_to_kick) { + should_wait |= kick_one_cpu(cpu, this_rq, ksyncs); + cpumask_clear_cpu(cpu, this_scx->cpus_to_kick); + cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); + } + + for_each_cpu(cpu, this_scx->cpus_to_kick_if_idle) { + kick_one_cpu_if_idle(cpu, this_rq); + cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); + } + + /* + * Can't wait in hardirq — kick_sync can't advance, deadlocking if + * CPUs wait for each other. Defer to kick_sync_wait_bal_cb(). + */ + if (should_wait) { + raw_spin_rq_lock(this_rq); + this_scx->kick_sync_pending = true; + resched_curr(this_rq); + raw_spin_rq_unlock(this_rq); + } +} + +/** + * print_scx_info - print out sched_ext scheduler state + * @log_lvl: the log level to use when printing + * @p: target task + * + * If a sched_ext scheduler is enabled, print the name and state of the + * scheduler. If @p is on sched_ext, print further information about the task. + * + * This function can be safely called on any task as long as the task_struct + * itself is accessible. While safe, this function isn't synchronized and may + * print out mixups or garbages of limited length. + */ +void print_scx_info(const char *log_lvl, struct task_struct *p) +{ + struct scx_sched *sch; + enum scx_enable_state state = scx_enable_state(); + const char *all = READ_ONCE(scx_switching_all) ? "+all" : ""; + char runnable_at_buf[22] = "?"; + struct sched_class *class; + unsigned long runnable_at; + + guard(rcu)(); + + sch = scx_task_sched_rcu(p); + + if (!sch) + return; + + /* + * Carefully check if the task was running on sched_ext, and then + * carefully copy the time it's been runnable, and its state. + */ + if (copy_from_kernel_nofault(&class, &p->sched_class, sizeof(class)) || + class != &ext_sched_class) { + printk("%sSched_ext: %s (%s%s)", log_lvl, sch->ops.name, + scx_enable_state_str[state], all); + return; + } + + if (!copy_from_kernel_nofault(&runnable_at, &p->scx.runnable_at, + sizeof(runnable_at))) + scnprintf(runnable_at_buf, sizeof(runnable_at_buf), "%+ldms", + jiffies_delta_msecs(runnable_at, jiffies)); + + /* print everything onto one line to conserve console space */ + printk("%sSched_ext: %s (%s%s), task: runnable_at=%s", + log_lvl, sch->ops.name, scx_enable_state_str[state], all, + runnable_at_buf); +} + +static int scx_pm_handler(struct notifier_block *nb, unsigned long event, void *ptr) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = rcu_dereference(scx_root); + if (!sch) + return NOTIFY_OK; + + /* + * SCX schedulers often have userspace components which are sometimes + * involved in critial scheduling paths. PM operations involve freezing + * userspace which can lead to scheduling misbehaviors including stalls. + * Let's bypass while PM operations are in progress. + */ + switch (event) { + case PM_HIBERNATION_PREPARE: + case PM_SUSPEND_PREPARE: + case PM_RESTORE_PREPARE: + scx_bypass(sch, true); + break; + case PM_POST_HIBERNATION: + case PM_POST_SUSPEND: + case PM_POST_RESTORE: + scx_bypass(sch, false); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block scx_pm_notifier = { + .notifier_call = scx_pm_handler, +}; + +void __init init_sched_ext_class(void) +{ + s32 cpu, v; + + /* + * The following is to prevent the compiler from optimizing out the enum + * definitions so that BPF scheduler implementations can use them + * through the generated vmlinux.h. + */ + WRITE_ONCE(v, SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | SCX_KICK_PREEMPT | + SCX_TG_ONLINE); + + scx_idle_init_masks(); + + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + int n = cpu_to_node(cpu); + + /* local_dsq's sch will be set during scx_root_enable() */ + BUG_ON(init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL)); + + INIT_LIST_HEAD(&rq->scx.runnable_list); + INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals); + + BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick, GFP_KERNEL, n)); + BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick_if_idle, GFP_KERNEL, n)); + BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_preempt, GFP_KERNEL, n)); + BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_wait, GFP_KERNEL, n)); + BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_sync, GFP_KERNEL, n)); + raw_spin_lock_init(&rq->scx.deferred_reenq_lock); + INIT_LIST_HEAD(&rq->scx.deferred_reenq_locals); + INIT_LIST_HEAD(&rq->scx.deferred_reenq_users); + rq->scx.deferred_irq_work = IRQ_WORK_INIT_HARD(deferred_irq_workfn); + rq->scx.kick_cpus_irq_work = IRQ_WORK_INIT_HARD(kick_cpus_irq_workfn); + + if (cpu_online(cpu)) + cpu_rq(cpu)->scx.flags |= SCX_RQ_ONLINE; + } + + register_sysrq_key('S', &sysrq_sched_ext_reset_op); + register_sysrq_key('D', &sysrq_sched_ext_dump_op); + INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); + +#ifdef CONFIG_EXT_SUB_SCHED + BUG_ON(rhashtable_init(&scx_sched_hash, &scx_sched_hash_params)); +#endif /* CONFIG_EXT_SUB_SCHED */ +} + + +/******************************************************************************** + * Helpers that can be called from the BPF scheduler. + */ +static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags) +{ + bool is_local = dsq_id == SCX_DSQ_LOCAL || + (dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON; + + if (*enq_flags & SCX_ENQ_IMMED) { + if (unlikely(!is_local)) { + scx_error(sch, "SCX_ENQ_IMMED on a non-local DSQ 0x%llx", dsq_id); + return false; + } + } else if ((sch->ops.flags & SCX_OPS_ALWAYS_ENQ_IMMED) && is_local) { + *enq_flags |= SCX_ENQ_IMMED; + } + + return true; +} + +static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p, + u64 dsq_id, u64 *enq_flags) +{ + lockdep_assert_irqs_disabled(); + + if (unlikely(!p)) { + scx_error(sch, "called with NULL task"); + return false; + } + + if (unlikely(*enq_flags & __SCX_ENQ_INTERNAL_MASK)) { + scx_error(sch, "invalid enq_flags 0x%llx", *enq_flags); + return false; + } + + /* see SCX_EV_INSERT_NOT_OWNED definition */ + if (unlikely(!scx_task_on_sched(sch, p))) { + __scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1); + return false; + } + + if (!scx_vet_enq_flags(sch, dsq_id, enq_flags)) + return false; + + return true; +} + +static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p, + u64 dsq_id, u64 enq_flags) +{ + struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; + struct task_struct *ddsp_task; + + ddsp_task = __this_cpu_read(direct_dispatch_task); + if (ddsp_task) { + mark_direct_dispatch(sch, ddsp_task, p, dsq_id, enq_flags); + return; + } + + if (unlikely(dspc->cursor >= sch->dsp_max_batch)) { + scx_error(sch, "dispatch buffer overflow"); + return; + } + + dspc->buf[dspc->cursor++] = (struct scx_dsp_buf_ent){ + .task = p, + .qseq = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK, + .dsq_id = dsq_id, + .enq_flags = enq_flags, + }; +} + +__bpf_kfunc_start_defs(); + +/** + * scx_bpf_dsq_insert - Insert a task into the FIFO queue of a DSQ + * @p: task_struct to insert + * @dsq_id: DSQ to insert into + * @slice: duration @p can run for in nsecs, 0 to keep the current value + * @enq_flags: SCX_ENQ_* + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Insert @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe to + * call this function spuriously. Can be called from ops.enqueue(), + * ops.select_cpu(), and ops.dispatch(). + * + * When called from ops.select_cpu() or ops.enqueue(), it's for direct dispatch + * and @p must match the task being enqueued. + * + * When called from ops.select_cpu(), @enq_flags and @dsp_id are stored, and @p + * will be directly inserted into the corresponding dispatch queue after + * ops.select_cpu() returns. If @p is inserted into SCX_DSQ_LOCAL, it will be + * inserted into the local DSQ of the CPU returned by ops.select_cpu(). + * @enq_flags are OR'd with the enqueue flags on the enqueue path before the + * task is inserted. + * + * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id + * and this function can be called upto ops.dispatch_max_batch times to insert + * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the + * remaining slots. scx_bpf_dsq_move_to_local() flushes the batch and resets the + * counter. + * + * This function doesn't have any locking restrictions and may be called under + * BPF locks (in the future when BPF introduces more flexible locking). + * + * @p is allowed to run for @slice. The scheduling path is triggered on slice + * exhaustion. If zero, the current residual slice is maintained. If + * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with + * scx_bpf_kick_cpu() to trigger scheduling. + * + * Returns %true on successful insertion, %false on failure. On the root + * scheduler, %false return triggers scheduler abort and the caller doesn't need + * to check the return value. + */ +__bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id, + u64 slice, u64 enq_flags, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return false; + + if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) + return false; + + if (slice) + p->scx.slice = slice; + else + p->scx.slice = p->scx.slice ?: 1; + + scx_dsq_insert_commit(sch, p, dsq_id, enq_flags); + + return true; +} + +/* + * COMPAT: Will be removed in v6.23 along with the ___v2 suffix. + */ +__bpf_kfunc void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, + u64 slice, u64 enq_flags, + const struct bpf_prog_aux *aux) +{ + scx_bpf_dsq_insert___v2(p, dsq_id, slice, enq_flags, aux); +} + +static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p, + u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) +{ + if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) + return false; + + if (slice) + p->scx.slice = slice; + else + p->scx.slice = p->scx.slice ?: 1; + + p->scx.dsq_vtime = vtime; + + scx_dsq_insert_commit(sch, p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); + + return true; +} + +struct scx_bpf_dsq_insert_vtime_args { + /* @p can't be packed together as KF_RCU is not transitive */ + u64 dsq_id; + u64 slice; + u64 vtime; + u64 enq_flags; +}; + +/** + * __scx_bpf_dsq_insert_vtime - Arg-wrapped vtime DSQ insertion + * @p: task_struct to insert + * @args: struct containing the rest of the arguments + * @args->dsq_id: DSQ to insert into + * @args->slice: duration @p can run for in nsecs, 0 to keep the current value + * @args->vtime: @p's ordering inside the vtime-sorted queue of the target DSQ + * @args->enq_flags: SCX_ENQ_* + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Wrapper kfunc that takes arguments via struct to work around BPF's 5 argument + * limit. BPF programs should use scx_bpf_dsq_insert_vtime() which is provided + * as an inline wrapper in common.bpf.h. + * + * Insert @p into the vtime priority queue of the DSQ identified by + * @args->dsq_id. Tasks queued into the priority queue are ordered by + * @args->vtime. All other aspects are identical to scx_bpf_dsq_insert(). + * + * @args->vtime ordering is according to time_before64() which considers + * wrapping. A numerically larger vtime may indicate an earlier position in the + * ordering and vice-versa. + * + * A DSQ can only be used as a FIFO or priority queue at any given time and this + * function must not be called on a DSQ which already has one or more FIFO tasks + * queued and vice-versa. Also, the built-in DSQs (SCX_DSQ_LOCAL and + * SCX_DSQ_GLOBAL) cannot be used as priority queues. + * + * Returns %true on successful insertion, %false on failure. On the root + * scheduler, %false return triggers scheduler abort and the caller doesn't need + * to check the return value. + */ +__bpf_kfunc bool +__scx_bpf_dsq_insert_vtime(struct task_struct *p, + struct scx_bpf_dsq_insert_vtime_args *args, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return false; + + return scx_dsq_insert_vtime(sch, p, args->dsq_id, args->slice, + args->vtime, args->enq_flags); +} + +/* + * COMPAT: Will be removed in v6.23. + */ +__bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, + u64 slice, u64 vtime, u64 enq_flags) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = rcu_dereference(scx_root); + if (unlikely(!sch)) + return; + +#ifdef CONFIG_EXT_SUB_SCHED + /* + * Disallow if any sub-scheds are attached. There is no way to tell + * which scheduler called us, just error out @p's scheduler. + */ + if (unlikely(!list_empty(&sch->children))) { + scx_error(scx_task_sched(p), "__scx_bpf_dsq_insert_vtime() must be used"); + return; + } +#endif + + scx_dsq_insert_vtime(sch, p, dsq_id, slice, vtime, enq_flags); +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(scx_kfunc_ids_enqueue_dispatch) +BTF_ID_FLAGS(func, scx_bpf_dsq_insert, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_insert___v2, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, __scx_bpf_dsq_insert_vtime, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_insert_vtime, KF_RCU) +BTF_KFUNCS_END(scx_kfunc_ids_enqueue_dispatch) + +static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_enqueue_dispatch, + .filter = scx_kfunc_context_filter, +}; + +static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, + struct task_struct *p, u64 dsq_id, u64 enq_flags) +{ + struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq; + struct scx_sched *sch; + struct rq *this_rq, *src_rq, *locked_rq; + bool dispatched = false; + bool in_balance; + unsigned long flags; + + /* + * The verifier considers an iterator slot initialized on any + * KF_ITER_NEW return, so a BPF program may legally reach here after + * bpf_iter_scx_dsq_new() failed and left @kit->dsq NULL. + */ + if (unlikely(!src_dsq)) + return false; + + sch = src_dsq->sched; + + if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags)) + return false; + + /* + * If the BPF scheduler keeps calling this function repeatedly, it can + * cause similar live-lock conditions as consume_dispatch_q(). + */ + if (unlikely(READ_ONCE(sch->aborting))) + return false; + + if (unlikely(!scx_task_on_sched(sch, p))) { + scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler", + p->comm, p->pid); + return false; + } + + /* + * Can be called from either ops.dispatch() locking this_rq() or any + * context where no rq lock is held. If latter, lock @p's task_rq which + * we'll likely need anyway. + */ + src_rq = task_rq(p); + + local_irq_save(flags); + this_rq = this_rq(); + in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE; + + if (in_balance) { + if (this_rq != src_rq) { + raw_spin_rq_unlock(this_rq); + raw_spin_rq_lock(src_rq); + } + } else { + raw_spin_rq_lock(src_rq); + } + + locked_rq = src_rq; + raw_spin_lock(&src_dsq->lock); + + /* did someone else get to it while we dropped the locks? */ + if (nldsq_cursor_lost_task(&kit->cursor, src_rq, src_dsq, p)) { + raw_spin_unlock(&src_dsq->lock); + goto out; + } + + /* @p is still on $src_dsq and stable, determine the destination */ + dst_dsq = find_dsq_for_dispatch(sch, this_rq, dsq_id, task_cpu(p)); + + /* + * Apply vtime and slice updates before moving so that the new time is + * visible before inserting into $dst_dsq. @p is still on $src_dsq but + * this is safe as we're locking it. + */ + if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME) + p->scx.dsq_vtime = kit->vtime; + if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE) + p->scx.slice = kit->slice; + + /* execute move */ + locked_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); + dispatched = true; +out: + if (in_balance) { + if (this_rq != locked_rq) { + raw_spin_rq_unlock(locked_rq); + raw_spin_rq_lock(this_rq); + } + } else { + raw_spin_rq_unlock_irqrestore(locked_rq, flags); + } + + kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE | + __SCX_DSQ_ITER_HAS_VTIME); + return dispatched; +} + +__bpf_kfunc_start_defs(); + +/** + * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Can only be called from ops.dispatch(). + */ +__bpf_kfunc u32 scx_bpf_dispatch_nr_slots(const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return 0; + + return sch->dsp_max_batch - __this_cpu_read(sch->pcpu->dsp_ctx.cursor); +} + +/** + * scx_bpf_dispatch_cancel - Cancel the latest dispatch + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Cancel the latest dispatch. Can be called multiple times to cancel further + * dispatches. Can only be called from ops.dispatch(). + */ +__bpf_kfunc void scx_bpf_dispatch_cancel(const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + struct scx_dsp_ctx *dspc; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return; + + dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; + + if (dspc->cursor > 0) + dspc->cursor--; + else + scx_error(sch, "dispatch buffer underflow"); +} + +/** + * scx_bpf_dsq_move_to_local - move a task from a DSQ to the current CPU's local DSQ + * @dsq_id: DSQ to move task from. Must be a user-created DSQ + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * @enq_flags: %SCX_ENQ_* + * + * Move a task from the non-local DSQ identified by @dsq_id to the current CPU's + * local DSQ for execution with @enq_flags applied. Can only be called from + * ops.dispatch(). + * + * Built-in DSQs (%SCX_DSQ_GLOBAL and %SCX_DSQ_LOCAL*) are not supported as + * sources. Local DSQs support reenqueueing (a task can be picked up for + * execution, dequeued for property changes, or reenqueued), but the BPF + * scheduler cannot directly iterate or move tasks from them. %SCX_DSQ_GLOBAL + * is similar but also doesn't support reenqueueing, as it maps to multiple + * per-node DSQs making the scope difficult to define; this may change in the + * future. + * + * This function flushes the in-flight dispatches from scx_bpf_dsq_insert() + * before trying to move from the specified DSQ. It may also grab rq locks and + * thus can't be called under any BPF locks. + * + * Returns %true if a task has been moved, %false if there isn't any task to + * move. + */ +__bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags, + const struct bpf_prog_aux *aux) +{ + struct scx_dispatch_q *dsq; + struct scx_sched *sch; + struct scx_dsp_ctx *dspc; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return false; + + if (!scx_vet_enq_flags(sch, SCX_DSQ_LOCAL, &enq_flags)) + return false; + + dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; + + flush_dispatch_buf(sch, dspc->rq); + + dsq = find_user_dsq(sch, dsq_id); + if (unlikely(!dsq)) { + scx_error(sch, "invalid DSQ ID 0x%016llx", dsq_id); + return false; + } + + if (consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) { + /* + * A successfully consumed task can be dequeued before it starts + * running while the CPU is trying to migrate other dispatched + * tasks. Bump nr_tasks to tell balance_one() to retry on empty + * local DSQ. + */ + dspc->nr_tasks++; + return true; + } else { + return false; + } +} + +/* + * COMPAT: ___v2 was introduced in v7.1. Remove this and ___v2 tag in the future. + */ +__bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id, const struct bpf_prog_aux *aux) +{ + return scx_bpf_dsq_move_to_local___v2(dsq_id, 0, aux); +} + +/** + * scx_bpf_dsq_move_set_slice - Override slice when moving between DSQs + * @it__iter: DSQ iterator in progress + * @slice: duration the moved task can run for in nsecs + * + * Override the slice of the next task that will be moved from @it__iter using + * scx_bpf_dsq_move[_vtime](). If this function is not called, the previous + * slice duration is kept. + */ +__bpf_kfunc void scx_bpf_dsq_move_set_slice(struct bpf_iter_scx_dsq *it__iter, + u64 slice) +{ + struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; + + kit->slice = slice; + kit->cursor.flags |= __SCX_DSQ_ITER_HAS_SLICE; +} + +/** + * scx_bpf_dsq_move_set_vtime - Override vtime when moving between DSQs + * @it__iter: DSQ iterator in progress + * @vtime: task's ordering inside the vtime-sorted queue of the target DSQ + * + * Override the vtime of the next task that will be moved from @it__iter using + * scx_bpf_dsq_move_vtime(). If this function is not called, the previous slice + * vtime is kept. If scx_bpf_dsq_move() is used to dispatch the next task, the + * override is ignored and cleared. + */ +__bpf_kfunc void scx_bpf_dsq_move_set_vtime(struct bpf_iter_scx_dsq *it__iter, + u64 vtime) +{ + struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; + + kit->vtime = vtime; + kit->cursor.flags |= __SCX_DSQ_ITER_HAS_VTIME; +} + +/** + * scx_bpf_dsq_move - Move a task from DSQ iteration to a DSQ + * @it__iter: DSQ iterator in progress + * @p: task to transfer + * @dsq_id: DSQ to move @p to + * @enq_flags: SCX_ENQ_* + * + * Transfer @p which is on the DSQ currently iterated by @it__iter to the DSQ + * specified by @dsq_id. All DSQs - local DSQs, global DSQ and user DSQs - can + * be the destination. + * + * For the transfer to be successful, @p must still be on the DSQ and have been + * queued before the DSQ iteration started. This function doesn't care whether + * @p was obtained from the DSQ iteration. @p just has to be on the DSQ and have + * been queued before the iteration started. + * + * @p's slice is kept by default. Use scx_bpf_dsq_move_set_slice() to update. + * + * Can be called from ops.dispatch() or any BPF context which doesn't hold a rq + * lock (e.g. BPF timers or SYSCALL programs). + * + * Returns %true if @p has been consumed, %false if @p had already been + * consumed, dequeued, or, for sub-scheds, @dsq_id points to a disallowed local + * DSQ. + */ +__bpf_kfunc bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter, + struct task_struct *p, u64 dsq_id, + u64 enq_flags) +{ + return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, + p, dsq_id, enq_flags); +} + +/** + * scx_bpf_dsq_move_vtime - Move a task from DSQ iteration to a PRIQ DSQ + * @it__iter: DSQ iterator in progress + * @p: task to transfer + * @dsq_id: DSQ to move @p to + * @enq_flags: SCX_ENQ_* + * + * Transfer @p which is on the DSQ currently iterated by @it__iter to the + * priority queue of the DSQ specified by @dsq_id. The destination must be a + * user DSQ as only user DSQs support priority queue. + * + * @p's slice and vtime are kept by default. Use scx_bpf_dsq_move_set_slice() + * and scx_bpf_dsq_move_set_vtime() to update. + * + * All other aspects are identical to scx_bpf_dsq_move(). See + * scx_bpf_dsq_insert_vtime() for more information on @vtime. + */ +__bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter, + struct task_struct *p, u64 dsq_id, + u64 enq_flags) +{ + return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, + p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); +} + +#ifdef CONFIG_EXT_SUB_SCHED +/** + * scx_bpf_sub_dispatch - Trigger dispatching on a child scheduler + * @cgroup_id: cgroup ID of the child scheduler to dispatch + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Allows a parent scheduler to trigger dispatching on one of its direct + * child schedulers. The child scheduler runs its dispatch operation to + * move tasks from dispatch queues to the local runqueue. + * + * Returns: true on success, false if cgroup_id is invalid, not a direct + * child, or caller lacks dispatch permission. + */ +__bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux) +{ + struct rq *this_rq = this_rq(); + struct scx_sched *parent, *child; + + guard(rcu)(); + parent = scx_prog_sched(aux); + if (unlikely(!parent)) + return false; + + child = scx_find_sub_sched(cgroup_id); + + if (unlikely(!child)) + return false; + + if (unlikely(scx_parent(child) != parent)) { + scx_error(parent, "trying to dispatch a distant sub-sched on cgroup %llu", + cgroup_id); + return false; + } + + return scx_dispatch_sched(child, this_rq, this_rq->scx.sub_dispatch_prev, + true); +} +#endif /* CONFIG_EXT_SUB_SCHED */ + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(scx_kfunc_ids_dispatch) +BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_dispatch_cancel, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local___v2, KF_IMPLICIT_ARGS) +/* scx_bpf_dsq_move*() also in scx_kfunc_ids_unlocked: callable from unlocked contexts */ +BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) +#ifdef CONFIG_EXT_SUB_SCHED +BTF_ID_FLAGS(func, scx_bpf_sub_dispatch, KF_IMPLICIT_ARGS) +#endif +BTF_KFUNCS_END(scx_kfunc_ids_dispatch) + +static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_dispatch, + .filter = scx_kfunc_context_filter, +}; + +__bpf_kfunc_start_defs(); + +/** + * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Iterate over all of the tasks currently enqueued on the local DSQ of the + * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of + * processed tasks. Can only be called from ops.cpu_release(). + */ +__bpf_kfunc u32 scx_bpf_reenqueue_local(const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + struct rq *rq; + + guard(rcu)(); + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return 0; + + rq = cpu_rq(smp_processor_id()); + lockdep_assert_rq_held(rq); + + return reenq_local(sch, rq, SCX_REENQ_ANY); +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(scx_kfunc_ids_cpu_release) +BTF_ID_FLAGS(func, scx_bpf_reenqueue_local, KF_IMPLICIT_ARGS) +BTF_KFUNCS_END(scx_kfunc_ids_cpu_release) + +static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_cpu_release, + .filter = scx_kfunc_context_filter, +}; + +__bpf_kfunc_start_defs(); + +/** + * scx_bpf_create_dsq - Create a custom DSQ + * @dsq_id: DSQ to create + * @node: NUMA node to allocate from + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Create a custom DSQ identified by @dsq_id. Can be called from any sleepable + * scx callback, and any BPF_PROG_TYPE_SYSCALL prog. + */ +__bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node, const struct bpf_prog_aux *aux) +{ + struct scx_dispatch_q *dsq; + struct scx_sched *sch; + s32 ret; + + if (unlikely(node >= (int)nr_node_ids || + (node < 0 && node != NUMA_NO_NODE))) + return -EINVAL; + + if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN)) + return -EINVAL; + + dsq = kmalloc_node(sizeof(*dsq), GFP_KERNEL, node); + if (!dsq) + return -ENOMEM; + + /* + * init_dsq() must be called in GFP_KERNEL context. Init it with NULL + * @sch and update afterwards. + */ + ret = init_dsq(dsq, dsq_id, NULL); + if (ret) { + kfree(dsq); + return ret; + } + + rcu_read_lock(); + + sch = scx_prog_sched(aux); + if (sch) { + dsq->sched = sch; + ret = rhashtable_lookup_insert_fast(&sch->dsq_hash, &dsq->hash_node, + dsq_hash_params); + } else { + ret = -ENODEV; + } + + rcu_read_unlock(); + if (ret) { + exit_dsq(dsq); + kfree(dsq); + } + return ret; +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(scx_kfunc_ids_unlocked) +BTF_ID_FLAGS(func, scx_bpf_create_dsq, KF_IMPLICIT_ARGS | KF_SLEEPABLE) +/* also in scx_kfunc_ids_dispatch: also callable from ops.dispatch() */ +BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) +/* also in scx_kfunc_ids_select_cpu: also callable from ops.select_cpu()/ops.enqueue() */ +BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU) +BTF_KFUNCS_END(scx_kfunc_ids_unlocked) + +static const struct btf_kfunc_id_set scx_kfunc_set_unlocked = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_unlocked, + .filter = scx_kfunc_context_filter, +}; + +__bpf_kfunc_start_defs(); + +/** + * scx_bpf_task_set_slice - Set task's time slice + * @p: task of interest + * @slice: time slice to set in nsecs + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Set @p's time slice to @slice. Returns %true on success, %false if the + * calling scheduler doesn't have authority over @p. + */ +__bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + sch = scx_prog_sched(aux); + if (unlikely(!sch || !scx_task_on_sched(sch, p))) + return false; + + p->scx.slice = slice; + return true; +} + +/** + * scx_bpf_task_set_dsq_vtime - Set task's virtual time for DSQ ordering + * @p: task of interest + * @vtime: virtual time to set + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Set @p's virtual time to @vtime. Returns %true on success, %false if the + * calling scheduler doesn't have authority over @p. + */ +__bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + sch = scx_prog_sched(aux); + if (unlikely(!sch || !scx_task_on_sched(sch, p))) + return false; + + p->scx.dsq_vtime = vtime; + return true; +} + +static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) +{ + struct rq *this_rq; + unsigned long irq_flags; + + local_irq_save(irq_flags); + + this_rq = this_rq(); + + /* + * While bypassing for PM ops, IRQ handling may not be online which can + * lead to irq_work_queue() malfunction such as infinite busy wait for + * IRQ status update. Suppress kicking. + */ + if (scx_bypassing(sch, cpu_of(this_rq))) + goto out; + + /* + * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting + * rq locks. We can probably be smarter and avoid bouncing if called + * from ops which don't hold a rq lock. + */ + if (flags & SCX_KICK_IDLE) { + struct rq *target_rq = cpu_rq(cpu); + + if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_WAIT))) + scx_error(sch, "PREEMPT/WAIT cannot be used with SCX_KICK_IDLE"); + + if (raw_spin_rq_trylock(target_rq)) { + if (can_skip_idle_kick(target_rq)) { + raw_spin_rq_unlock(target_rq); + goto out; + } + raw_spin_rq_unlock(target_rq); + } + cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick_if_idle); + } else { + cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick); + + if (flags & SCX_KICK_PREEMPT) + cpumask_set_cpu(cpu, this_rq->scx.cpus_to_preempt); + if (flags & SCX_KICK_WAIT) + cpumask_set_cpu(cpu, this_rq->scx.cpus_to_wait); + } + + irq_work_queue(&this_rq->scx.kick_cpus_irq_work); +out: + local_irq_restore(irq_flags); +} + +/** + * scx_bpf_kick_cpu - Trigger reschedule on a CPU + * @cpu: cpu to kick + * @flags: %SCX_KICK_* flags + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or + * trigger rescheduling on a busy CPU. This can be called from any online + * scx_ops operation and the actual kicking is performed asynchronously through + * an irq work. + */ +__bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + sch = scx_prog_sched(aux); + if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) + scx_kick_cpu(sch, cpu, flags); +} + +/** + * scx_bpf_kick_cid - Trigger reschedule on the CPU mapped to @cid + * @cid: cid to kick + * @flags: %SCX_KICK_* flags + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * cid-addressed equivalent of scx_bpf_kick_cpu(). Return 0 on success, + * -errno otherwise. + */ +__bpf_kfunc s32 scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -ENODEV; + cpu = scx_cid_to_cpu(sch, cid); + if (cpu < 0) + return cpu; + scx_kick_cpu(sch, cpu, flags); + return 0; +} + +/** + * scx_bpf_dsq_nr_queued - Return the number of queued tasks + * @dsq_id: id of the DSQ + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Return the number of tasks in the DSQ matching @dsq_id. If not found, + * -%ENOENT is returned. + */ +__bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + struct scx_dispatch_q *dsq; + s32 ret; + + preempt_disable(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) { + ret = -ENODEV; + goto out; + } + + if (dsq_id == SCX_DSQ_LOCAL) { + ret = READ_ONCE(this_rq()->scx.local_dsq.nr); + goto out; + } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { + s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK); + + if (scx_cpu_valid(sch, cpu, NULL)) { + ret = READ_ONCE(cpu_rq(cpu)->scx.local_dsq.nr); + goto out; + } + } else { + dsq = find_user_dsq(sch, dsq_id); + if (dsq) { + ret = READ_ONCE(dsq->nr); + goto out; + } + } + ret = -ENOENT; +out: + preempt_enable(); + return ret; +} + +/** + * scx_bpf_destroy_dsq - Destroy a custom DSQ + * @dsq_id: DSQ to destroy + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with + * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is + * empty and no further tasks are dispatched to it. Ignored if called on a DSQ + * which doesn't exist. Can be called from any online scx_ops operations. + */ +__bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + sch = scx_prog_sched(aux); + if (sch) + destroy_dsq(sch, dsq_id); +} + +/** + * bpf_iter_scx_dsq_new - Create a DSQ iterator + * @it: iterator to initialize + * @dsq_id: DSQ to iterate + * @flags: %SCX_DSQ_ITER_* + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Initialize BPF iterator @it which can be used with bpf_for_each() to walk + * tasks in the DSQ specified by @dsq_id. Iteration using @it only includes + * tasks which are already queued when this function is invoked. + */ +__bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, + u64 flags, const struct bpf_prog_aux *aux) +{ + struct bpf_iter_scx_dsq_kern *kit = (void *)it; + struct scx_sched *sch; + + BUILD_BUG_ON(sizeof(struct bpf_iter_scx_dsq_kern) > + sizeof(struct bpf_iter_scx_dsq)); + BUILD_BUG_ON(__alignof__(struct bpf_iter_scx_dsq_kern) != + __alignof__(struct bpf_iter_scx_dsq)); + BUILD_BUG_ON(__SCX_DSQ_ITER_ALL_FLAGS & + ((1U << __SCX_DSQ_LNODE_PRIV_SHIFT) - 1)); + + /* + * next() and destroy() will be called regardless of the return value. + * Always clear $kit->dsq. + */ + kit->dsq = NULL; + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -ENODEV; + + if (flags & ~__SCX_DSQ_ITER_USER_FLAGS) + return -EINVAL; + + kit->dsq = find_user_dsq(sch, dsq_id); + if (!kit->dsq) + return -ENOENT; + + kit->cursor = INIT_DSQ_LIST_CURSOR(kit->cursor, kit->dsq, flags); + + return 0; +} + +/** + * bpf_iter_scx_dsq_next - Progress a DSQ iterator + * @it: iterator to progress + * + * Return the next task. See bpf_iter_scx_dsq_new(). + */ +__bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) +{ + struct bpf_iter_scx_dsq_kern *kit = (void *)it; + + if (!kit->dsq) + return NULL; + + guard(raw_spinlock_irqsave)(&kit->dsq->lock); + + return nldsq_cursor_next_task(&kit->cursor, kit->dsq); +} + +/** + * bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator + * @it: iterator to destroy + * + * Undo scx_iter_scx_dsq_new(). + */ +__bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) +{ + struct bpf_iter_scx_dsq_kern *kit = (void *)it; + + if (!kit->dsq) + return; + + if (!list_empty(&kit->cursor.node)) { + unsigned long flags; + + raw_spin_lock_irqsave(&kit->dsq->lock, flags); + list_del_init(&kit->cursor.node); + raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); + } + kit->dsq = NULL; +} + +/** + * scx_bpf_dsq_peek - Lockless peek at the first element. + * @dsq_id: DSQ to examine. + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Read the first element in the DSQ. This is semantically equivalent to using + * the DSQ iterator, but is lockfree. Of course, like any lockless operation, + * this provides only a point-in-time snapshot, and the contents may change + * by the time any subsequent locking operation reads the queue. + * + * Returns the pointer, or NULL indicates an empty queue OR internal error. + */ +__bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + struct scx_dispatch_q *dsq; + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return NULL; + + if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN)) { + scx_error(sch, "peek disallowed on builtin DSQ 0x%llx", dsq_id); + return NULL; + } + + dsq = find_user_dsq(sch, dsq_id); + if (unlikely(!dsq)) { + scx_error(sch, "peek on non-existent DSQ 0x%llx", dsq_id); + return NULL; + } + + return rcu_dereference(dsq->first_task); +} + +/** + * scx_bpf_dsq_reenq - Re-enqueue tasks on a DSQ + * @dsq_id: DSQ to re-enqueue + * @reenq_flags: %SCX_RENQ_* + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Iterate over all of the tasks currently enqueued on the DSQ identified by + * @dsq_id, and re-enqueue them in the BPF scheduler. The following DSQs are + * supported: + * + * - Local DSQs (%SCX_DSQ_LOCAL or %SCX_DSQ_LOCAL_ON | $cpu) + * - User DSQs + * + * Re-enqueues are performed asynchronously. Can be called from anywhere. + */ +__bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + struct scx_dispatch_q *dsq; + + guard(preempt)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return; + + if (unlikely(reenq_flags & ~__SCX_REENQ_USER_MASK)) { + scx_error(sch, "invalid SCX_REENQ flags 0x%llx", reenq_flags); + return; + } + + /* not specifying any filter bits is the same as %SCX_REENQ_ANY */ + if (!(reenq_flags & __SCX_REENQ_FILTER_MASK)) + reenq_flags |= SCX_REENQ_ANY; + + dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, smp_processor_id()); + schedule_dsq_reenq(sch, dsq, reenq_flags, scx_locked_rq()); +} + +/** + * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Iterate over all of the tasks currently enqueued on the local DSQ of the + * caller's CPU, and re-enqueue them in the BPF scheduler. Can be called from + * anywhere. + * + * This is now a special case of scx_bpf_dsq_reenq() and may be removed in the + * future. + */ +__bpf_kfunc void scx_bpf_reenqueue_local___v2(const struct bpf_prog_aux *aux) +{ + scx_bpf_dsq_reenq(SCX_DSQ_LOCAL, 0, aux); +} + +__bpf_kfunc_end_defs(); + +__printf(5, 0) +static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf, + size_t line_size, char *fmt, unsigned long long *data, + u32 data__sz) +{ + struct bpf_bprintf_data bprintf_data = { .get_bin_args = true }; + s32 ret; + + if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || + (data__sz && !data)) { + scx_error(sch, "invalid data=%p and data__sz=%u", (void *)data, data__sz); + return -EINVAL; + } + + ret = copy_from_kernel_nofault(data_buf, data, data__sz); + if (ret < 0) { + scx_error(sch, "failed to read data fields (%d)", ret); + return ret; + } + + ret = bpf_bprintf_prepare(fmt, UINT_MAX, data_buf, data__sz / 8, + &bprintf_data); + if (ret < 0) { + scx_error(sch, "format preparation failed (%d)", ret); + return ret; + } + + ret = bstr_printf(line_buf, line_size, fmt, + bprintf_data.bin_args); + bpf_bprintf_cleanup(&bprintf_data); + if (ret < 0) { + scx_error(sch, "(\"%s\", %p, %u) failed to format", fmt, data, data__sz); + return ret; + } + + return ret; +} + +__printf(3, 0) +static s32 bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf, + char *fmt, unsigned long long *data, u32 data__sz) +{ + return __bstr_format(sch, buf->data, buf->line, sizeof(buf->line), + fmt, data, data__sz); +} + +__bpf_kfunc_start_defs(); + +/** + * scx_bpf_exit_bstr - Gracefully exit the BPF scheduler. + * @exit_code: Exit value to pass to user space via struct scx_exit_info. + * @fmt: error message format string + * @data: format string parameters packaged using ___bpf_fill() macro + * @data__sz: @data len, must end in '__sz' for the verifier + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Indicate that the BPF scheduler wants to exit gracefully, and initiate ops + * disabling. + */ +__printf(2, 0) +__bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt, + unsigned long long *data, u32 data__sz, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + unsigned long flags; + + raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); + sch = scx_prog_sched(aux); + if (likely(sch) && + bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) + scx_exit(sch, SCX_EXIT_UNREG_BPF, exit_code, "%s", scx_exit_bstr_buf.line); + raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); +} + +/** + * scx_bpf_error_bstr - Indicate fatal error + * @fmt: error message format string + * @data: format string parameters packaged using ___bpf_fill() macro + * @data__sz: @data len, must end in '__sz' for the verifier + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Indicate that the BPF scheduler encountered a fatal error and initiate ops + * disabling. + */ +__printf(1, 0) +__bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data, + u32 data__sz, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + unsigned long flags; + + raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); + sch = scx_prog_sched(aux); + if (likely(sch) && + bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) + scx_exit(sch, SCX_EXIT_ERROR_BPF, 0, "%s", scx_exit_bstr_buf.line); + raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); +} + +/** + * scx_bpf_dump_bstr - Generate extra debug dump specific to the BPF scheduler + * @fmt: format string + * @data: format string parameters packaged using ___bpf_fill() macro + * @data__sz: @data len, must end in '__sz' for the verifier + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * To be called through scx_bpf_dump() helper from ops.dump(), dump_cpu() and + * dump_task() to generate extra debug dump specific to the BPF scheduler. + * + * The extra dump may be multiple lines. A single line may be split over + * multiple calls. The last line is automatically terminated. + */ +__printf(1, 0) +__bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, + u32 data__sz, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + struct scx_dump_data *dd = &scx_dump_data; + struct scx_bstr_buf *buf = &dd->buf; + s32 ret; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return; + + if (raw_smp_processor_id() != dd->cpu) { + scx_error(sch, "scx_bpf_dump() must only be called from ops.dump() and friends"); + return; + } + + /* append the formatted string to the line buf */ + ret = __bstr_format(sch, buf->data, buf->line + dd->cursor, + sizeof(buf->line) - dd->cursor, fmt, data, data__sz); + if (ret < 0) { + dump_line(dd->s, "%s[!] (\"%s\", %p, %u) failed to format (%d)", + dd->prefix, fmt, data, data__sz, ret); + return; + } + + dd->cursor += ret; + dd->cursor = min_t(s32, dd->cursor, sizeof(buf->line)); + + if (!dd->cursor) + return; + + /* + * If the line buf overflowed or ends in a newline, flush it into the + * dump. This is to allow the caller to generate a single line over + * multiple calls. As ops_dump_flush() can also handle multiple lines in + * the line buf, the only case which can lead to an unexpected + * truncation is when the caller keeps generating newlines in the middle + * instead of the end consecutively. Don't do that. + */ + if (dd->cursor >= sizeof(buf->line) || buf->line[dd->cursor - 1] == '\n') + ops_dump_flush(); +} + +/** + * scx_bpf_cpuperf_cap - Query the maximum relative capacity of a CPU + * @cpu: CPU of interest + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Return the maximum relative capacity of @cpu in relation to the most + * performant CPU in the system. The return value is in the range [1, + * %SCX_CPUPERF_ONE]. See scx_bpf_cpuperf_cur(). + */ +__bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) + return arch_scale_cpu_capacity(cpu); + else + return SCX_CPUPERF_ONE; +} + +/** + * scx_bpf_cidperf_cap - Query the maximum relative capacity of the CPU at @cid + * @cid: cid of the CPU to query + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * cid-addressed equivalent of scx_bpf_cpuperf_cap(). + */ +__bpf_kfunc u32 scx_bpf_cidperf_cap(s32 cid, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return SCX_CPUPERF_ONE; + cpu = scx_cid_to_cpu(sch, cid); + if (cpu < 0) + return SCX_CPUPERF_ONE; + return arch_scale_cpu_capacity(cpu); +} + +/** + * scx_bpf_cpuperf_cur - Query the current relative performance of a CPU + * @cpu: CPU of interest + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Return the current relative performance of @cpu in relation to its maximum. + * The return value is in the range [1, %SCX_CPUPERF_ONE]. + * + * The current performance level of a CPU in relation to the maximum performance + * available in the system can be calculated as follows: + * + * scx_bpf_cpuperf_cap() * scx_bpf_cpuperf_cur() / %SCX_CPUPERF_ONE + * + * The result is in the range [1, %SCX_CPUPERF_ONE]. + */ +__bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) + return arch_scale_freq_capacity(cpu); + else + return SCX_CPUPERF_ONE; +} + +/** + * scx_bpf_cidperf_cur - Query the current performance of the CPU at @cid + * @cid: cid of the CPU to query + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * cid-addressed equivalent of scx_bpf_cpuperf_cur(). + */ +__bpf_kfunc u32 scx_bpf_cidperf_cur(s32 cid, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return SCX_CPUPERF_ONE; + cpu = scx_cid_to_cpu(sch, cid); + if (cpu < 0) + return SCX_CPUPERF_ONE; + return arch_scale_freq_capacity(cpu); +} + +/** + * scx_bpf_cpuperf_set - Set the relative performance target of a CPU + * @cpu: CPU of interest + * @perf: target performance level [0, %SCX_CPUPERF_ONE] + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Set the target performance level of @cpu to @perf. @perf is in linear + * relative scale between 0 and %SCX_CPUPERF_ONE. This determines how the + * schedutil cpufreq governor chooses the target frequency. + * + * The actual performance level chosen, CPU grouping, and the overhead and + * latency of the operations are dependent on the hardware and cpufreq driver in + * use. Consult hardware and cpufreq documentation for more information. The + * current performance level can be monitored using scx_bpf_cpuperf_cur(). + */ +__bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return; + + if (unlikely(perf > SCX_CPUPERF_ONE)) { + scx_error(sch, "Invalid cpuperf target %u for CPU %d", perf, cpu); + return; + } + + if (scx_cpu_valid(sch, cpu, NULL)) { + struct rq *rq = cpu_rq(cpu), *locked_rq = scx_locked_rq(); + struct rq_flags rf; + + /* + * When called with an rq lock held, restrict the operation + * to the corresponding CPU to prevent ABBA deadlocks. + */ + if (locked_rq && rq != locked_rq) { + scx_error(sch, "Invalid target CPU %d", cpu); + return; + } + + /* + * If no rq lock is held, allow to operate on any CPU by + * acquiring the corresponding rq lock. + */ + if (!locked_rq) { + rq_lock_irqsave(rq, &rf); + update_rq_clock(rq); + } + + rq->scx.cpuperf_target = perf; + cpufreq_update_util(rq, 0); + + if (!locked_rq) + rq_unlock_irqrestore(rq, &rf); + } +} + +/** + * scx_bpf_cidperf_set - Set the performance target of the CPU at @cid + * @cid: cid of the CPU to target + * @perf: target performance level [0, %SCX_CPUPERF_ONE] + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * cid-addressed equivalent of scx_bpf_cpuperf_set(). + */ +__bpf_kfunc void scx_bpf_cidperf_set(s32 cid, u32 perf, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return; + cpu = scx_cid_to_cpu(sch, cid); + if (cpu < 0) + return; + scx_bpf_cpuperf_set(cpu, perf, aux); +} + +/** + * scx_bpf_nr_node_ids - Return the number of possible node IDs + * + * All valid node IDs in the system are smaller than the returned value. + */ +__bpf_kfunc u32 scx_bpf_nr_node_ids(void) +{ + return nr_node_ids; +} + +/** + * scx_bpf_nr_cpu_ids - Return the number of possible CPU IDs + * + * All valid CPU IDs in the system are smaller than the returned value. + */ +__bpf_kfunc u32 scx_bpf_nr_cpu_ids(void) +{ + return nr_cpu_ids; +} + +/** + * scx_bpf_nr_cids - Return the size of the cid space + * + * Equals num_possible_cpus(). All valid cids are in [0, return value). + */ +__bpf_kfunc u32 scx_bpf_nr_cids(void) +{ + return num_possible_cpus(); +} + +/** + * scx_bpf_nr_online_cids - Return current count of online CPUs in cid space + * + * Return num_online_cpus(). The standard model restarts the scheduler on + * hotplug, which lets schedulers treat [0, nr_online_cids) as the online + * range. Schedulers that prefer to handle hotplug without a restart should + * install a custom mapping via scx_bpf_cid_override() and track onlining + * through the ops.cid_online / ops.cid_offline callbacks. + */ +__bpf_kfunc u32 scx_bpf_nr_online_cids(void) +{ + return num_online_cpus(); +} + +/** + * scx_bpf_this_cid - Return the cid of the CPU this program is running on + * + * cid-addressed equivalent of bpf_get_smp_processor_id() for scx programs. + * The current cpu is trivially valid, so this is just a table lookup. Return + * -EINVAL if called from a non-SCX program before any scheduler has ever + * been enabled (the cid table is still unallocated at that point). + */ +__bpf_kfunc s32 scx_bpf_this_cid(void) +{ + s16 *tbl = READ_ONCE(scx_cpu_to_cid_tbl); + + if (!tbl) + return -EINVAL; + return tbl[raw_smp_processor_id()]; +} + +/** + * scx_bpf_get_possible_cpumask - Get a referenced kptr to cpu_possible_mask + */ +__bpf_kfunc const struct cpumask *scx_bpf_get_possible_cpumask(void) +{ + return cpu_possible_mask; +} + +/** + * scx_bpf_get_online_cpumask - Get a referenced kptr to cpu_online_mask + */ +__bpf_kfunc const struct cpumask *scx_bpf_get_online_cpumask(void) +{ + return cpu_online_mask; +} + +/** + * scx_bpf_put_cpumask - Release a possible/online cpumask + * @cpumask: cpumask to release + */ +__bpf_kfunc void scx_bpf_put_cpumask(const struct cpumask *cpumask) +{ + /* + * Empty function body because we aren't actually acquiring or releasing + * a reference to a global cpumask, which is read-only in the caller and + * is never released. The acquire / release semantics here are just used + * to make the cpumask is a trusted pointer in the caller. + */ +} + +/** + * scx_bpf_task_running - Is task currently running? + * @p: task of interest + */ +__bpf_kfunc bool scx_bpf_task_running(const struct task_struct *p) +{ + return task_rq(p)->curr == p; +} + +/** + * scx_bpf_task_cpu - CPU a task is currently associated with + * @p: task of interest + */ +__bpf_kfunc s32 scx_bpf_task_cpu(const struct task_struct *p) +{ + return task_cpu(p); +} + +/** + * scx_bpf_task_cid - cid a task is currently associated with + * @p: task of interest + * + * cid-addressed equivalent of scx_bpf_task_cpu(). task_cpu(p) is always a + * valid cpu, so this is just a table lookup. Return -EINVAL if called from + * a non-SCX program before any scheduler has ever been enabled. + */ +__bpf_kfunc s32 scx_bpf_task_cid(const struct task_struct *p) +{ + s16 *tbl = READ_ONCE(scx_cpu_to_cid_tbl); + + if (!tbl) + return -EINVAL; + return tbl[task_cpu(p)]; +} + +/** + * scx_bpf_cpu_rq - Fetch the rq of a CPU + * @cpu: CPU of the rq + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + */ +__bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return NULL; + + if (!scx_cpu_valid(sch, cpu, NULL)) + return NULL; + + if (!sch->warned_deprecated_rq) { + printk_deferred(KERN_WARNING "sched_ext: %s() is deprecated; " + "use scx_bpf_locked_rq() when holding rq lock " + "or scx_bpf_cpu_curr() to read remote curr safely.\n", __func__); + sch->warned_deprecated_rq = true; + } + + return cpu_rq(cpu); +} + +/** + * scx_bpf_locked_rq - Return the rq currently locked by SCX + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Returns the rq if a rq lock is currently held by SCX. + * Otherwise emits an error and returns NULL. + */ +__bpf_kfunc struct rq *scx_bpf_locked_rq(const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + struct rq *rq; + + guard(preempt)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return NULL; + + rq = scx_locked_rq(); + if (!rq) { + scx_error(sch, "accessing rq without holding rq lock"); + return NULL; + } + + return rq; +} + +/** + * scx_bpf_cpu_curr - Return remote CPU's curr task + * @cpu: CPU of interest + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Callers must hold RCU read lock (KF_RCU). + */ +__bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return NULL; + + if (!scx_cpu_valid(sch, cpu, NULL)) + return NULL; + + return rcu_dereference(cpu_rq(cpu)->curr); +} + +/** + * scx_bpf_cid_curr - Return the curr task on the CPU at @cid + * @cid: cid of interest + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * cid-addressed equivalent of scx_bpf_cpu_curr(). Callers must hold RCU + * read lock (KF_RCU). + */ +__bpf_kfunc struct task_struct *scx_bpf_cid_curr(s32 cid, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return NULL; + cpu = scx_cid_to_cpu(sch, cid); + if (cpu < 0) + return NULL; + return rcu_dereference(cpu_rq(cpu)->curr); +} + +/** + * scx_bpf_tid_to_task - Look up a task by its scx tid + * @tid: task ID previously read from p->scx.tid + * + * Returns the task with the given tid, or NULL if no such task exists. The + * returned pointer is valid until the end of the current RCU read section + * (KF_RCU_PROTECTED). Requires SCX_OPS_TID_TO_TASK to be set on the root + * scheduler; otherwise an error is raised and NULL returned. + */ +__bpf_kfunc struct task_struct *scx_bpf_tid_to_task(u64 tid) +{ + struct sched_ext_entity *scx; + + if (!scx_tid_to_task_enabled()) { + struct scx_sched *sch = rcu_dereference(scx_root); + + if (sch) + scx_error(sch, "scx_bpf_tid_to_task() called without SCX_OPS_TID_TO_TASK"); + return NULL; + } + + scx = rhashtable_lookup(&scx_tid_hash, &tid, scx_tid_hash_params); + if (!scx) + return NULL; + + return container_of(scx, struct task_struct, scx); +} + +/** + * scx_bpf_now - Returns a high-performance monotonically non-decreasing + * clock for the current CPU. The clock returned is in nanoseconds. + * + * It provides the following properties: + * + * 1) High performance: Many BPF schedulers call bpf_ktime_get_ns() frequently + * to account for execution time and track tasks' runtime properties. + * Unfortunately, in some hardware platforms, bpf_ktime_get_ns() -- which + * eventually reads a hardware timestamp counter -- is neither performant nor + * scalable. scx_bpf_now() aims to provide a high-performance clock by + * using the rq clock in the scheduler core whenever possible. + * + * 2) High enough resolution for the BPF scheduler use cases: In most BPF + * scheduler use cases, the required clock resolution is lower than the most + * accurate hardware clock (e.g., rdtsc in x86). scx_bpf_now() basically + * uses the rq clock in the scheduler core whenever it is valid. It considers + * that the rq clock is valid from the time the rq clock is updated + * (update_rq_clock) until the rq is unlocked (rq_unpin_lock). + * + * 3) Monotonically non-decreasing clock for the same CPU: scx_bpf_now() + * guarantees the clock never goes backward when comparing them in the same + * CPU. On the other hand, when comparing clocks in different CPUs, there + * is no such guarantee -- the clock can go backward. It provides a + * monotonically *non-decreasing* clock so that it would provide the same + * clock values in two different scx_bpf_now() calls in the same CPU + * during the same period of when the rq clock is valid. + */ +__bpf_kfunc u64 scx_bpf_now(void) +{ + struct rq *rq; + u64 clock; + + preempt_disable(); + + rq = this_rq(); + if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) { + /* + * If the rq clock is valid, use the cached rq clock. + * + * Note that scx_bpf_now() is re-entrant between a process + * context and an interrupt context (e.g., timer interrupt). + * However, we don't need to consider the race between them + * because such race is not observable from a caller. + */ + clock = READ_ONCE(rq->scx.clock); + } else { + /* + * Otherwise, return a fresh rq clock. + * + * The rq clock is updated outside of the rq lock. + * In this case, keep the updated rq clock invalid so the next + * kfunc call outside the rq lock gets a fresh rq clock. + */ + clock = sched_clock_cpu(cpu_of(rq)); + } + + preempt_enable(); + + return clock; +} + +static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *events) +{ + struct scx_event_stats *e_cpu; + int cpu; + + /* Aggregate per-CPU event counters into @events. */ + memset(events, 0, sizeof(*events)); + for_each_possible_cpu(cpu) { + e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats; + scx_agg_event(events, e_cpu, SCX_EV_SELECT_CPU_FALLBACK); + scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); + scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_KEEP_LAST); + scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_EXITING); + scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); + scx_agg_event(events, e_cpu, SCX_EV_REENQ_IMMED); + scx_agg_event(events, e_cpu, SCX_EV_REENQ_LOCAL_REPEAT); + scx_agg_event(events, e_cpu, SCX_EV_REFILL_SLICE_DFL); + scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DURATION); + scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DISPATCH); + scx_agg_event(events, e_cpu, SCX_EV_BYPASS_ACTIVATE); + scx_agg_event(events, e_cpu, SCX_EV_INSERT_NOT_OWNED); + scx_agg_event(events, e_cpu, SCX_EV_SUB_BYPASS_DISPATCH); + } +} + +/* + * scx_bpf_events - Get a system-wide event counter to + * @events: output buffer from a BPF program + * @events__sz: @events len, must end in '__sz'' for the verifier + */ +__bpf_kfunc void scx_bpf_events(struct scx_event_stats *events, + size_t events__sz) +{ + struct scx_sched *sch; + struct scx_event_stats e_sys; + + rcu_read_lock(); + sch = rcu_dereference(scx_root); + if (sch) + scx_read_events(sch, &e_sys); + else + memset(&e_sys, 0, sizeof(e_sys)); + rcu_read_unlock(); + + /* + * We cannot entirely trust a BPF-provided size since a BPF program + * might be compiled against a different vmlinux.h, of which + * scx_event_stats would be larger (a newer vmlinux.h) or smaller + * (an older vmlinux.h). Hence, we use the smaller size to avoid + * memory corruption. + */ + events__sz = min(events__sz, sizeof(*events)); + memcpy(events, &e_sys, events__sz); +} + +#ifdef CONFIG_CGROUP_SCHED +/** + * scx_bpf_task_cgroup - Return the sched cgroup of a task + * @p: task of interest + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with + * from the scheduler's POV. SCX operations should use this function to + * determine @p's current cgroup as, unlike following @p->cgroups, + * @p->sched_task_group is stable for the duration of the SCX op. See + * SCX_CALL_OP_TASK() for details. + */ +__bpf_kfunc struct cgroup *scx_bpf_task_cgroup(struct task_struct *p, + const struct bpf_prog_aux *aux) +{ + struct task_group *tg = p->sched_task_group; + struct cgroup *cgrp = &cgrp_dfl_root.cgrp; + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + goto out; + + if (!scx_kf_arg_task_ok(sch, p)) + goto out; + + cgrp = tg_cgrp(tg); + +out: + cgroup_get(cgrp); + return cgrp; +} +#endif /* CONFIG_CGROUP_SCHED */ + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(scx_kfunc_ids_any) +BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS | KF_RCU); +BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS | KF_RCU); +BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_kick_cid, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_destroy_dsq, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_dsq_peek, KF_IMPLICIT_ARGS | KF_RCU_PROTECTED | KF_RET_NULL) +BTF_ID_FLAGS(func, scx_bpf_dsq_reenq, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_reenqueue_local___v2, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, bpf_iter_scx_dsq_new, KF_IMPLICIT_ARGS | KF_ITER_NEW | KF_RCU_PROTECTED) +BTF_ID_FLAGS(func, bpf_iter_scx_dsq_next, KF_ITER_NEXT | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_iter_scx_dsq_destroy, KF_ITER_DESTROY) +BTF_ID_FLAGS(func, scx_bpf_exit_bstr, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_dump_bstr, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cidperf_cap, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cidperf_cur, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cidperf_set, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_nr_node_ids) +BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids) +BTF_ID_FLAGS(func, scx_bpf_nr_cids) +BTF_ID_FLAGS(func, scx_bpf_nr_online_cids) +BTF_ID_FLAGS(func, scx_bpf_this_cid) +BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE) +BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_task_cid, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_IMPLICIT_ARGS | KF_RET_NULL) +BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) +BTF_ID_FLAGS(func, scx_bpf_cid_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) +BTF_ID_FLAGS(func, scx_bpf_tid_to_task, KF_RET_NULL | KF_RCU_PROTECTED) +BTF_ID_FLAGS(func, scx_bpf_now) +BTF_ID_FLAGS(func, scx_bpf_events) +#ifdef CONFIG_CGROUP_SCHED +BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_IMPLICIT_ARGS | KF_RCU | KF_ACQUIRE) +#endif +BTF_KFUNCS_END(scx_kfunc_ids_any) + +static const struct btf_kfunc_id_set scx_kfunc_set_any = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_any, + .filter = scx_kfunc_context_filter, +}; + +/* + * cpu-form kfuncs that are forbidden from cid-form schedulers + * (bpf_sched_ext_ops_cid). Programs targeting the cid struct_ops type must + * use the cid-form alternative (cid/cmask kfuncs). + * + * Membership overlaps with scx_kfunc_ids_{any,idle,select_cpu}; the filter + * tests this set independently and rejects matches before the per-op + * allow-list check runs. + * + * pahole/resolve_btfids scans every BTF_ID_FLAGS() at build time and + * intersects flags across duplicate entries, so each entry must carry the + * same flags as the kfunc's primary declaration; otherwise the flags get + * dropped globally. + */ +BTF_KFUNCS_START(scx_kfunc_ids_cpu_only) +BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) +BTF_ID_FLAGS(func, scx_bpf_cpu_node, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE) +BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) +BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) +BTF_KFUNCS_END(scx_kfunc_ids_cpu_only) + +/* + * Per-op kfunc allow flags. Each bit corresponds to a context-sensitive kfunc + * group; an op may permit zero or more groups, with the union expressed in + * scx_kf_allow_flags[]. The verifier-time filter (scx_kfunc_context_filter()) + * consults this table to decide whether a context-sensitive kfunc is callable + * from a given SCX op. + */ +enum scx_kf_allow_flags { + SCX_KF_ALLOW_UNLOCKED = 1 << 0, + SCX_KF_ALLOW_INIT = 1 << 1, + SCX_KF_ALLOW_CPU_RELEASE = 1 << 2, + SCX_KF_ALLOW_DISPATCH = 1 << 3, + SCX_KF_ALLOW_ENQUEUE = 1 << 4, + SCX_KF_ALLOW_SELECT_CPU = 1 << 5, +}; + +/* + * Map each SCX op to the union of kfunc groups it permits, indexed by + * SCX_OP_IDX(op). Ops not listed only permit kfuncs that are not + * context-sensitive. + */ +static const u32 scx_kf_allow_flags[] = { + [SCX_OP_IDX(select_cpu)] = SCX_KF_ALLOW_SELECT_CPU | SCX_KF_ALLOW_ENQUEUE, + [SCX_OP_IDX(enqueue)] = SCX_KF_ALLOW_SELECT_CPU | SCX_KF_ALLOW_ENQUEUE, + [SCX_OP_IDX(dispatch)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH, + [SCX_OP_IDX(cpu_release)] = SCX_KF_ALLOW_CPU_RELEASE, + [SCX_OP_IDX(init_task)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(dump)] = SCX_KF_ALLOW_UNLOCKED, +#ifdef CONFIG_EXT_GROUP_SCHED + [SCX_OP_IDX(cgroup_init)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(cgroup_exit)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(cgroup_prep_move)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(cgroup_cancel_move)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(cgroup_set_weight)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(cgroup_set_bandwidth)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(cgroup_set_idle)] = SCX_KF_ALLOW_UNLOCKED, +#endif /* CONFIG_EXT_GROUP_SCHED */ + [SCX_OP_IDX(sub_attach)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(sub_detach)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(cpu_online)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(cpu_offline)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(init)] = SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT, + [SCX_OP_IDX(exit)] = SCX_KF_ALLOW_UNLOCKED, +}; + +/* + * Verifier-time filter for SCX kfuncs. Registered via the .filter field on + * each per-group btf_kfunc_id_set. The BPF core invokes this for every kfunc + * call in the registered hook (BPF_PROG_TYPE_STRUCT_OPS or + * BPF_PROG_TYPE_SYSCALL), regardless of which set originally introduced the + * kfunc - so the filter must short-circuit on kfuncs it doesn't govern by + * falling through to "allow" when none of the SCX sets contain the kfunc. + */ +int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) +{ + bool in_unlocked = btf_id_set8_contains(&scx_kfunc_ids_unlocked, kfunc_id); + bool in_init = btf_id_set8_contains(&scx_kfunc_ids_init, kfunc_id); + bool in_select_cpu = btf_id_set8_contains(&scx_kfunc_ids_select_cpu, kfunc_id); + bool in_enqueue = btf_id_set8_contains(&scx_kfunc_ids_enqueue_dispatch, kfunc_id); + bool in_dispatch = btf_id_set8_contains(&scx_kfunc_ids_dispatch, kfunc_id); + bool in_cpu_release = btf_id_set8_contains(&scx_kfunc_ids_cpu_release, kfunc_id); + bool in_idle = btf_id_set8_contains(&scx_kfunc_ids_idle, kfunc_id); + bool in_any = btf_id_set8_contains(&scx_kfunc_ids_any, kfunc_id); + bool in_cpu_only = btf_id_set8_contains(&scx_kfunc_ids_cpu_only, kfunc_id); + u32 moff, flags; + + /* Not an SCX kfunc - allow. */ + if (!(in_unlocked || in_init || in_select_cpu || in_enqueue || in_dispatch || + in_cpu_release || in_idle || in_any)) + return 0; + + /* SYSCALL progs (e.g. BPF test_run()) may call unlocked and select_cpu kfuncs. */ + if (prog->type == BPF_PROG_TYPE_SYSCALL) + return (in_unlocked || in_select_cpu || in_idle || in_any) ? 0 : -EACCES; + + if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) + return (in_any || in_idle) ? 0 : -EACCES; + + /* + * add_subprog_and_kfunc() collects all kfunc calls, including dead code + * guarded by bpf_ksym_exists(), before check_attach_btf_id() sets + * prog->aux->st_ops. Allow all kfuncs when st_ops is not yet set; + * do_check_main() re-runs the filter with st_ops set and enforces the + * actual restrictions. + */ + if (!prog->aux->st_ops) + return 0; + + /* + * Non-SCX struct_ops: SCX kfuncs are not permitted. + * + * Both bpf_sched_ext_ops (cpu-form) and bpf_sched_ext_ops_cid + * (cid-form) are valid SCX struct_ops. Member offsets match between + * the two (verified by BUILD_BUG_ON in scx_init()), so the shared + * scx_kf_allow_flags[] table indexed by SCX_MOFF_IDX(moff) applies to + * both. + */ + if (prog->aux->st_ops != &bpf_sched_ext_ops && + prog->aux->st_ops != &bpf_sched_ext_ops_cid) + return -EACCES; + + /* + * cid-form schedulers must use cid/cmask kfuncs. cid and cpu are both + * small s32s and trivially confused, so cpu-only kfuncs are rejected at + * load time. The reverse (cpu-form calling cid-form kfuncs) is + * intentionally permissive to ease gradual cpumask -> cid migration. + */ + if (prog->aux->st_ops == &bpf_sched_ext_ops_cid && in_cpu_only) + return -EACCES; + + /* SCX struct_ops: check the per-op allow list. */ + if (in_any || in_idle) + return 0; + + moff = prog->aux->attach_st_ops_member_off; + flags = scx_kf_allow_flags[SCX_MOFF_IDX(moff)]; + + if ((flags & SCX_KF_ALLOW_UNLOCKED) && in_unlocked) + return 0; + if ((flags & SCX_KF_ALLOW_INIT) && in_init) + return 0; + if ((flags & SCX_KF_ALLOW_CPU_RELEASE) && in_cpu_release) + return 0; + if ((flags & SCX_KF_ALLOW_DISPATCH) && in_dispatch) + return 0; + if ((flags & SCX_KF_ALLOW_ENQUEUE) && in_enqueue) + return 0; + if ((flags & SCX_KF_ALLOW_SELECT_CPU) && in_select_cpu) + return 0; + + return -EACCES; +} + +static int __init scx_init(void) +{ + int ret; + + /* + * sched_ext_ops_cid mirrors sched_ext_ops up to and including @priv. + * Both bpf_scx_init_member() and bpf_scx_check_member() use offsets + * from struct sched_ext_ops; sched_ext_ops_cid relies on those offsets + * matching for the shared fields. Catch any drift at boot. + */ +#define CID_OFFSET_MATCH(cpu_field, cid_field) \ + BUILD_BUG_ON(offsetof(struct sched_ext_ops, cpu_field) != \ + offsetof(struct sched_ext_ops_cid, cid_field)) + /* data fields used by bpf_scx_init_member() */ + CID_OFFSET_MATCH(dispatch_max_batch, dispatch_max_batch); + CID_OFFSET_MATCH(flags, flags); + CID_OFFSET_MATCH(name, name); + CID_OFFSET_MATCH(timeout_ms, timeout_ms); + CID_OFFSET_MATCH(exit_dump_len, exit_dump_len); + CID_OFFSET_MATCH(hotplug_seq, hotplug_seq); + CID_OFFSET_MATCH(sub_cgroup_id, sub_cgroup_id); + /* shared callbacks: the union view requires byte-for-byte offset match */ + CID_OFFSET_MATCH(enqueue, enqueue); + CID_OFFSET_MATCH(dequeue, dequeue); + CID_OFFSET_MATCH(dispatch, dispatch); + CID_OFFSET_MATCH(tick, tick); + CID_OFFSET_MATCH(runnable, runnable); + CID_OFFSET_MATCH(running, running); + CID_OFFSET_MATCH(stopping, stopping); + CID_OFFSET_MATCH(quiescent, quiescent); + CID_OFFSET_MATCH(yield, yield); + CID_OFFSET_MATCH(core_sched_before, core_sched_before); + CID_OFFSET_MATCH(set_weight, set_weight); + CID_OFFSET_MATCH(update_idle, update_idle); + CID_OFFSET_MATCH(init_task, init_task); + CID_OFFSET_MATCH(exit_task, exit_task); + CID_OFFSET_MATCH(enable, enable); + CID_OFFSET_MATCH(disable, disable); + CID_OFFSET_MATCH(dump, dump); + CID_OFFSET_MATCH(dump_task, dump_task); + CID_OFFSET_MATCH(sub_attach, sub_attach); + CID_OFFSET_MATCH(sub_detach, sub_detach); + CID_OFFSET_MATCH(init, init); + CID_OFFSET_MATCH(exit, exit); +#ifdef CONFIG_EXT_GROUP_SCHED + CID_OFFSET_MATCH(cgroup_init, cgroup_init); + CID_OFFSET_MATCH(cgroup_exit, cgroup_exit); + CID_OFFSET_MATCH(cgroup_prep_move, cgroup_prep_move); + CID_OFFSET_MATCH(cgroup_move, cgroup_move); + CID_OFFSET_MATCH(cgroup_cancel_move, cgroup_cancel_move); + CID_OFFSET_MATCH(cgroup_set_weight, cgroup_set_weight); + CID_OFFSET_MATCH(cgroup_set_bandwidth, cgroup_set_bandwidth); + CID_OFFSET_MATCH(cgroup_set_idle, cgroup_set_idle); +#endif + /* renamed callbacks must occupy the same slot as their cpu-form sibling */ + CID_OFFSET_MATCH(select_cpu, select_cid); + CID_OFFSET_MATCH(set_cpumask, set_cmask); + CID_OFFSET_MATCH(cpu_online, cid_online); + CID_OFFSET_MATCH(cpu_offline, cid_offline); + CID_OFFSET_MATCH(dump_cpu, dump_cid); + /* @priv tail must align since both share the same data block */ + CID_OFFSET_MATCH(priv, priv); + /* + * cid-form must end exactly at @priv - validate_ops() skips + * cpu_acquire/cpu_release for cid-form because reading those fields + * past the BPF allocation would be UB. + */ + BUILD_BUG_ON(offsetof(struct sched_ext_ops_cid, __end) != + offsetofend(struct sched_ext_ops, priv)); +#undef CID_OFFSET_MATCH + + /* + * kfunc registration can't be done from init_sched_ext_class() as + * register_btf_kfunc_id_set() needs most of the system to be up. + * + * Some kfuncs are context-sensitive and can only be called from + * specific SCX ops. They are grouped into per-context BTF sets, each + * registered with scx_kfunc_context_filter as its .filter callback. The + * BPF core dedups identical filter pointers per hook + * (btf_populate_kfunc_set()), so the filter is invoked exactly once per + * kfunc lookup; it consults scx_kf_allow_flags[] to enforce per-op + * restrictions at verify time. + */ + if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &scx_kfunc_set_enqueue_dispatch)) || + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &scx_kfunc_set_dispatch)) || + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &scx_kfunc_set_cpu_release)) || + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &scx_kfunc_set_unlocked)) || + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, + &scx_kfunc_set_unlocked)) || + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &scx_kfunc_set_any)) || + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, + &scx_kfunc_set_any)) || + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, + &scx_kfunc_set_any))) { + pr_err("sched_ext: Failed to register kfunc sets (%d)\n", ret); + return ret; + } + + ret = scx_idle_init(); + if (ret) { + pr_err("sched_ext: Failed to initialize idle tracking (%d)\n", ret); + return ret; + } + + ret = scx_cid_kfunc_init(); + if (ret) { + pr_err("sched_ext: Failed to register cid kfuncs (%d)\n", ret); + return ret; + } + + ret = register_bpf_struct_ops(&bpf_sched_ext_ops, sched_ext_ops); + if (ret) { + pr_err("sched_ext: Failed to register struct_ops (%d)\n", ret); + return ret; + } + + ret = register_bpf_struct_ops(&bpf_sched_ext_ops_cid, sched_ext_ops_cid); + if (ret) { + pr_err("sched_ext: Failed to register cid struct_ops (%d)\n", ret); + return ret; + } + + ret = register_pm_notifier(&scx_pm_notifier); + if (ret) { + pr_err("sched_ext: Failed to register PM notifier (%d)\n", ret); + return ret; + } + + scx_kset = kset_create_and_add("sched_ext", &scx_uevent_ops, kernel_kobj); + if (!scx_kset) { + pr_err("sched_ext: Failed to create /sys/kernel/sched_ext\n"); + return -ENOMEM; + } + + ret = sysfs_create_group(&scx_kset->kobj, &scx_global_attr_group); + if (ret < 0) { + pr_err("sched_ext: Failed to add global attributes\n"); + return ret; + } + + return 0; +} +__initcall(scx_init); diff --git a/kernel/sched/ext/ext.h b/kernel/sched/ext/ext.h new file mode 100644 index 000000000000..0b7fc46aee08 --- /dev/null +++ b/kernel/sched/ext/ext.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + */ +#ifdef CONFIG_SCHED_CLASS_EXT + +void scx_tick(struct rq *rq); +void init_scx_entity(struct sched_ext_entity *scx); +void scx_pre_fork(struct task_struct *p); +int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs); +void scx_post_fork(struct task_struct *p); +void scx_cancel_fork(struct task_struct *p); +bool scx_can_stop_tick(struct rq *rq); +void scx_rq_activate(struct rq *rq); +void scx_rq_deactivate(struct rq *rq); +int scx_check_setscheduler(struct task_struct *p, int policy); +bool task_should_scx(int policy); +bool scx_allow_ttwu_queue(const struct task_struct *p); +void init_sched_ext_class(void); + +static inline u32 scx_cpuperf_target(s32 cpu) +{ + if (scx_enabled()) + return cpu_rq(cpu)->scx.cpuperf_target; + else + return 0; +} + +static inline bool task_on_scx(const struct task_struct *p) +{ + return scx_enabled() && p->sched_class == &ext_sched_class; +} + +#ifdef CONFIG_SCHED_CORE +bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, + bool in_fi); +#endif + +#else /* CONFIG_SCHED_CLASS_EXT */ + +static inline void scx_tick(struct rq *rq) {} +static inline void scx_pre_fork(struct task_struct *p) {} +static inline int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs) { return 0; } +static inline void scx_post_fork(struct task_struct *p) {} +static inline void scx_cancel_fork(struct task_struct *p) {} +static inline u32 scx_cpuperf_target(s32 cpu) { return 0; } +static inline bool scx_can_stop_tick(struct rq *rq) { return true; } +static inline void scx_rq_activate(struct rq *rq) {} +static inline void scx_rq_deactivate(struct rq *rq) {} +static inline int scx_check_setscheduler(struct task_struct *p, int policy) { return 0; } +static inline bool task_on_scx(const struct task_struct *p) { return false; } +static inline bool scx_allow_ttwu_queue(const struct task_struct *p) { return true; } +static inline void init_sched_ext_class(void) {} + +#endif /* CONFIG_SCHED_CLASS_EXT */ + +#ifdef CONFIG_SCHED_CLASS_EXT +void __scx_update_idle(struct rq *rq, bool idle, bool do_notify); + +static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) +{ + if (scx_enabled()) + __scx_update_idle(rq, idle, do_notify); +} +#else +static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) {} +#endif + +#ifdef CONFIG_CGROUP_SCHED +#ifdef CONFIG_EXT_GROUP_SCHED +void scx_tg_init(struct task_group *tg); +int scx_tg_online(struct task_group *tg); +void scx_tg_offline(struct task_group *tg); +int scx_cgroup_can_attach(struct cgroup_taskset *tset); +void scx_cgroup_move_task(struct task_struct *p); +void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); +void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); +void scx_group_set_idle(struct task_group *tg, bool idle); +void scx_group_set_bandwidth(struct task_group *tg, u64 period_us, u64 quota_us, u64 burst_us); +#else /* CONFIG_EXT_GROUP_SCHED */ +static inline void scx_tg_init(struct task_group *tg) {} +static inline int scx_tg_online(struct task_group *tg) { return 0; } +static inline void scx_tg_offline(struct task_group *tg) {} +static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } +static inline void scx_cgroup_move_task(struct task_struct *p) {} +static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} +static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} +static inline void scx_group_set_idle(struct task_group *tg, bool idle) {} +static inline void scx_group_set_bandwidth(struct task_group *tg, u64 period_us, u64 quota_us, u64 burst_us) {} +#endif /* CONFIG_EXT_GROUP_SCHED */ +#endif /* CONFIG_CGROUP_SCHED */ diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c new file mode 100644 index 000000000000..2077373d8da3 --- /dev/null +++ b/kernel/sched/ext/idle.c @@ -0,0 +1,1511 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Built-in idle CPU tracking policy. + * + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + * Copyright (c) 2024 Andrea Righi + */ + +/* Enable/disable built-in idle CPU selection policy */ +static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); + +/* Enable/disable per-node idle cpumasks */ +static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_per_node); + +/* Enable/disable LLC aware optimizations */ +static DEFINE_STATIC_KEY_FALSE(scx_selcpu_topo_llc); + +/* Enable/disable NUMA aware optimizations */ +static DEFINE_STATIC_KEY_FALSE(scx_selcpu_topo_numa); + +/* + * cpumasks to track idle CPUs within each NUMA node. + * + * If SCX_OPS_BUILTIN_IDLE_PER_NODE is not enabled, a single global cpumask + * from is used to track all the idle CPUs in the system. + */ +struct scx_idle_cpus { + cpumask_var_t cpu; + cpumask_var_t smt; +}; + +/* + * Global host-wide idle cpumasks (used when SCX_OPS_BUILTIN_IDLE_PER_NODE + * is not enabled). + */ +static struct scx_idle_cpus scx_idle_global_masks; + +/* + * Per-node idle cpumasks. + */ +static struct scx_idle_cpus **scx_idle_node_masks; + +/* + * Local per-CPU cpumasks (used to generate temporary idle cpumasks). + */ +static DEFINE_PER_CPU(cpumask_var_t, local_idle_cpumask); +static DEFINE_PER_CPU(cpumask_var_t, local_llc_idle_cpumask); +static DEFINE_PER_CPU(cpumask_var_t, local_numa_idle_cpumask); + +/* + * Return the idle masks associated to a target @node. + * + * NUMA_NO_NODE identifies the global idle cpumask. + */ +static struct scx_idle_cpus *idle_cpumask(int node) +{ + return node == NUMA_NO_NODE ? &scx_idle_global_masks : scx_idle_node_masks[node]; +} + +/* + * Returns the NUMA node ID associated with a @cpu, or NUMA_NO_NODE if + * per-node idle cpumasks are disabled. + */ +static int scx_cpu_node_if_enabled(int cpu) +{ + if (!static_branch_maybe(CONFIG_NUMA, &scx_builtin_idle_per_node)) + return NUMA_NO_NODE; + + return cpu_to_node(cpu); +} + +static bool scx_idle_test_and_clear_cpu(int cpu) +{ + int node = scx_cpu_node_if_enabled(cpu); + struct cpumask *idle_cpus = idle_cpumask(node)->cpu; + + /* + * SMT mask should be cleared whether we can claim @cpu or not. The SMT + * cluster is not wholly idle either way. This also prevents + * scx_pick_idle_cpu() from getting caught in an infinite loop. + */ + if (sched_smt_active()) { + const struct cpumask *smt = cpu_smt_mask(cpu); + struct cpumask *idle_smts = idle_cpumask(node)->smt; + + /* + * If offline, @cpu is not its own sibling and + * scx_pick_idle_cpu() can get caught in an infinite loop as + * @cpu is never cleared from the idle SMT mask. Ensure that + * @cpu is eventually cleared. + * + * NOTE: Use cpumask_intersects() and cpumask_test_cpu() to + * reduce memory writes, which may help alleviate cache + * coherence pressure. + */ + if (cpumask_intersects(smt, idle_smts)) + cpumask_andnot(idle_smts, idle_smts, smt); + else if (cpumask_test_cpu(cpu, idle_smts)) + __cpumask_clear_cpu(cpu, idle_smts); + } + + return cpumask_test_and_clear_cpu(cpu, idle_cpus); +} + +/* + * Pick an idle CPU in a specific NUMA node. + */ +static s32 pick_idle_cpu_in_node(const struct cpumask *cpus_allowed, int node, u64 flags) +{ + int cpu; + +retry: + if (sched_smt_active()) { + cpu = cpumask_any_and_distribute(idle_cpumask(node)->smt, cpus_allowed); + if (cpu < nr_cpu_ids) + goto found; + + if (flags & SCX_PICK_IDLE_CORE) + return -EBUSY; + } + + cpu = cpumask_any_and_distribute(idle_cpumask(node)->cpu, cpus_allowed); + if (cpu >= nr_cpu_ids) + return -EBUSY; + +found: + if (scx_idle_test_and_clear_cpu(cpu)) + return cpu; + else + goto retry; +} + +#ifdef CONFIG_NUMA +/* + * Tracks nodes that have not yet been visited when searching for an idle + * CPU across all available nodes. + */ +static DEFINE_PER_CPU(nodemask_t, per_cpu_unvisited); + +/* + * Search for an idle CPU across all nodes, excluding @node. + */ +static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, int node, u64 flags) +{ + nodemask_t *unvisited; + s32 cpu = -EBUSY; + + preempt_disable(); + unvisited = this_cpu_ptr(&per_cpu_unvisited); + + /* + * Restrict the search to the online nodes (excluding the current + * node that has been visited already). + */ + nodes_copy(*unvisited, node_states[N_ONLINE]); + node_clear(node, *unvisited); + + /* + * Traverse all nodes in order of increasing distance, starting + * from @node. + * + * This loop is O(N^2), with N being the amount of NUMA nodes, + * which might be quite expensive in large NUMA systems. However, + * this complexity comes into play only when a scheduler enables + * SCX_OPS_BUILTIN_IDLE_PER_NODE and it's requesting an idle CPU + * without specifying a target NUMA node, so it shouldn't be a + * bottleneck is most cases. + * + * As a future optimization we may want to cache the list of nodes + * in a per-node array, instead of actually traversing them every + * time. + */ + for_each_node_numadist(node, *unvisited) { + cpu = pick_idle_cpu_in_node(cpus_allowed, node, flags); + if (cpu >= 0) + break; + } + preempt_enable(); + + return cpu; +} +#else +static inline s32 +pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, int node, u64 flags) +{ + return -EBUSY; +} +#endif + +/* + * Find an idle CPU in the system, starting from @node. + */ +static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed, int node, u64 flags) +{ + s32 cpu; + + /* + * Always search in the starting node first (this is an + * optimization that can save some cycles even when the search is + * not limited to a single node). + */ + cpu = pick_idle_cpu_in_node(cpus_allowed, node, flags); + if (cpu >= 0) + return cpu; + + /* + * Stop the search if we are using only a single global cpumask + * (NUMA_NO_NODE) or if the search is restricted to the first node + * only. + */ + if (node == NUMA_NO_NODE || flags & SCX_PICK_IDLE_IN_NODE) + return -EBUSY; + + /* + * Extend the search to the other online nodes. + */ + return pick_idle_cpu_from_online_nodes(cpus_allowed, node, flags); +} + +/* + * Return the amount of CPUs in the same LLC domain of @cpu (or zero if the LLC + * domain is not defined). + */ +static unsigned int llc_weight(s32 cpu) +{ + struct sched_domain *sd; + + sd = rcu_dereference(per_cpu(sd_llc, cpu)); + if (!sd) + return 0; + + return sd->span_weight; +} + +/* + * Return the cpumask representing the LLC domain of @cpu (or NULL if the LLC + * domain is not defined). + */ +static struct cpumask *llc_span(s32 cpu) +{ + struct sched_domain *sd; + + sd = rcu_dereference(per_cpu(sd_llc, cpu)); + if (!sd) + return NULL; + + return sched_domain_span(sd); +} + +/* + * Return the amount of CPUs in the same NUMA domain of @cpu (or zero if the + * NUMA domain is not defined). + */ +static unsigned int numa_weight(s32 cpu) +{ + struct sched_domain *sd; + struct sched_group *sg; + + sd = rcu_dereference(per_cpu(sd_numa, cpu)); + if (!sd) + return 0; + sg = sd->groups; + if (!sg) + return 0; + + return sg->group_weight; +} + +/* + * Return the cpumask representing the NUMA domain of @cpu (or NULL if the NUMA + * domain is not defined). + */ +static struct cpumask *numa_span(s32 cpu) +{ + struct sched_domain *sd; + struct sched_group *sg; + + sd = rcu_dereference(per_cpu(sd_numa, cpu)); + if (!sd) + return NULL; + sg = sd->groups; + if (!sg) + return NULL; + + return sched_group_span(sg); +} + +/* + * Return true if the LLC domains do not perfectly overlap with the NUMA + * domains, false otherwise. + */ +static bool llc_numa_mismatch(void) +{ + int cpu; + + /* + * We need to scan all online CPUs to verify whether their scheduling + * domains overlap. + * + * While it is rare to encounter architectures with asymmetric NUMA + * topologies, CPU hotplugging or virtualized environments can result + * in asymmetric configurations. + * + * For example: + * + * NUMA 0: + * - LLC 0: cpu0..cpu7 + * - LLC 1: cpu8..cpu15 [offline] + * + * NUMA 1: + * - LLC 0: cpu16..cpu23 + * - LLC 1: cpu24..cpu31 + * + * In this case, if we only check the first online CPU (cpu0), we might + * incorrectly assume that the LLC and NUMA domains are fully + * overlapping, which is incorrect (as NUMA 1 has two distinct LLC + * domains). + */ + for_each_online_cpu(cpu) + if (llc_weight(cpu) != numa_weight(cpu)) + return true; + + return false; +} + +/* + * Initialize topology-aware scheduling. + * + * Detect if the system has multiple LLC or multiple NUMA domains and enable + * cache-aware / NUMA-aware scheduling optimizations in the default CPU idle + * selection policy. + * + * Assumption: the kernel's internal topology representation assumes that each + * CPU belongs to a single LLC domain, and that each LLC domain is entirely + * contained within a single NUMA node. + */ +void scx_idle_update_selcpu_topology(struct sched_ext_ops *ops) +{ + bool enable_llc = false, enable_numa = false; + unsigned int nr_cpus; + s32 cpu = cpumask_first(cpu_online_mask); + + /* + * Enable LLC domain optimization only when there are multiple LLC + * domains among the online CPUs. If all online CPUs are part of a + * single LLC domain, the idle CPU selection logic can choose any + * online CPU without bias. + * + * Note that it is sufficient to check the LLC domain of the first + * online CPU to determine whether a single LLC domain includes all + * CPUs. + */ + rcu_read_lock(); + nr_cpus = llc_weight(cpu); + if (nr_cpus > 0) { + if (nr_cpus < num_online_cpus()) + enable_llc = true; + pr_debug("sched_ext: LLC=%*pb weight=%u\n", + cpumask_pr_args(llc_span(cpu)), llc_weight(cpu)); + } + + /* + * Enable NUMA optimization only when there are multiple NUMA domains + * among the online CPUs and the NUMA domains don't perfectly overlap + * with the LLC domains. + * + * If all CPUs belong to the same NUMA node and the same LLC domain, + * enabling both NUMA and LLC optimizations is unnecessary, as checking + * for an idle CPU in the same domain twice is redundant. + * + * If SCX_OPS_BUILTIN_IDLE_PER_NODE is enabled ignore the NUMA + * optimization, as we would naturally select idle CPUs within + * specific NUMA nodes querying the corresponding per-node cpumask. + */ + if (!(ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE)) { + nr_cpus = numa_weight(cpu); + if (nr_cpus > 0) { + if (nr_cpus < num_online_cpus() && llc_numa_mismatch()) + enable_numa = true; + pr_debug("sched_ext: NUMA=%*pb weight=%u\n", + cpumask_pr_args(numa_span(cpu)), nr_cpus); + } + } + rcu_read_unlock(); + + pr_debug("sched_ext: LLC idle selection %s\n", + str_enabled_disabled(enable_llc)); + pr_debug("sched_ext: NUMA idle selection %s\n", + str_enabled_disabled(enable_numa)); + + if (enable_llc) + static_branch_enable_cpuslocked(&scx_selcpu_topo_llc); + else + static_branch_disable_cpuslocked(&scx_selcpu_topo_llc); + if (enable_numa) + static_branch_enable_cpuslocked(&scx_selcpu_topo_numa); + else + static_branch_disable_cpuslocked(&scx_selcpu_topo_numa); +} + +/* + * Return true if @p can run on all possible CPUs, false otherwise. + */ +static inline bool task_affinity_all(const struct task_struct *p) +{ + return p->nr_cpus_allowed >= num_possible_cpus(); +} + +/* + * Built-in CPU idle selection policy: + * + * 1. Prioritize full-idle cores: + * - always prioritize CPUs from fully idle cores (both logical CPUs are + * idle) to avoid interference caused by SMT. + * + * 2. Reuse the same CPU: + * - prefer the last used CPU to take advantage of cached data (L1, L2) and + * branch prediction optimizations. + * + * 3. Prefer @prev_cpu's SMT sibling: + * - if @prev_cpu is busy and no fully idle core is available, try to + * place the task on an idle SMT sibling of @prev_cpu; keeping the + * task on the same core makes migration cheaper, preserves L1 cache + * locality and reduces wakeup latency. + * + * 4. Pick a CPU within the same LLC (Last-Level Cache): + * - if the above conditions aren't met, pick a CPU that shares the same + * LLC, if the LLC domain is a subset of @cpus_allowed, to maintain + * cache locality. + * + * 5. Pick a CPU within the same NUMA node, if enabled: + * - choose a CPU from the same NUMA node, if the node cpumask is a + * subset of @cpus_allowed, to reduce memory access latency. + * + * 6. Pick any idle CPU within the @cpus_allowed domain. + * + * Step 4 and 5 are performed only if the system has, respectively, + * multiple LLCs / multiple NUMA nodes (see scx_selcpu_topo_llc and + * scx_selcpu_topo_numa) and they don't contain the same subset of CPUs. + * + * If %SCX_OPS_BUILTIN_IDLE_PER_NODE is enabled, the search will always + * begin in @prev_cpu's node and proceed to other nodes in order of + * increasing distance. + * + * Return the picked CPU if idle, or a negative value otherwise. + * + * NOTE: tasks that can only run on 1 CPU are excluded by this logic, because + * we never call ops.select_cpu() for them, see select_task_rq(). + */ +s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, + const struct cpumask *cpus_allowed, u64 flags) +{ + const struct cpumask *llc_cpus = NULL, *numa_cpus = NULL; + const struct cpumask *allowed = cpus_allowed ?: p->cpus_ptr; + int node = scx_cpu_node_if_enabled(prev_cpu); + bool is_prev_allowed; + s32 cpu; + + preempt_disable(); + + /* + * Determine the subset of CPUs usable by @p within @cpus_allowed. + */ + if (allowed != p->cpus_ptr) { + struct cpumask *local_cpus = this_cpu_cpumask_var_ptr(local_idle_cpumask); + + if (task_affinity_all(p)) { + allowed = cpus_allowed; + } else if (cpumask_and(local_cpus, cpus_allowed, p->cpus_ptr)) { + allowed = local_cpus; + } else { + cpu = -EBUSY; + goto out_enable; + } + } + + /* + * Check whether @prev_cpu is still within the allowed set. If not, + * we can still try selecting a nearby CPU. + */ + is_prev_allowed = cpumask_test_cpu(prev_cpu, allowed); + + /* + * This is necessary to protect llc_cpus. + */ + rcu_read_lock(); + + /* + * Determine the subset of CPUs that the task can use in its + * current LLC and node. + * + * If the task can run on all CPUs, use the node and LLC cpumasks + * directly. + */ + if (static_branch_maybe(CONFIG_NUMA, &scx_selcpu_topo_numa)) { + struct cpumask *local_cpus = this_cpu_cpumask_var_ptr(local_numa_idle_cpumask); + const struct cpumask *cpus = numa_span(prev_cpu); + + if (allowed == p->cpus_ptr && task_affinity_all(p)) + numa_cpus = cpus; + else if (cpus && cpumask_and(local_cpus, allowed, cpus)) + numa_cpus = local_cpus; + } + + if (static_branch_maybe(CONFIG_SCHED_MC, &scx_selcpu_topo_llc)) { + struct cpumask *local_cpus = this_cpu_cpumask_var_ptr(local_llc_idle_cpumask); + const struct cpumask *cpus = llc_span(prev_cpu); + + if (allowed == p->cpus_ptr && task_affinity_all(p)) + llc_cpus = cpus; + else if (cpus && cpumask_and(local_cpus, allowed, cpus)) + llc_cpus = local_cpus; + } + + /* + * If WAKE_SYNC, try to migrate the wakee to the waker's CPU. + */ + if (wake_flags & SCX_WAKE_SYNC) { + int waker_node; + + /* + * If the waker's CPU is cache affine and prev_cpu is idle, + * then avoid a migration. + */ + cpu = smp_processor_id(); + if (is_prev_allowed && cpus_share_cache(cpu, prev_cpu) && + scx_idle_test_and_clear_cpu(prev_cpu)) { + cpu = prev_cpu; + goto out_unlock; + } + + /* + * If the waker's local DSQ is empty, and the system is under + * utilized, try to wake up @p to the local DSQ of the waker. + * + * Checking only for an empty local DSQ is insufficient as it + * could give the wakee an unfair advantage when the system is + * oversaturated. + * + * Checking only for the presence of idle CPUs is also + * insufficient as the local DSQ of the waker could have tasks + * piled up on it even if there is an idle core elsewhere on + * the system. + */ + waker_node = scx_cpu_node_if_enabled(cpu); + if (!(current->flags & PF_EXITING) && + cpu_rq(cpu)->scx.local_dsq.nr == 0 && + (!(flags & SCX_PICK_IDLE_IN_NODE) || (waker_node == node)) && + !cpumask_empty(idle_cpumask(waker_node)->cpu)) { + if (cpumask_test_cpu(cpu, allowed)) + goto out_unlock; + } + } + + /* + * If CPU has SMT, any wholly idle CPU is likely a better pick than + * partially idle @prev_cpu. + */ + if (sched_smt_active()) { + /* + * Keep using @prev_cpu if it's part of a fully idle core. + */ + if (is_prev_allowed && + cpumask_test_cpu(prev_cpu, idle_cpumask(node)->smt) && + scx_idle_test_and_clear_cpu(prev_cpu)) { + cpu = prev_cpu; + goto out_unlock; + } + + /* + * Search for any fully idle core in the same LLC domain. + */ + if (llc_cpus) { + cpu = pick_idle_cpu_in_node(llc_cpus, node, SCX_PICK_IDLE_CORE); + if (cpu >= 0) + goto out_unlock; + } + + /* + * Search for any fully idle core in the same NUMA node. + */ + if (numa_cpus) { + cpu = pick_idle_cpu_in_node(numa_cpus, node, SCX_PICK_IDLE_CORE); + if (cpu >= 0) + goto out_unlock; + } + + /* + * Search for any full-idle core usable by the task. + * + * If the node-aware idle CPU selection policy is enabled + * (%SCX_OPS_BUILTIN_IDLE_PER_NODE), the search will always + * begin in prev_cpu's node and proceed to other nodes in + * order of increasing distance. + */ + cpu = scx_pick_idle_cpu(allowed, node, flags | SCX_PICK_IDLE_CORE); + if (cpu >= 0) + goto out_unlock; + + /* + * Give up if we're strictly looking for a full-idle SMT + * core. + */ + if (flags & SCX_PICK_IDLE_CORE) { + cpu = -EBUSY; + goto out_unlock; + } + } + + /* + * Use @prev_cpu if it's idle. + */ + if (is_prev_allowed && scx_idle_test_and_clear_cpu(prev_cpu)) { + cpu = prev_cpu; + goto out_unlock; + } + + /* + * Use @prev_cpu's sibling if it's idle. + */ + if (sched_smt_active()) { + for_each_cpu_and(cpu, cpu_smt_mask(prev_cpu), allowed) { + if (cpu == prev_cpu) + continue; + if (scx_idle_test_and_clear_cpu(cpu)) + goto out_unlock; + } + } + + /* + * Search for any idle CPU in the same LLC domain. + */ + if (llc_cpus) { + cpu = pick_idle_cpu_in_node(llc_cpus, node, 0); + if (cpu >= 0) + goto out_unlock; + } + + /* + * Search for any idle CPU in the same NUMA node. + */ + if (numa_cpus) { + cpu = pick_idle_cpu_in_node(numa_cpus, node, 0); + if (cpu >= 0) + goto out_unlock; + } + + /* + * Search for any idle CPU usable by the task. + * + * If the node-aware idle CPU selection policy is enabled + * (%SCX_OPS_BUILTIN_IDLE_PER_NODE), the search will always begin + * in prev_cpu's node and proceed to other nodes in order of + * increasing distance. + */ + cpu = scx_pick_idle_cpu(allowed, node, flags); + +out_unlock: + rcu_read_unlock(); +out_enable: + preempt_enable(); + + return cpu; +} + +/* + * Initialize global and per-node idle cpumasks. + */ +void scx_idle_init_masks(void) +{ + int i; + + /* Allocate global idle cpumasks */ + BUG_ON(!alloc_cpumask_var(&scx_idle_global_masks.cpu, GFP_KERNEL)); + BUG_ON(!alloc_cpumask_var(&scx_idle_global_masks.smt, GFP_KERNEL)); + + /* Allocate per-node idle cpumasks (use nr_node_ids for non-contiguous NUMA nodes) */ + scx_idle_node_masks = kzalloc_objs(*scx_idle_node_masks, nr_node_ids); + BUG_ON(!scx_idle_node_masks); + + for_each_node(i) { + scx_idle_node_masks[i] = kzalloc_node(sizeof(**scx_idle_node_masks), + GFP_KERNEL, i); + BUG_ON(!scx_idle_node_masks[i]); + + BUG_ON(!alloc_cpumask_var_node(&scx_idle_node_masks[i]->cpu, GFP_KERNEL, i)); + BUG_ON(!alloc_cpumask_var_node(&scx_idle_node_masks[i]->smt, GFP_KERNEL, i)); + } + + /* Allocate local per-cpu idle cpumasks */ + for_each_possible_cpu(i) { + BUG_ON(!alloc_cpumask_var_node(&per_cpu(local_idle_cpumask, i), + GFP_KERNEL, cpu_to_node(i))); + BUG_ON(!alloc_cpumask_var_node(&per_cpu(local_llc_idle_cpumask, i), + GFP_KERNEL, cpu_to_node(i))); + BUG_ON(!alloc_cpumask_var_node(&per_cpu(local_numa_idle_cpumask, i), + GFP_KERNEL, cpu_to_node(i))); + } +} + +static void update_builtin_idle(int cpu, bool idle) +{ + int node = scx_cpu_node_if_enabled(cpu); + struct cpumask *idle_cpus = idle_cpumask(node)->cpu; + + assign_cpu(cpu, idle_cpus, idle); + + if (sched_smt_active()) { + const struct cpumask *smt = cpu_smt_mask(cpu); + struct cpumask *idle_smts = idle_cpumask(node)->smt; + + if (idle) { + /* + * idle_smt handling is racy but that's fine as it's + * only for optimization and self-correcting. + */ + if (!cpumask_subset(smt, idle_cpus)) + return; + cpumask_or(idle_smts, idle_smts, smt); + } else { + cpumask_andnot(idle_smts, idle_smts, smt); + } + } +} + +/* + * Update the idle state of a CPU to @idle. + * + * If @do_notify is true, ops.update_idle() is invoked to notify the scx + * scheduler of an actual idle state transition (idle to busy or vice + * versa). If @do_notify is false, only the idle state in the idle masks is + * refreshed without invoking ops.update_idle(). + * + * This distinction is necessary, because an idle CPU can be "reserved" and + * awakened via scx_bpf_pick_idle_cpu() + scx_bpf_kick_cpu(), marking it as + * busy even if no tasks are dispatched. In this case, the CPU may return + * to idle without a true state transition. Refreshing the idle masks + * without invoking ops.update_idle() ensures accurate idle state tracking + * while avoiding unnecessary updates and maintaining balanced state + * transitions. + */ +void __scx_update_idle(struct rq *rq, bool idle, bool do_notify) +{ + struct scx_sched *sch = scx_root; + int cpu = cpu_of(rq); + + lockdep_assert_rq_held(rq); + + /* + * Update the idle masks: + * - for real idle transitions (do_notify == true) + * - for idle-to-idle transitions (indicated by the previous task + * being the idle thread, managed by pick_task_idle()) + * + * Skip updating idle masks if the previous task is not the idle + * thread, since set_next_task_idle() has already handled it when + * transitioning from a task to the idle thread (calling this + * function with do_notify == true). + * + * In this way we can avoid updating the idle masks twice, + * unnecessarily. + */ + if (static_branch_likely(&scx_builtin_idle_enabled)) + if (do_notify || is_idle_task(rq->curr)) + update_builtin_idle(cpu, idle); + + /* + * Trigger ops.update_idle() only when transitioning from a task to + * the idle thread and vice versa. + * + * Idle transitions are indicated by do_notify being set to true, + * managed by put_prev_task_idle()/set_next_task_idle(). + * + * This must come after builtin idle update so that BPF schedulers can + * create interlocking between ops.update_idle() and ops.enqueue() - + * either enqueue() sees the idle bit or update_idle() sees the task + * that enqueue() queued. + */ + if (SCX_HAS_OP(sch, update_idle) && do_notify && + !scx_bypassing(sch, cpu_of(rq))) + SCX_CALL_OP(sch, update_idle, rq, scx_cpu_arg(cpu_of(rq)), idle); +} + +static void reset_idle_masks(struct sched_ext_ops *ops) +{ + int node; + + /* + * Consider all online cpus idle. Should converge to the actual state + * quickly. + */ + if (!(ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE)) { + cpumask_copy(idle_cpumask(NUMA_NO_NODE)->cpu, cpu_online_mask); + cpumask_copy(idle_cpumask(NUMA_NO_NODE)->smt, cpu_online_mask); + return; + } + + for_each_node(node) { + const struct cpumask *node_mask = cpumask_of_node(node); + + cpumask_and(idle_cpumask(node)->cpu, cpu_online_mask, node_mask); + cpumask_and(idle_cpumask(node)->smt, cpu_online_mask, node_mask); + } +} + +void scx_idle_enable(struct sched_ext_ops *ops) +{ + if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) + static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); + else + static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); + + if (ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE) + static_branch_enable_cpuslocked(&scx_builtin_idle_per_node); + else + static_branch_disable_cpuslocked(&scx_builtin_idle_per_node); + + reset_idle_masks(ops); +} + +void scx_idle_disable(void) +{ + static_branch_disable(&scx_builtin_idle_enabled); + static_branch_disable(&scx_builtin_idle_per_node); +} + +/******************************************************************************** + * Helpers that can be called from the BPF scheduler. + */ + +static int validate_node(struct scx_sched *sch, int node) +{ + if (!static_branch_likely(&scx_builtin_idle_per_node)) { + scx_error(sch, "per-node idle tracking is disabled"); + return -EOPNOTSUPP; + } + + /* Return no entry for NUMA_NO_NODE (not a critical scx error) */ + if (node == NUMA_NO_NODE) + return -ENOENT; + + /* Make sure node is in a valid range */ + if (node < 0 || node >= nr_node_ids) { + scx_error(sch, "invalid node %d", node); + return -EINVAL; + } + + /* Make sure the node is part of the set of possible nodes */ + if (!node_possible(node)) { + scx_error(sch, "unavailable node %d", node); + return -EINVAL; + } + + return node; +} + +__bpf_kfunc_start_defs(); + +static bool check_builtin_idle_enabled(struct scx_sched *sch) +{ + if (static_branch_likely(&scx_builtin_idle_enabled)) + return true; + + scx_error(sch, "built-in idle tracking is disabled"); + return false; +} + +/* + * Determine whether @p is a migration-disabled task in the context of BPF + * code. + * + * We can't simply check whether @p->migration_disabled is set in a + * sched_ext callback, because the BPF prolog (__bpf_prog_enter) may disable + * migration for the current task while running BPF code. + * + * Since the BPF prolog calls migrate_disable() only when CONFIG_PREEMPT_RCU + * is enabled (via rcu_read_lock_dont_migrate()), migration_disabled == 1 for + * the current task is ambiguous only in that case: it could be from the BPF + * prolog rather than a real migrate_disable() call. + * + * Without CONFIG_PREEMPT_RCU, the BPF prolog never calls migrate_disable(), + * so migration_disabled == 1 always means the task is truly + * migration-disabled. + * + * Therefore, when migration_disabled == 1 and CONFIG_PREEMPT_RCU is enabled, + * check whether @p is the current task or not: if it is, then migration was + * not disabled before entering the callback, otherwise migration was disabled. + * + * Returns true if @p is migration-disabled, false otherwise. + */ +static bool is_bpf_migration_disabled(const struct task_struct *p) +{ + if (p->migration_disabled == 1) { + if (IS_ENABLED(CONFIG_PREEMPT_RCU)) + return p != current; + return true; + } + return p->migration_disabled; +} + +static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p, + s32 prev_cpu, u64 wake_flags, + const struct cpumask *allowed, u64 flags) +{ + unsigned long irq_flags; + bool we_locked = false; + s32 cpu; + + if (!scx_cpu_valid(sch, prev_cpu, NULL)) + return -EINVAL; + + if (!check_builtin_idle_enabled(sch)) + return -EBUSY; + + /* + * Accessing p->cpus_ptr / p->nr_cpus_allowed needs either @p's rq + * lock or @p's pi_lock. Three cases: + * + * - inside ops.select_cpu(): try_to_wake_up() holds the wake-up + * task's pi_lock; the wake-up task is recorded in kf_tasks[0] + * by SCX_CALL_OP_TASK_RET(). + * - other rq-locked SCX op: scx_locked_rq() points at the held rq. + * - truly unlocked (UNLOCKED ops, SYSCALL, non-SCX struct_ops): + * nothing held, take pi_lock ourselves. + * + * In the first two cases, BPF schedulers may pass an arbitrary task + * that the held lock doesn't cover. Refuse those. + */ + if (this_rq()->scx.in_select_cpu) { + if (!scx_kf_arg_task_ok(sch, p)) + return -EINVAL; + lockdep_assert_held(&p->pi_lock); + } else if (scx_locked_rq()) { + if (task_rq(p) != scx_locked_rq()) + goto cross_task; + } else { + raw_spin_lock_irqsave(&p->pi_lock, irq_flags); + we_locked = true; + } + + /* + * This may also be called from ops.enqueue(), so we need to handle + * per-CPU tasks as well. For these tasks, we can skip all idle CPU + * selection optimizations and simply check whether the previously + * used CPU is idle and within the allowed cpumask. + */ + if (p->nr_cpus_allowed == 1 || is_bpf_migration_disabled(p)) { + if (cpumask_test_cpu(prev_cpu, allowed ?: p->cpus_ptr) && + scx_idle_test_and_clear_cpu(prev_cpu)) + cpu = prev_cpu; + else + cpu = -EBUSY; + } else { + cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, + allowed ?: p->cpus_ptr, flags); + } + + if (we_locked) + raw_spin_unlock_irqrestore(&p->pi_lock, irq_flags); + + return cpu; + +cross_task: + scx_error(sch, "select_cpu kfunc called cross-task on %s[%d]", + p->comm, p->pid); + return -EINVAL; +} + +/** + * scx_bpf_cpu_node - Return the NUMA node the given @cpu belongs to, or + * trigger an error if @cpu is invalid + * @cpu: target CPU + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + */ +__bpf_kfunc s32 scx_bpf_cpu_node(s32 cpu, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch) || !scx_cpu_valid(sch, cpu, NULL)) + return NUMA_NO_NODE; + return cpu_to_node(cpu); +} + +/** + * scx_bpf_select_cpu_dfl - The default implementation of ops.select_cpu() + * @p: task_struct to select a CPU for + * @prev_cpu: CPU @p was on previously + * @wake_flags: %SCX_WAKE_* flags + * @is_idle: out parameter indicating whether the returned CPU is idle + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Can be called from ops.select_cpu(), ops.enqueue(), or from an unlocked + * context such as a BPF test_run() call, as long as built-in CPU selection + * is enabled: ops.update_idle() is missing or %SCX_OPS_KEEP_BUILTIN_IDLE + * is set. + * + * Returns the picked CPU with *@is_idle indicating whether the picked CPU is + * currently idle and thus a good candidate for direct dispatching. + */ +__bpf_kfunc s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, + u64 wake_flags, bool *is_idle, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -ENODEV; + + cpu = select_cpu_from_kfunc(sch, p, prev_cpu, wake_flags, NULL, 0); + if (cpu >= 0) { + *is_idle = true; + return cpu; + } + *is_idle = false; + return prev_cpu; +} + +struct scx_bpf_select_cpu_and_args { + /* @p and @cpus_allowed can't be packed together as KF_RCU is not transitive */ + s32 prev_cpu; + u64 wake_flags; + u64 flags; +}; + +/** + * __scx_bpf_select_cpu_and - Arg-wrapped CPU selection with cpumask + * @p: task_struct to select a CPU for + * @cpus_allowed: cpumask of allowed CPUs + * @args: struct containing the rest of the arguments + * @args->prev_cpu: CPU @p was on previously + * @args->wake_flags: %SCX_WAKE_* flags + * @args->flags: %SCX_PICK_IDLE* flags + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Wrapper kfunc that takes arguments via struct to work around BPF's 5 argument + * limit. BPF programs should use scx_bpf_select_cpu_and() which is provided + * as an inline wrapper in common.bpf.h. + * + * Can be called from ops.select_cpu(), ops.enqueue(), or from an unlocked + * context such as a BPF test_run() call, as long as built-in CPU selection + * is enabled: ops.update_idle() is missing or %SCX_OPS_KEEP_BUILTIN_IDLE + * is set. + * + * @p, @args->prev_cpu and @args->wake_flags match ops.select_cpu(). + * + * Returns the selected idle CPU, which will be automatically awakened upon + * returning from ops.select_cpu() and can be used for direct dispatch, or + * a negative value if no idle CPU is available. + */ +__bpf_kfunc s32 +__scx_bpf_select_cpu_and(struct task_struct *p, const struct cpumask *cpus_allowed, + struct scx_bpf_select_cpu_and_args *args, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -ENODEV; + + return select_cpu_from_kfunc(sch, p, args->prev_cpu, args->wake_flags, + cpus_allowed, args->flags); +} + +/* + * COMPAT: Will be removed in v6.22. + */ +__bpf_kfunc s32 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64 wake_flags, + const struct cpumask *cpus_allowed, u64 flags) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = rcu_dereference(scx_root); + if (unlikely(!sch)) + return -ENODEV; + +#ifdef CONFIG_EXT_SUB_SCHED + /* + * Disallow if any sub-scheds are attached. There is no way to tell + * which scheduler called us, just error out @p's scheduler. + */ + if (unlikely(!list_empty(&sch->children))) { + scx_error(scx_task_sched(p), "__scx_bpf_select_cpu_and() must be used"); + return -EINVAL; + } +#endif + + return select_cpu_from_kfunc(sch, p, prev_cpu, wake_flags, + cpus_allowed, flags); +} + +/** + * scx_bpf_get_idle_cpumask_node - Get a referenced kptr to the + * idle-tracking per-CPU cpumask of a target NUMA node. + * @node: target NUMA node + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Returns an empty cpumask if idle tracking is not enabled, if @node is + * not valid, or running on a UP kernel. In this case the actual error will + * be reported to the BPF scheduler via scx_error(). + */ +__bpf_kfunc const struct cpumask * +scx_bpf_get_idle_cpumask_node(s32 node, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return cpu_none_mask; + + node = validate_node(sch, node); + if (node < 0) + return cpu_none_mask; + + return idle_cpumask(node)->cpu; +} + +/** + * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking + * per-CPU cpumask. + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Returns an empty mask if idle tracking is not enabled, or running on a + * UP kernel. + */ +__bpf_kfunc const struct cpumask *scx_bpf_get_idle_cpumask(const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return cpu_none_mask; + + if (static_branch_unlikely(&scx_builtin_idle_per_node)) { + scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE enabled"); + return cpu_none_mask; + } + + if (!check_builtin_idle_enabled(sch)) + return cpu_none_mask; + + return idle_cpumask(NUMA_NO_NODE)->cpu; +} + +/** + * scx_bpf_get_idle_smtmask_node - Get a referenced kptr to the + * idle-tracking, per-physical-core cpumask of a target NUMA node. Can be + * used to determine if an entire physical core is free. + * @node: target NUMA node + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Returns an empty cpumask if idle tracking is not enabled, if @node is + * not valid, or running on a UP kernel. In this case the actual error will + * be reported to the BPF scheduler via scx_error(). + */ +__bpf_kfunc const struct cpumask * +scx_bpf_get_idle_smtmask_node(s32 node, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return cpu_none_mask; + + node = validate_node(sch, node); + if (node < 0) + return cpu_none_mask; + + if (sched_smt_active()) + return idle_cpumask(node)->smt; + else + return idle_cpumask(node)->cpu; +} + +/** + * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, + * per-physical-core cpumask. Can be used to determine if an entire physical + * core is free. + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Returns an empty mask if idle tracking is not enabled, or running on a + * UP kernel. + */ +__bpf_kfunc const struct cpumask *scx_bpf_get_idle_smtmask(const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return cpu_none_mask; + + if (static_branch_unlikely(&scx_builtin_idle_per_node)) { + scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE enabled"); + return cpu_none_mask; + } + + if (!check_builtin_idle_enabled(sch)) + return cpu_none_mask; + + if (sched_smt_active()) + return idle_cpumask(NUMA_NO_NODE)->smt; + else + return idle_cpumask(NUMA_NO_NODE)->cpu; +} + +/** + * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to + * either the percpu, or SMT idle-tracking cpumask. + * @idle_mask: &cpumask to use + */ +__bpf_kfunc void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) +{ + /* + * Empty function body because we aren't actually acquiring or releasing + * a reference to a global idle cpumask, which is read-only in the + * caller and is never released. The acquire / release semantics here + * are just used to make the cpumask a trusted pointer in the caller. + */ +} + +/** + * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state + * @cpu: cpu to test and clear idle for + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Returns %true if @cpu was idle and its idle state was successfully cleared. + * %false otherwise. + * + * Unavailable if ops.update_idle() is implemented and + * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. + */ +__bpf_kfunc bool scx_bpf_test_and_clear_cpu_idle(s32 cpu, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return false; + + if (!check_builtin_idle_enabled(sch)) + return false; + + if (!scx_cpu_valid(sch, cpu, NULL)) + return false; + + return scx_idle_test_and_clear_cpu(cpu); +} + +/** + * scx_bpf_pick_idle_cpu_node - Pick and claim an idle cpu from @node + * @cpus_allowed: Allowed cpumask + * @node: target NUMA node + * @flags: %SCX_PICK_IDLE_* flags + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Pick and claim an idle cpu in @cpus_allowed from the NUMA node @node. + * + * Returns the picked idle cpu number on success, or -%EBUSY if no matching + * cpu was found. + * + * The search starts from @node and proceeds to other online NUMA nodes in + * order of increasing distance (unless SCX_PICK_IDLE_IN_NODE is specified, + * in which case the search is limited to the target @node). + * + * Always returns an error if ops.update_idle() is implemented and + * %SCX_OPS_KEEP_BUILTIN_IDLE is not set, or if + * %SCX_OPS_BUILTIN_IDLE_PER_NODE is not set. + */ +__bpf_kfunc s32 scx_bpf_pick_idle_cpu_node(const struct cpumask *cpus_allowed, + s32 node, u64 flags, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -ENODEV; + + node = validate_node(sch, node); + if (node < 0) + return node; + + return scx_pick_idle_cpu(cpus_allowed, node, flags); +} + +/** + * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu + * @cpus_allowed: Allowed cpumask + * @flags: %SCX_PICK_IDLE_CPU_* flags + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Pick and claim an idle cpu in @cpus_allowed. Returns the picked idle cpu + * number on success. -%EBUSY if no matching cpu was found. + * + * Idle CPU tracking may race against CPU scheduling state transitions. For + * example, this function may return -%EBUSY as CPUs are transitioning into the + * idle state. If the caller then assumes that there will be dispatch events on + * the CPUs as they were all busy, the scheduler may end up stalling with CPUs + * idling while there are pending tasks. Use scx_bpf_pick_any_cpu() and + * scx_bpf_kick_cpu() to guarantee that there will be at least one dispatch + * event in the near future. + * + * Unavailable if ops.update_idle() is implemented and + * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. + * + * Always returns an error if %SCX_OPS_BUILTIN_IDLE_PER_NODE is set, use + * scx_bpf_pick_idle_cpu_node() instead. + */ +__bpf_kfunc s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed, + u64 flags, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -ENODEV; + + if (static_branch_maybe(CONFIG_NUMA, &scx_builtin_idle_per_node)) { + scx_error(sch, "per-node idle tracking is enabled"); + return -EBUSY; + } + + if (!check_builtin_idle_enabled(sch)) + return -EBUSY; + + return scx_pick_idle_cpu(cpus_allowed, NUMA_NO_NODE, flags); +} + +/** + * scx_bpf_pick_any_cpu_node - Pick and claim an idle cpu if available + * or pick any CPU from @node + * @cpus_allowed: Allowed cpumask + * @node: target NUMA node + * @flags: %SCX_PICK_IDLE_CPU_* flags + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any + * CPU in @cpus_allowed. Guaranteed to succeed and returns the picked idle cpu + * number if @cpus_allowed is not empty. -%EBUSY is returned if @cpus_allowed is + * empty. + * + * The search starts from @node and proceeds to other online NUMA nodes in + * order of increasing distance (unless %SCX_PICK_IDLE_IN_NODE is specified, + * in which case the search is limited to the target @node, regardless of + * the CPU idle state). + * + * If ops.update_idle() is implemented and %SCX_OPS_KEEP_BUILTIN_IDLE is not + * set, this function can't tell which CPUs are idle and will always pick any + * CPU. + */ +__bpf_kfunc s32 scx_bpf_pick_any_cpu_node(const struct cpumask *cpus_allowed, + s32 node, u64 flags, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -ENODEV; + + node = validate_node(sch, node); + if (node < 0) + return node; + + cpu = scx_pick_idle_cpu(cpus_allowed, node, flags); + if (cpu >= 0) + return cpu; + + if (flags & SCX_PICK_IDLE_IN_NODE) + cpu = cpumask_any_and_distribute(cpumask_of_node(node), cpus_allowed); + else + cpu = cpumask_any_distribute(cpus_allowed); + if (cpu < nr_cpu_ids) + return cpu; + else + return -EBUSY; +} + +/** + * scx_bpf_pick_any_cpu - Pick and claim an idle cpu if available or pick any CPU + * @cpus_allowed: Allowed cpumask + * @flags: %SCX_PICK_IDLE_CPU_* flags + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any + * CPU in @cpus_allowed. Guaranteed to succeed and returns the picked idle cpu + * number if @cpus_allowed is not empty. -%EBUSY is returned if @cpus_allowed is + * empty. + * + * If ops.update_idle() is implemented and %SCX_OPS_KEEP_BUILTIN_IDLE is not + * set, this function can't tell which CPUs are idle and will always pick any + * CPU. + * + * Always returns an error if %SCX_OPS_BUILTIN_IDLE_PER_NODE is set, use + * scx_bpf_pick_any_cpu_node() instead. + */ +__bpf_kfunc s32 scx_bpf_pick_any_cpu(const struct cpumask *cpus_allowed, + u64 flags, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -ENODEV; + + if (static_branch_maybe(CONFIG_NUMA, &scx_builtin_idle_per_node)) { + scx_error(sch, "per-node idle tracking is enabled"); + return -EBUSY; + } + + if (static_branch_likely(&scx_builtin_idle_enabled)) { + cpu = scx_pick_idle_cpu(cpus_allowed, NUMA_NO_NODE, flags); + if (cpu >= 0) + return cpu; + } + + cpu = cpumask_any_distribute(cpus_allowed); + if (cpu < nr_cpu_ids) + return cpu; + else + return -EBUSY; +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(scx_kfunc_ids_idle) +BTF_ID_FLAGS(func, scx_bpf_cpu_node, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) +BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_IMPLICIT_ARGS | KF_RCU) +BTF_KFUNCS_END(scx_kfunc_ids_idle) + +static const struct btf_kfunc_id_set scx_kfunc_set_idle = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_idle, + .filter = scx_kfunc_context_filter, +}; + +/* + * The select_cpu kfuncs internally call task_rq_lock() when invoked from an + * rq-unlocked context, and thus cannot be safely called from arbitrary tracing + * contexts where @p's pi_lock state is unknown. Keep them out of + * BPF_PROG_TYPE_TRACING by registering them in their own set which is exposed + * only to STRUCT_OPS and SYSCALL programs. + * + * These kfuncs are also members of scx_kfunc_ids_unlocked (see ext.c) because + * they're callable from unlocked contexts in addition to ops.select_cpu() and + * ops.enqueue(). + */ +BTF_KFUNCS_START(scx_kfunc_ids_select_cpu) +BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU) +BTF_KFUNCS_END(scx_kfunc_ids_select_cpu) + +static const struct btf_kfunc_id_set scx_kfunc_set_select_cpu = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_select_cpu, + .filter = scx_kfunc_context_filter, +}; + +int scx_idle_init(void) +{ + return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_idle) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_idle) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_idle) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_select_cpu) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_select_cpu); +} diff --git a/kernel/sched/ext/idle.h b/kernel/sched/ext/idle.h new file mode 100644 index 000000000000..8d169d3bbdf9 --- /dev/null +++ b/kernel/sched/ext/idle.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + * Copyright (c) 2024 Andrea Righi + */ +#ifndef _KERNEL_SCHED_EXT_IDLE_H +#define _KERNEL_SCHED_EXT_IDLE_H + +struct sched_ext_ops; + +extern struct btf_id_set8 scx_kfunc_ids_idle; +extern struct btf_id_set8 scx_kfunc_ids_select_cpu; + +void scx_idle_update_selcpu_topology(struct sched_ext_ops *ops); +void scx_idle_init_masks(void); + +s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, + const struct cpumask *cpus_allowed, u64 flags); +void scx_idle_enable(struct sched_ext_ops *ops); +void scx_idle_disable(void); +int scx_idle_init(void); + +#endif /* _KERNEL_SCHED_EXT_IDLE_H */ diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h new file mode 100644 index 000000000000..b04701190b23 --- /dev/null +++ b/kernel/sched/ext/internal.h @@ -0,0 +1,1653 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Copyright (c) 2025 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2025 Tejun Heo + */ +#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) +#define SCX_MOFF_IDX(moff) ((moff) / sizeof(void (*)(void))) + +enum scx_exit_kind { + SCX_EXIT_NONE, + SCX_EXIT_DONE, + + SCX_EXIT_UNREG = 64, /* user-space initiated unregistration */ + SCX_EXIT_UNREG_BPF, /* BPF-initiated unregistration */ + SCX_EXIT_UNREG_KERN, /* kernel-initiated unregistration */ + SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ + SCX_EXIT_PARENT, /* parent exiting */ + + SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ + SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ + SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ +}; + +/* + * An exit code can be specified when exiting with scx_bpf_exit() or scx_exit(), + * corresponding to exit_kind UNREG_BPF and UNREG_KERN respectively. The codes + * are 64bit of the format: + * + * Bits: [63 .. 48 47 .. 32 31 .. 0] + * [ SYS ACT ] [ SYS RSN ] [ USR ] + * + * SYS ACT: System-defined exit actions + * SYS RSN: System-defined exit reasons + * USR : User-defined exit codes and reasons + * + * Using the above, users may communicate intention and context by ORing system + * actions and/or system reasons with a user-defined exit code. + */ +enum scx_exit_code { + /* Reasons */ + SCX_ECODE_RSN_HOTPLUG = 1LLU << 32, + SCX_ECODE_RSN_CGROUP_OFFLINE = 2LLU << 32, + + /* Actions */ + SCX_ECODE_ACT_RESTART = 1LLU << 48, +}; + +enum scx_exit_flags { + /* + * ops.exit() may be called even if the loading failed before ops.init() + * finishes successfully. This is because ops.exit() allows rich exit + * info communication. The following flag indicates whether ops.init() + * finished successfully. + */ + SCX_EFLAG_INITIALIZED = 1LLU << 0, +}; + +/* + * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is + * being disabled. + */ +struct scx_exit_info { + /* %SCX_EXIT_* - broad category of the exit reason */ + enum scx_exit_kind kind; + + /* + * CPU that initiated the exit, valid once @kind has been set. + * Negative if the exit path didn't identify a CPU. + */ + s32 exit_cpu; + + /* exit code if gracefully exiting */ + s64 exit_code; + + /* %SCX_EFLAG_* */ + u64 flags; + + /* textual representation of the above */ + const char *reason; + + /* backtrace if exiting due to an error */ + unsigned long *bt; + u32 bt_len; + + /* informational message */ + char *msg; + + /* debug dump */ + char *dump; +}; + +/* sched_ext_ops.flags */ +enum scx_ops_flags { + /* + * Keep built-in idle tracking even if ops.update_idle() is implemented. + */ + SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, + + /* + * By default, if there are no other task to run on the CPU, ext core + * keeps running the current task even after its slice expires. If this + * flag is specified, such tasks are passed to ops.enqueue() with + * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. + */ + SCX_OPS_ENQ_LAST = 1LLU << 1, + + /* + * An exiting task may schedule after PF_EXITING is set. In such cases, + * bpf_task_from_pid() may not be able to find the task and if the BPF + * scheduler depends on pid lookup for dispatching, the task will be + * lost leading to various issues including RCU grace period stalls. + * + * To mask this problem, by default, unhashed tasks are automatically + * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't + * depend on pid lookups and wants to handle these tasks directly, the + * following flag can be used. With %SCX_OPS_TID_TO_TASK, + * scx_bpf_tid_to_task() can find exiting tasks reliably. + */ + SCX_OPS_ENQ_EXITING = 1LLU << 2, + + /* + * If set, only tasks with policy set to SCHED_EXT are attached to + * sched_ext. If clear, SCHED_NORMAL tasks are also included. + */ + SCX_OPS_SWITCH_PARTIAL = 1LLU << 3, + + /* + * A migration disabled task can only execute on its current CPU. By + * default, such tasks are automatically put on the CPU's local DSQ with + * the default slice on enqueue. If this ops flag is set, they also go + * through ops.enqueue(). + * + * A migration disabled task never invokes ops.select_cpu() as it can + * only select the current CPU. Also, p->cpus_ptr will only contain its + * current CPU while p->nr_cpus_allowed keeps tracking p->user_cpus_ptr + * and thus may disagree with cpumask_weight(p->cpus_ptr). + */ + SCX_OPS_ENQ_MIGRATION_DISABLED = 1LLU << 4, + + /* + * Queued wakeup (ttwu_queue) is a wakeup optimization that invokes + * ops.enqueue() on the ops.select_cpu() selected or the wakee's + * previous CPU via IPI (inter-processor interrupt) to reduce cacheline + * transfers. When this optimization is enabled, ops.select_cpu() is + * skipped in some cases (when racing against the wakee switching out). + * As the BPF scheduler may depend on ops.select_cpu() being invoked + * during wakeups, queued wakeup is disabled by default. + * + * If this ops flag is set, queued wakeup optimization is enabled and + * the BPF scheduler must be able to handle ops.enqueue() invoked on the + * wakee's CPU without preceding ops.select_cpu() even for tasks which + * may be executed on multiple CPUs. + */ + SCX_OPS_ALLOW_QUEUED_WAKEUP = 1LLU << 5, + + /* + * If set, enable per-node idle cpumasks. If clear, use a single global + * flat idle cpumask. + */ + SCX_OPS_BUILTIN_IDLE_PER_NODE = 1LLU << 6, + + /* + * If set, %SCX_ENQ_IMMED is assumed to be set on all local DSQ + * enqueues. + */ + SCX_OPS_ALWAYS_ENQ_IMMED = 1LLU << 7, + + /* + * Maintain a mapping from p->scx.tid to task_struct so the BPF + * scheduler can recover task pointers from stored tids via + * scx_bpf_tid_to_task(). + * + * Only the root scheduler turns this on. A sub-sched may set the flag + * to declare a dependency on the lookup; if the root scheduler hasn't + * enabled it, attaching the sub-sched is rejected. + */ + SCX_OPS_TID_TO_TASK = 1LLU << 8, + + SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | + SCX_OPS_ENQ_LAST | + SCX_OPS_ENQ_EXITING | + SCX_OPS_ENQ_MIGRATION_DISABLED | + SCX_OPS_ALLOW_QUEUED_WAKEUP | + SCX_OPS_SWITCH_PARTIAL | + SCX_OPS_BUILTIN_IDLE_PER_NODE | + SCX_OPS_ALWAYS_ENQ_IMMED | + SCX_OPS_TID_TO_TASK, + + /* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */ + __SCX_OPS_INTERNAL_MASK = 0xffLLU << 56, + + SCX_OPS_HAS_CPU_PREEMPT = 1LLU << 56, +}; + +/* argument container for ops.init_task() */ +struct scx_init_task_args { + /* + * Set if ops.init_task() is being invoked on the fork path, as opposed + * to the scheduler transition path. + */ + bool fork; +#ifdef CONFIG_EXT_GROUP_SCHED + /* the cgroup the task is joining */ + struct cgroup *cgroup; +#endif +}; + +/* argument container for ops.exit_task() */ +struct scx_exit_task_args { + /* Whether the task exited before running on sched_ext. */ + bool cancelled; +}; + +/* argument container for ops.cgroup_init() */ +struct scx_cgroup_init_args { + /* the weight of the cgroup [1..10000] */ + u32 weight; + + /* bandwidth control parameters from cpu.max and cpu.max.burst */ + u64 bw_period_us; + u64 bw_quota_us; + u64 bw_burst_us; +}; + +enum scx_cpu_preempt_reason { + /* next task is being scheduled by &sched_class_rt */ + SCX_CPU_PREEMPT_RT, + /* next task is being scheduled by &sched_class_dl */ + SCX_CPU_PREEMPT_DL, + /* next task is being scheduled by &sched_class_stop */ + SCX_CPU_PREEMPT_STOP, + /* unknown reason for SCX being preempted */ + SCX_CPU_PREEMPT_UNKNOWN, +}; + +/* + * Argument container for ops.cpu_acquire(). Currently empty, but may be + * expanded in the future. + */ +struct scx_cpu_acquire_args {}; + +/* argument container for ops.cpu_release() */ +struct scx_cpu_release_args { + /* the reason the CPU was preempted */ + enum scx_cpu_preempt_reason reason; + + /* the task that's going to be scheduled on the CPU */ + struct task_struct *task; +}; + +/* informational context provided to dump operations */ +struct scx_dump_ctx { + enum scx_exit_kind kind; + s64 exit_code; + const char *reason; + u64 at_ns; + u64 at_jiffies; +}; + +/* argument container for ops.sub_attach() */ +struct scx_sub_attach_args { + struct sched_ext_ops *ops; + char *cgroup_path; +}; + +/* argument container for ops.sub_detach() */ +struct scx_sub_detach_args { + struct sched_ext_ops *ops; + char *cgroup_path; +}; + +/** + * struct sched_ext_ops - Operation table for BPF scheduler implementation + * + * A BPF scheduler can implement an arbitrary scheduling policy by + * implementing and loading operations in this table. Note that a userland + * scheduling policy can also be implemented using the BPF scheduler + * as a shim layer. + */ +struct sched_ext_ops { + /** + * @select_cpu: Pick the target CPU for a task which is being woken up + * @p: task being woken up + * @prev_cpu: the cpu @p was on before sleeping + * @wake_flags: SCX_WAKE_* + * + * Decision made here isn't final. @p may be moved to any CPU while it + * is getting dispatched for execution later. However, as @p is not on + * the rq at this point, getting the eventual execution CPU right here + * saves a small bit of overhead down the line. + * + * If an idle CPU is returned, the CPU is kicked and will try to + * dispatch. While an explicit custom mechanism can be added, + * select_cpu() serves as the default way to wake up idle CPUs. + * + * @p may be inserted into a DSQ directly by calling + * scx_bpf_dsq_insert(). If so, the ops.enqueue() will be skipped. + * Directly inserting into %SCX_DSQ_LOCAL will put @p in the local DSQ + * of the CPU returned by this operation. + * + * Note that select_cpu() is never called for tasks that can only run + * on a single CPU or tasks with migration disabled, as they don't have + * the option to select a different CPU. See select_task_rq() for + * details. + */ + s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); + + /** + * @enqueue: Enqueue a task on the BPF scheduler + * @p: task being enqueued + * @enq_flags: %SCX_ENQ_* + * + * @p is ready to run. Insert directly into a DSQ by calling + * scx_bpf_dsq_insert() or enqueue on the BPF scheduler. If not directly + * inserted, the bpf scheduler owns @p and if it fails to dispatch @p, + * the task will stall. + * + * If @p was inserted into a DSQ from ops.select_cpu(), this callback is + * skipped. + */ + void (*enqueue)(struct task_struct *p, u64 enq_flags); + + /** + * @dequeue: Remove a task from the BPF scheduler + * @p: task being dequeued + * @deq_flags: %SCX_DEQ_* + * + * Remove @p from the BPF scheduler. This is usually called to isolate + * the task while updating its scheduling properties (e.g. priority). + * + * The ext core keeps track of whether the BPF side owns a given task or + * not and can gracefully ignore spurious dispatches from BPF side, + * which makes it safe to not implement this method. However, depending + * on the scheduling logic, this can lead to confusing behaviors - e.g. + * scheduling position not being updated across a priority change. + */ + void (*dequeue)(struct task_struct *p, u64 deq_flags); + + /** + * @dispatch: Dispatch tasks from the BPF scheduler and/or user DSQs + * @cpu: CPU to dispatch tasks for + * @prev: previous task being switched out + * + * Called when a CPU's local dsq is empty. The operation should dispatch + * one or more tasks from the BPF scheduler into the DSQs using + * scx_bpf_dsq_insert() and/or move from user DSQs into the local DSQ + * using scx_bpf_dsq_move_to_local(). + * + * The maximum number of times scx_bpf_dsq_insert() can be called + * without an intervening scx_bpf_dsq_move_to_local() is specified by + * ops.dispatch_max_batch. See the comments on top of the two functions + * for more details. + * + * When not %NULL, @prev is an SCX task with its slice depleted. If + * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in + * @prev->scx.flags, it is not enqueued yet and will be enqueued after + * ops.dispatch() returns. To keep executing @prev, return without + * dispatching or moving any tasks. Also see %SCX_OPS_ENQ_LAST. + */ + void (*dispatch)(s32 cpu, struct task_struct *prev); + + /** + * @tick: Periodic tick + * @p: task running currently + * + * This operation is called every 1/HZ seconds on CPUs which are + * executing an SCX task. Setting @p->scx.slice to 0 will trigger an + * immediate dispatch cycle on the CPU. + */ + void (*tick)(struct task_struct *p); + + /** + * @runnable: A task is becoming runnable on its associated CPU + * @p: task becoming runnable + * @enq_flags: %SCX_ENQ_* + * + * This and the following three functions can be used to track a task's + * execution state transitions. A task becomes ->runnable() on a CPU, + * and then goes through one or more ->running() and ->stopping() pairs + * as it runs on the CPU, and eventually becomes ->quiescent() when it's + * done running on the CPU. + * + * @p is becoming runnable on the CPU because it's + * + * - waking up (%SCX_ENQ_WAKEUP) + * - being moved from another CPU + * - being restored after temporarily taken off the queue for an + * attribute change. + * + * This and ->enqueue() are related but not coupled. This operation + * notifies @p's state transition and may not be followed by ->enqueue() + * e.g. when @p is being dispatched to a remote CPU, or when @p is + * being enqueued on a CPU experiencing a hotplug event. Likewise, a + * task may be ->enqueue()'d without being preceded by this operation + * e.g. after exhausting its slice. + */ + void (*runnable)(struct task_struct *p, u64 enq_flags); + + /** + * @running: A task is starting to run on its associated CPU + * @p: task starting to run + * + * Note that this callback may be called from a CPU other than the + * one the task is going to run on. This can happen when a task + * property is changed (i.e., affinity), since scx_next_task_scx(), + * which triggers this callback, may run on a CPU different from + * the task's assigned CPU. + * + * Therefore, always use scx_bpf_task_cpu(@p) to determine the + * target CPU the task is going to use. + * + * See ->runnable() for explanation on the task state notifiers. + */ + void (*running)(struct task_struct *p); + + /** + * @stopping: A task is stopping execution + * @p: task stopping to run + * @runnable: is task @p still runnable? + * + * Note that this callback may be called from a CPU other than the + * one the task was running on. This can happen when a task + * property is changed (i.e., affinity), since dequeue_task_scx(), + * which triggers this callback, may run on a CPU different from + * the task's assigned CPU. + * + * Therefore, always use scx_bpf_task_cpu(@p) to retrieve the CPU + * the task was running on. + * + * See ->runnable() for explanation on the task state notifiers. If + * !@runnable, ->quiescent() will be invoked after this operation + * returns. + */ + void (*stopping)(struct task_struct *p, bool runnable); + + /** + * @quiescent: A task is becoming not runnable on its associated CPU + * @p: task becoming not runnable + * @deq_flags: %SCX_DEQ_* + * + * See ->runnable() for explanation on the task state notifiers. + * + * @p is becoming quiescent on the CPU because it's + * + * - sleeping (%SCX_DEQ_SLEEP) + * - being moved to another CPU + * - being temporarily taken off the queue for an attribute change + * (%SCX_DEQ_SAVE) + * + * This and ->dequeue() are related but not coupled. This operation + * notifies @p's state transition and may not be preceded by ->dequeue() + * e.g. when @p is being dispatched to a remote CPU. + */ + void (*quiescent)(struct task_struct *p, u64 deq_flags); + + /** + * @yield: Yield CPU + * @from: yielding task + * @to: optional yield target task + * + * If @to is NULL, @from is yielding the CPU to other runnable tasks. + * The BPF scheduler should ensure that other available tasks are + * dispatched before the yielding task. Return value is ignored in this + * case. + * + * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf + * scheduler can implement the request, return %true; otherwise, %false. + */ + bool (*yield)(struct task_struct *from, struct task_struct *to); + + /** + * @core_sched_before: Task ordering for core-sched + * @a: task A + * @b: task B + * + * Used by core-sched to determine the ordering between two tasks. See + * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on + * core-sched. + * + * Both @a and @b are runnable and may or may not currently be queued on + * the BPF scheduler. Should return %true if @a should run before @b. + * %false if there's no required ordering or @b should run before @a. + * + * If not specified, the default is ordering them according to when they + * became runnable. + */ + bool (*core_sched_before)(struct task_struct *a, struct task_struct *b); + + /** + * @set_weight: Set task weight + * @p: task to set weight for + * @weight: new weight [1..10000] + * + * Update @p's weight to @weight. + */ + void (*set_weight)(struct task_struct *p, u32 weight); + + /** + * @set_cpumask: Set CPU affinity + * @p: task to set CPU affinity for + * @cpumask: cpumask of cpus that @p can run on + * + * Update @p's CPU affinity to @cpumask. + */ + void (*set_cpumask)(struct task_struct *p, + const struct cpumask *cpumask); + + /** + * @update_idle: Update the idle state of a CPU + * @cpu: CPU to update the idle state for + * @idle: whether entering or exiting the idle state + * + * This operation is called when @rq's CPU goes or leaves the idle + * state. By default, implementing this operation disables the built-in + * idle CPU tracking and the following helpers become unavailable: + * + * - scx_bpf_select_cpu_dfl() + * - scx_bpf_select_cpu_and() + * - scx_bpf_test_and_clear_cpu_idle() + * - scx_bpf_pick_idle_cpu() + * + * The user also must implement ops.select_cpu() as the default + * implementation relies on scx_bpf_select_cpu_dfl(). + * + * Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle + * tracking. + */ + void (*update_idle)(s32 cpu, bool idle); + + /** + * @init_task: Initialize a task to run in a BPF scheduler + * @p: task to initialize for BPF scheduling + * @args: init arguments, see the struct definition + * + * Either we're loading a BPF scheduler or a new task is being forked. + * Initialize @p for BPF scheduling. This operation may block and can + * be used for allocations, and is called exactly once for a task. + * + * Return 0 for success, -errno for failure. An error return while + * loading will abort loading of the BPF scheduler. During a fork, it + * will abort that specific fork. + */ + s32 (*init_task)(struct task_struct *p, struct scx_init_task_args *args); + + /** + * @exit_task: Exit a previously-running task from the system + * @p: task to exit + * @args: exit arguments, see the struct definition + * + * @p is exiting or the BPF scheduler is being unloaded. Perform any + * necessary cleanup for @p. + */ + void (*exit_task)(struct task_struct *p, struct scx_exit_task_args *args); + + /** + * @enable: Enable BPF scheduling for a task + * @p: task to enable BPF scheduling for + * + * Enable @p for BPF scheduling. enable() is called on @p any time it + * enters SCX, and is always paired with a matching disable(). + */ + void (*enable)(struct task_struct *p); + + /** + * @disable: Disable BPF scheduling for a task + * @p: task to disable BPF scheduling for + * + * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. + * Disable BPF scheduling for @p. A disable() call is always matched + * with a prior enable() call. + */ + void (*disable)(struct task_struct *p); + + /** + * @dump: Dump BPF scheduler state on error + * @ctx: debug dump context + * + * Use scx_bpf_dump() to generate BPF scheduler specific debug dump. + */ + void (*dump)(struct scx_dump_ctx *ctx); + + /** + * @dump_cpu: Dump BPF scheduler state for a CPU on error + * @ctx: debug dump context + * @cpu: CPU to generate debug dump for + * @idle: @cpu is currently idle without any runnable tasks + * + * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for + * @cpu. If @idle is %true and this operation doesn't produce any + * output, @cpu is skipped for dump. + */ + void (*dump_cpu)(struct scx_dump_ctx *ctx, s32 cpu, bool idle); + + /** + * @dump_task: Dump BPF scheduler state for a runnable task on error + * @ctx: debug dump context + * @p: runnable task to generate debug dump for + * + * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for + * @p. + */ + void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p); + +#ifdef CONFIG_EXT_GROUP_SCHED + /** + * @cgroup_init: Initialize a cgroup + * @cgrp: cgroup being initialized + * @args: init arguments, see the struct definition + * + * Either the BPF scheduler is being loaded or @cgrp created, initialize + * @cgrp for sched_ext. This operation may block. + * + * Return 0 for success, -errno for failure. An error return while + * loading will abort loading of the BPF scheduler. During cgroup + * creation, it will abort the specific cgroup creation. + */ + s32 (*cgroup_init)(struct cgroup *cgrp, + struct scx_cgroup_init_args *args); + + /** + * @cgroup_exit: Exit a cgroup + * @cgrp: cgroup being exited + * + * Either the BPF scheduler is being unloaded or @cgrp destroyed, exit + * @cgrp for sched_ext. This operation my block. + */ + void (*cgroup_exit)(struct cgroup *cgrp); + + /** + * @cgroup_prep_move: Prepare a task to be moved to a different cgroup + * @p: task being moved + * @from: cgroup @p is being moved from + * @to: cgroup @p is being moved to + * + * Prepare @p for move from cgroup @from to @to. This operation may + * block and can be used for allocations. + * + * Return 0 for success, -errno for failure. An error return aborts the + * migration. + */ + s32 (*cgroup_prep_move)(struct task_struct *p, + struct cgroup *from, struct cgroup *to); + + /** + * @cgroup_move: Commit cgroup move + * @p: task being moved + * @from: cgroup @p is being moved from + * @to: cgroup @p is being moved to + * + * Commit the move. @p is dequeued during this operation. + */ + void (*cgroup_move)(struct task_struct *p, + struct cgroup *from, struct cgroup *to); + + /** + * @cgroup_cancel_move: Cancel cgroup move + * @p: task whose cgroup move is being canceled + * @from: cgroup @p was being moved from + * @to: cgroup @p was being moved to + * + * @p was cgroup_prep_move()'d but failed before reaching cgroup_move(). + * Undo the preparation. + */ + void (*cgroup_cancel_move)(struct task_struct *p, + struct cgroup *from, struct cgroup *to); + + /** + * @cgroup_set_weight: A cgroup's weight is being changed + * @cgrp: cgroup whose weight is being updated + * @weight: new weight [1..10000] + * + * Update @cgrp's weight to @weight. + */ + void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); + + /** + * @cgroup_set_bandwidth: A cgroup's bandwidth is being changed + * @cgrp: cgroup whose bandwidth is being updated + * @period_us: bandwidth control period + * @quota_us: bandwidth control quota + * @burst_us: bandwidth control burst + * + * Update @cgrp's bandwidth control parameters. This is from the cpu.max + * cgroup interface. + * + * @quota_us / @period_us determines the CPU bandwidth @cgrp is entitled + * to. For example, if @period_us is 1_000_000 and @quota_us is + * 2_500_000. @cgrp is entitled to 2.5 CPUs. @burst_us can be + * interpreted in the same fashion and specifies how much @cgrp can + * burst temporarily. The specific control mechanism and thus the + * interpretation of @period_us and burstiness is up to the BPF + * scheduler. + */ + void (*cgroup_set_bandwidth)(struct cgroup *cgrp, + u64 period_us, u64 quota_us, u64 burst_us); + + /** + * @cgroup_set_idle: A cgroup's idle state is being changed + * @cgrp: cgroup whose idle state is being updated + * @idle: whether the cgroup is entering or exiting idle state + * + * Update @cgrp's idle state to @idle. This callback is invoked when + * a cgroup transitions between idle and non-idle states, allowing the + * BPF scheduler to adjust its behavior accordingly. + */ + void (*cgroup_set_idle)(struct cgroup *cgrp, bool idle); + +#endif /* CONFIG_EXT_GROUP_SCHED */ + + /** + * @sub_attach: Attach a sub-scheduler + * @args: argument container, see the struct definition + * + * Return 0 to accept the sub-scheduler. -errno to reject. + */ + s32 (*sub_attach)(struct scx_sub_attach_args *args); + + /** + * @sub_detach: Detach a sub-scheduler + * @args: argument container, see the struct definition + */ + void (*sub_detach)(struct scx_sub_detach_args *args); + + /* + * All online ops must come before ops.cpu_online(). + */ + + /** + * @cpu_online: A CPU became online + * @cpu: CPU which just came up + * + * @cpu just came online. @cpu will not call ops.enqueue() or + * ops.dispatch(), nor run tasks associated with other CPUs beforehand. + */ + void (*cpu_online)(s32 cpu); + + /** + * @cpu_offline: A CPU is going offline + * @cpu: CPU which is going offline + * + * @cpu is going offline. @cpu will not call ops.enqueue() or + * ops.dispatch(), nor run tasks associated with other CPUs afterwards. + */ + void (*cpu_offline)(s32 cpu); + + /* + * All CPU hotplug ops must come before ops.init(). + */ + + /** + * @init: Initialize the BPF scheduler + */ + s32 (*init)(void); + + /** + * @exit: Clean up after the BPF scheduler + * @info: Exit info + * + * ops.exit() is also called on ops.init() failure, which is a bit + * unusual. This is to allow rich reporting through @info on how + * ops.init() failed. + */ + void (*exit)(struct scx_exit_info *info); + + /* + * Data fields must comes after all ops fields. + */ + + /** + * @dispatch_max_batch: Max nr of tasks that dispatch() can dispatch + */ + u32 dispatch_max_batch; + + /** + * @flags: %SCX_OPS_* flags + */ + u64 flags; + + /** + * @timeout_ms: The maximum amount of time, in milliseconds, that a + * runnable task should be able to wait before being scheduled. The + * maximum timeout may not exceed the default timeout of 30 seconds. + * + * Defaults to the maximum allowed timeout value of 30 seconds. + */ + u32 timeout_ms; + + /** + * @exit_dump_len: scx_exit_info.dump buffer length. If 0, the default + * value of 32768 is used. + */ + u32 exit_dump_len; + + /** + * @hotplug_seq: A sequence number that may be set by the scheduler to + * detect when a hotplug event has occurred during the loading process. + * If 0, no detection occurs. Otherwise, the scheduler will fail to + * load if the sequence number does not match @scx_hotplug_seq on the + * enable path. + */ + u64 hotplug_seq; + + /** + * @cgroup_id: When >1, attach the scheduler as a sub-scheduler on the + * specified cgroup. + */ + u64 sub_cgroup_id; + + /** + * @name: BPF scheduler's name + * + * Must be a non-zero valid BPF object name including only isalnum(), + * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the + * BPF scheduler is enabled. + */ + char name[SCX_OPS_NAME_LEN]; + + /* internal use only, must be NULL */ + void __rcu *priv; + + /* + * Deprecated callbacks. Kept at the end of the struct so the cid-form + * struct (sched_ext_ops_cid) can omit them without affecting the + * shared field offsets. Use SCX_ENQ_IMMED instead. Sitting past + * SCX_OPI_END means has_op doesn't cover them, so SCX_HAS_OP() cannot + * be used; callers must test sch->ops.cpu_acquire / cpu_release + * directly. + */ + + /** + * @cpu_acquire: A CPU is becoming available to the BPF scheduler + * @cpu: The CPU being acquired by the BPF scheduler. + * @args: Acquire arguments, see the struct definition. + * + * A CPU that was previously released from the BPF scheduler is now once + * again under its control. Deprecated; use SCX_ENQ_IMMED instead. + */ + void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); + + /** + * @cpu_release: A CPU is taken away from the BPF scheduler + * @cpu: The CPU being released by the BPF scheduler. + * @args: Release arguments, see the struct definition. + * + * The specified CPU is no longer under the control of the BPF + * scheduler. This could be because it was preempted by a higher + * priority sched_class, though there may be other reasons as well. The + * caller should consult @args->reason to determine the cause. + * Deprecated; use SCX_ENQ_IMMED instead. + */ + void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); +}; + +/** + * struct sched_ext_ops_cid - cid-form alternative to struct sched_ext_ops + * + * Mirrors struct sched_ext_ops with cpu/cpumask substituted with cid/cmask + * where applicable. Layout up to and including @priv matches sched_ext_ops + * byte-for-byte (verified by BUILD_BUG_ON checks at scx_init() time) so + * shared field offsets work for both struct types in bpf_scx_init_member() + * and bpf_scx_check_member(). The deprecated cpu_acquire/cpu_release + * callbacks at the tail of sched_ext_ops are omitted here entirely. + * + * Differences from sched_ext_ops: + * - select_cpu -> select_cid (returns cid) + * - dispatch -> dispatch (cpu arg is now cid) + * - update_idle -> update_idle (cpu arg is now cid) + * - set_cpumask -> set_cmask (cmask instead of cpumask) + * - cpu_online -> cid_online + * - cpu_offline -> cid_offline + * - dump_cpu -> dump_cid + * - cpu_acquire/cpu_release -> not present (deprecated in sched_ext_ops) + * + * BPF schedulers using this type cannot call cpu-form scx_bpf_* kfuncs; + * use the cid-form variants instead. Enforced at BPF verifier time via + * scx_kfunc_context_filter() branching on prog->aux->st_ops. + * + * See sched_ext_ops for callback documentation. + */ +struct sched_ext_ops_cid { + s32 (*select_cid)(struct task_struct *p, s32 prev_cid, u64 wake_flags); + void (*enqueue)(struct task_struct *p, u64 enq_flags); + void (*dequeue)(struct task_struct *p, u64 deq_flags); + void (*dispatch)(s32 cid, struct task_struct *prev); + void (*tick)(struct task_struct *p); + void (*runnable)(struct task_struct *p, u64 enq_flags); + void (*running)(struct task_struct *p); + void (*stopping)(struct task_struct *p, bool runnable); + void (*quiescent)(struct task_struct *p, u64 deq_flags); + bool (*yield)(struct task_struct *from, struct task_struct *to); + bool (*core_sched_before)(struct task_struct *a, + struct task_struct *b); + void (*set_weight)(struct task_struct *p, u32 weight); + void (*set_cmask)(struct task_struct *p, + const struct scx_cmask *cmask); + void (*update_idle)(s32 cid, bool idle); + s32 (*init_task)(struct task_struct *p, + struct scx_init_task_args *args); + void (*exit_task)(struct task_struct *p, + struct scx_exit_task_args *args); + void (*enable)(struct task_struct *p); + void (*disable)(struct task_struct *p); + void (*dump)(struct scx_dump_ctx *ctx); + void (*dump_cid)(struct scx_dump_ctx *ctx, s32 cid, bool idle); + void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p); +#ifdef CONFIG_EXT_GROUP_SCHED + s32 (*cgroup_init)(struct cgroup *cgrp, + struct scx_cgroup_init_args *args); + void (*cgroup_exit)(struct cgroup *cgrp); + s32 (*cgroup_prep_move)(struct task_struct *p, + struct cgroup *from, struct cgroup *to); + void (*cgroup_move)(struct task_struct *p, + struct cgroup *from, struct cgroup *to); + void (*cgroup_cancel_move)(struct task_struct *p, + struct cgroup *from, struct cgroup *to); + void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); + void (*cgroup_set_bandwidth)(struct cgroup *cgrp, + u64 period_us, u64 quota_us, u64 burst_us); + void (*cgroup_set_idle)(struct cgroup *cgrp, bool idle); +#endif /* CONFIG_EXT_GROUP_SCHED */ + s32 (*sub_attach)(struct scx_sub_attach_args *args); + void (*sub_detach)(struct scx_sub_detach_args *args); + void (*cid_online)(s32 cid); + void (*cid_offline)(s32 cid); + s32 (*init)(void); + void (*exit)(struct scx_exit_info *info); + + /* Data fields - must match sched_ext_ops layout exactly */ + u32 dispatch_max_batch; + u64 flags; + u32 timeout_ms; + u32 exit_dump_len; + u64 hotplug_seq; + u64 sub_cgroup_id; + char name[SCX_OPS_NAME_LEN]; + + /* internal use only, must be NULL */ + void __rcu *priv; + + /* layout end anchor for the BUILD_BUG_ON in scx_init(); keep last */ + char __end[0]; +}; + +enum scx_opi { + SCX_OPI_BEGIN = 0, + SCX_OPI_NORMAL_BEGIN = 0, + SCX_OPI_NORMAL_END = SCX_OP_IDX(cpu_online), + SCX_OPI_CPU_HOTPLUG_BEGIN = SCX_OP_IDX(cpu_online), + SCX_OPI_CPU_HOTPLUG_END = SCX_OP_IDX(init), + SCX_OPI_END = SCX_OP_IDX(init), +}; + +/* + * Collection of event counters. Event types are placed in descending order. + */ +struct scx_event_stats { + /* + * If ops.select_cpu() returns a CPU which can't be used by the task, + * the core scheduler code silently picks a fallback CPU. + */ + s64 SCX_EV_SELECT_CPU_FALLBACK; + + /* + * When dispatching to a local DSQ, the CPU may have gone offline in + * the meantime. In this case, the task is bounced to the global DSQ. + */ + s64 SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE; + + /* + * If SCX_OPS_ENQ_LAST is not set, the number of times that a task + * continued to run because there were no other tasks on the CPU. + */ + s64 SCX_EV_DISPATCH_KEEP_LAST; + + /* + * If SCX_OPS_ENQ_EXITING is not set, the number of times that a task + * is dispatched to a local DSQ when exiting. + */ + s64 SCX_EV_ENQ_SKIP_EXITING; + + /* + * If SCX_OPS_ENQ_MIGRATION_DISABLED is not set, the number of times a + * migration disabled task skips ops.enqueue() and is dispatched to its + * local DSQ. + */ + s64 SCX_EV_ENQ_SKIP_MIGRATION_DISABLED; + + /* + * The number of times a task, enqueued on a local DSQ with + * SCX_ENQ_IMMED, was re-enqueued because the CPU was not available for + * immediate execution. + */ + s64 SCX_EV_REENQ_IMMED; + + /* + * The number of times a reenq of local DSQ caused another reenq of + * local DSQ. This can happen when %SCX_ENQ_IMMED races against a higher + * priority class task even if the BPF scheduler always satisfies the + * prerequisites for %SCX_ENQ_IMMED at the time of enqueue. However, + * that scenario is very unlikely and this count going up regularly + * indicates that the BPF scheduler is handling %SCX_ENQ_REENQ + * incorrectly causing recursive reenqueues. + */ + s64 SCX_EV_REENQ_LOCAL_REPEAT; + + /* + * Total number of times a task's time slice was refilled with the + * default value (SCX_SLICE_DFL). + */ + s64 SCX_EV_REFILL_SLICE_DFL; + + /* + * The total duration of bypass modes in nanoseconds. + */ + s64 SCX_EV_BYPASS_DURATION; + + /* + * The number of tasks dispatched in the bypassing mode. + */ + s64 SCX_EV_BYPASS_DISPATCH; + + /* + * The number of times the bypassing mode has been activated. + */ + s64 SCX_EV_BYPASS_ACTIVATE; + + /* + * The number of times the scheduler attempted to insert a task that it + * doesn't own into a DSQ. Such attempts are ignored. + * + * As BPF schedulers are allowed to ignore dequeues, it's difficult to + * tell whether such an attempt is from a scheduler malfunction or an + * ignored dequeue around sub-sched enabling. If this count keeps going + * up regardless of sub-sched enabling, it likely indicates a bug in the + * scheduler. + */ + s64 SCX_EV_INSERT_NOT_OWNED; + + /* + * The number of times tasks from bypassing descendants are scheduled + * from sub_bypass_dsq's. + */ + s64 SCX_EV_SUB_BYPASS_DISPATCH; +}; + +struct scx_sched; + +enum scx_sched_pcpu_flags { + SCX_SCHED_PCPU_BYPASSING = 1LLU << 0, +}; + +/* dispatch buf */ +struct scx_dsp_buf_ent { + struct task_struct *task; + unsigned long qseq; + u64 dsq_id; + u64 enq_flags; +}; + +struct scx_dsp_ctx { + struct rq *rq; + u32 cursor; + u32 nr_tasks; + struct scx_dsp_buf_ent buf[]; +}; + +struct scx_deferred_reenq_local { + struct list_head node; + u64 flags; + u64 seq; + u32 cnt; +}; + +struct scx_sched_pcpu { + struct scx_sched *sch; + u64 flags; /* protected by rq lock */ + + /* + * The event counters are in a per-CPU variable to minimize the + * accounting overhead. A system-wide view on the event counter is + * constructed when requested by scx_bpf_events(). + */ + struct scx_event_stats event_stats; + + struct scx_deferred_reenq_local deferred_reenq_local; + struct scx_dispatch_q bypass_dsq; +#ifdef CONFIG_EXT_SUB_SCHED + u32 bypass_host_seq; +#endif + + /* must be the last entry - contains flex array */ + struct scx_dsp_ctx dsp_ctx; +}; + +struct scx_sched_pnode { + struct scx_dispatch_q global_dsq; +}; + +struct scx_sched { + /* + * cpu-form and cid-form ops share field offsets up to .priv (verified + * by BUILD_BUG_ON in scx_init()). The anonymous union lets the kernel + * access either view of the same storage without function-pointer + * casts: use .ops for cpu-form and shared fields, .ops_cid for the + * cid-renamed callbacks (set_cmask, select_cid, cid_online, ...). + */ + union { + struct sched_ext_ops ops; + struct sched_ext_ops_cid ops_cid; + }; + bool is_cid_type; /* true if registered via bpf_sched_ext_ops_cid */ + + /* + * Arena map auto-discovered from member progs at struct_ops attach. + * cid-form schedulers must use exactly one arena across all member + * progs. NULL on cpu-form. + * + * @arena_pool sub-allocates @arena_map. Each gen_pool chunk is added + * at the kernel-side mapping address. @arena_kern_base is the start + * of the arena's kern_vm range. See scx_arena_to_kaddr() and + * scx_kaddr_to_arena(). + */ + struct bpf_map *arena_map; + struct gen_pool *arena_pool; + uintptr_t arena_kern_base; + + /* + * Per-CPU arena cmask used by scx_call_op_set_cpumask() to hand a cmask + * to ops_cid.set_cmask(). The kernel writes through the stored kern_va + * and hands BPF its arena pointer via scx_kaddr_to_arena(). + */ + struct scx_cmask * __percpu *set_cmask_scratch; + + DECLARE_BITMAP(has_op, SCX_OPI_END); + + /* + * Dispatch queues. + * + * The global DSQ (%SCX_DSQ_GLOBAL) is split per-node for scalability. + * This is to avoid live-locking in bypass mode where all tasks are + * dispatched to %SCX_DSQ_GLOBAL and all CPUs consume from it. If + * per-node split isn't sufficient, it can be further split. + */ + struct rhashtable dsq_hash; + struct scx_sched_pnode **pnode; + struct scx_sched_pcpu __percpu *pcpu; + + u64 slice_dfl; + u64 bypass_timestamp; + s32 bypass_depth; + + /* bypass dispatch path enable state, see bypass_dsp_enabled() */ + unsigned long bypass_dsp_claim; + atomic_t bypass_dsp_enable_depth; + + bool aborting; + bool dump_disabled; /* protected by scx_dump_lock */ + u32 dsp_max_batch; + s32 level; + + /* + * Updates to the following warned bitfields can race causing RMW issues + * but it doesn't really matter. + */ + bool warned_zero_slice:1; + bool warned_deprecated_rq:1; + bool warned_unassoc_progs:1; + + struct list_head all; + +#ifdef CONFIG_EXT_SUB_SCHED + struct rhash_head hash_node; + + struct list_head children; + struct list_head sibling; + struct cgroup *cgrp; + char *cgrp_path; + struct kset *sub_kset; + + bool sub_attached; +#endif /* CONFIG_EXT_SUB_SCHED */ + + /* + * The maximum amount of time in jiffies that a task may be runnable + * without being scheduled on a CPU. If this timeout is exceeded, it + * will trigger scx_error(). + */ + unsigned long watchdog_timeout; + + atomic_t exit_kind; + struct scx_exit_info *exit_info; + + struct kobject kobj; + + struct kthread_worker *helper; + struct irq_work disable_irq_work; + struct kthread_work disable_work; + struct timer_list bypass_lb_timer; + cpumask_var_t bypass_lb_donee_cpumask; + cpumask_var_t bypass_lb_resched_cpumask; + struct rcu_work rcu_work; + + /* all ancestors including self */ + struct scx_sched *ancestors[]; +}; + +/** + * scx_arena_to_kaddr - Translate a BPF-arena pointer to its kernel address + * @sch: scheduler whose arena hosts @bpf_ptr + * @bpf_ptr: BPF-arena pointer, only the low 32 bits are used + * + * The (u32) cast normalizes any input into the arena's 4 GiB kern_vm range, + * which combined with scratch-page fault recovery makes the returned pointer + * safe to dereference up to GUARD_SZ / 2 past the intended object. Accesses + * larger than GUARD_SZ / 2 must be explicitly bounds-checked. + */ +static inline void *scx_arena_to_kaddr(struct scx_sched *sch, const void *bpf_ptr) +{ + return (void *)(sch->arena_kern_base + (u32)(uintptr_t)bpf_ptr); +} + +/** + * scx_kaddr_to_arena - Translate a kernel arena address to its BPF form + * @sch: scheduler whose arena hosts @kaddr + * @kaddr: kernel-side arena address, supplied by trusted kernel code + */ +static inline void *scx_kaddr_to_arena(struct scx_sched *sch, const void *kaddr) +{ + return (void *)((uintptr_t)kaddr - sch->arena_kern_base); +} + +enum scx_wake_flags { + /* expose select WF_* flags as enums */ + SCX_WAKE_FORK = WF_FORK, + SCX_WAKE_TTWU = WF_TTWU, + SCX_WAKE_SYNC = WF_SYNC, +}; + +enum scx_enq_flags { + /* expose select ENQUEUE_* flags as enums */ + SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, + SCX_ENQ_HEAD = ENQUEUE_HEAD, + SCX_ENQ_CPU_SELECTED = ENQUEUE_RQ_SELECTED, + + /* high 32bits are SCX specific */ + + /* + * Set the following to trigger preemption when calling + * scx_bpf_dsq_insert() with a local dsq as the target. The slice of the + * current task is cleared to zero and the CPU is kicked into the + * scheduling path. Implies %SCX_ENQ_HEAD. + */ + SCX_ENQ_PREEMPT = 1LLU << 32, + + /* + * Only allowed on local DSQs. Guarantees that the task either gets + * on the CPU immediately and stays on it, or gets reenqueued back + * to the BPF scheduler. It will never linger on a local DSQ or be + * silently put back after preemption. + * + * The protection persists until the next fresh enqueue - it + * survives SAVE/RESTORE cycles, slice extensions and preemption. + * If the task can't stay on the CPU for any reason, it gets + * reenqueued back to the BPF scheduler. + * + * Exiting and migration-disabled tasks bypass ops.enqueue() and + * are placed directly on a local DSQ without IMMED protection + * unless %SCX_OPS_ENQ_EXITING and %SCX_OPS_ENQ_MIGRATION_DISABLED + * are set respectively. + */ + SCX_ENQ_IMMED = 1LLU << 33, + + /* + * The task being enqueued was previously enqueued on a DSQ, but was + * removed and is being re-enqueued. See SCX_TASK_REENQ_* flags to find + * out why a given task is being reenqueued. + */ + SCX_ENQ_REENQ = 1LLU << 40, + + /* + * The task being enqueued is the only task available for the cpu. By + * default, ext core keeps executing such tasks but when + * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with the + * %SCX_ENQ_LAST flag set. + * + * The BPF scheduler is responsible for triggering a follow-up + * scheduling event. Otherwise, Execution may stall. + */ + SCX_ENQ_LAST = 1LLU << 41, + + /* high 8 bits are internal */ + __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, + + SCX_ENQ_CLEAR_OPSS = 1LLU << 56, + SCX_ENQ_DSQ_PRIQ = 1LLU << 57, + SCX_ENQ_NESTED = 1LLU << 58, + SCX_ENQ_GDSQ_FALLBACK = 1LLU << 59, /* fell back to global DSQ */ +}; + +enum scx_deq_flags { + /* expose select DEQUEUE_* flags as enums */ + SCX_DEQ_SLEEP = DEQUEUE_SLEEP, + + /* high 32bits are SCX specific */ + + /* + * The generic core-sched layer decided to execute the task even though + * it hasn't been dispatched yet. Dequeue from the BPF side. + */ + SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, + + /* + * The task is being dequeued due to a property change (e.g., + * sched_setaffinity(), sched_setscheduler(), set_user_nice(), + * etc.). + */ + SCX_DEQ_SCHED_CHANGE = 1LLU << 33, +}; + +enum scx_reenq_flags { + /* low 16bits determine which tasks should be reenqueued */ + SCX_REENQ_ANY = 1LLU << 0, /* all tasks */ + + __SCX_REENQ_FILTER_MASK = 0xffffLLU, + + __SCX_REENQ_USER_MASK = SCX_REENQ_ANY, + + /* bits 32-35 used by task_should_reenq() */ + SCX_REENQ_TSR_RQ_OPEN = 1LLU << 32, + SCX_REENQ_TSR_NOT_FIRST = 1LLU << 33, + + __SCX_REENQ_TSR_MASK = 0xfLLU << 32, +}; + +enum scx_pick_idle_cpu_flags { + SCX_PICK_IDLE_CORE = 1LLU << 0, /* pick a CPU whose SMT siblings are also idle */ + SCX_PICK_IDLE_IN_NODE = 1LLU << 1, /* pick a CPU in the same target NUMA node */ +}; + +enum scx_kick_flags { + /* + * Kick the target CPU if idle. Guarantees that the target CPU goes + * through at least one full scheduling cycle before going idle. If the + * target CPU can be determined to be currently not idle and going to go + * through a scheduling cycle before going idle, noop. + */ + SCX_KICK_IDLE = 1LLU << 0, + + /* + * Preempt the current task and execute the dispatch path. If the + * current task of the target CPU is an SCX task, its ->scx.slice is + * cleared to zero before the scheduling path is invoked so that the + * task expires and the dispatch path is invoked. + */ + SCX_KICK_PREEMPT = 1LLU << 1, + + /* + * The scx_bpf_kick_cpu() call will return after the current SCX task of + * the target CPU switches out. This can be used to implement e.g. core + * scheduling. This has no effect if the current task on the target CPU + * is not on SCX. + */ + SCX_KICK_WAIT = 1LLU << 2, +}; + +enum scx_tg_flags { + SCX_TG_ONLINE = 1U << 0, + SCX_TG_INITED = 1U << 1, +}; + +enum scx_enable_state { + SCX_ENABLING, + SCX_ENABLED, + SCX_DISABLING, + SCX_DISABLED, +}; + +static const char *scx_enable_state_str[] = { + [SCX_ENABLING] = "enabling", + [SCX_ENABLED] = "enabled", + [SCX_DISABLING] = "disabling", + [SCX_DISABLED] = "disabled", +}; + +/* + * Task Ownership State Machine (sched_ext_entity->ops_state) + * + * The sched_ext core uses this state machine to track task ownership + * between the SCX core and the BPF scheduler. This allows the BPF + * scheduler to dispatch tasks without strict ordering requirements, while + * the SCX core safely rejects invalid dispatches. + * + * State Transitions + * + * .------------> NONE (owned by SCX core) + * | | ^ + * | enqueue | | direct dispatch + * | v | + * | QUEUEING -------' + * | | + * | enqueue | + * | completes | + * | v + * | QUEUED (owned by BPF scheduler) + * | | + * | dispatch | + * | | + * | v + * | DISPATCHING + * | | + * | dispatch | + * | completes | + * `---------------' + * + * State Descriptions + * + * - %SCX_OPSS_NONE: + * Task is owned by the SCX core. It's either on a run queue, running, + * or being manipulated by the core scheduler. The BPF scheduler has no + * claim on this task. + * + * - %SCX_OPSS_QUEUEING: + * Transitional state while transferring a task from the SCX core to + * the BPF scheduler. The task's rq lock is held during this state. + * Since QUEUEING is both entered and exited under the rq lock, dequeue + * can never observe this state (it would be a BUG). When finishing a + * dispatch, if the task is still in %SCX_OPSS_QUEUEING the completion + * path busy-waits for it to leave this state (via wait_ops_state()) + * before retrying. + * + * - %SCX_OPSS_QUEUED: + * Task is owned by the BPF scheduler. It's on a DSQ (dispatch queue) + * and the BPF scheduler is responsible for dispatching it. A QSEQ + * (queue sequence number) is embedded in this state to detect + * dispatch/dequeue races: if a task is dequeued and re-enqueued, the + * QSEQ changes and any in-flight dispatch operations targeting the old + * QSEQ are safely ignored. + * + * - %SCX_OPSS_DISPATCHING: + * Transitional state while transferring a task from the BPF scheduler + * back to the SCX core. This state indicates the BPF scheduler has + * selected the task for execution. When dequeue needs to take the task + * off a DSQ and it is still in %SCX_OPSS_DISPATCHING, the dequeue path + * busy-waits for it to leave this state (via wait_ops_state()) before + * proceeding. Exits to %SCX_OPSS_NONE when dispatch completes. + * + * Memory Ordering + * + * Transitions out of %SCX_OPSS_QUEUEING and %SCX_OPSS_DISPATCHING into + * %SCX_OPSS_NONE or %SCX_OPSS_QUEUED must use atomic_long_set_release() + * and waiters must use atomic_long_read_acquire(). This ensures proper + * synchronization between concurrent operations. + * + * Cross-CPU Task Migration + * + * When moving a task in the %SCX_OPSS_DISPATCHING state, we can't simply + * grab the target CPU's rq lock because a concurrent dequeue might be + * waiting on %SCX_OPSS_DISPATCHING while holding the source rq lock + * (deadlock). + * + * The sched_ext core uses a "lock dancing" protocol coordinated by + * p->scx.holding_cpu. When moving a task to a different rq: + * + * 1. Verify task can be moved (CPU affinity, migration_disabled, etc.) + * 2. Set p->scx.holding_cpu to the current CPU + * 3. Set task state to %SCX_OPSS_NONE; dequeue waits while DISPATCHING + * is set, so clearing DISPATCHING first prevents the circular wait + * (safe to lock the rq we need) + * 4. Unlock the current CPU's rq + * 5. Lock src_rq (where the task currently lives) + * 6. Verify p->scx.holding_cpu == current CPU, if not, dequeue won the + * race (dequeue clears holding_cpu to -1 when it takes the task), in + * this case migration is aborted + * 7. If src_rq == dst_rq: clear holding_cpu and enqueue directly + * into dst_rq's local DSQ (no lock swap needed) + * 8. Otherwise: call move_remote_task_to_local_dsq(), which releases + * src_rq, locks dst_rq, and performs the deactivate/activate + * migration cycle (dst_rq is held on return) + * 9. Unlock dst_rq and re-lock the current CPU's rq to restore + * the lock state expected by the caller + * + * If any verification fails, abort the migration. + * + * This state tracking allows the BPF scheduler to try to dispatch any task + * at any time regardless of its state. The SCX core can safely + * reject/ignore invalid dispatches, simplifying the BPF scheduler + * implementation. + */ +enum scx_ops_state { + SCX_OPSS_NONE, /* owned by the SCX core */ + SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ + SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ + SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ + + /* + * QSEQ brands each QUEUED instance so that, when dispatch races + * dequeue/requeue, the dispatcher can tell whether it still has a claim + * on the task being dispatched. + * + * As some 32bit archs can't do 64bit store_release/load_acquire, + * p->scx.ops_state is atomic_long_t which leaves 30 bits for QSEQ on + * 32bit machines. The dispatch race window QSEQ protects is very narrow + * and runs with IRQ disabled. 30 bits should be sufficient. + */ + SCX_OPSS_QSEQ_SHIFT = 2, +}; + +/* Use macros to ensure that the type is unsigned long for the masks */ +#define SCX_OPSS_STATE_MASK ((1LU << SCX_OPSS_QSEQ_SHIFT) - 1) +#define SCX_OPSS_QSEQ_MASK (~SCX_OPSS_STATE_MASK) + +extern struct scx_sched __rcu *scx_root; +DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); + +/* + * True when the currently loaded scheduler hierarchy is cid-form. All scheds + * in a hierarchy share one form, so this single key tells callsites which + * view to use without per-sch dereferences. Use scx_is_cid_type() to test. + */ +DECLARE_STATIC_KEY_FALSE(__scx_is_cid_type); + +int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id); + +bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where); + +__printf(5, 0) bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind, + s64 exit_code, s32 exit_cpu, const char *fmt, + va_list args); +__printf(5, 6) bool __scx_exit(struct scx_sched *sch, enum scx_exit_kind kind, + s64 exit_code, s32 exit_cpu, const char *fmt, ...); + +#define scx_exit(sch, kind, exit_code, fmt, args...) \ + __scx_exit(sch, kind, exit_code, raw_smp_processor_id(), fmt, ##args) +#define scx_error(sch, fmt, args...) \ + scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) +#define scx_verror(sch, fmt, args) \ + scx_vexit((sch), SCX_EXIT_ERROR, 0, raw_smp_processor_id(), fmt, args) + +/* + * Return the rq currently locked from an scx callback, or NULL if no rq is + * locked. + */ +static inline struct rq *scx_locked_rq(void) +{ + return __this_cpu_read(scx_locked_rq_state); +} + +static inline bool scx_bypassing(struct scx_sched *sch, s32 cpu) +{ + return unlikely(per_cpu_ptr(sch->pcpu, cpu)->flags & + SCX_SCHED_PCPU_BYPASSING); +} + +#ifdef CONFIG_EXT_SUB_SCHED +/** + * scx_task_sched - Find scx_sched scheduling a task + * @p: task of interest + * + * Return @p's scheduler instance. Must be called with @p's pi_lock or rq lock + * held. + */ +static inline struct scx_sched *scx_task_sched(const struct task_struct *p) +{ + return rcu_dereference_protected(p->scx.sched, + lockdep_is_held(&p->pi_lock) || + lockdep_is_held(__rq_lockp(task_rq(p)))); +} + +/** + * scx_task_sched_rcu - Find scx_sched scheduling a task + * @p: task of interest + * + * Return @p's scheduler instance. The returned scx_sched is RCU protected. + */ +static inline struct scx_sched *scx_task_sched_rcu(const struct task_struct *p) +{ + return rcu_dereference_all(p->scx.sched); +} + +/** + * scx_task_on_sched - Is a task on the specified sched? + * @sch: sched to test against + * @p: task of interest + * + * Returns %true if @p is on @sch, %false otherwise. + */ +static inline bool scx_task_on_sched(struct scx_sched *sch, + const struct task_struct *p) +{ + return rcu_access_pointer(p->scx.sched) == sch; +} + +/** + * scx_prog_sched - Find scx_sched associated with a BPF prog + * @aux: aux passed in from BPF to a kfunc + * + * To be called from kfuncs. Return the scheduler instance associated with the + * BPF program given the implicit kfunc argument aux. The returned scx_sched is + * RCU protected. + */ +static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) +{ + struct sched_ext_ops *ops; + struct scx_sched *root; + + ops = bpf_prog_get_assoc_struct_ops(aux); + if (likely(ops)) + return rcu_dereference_all(ops->priv); + + root = rcu_dereference_all(scx_root); + if (root) { + /* + * COMPAT-v6.19: Schedulers built before sub-sched support was + * introduced may have unassociated non-struct_ops programs. + */ + if (!root->ops.sub_attach) + return root; + + if (!root->warned_unassoc_progs) { + printk_deferred(KERN_WARNING "sched_ext: Unassociated program %s (id %d)\n", + aux->name, aux->id); + root->warned_unassoc_progs = true; + } + } + + return NULL; +} +#else /* CONFIG_EXT_SUB_SCHED */ +static inline struct scx_sched *scx_task_sched(const struct task_struct *p) +{ + return rcu_dereference_protected(scx_root, + lockdep_is_held(&p->pi_lock) || + lockdep_is_held(__rq_lockp(task_rq(p)))); +} + +static inline struct scx_sched *scx_task_sched_rcu(const struct task_struct *p) +{ + return rcu_dereference_all(scx_root); +} + +static inline bool scx_task_on_sched(struct scx_sched *sch, + const struct task_struct *p) +{ + return true; +} + +static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) +{ + return rcu_dereference_all(scx_root); +} +#endif /* CONFIG_EXT_SUB_SCHED */ diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h new file mode 100644 index 000000000000..8b3527e21fca --- /dev/null +++ b/kernel/sched/ext/types.h @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Early sched_ext type definitions. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ +#ifndef _KERNEL_SCHED_EXT_TYPES_H +#define _KERNEL_SCHED_EXT_TYPES_H + +enum scx_consts { + SCX_DSP_DFL_MAX_BATCH = 32, + SCX_DSP_MAX_LOOPS = 32, + SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, + + /* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */ + SCX_TID_CHUNK = 1024, + + SCX_EXIT_BT_LEN = 64, + SCX_EXIT_MSG_LEN = 1024, + SCX_EXIT_DUMP_DFL_LEN = 32768, + + SCX_CPUPERF_ONE = SCHED_CAPACITY_SCALE, + + /* + * Iterating all tasks may take a while. Periodically drop + * scx_tasks_lock to avoid causing e.g. CSD and RCU stalls. + */ + SCX_TASK_ITER_BATCH = 32, + + SCX_BYPASS_HOST_NTH = 2, + + SCX_BYPASS_LB_DFL_INTV_US = 500 * USEC_PER_MSEC, + SCX_BYPASS_LB_DONOR_PCT = 125, + SCX_BYPASS_LB_MIN_DELTA_DIV = 4, + SCX_BYPASS_LB_BATCH = 256, + + SCX_REENQ_LOCAL_MAX_REPEAT = 256, + + SCX_SUB_MAX_DEPTH = 4, +}; + +/* + * Per-cid topology info. For each topology level (core, LLC, node), records + * the first cid in the unit and its global index. Global indices are + * consecutive integers assigned in cid-walk order, so e.g. core_idx ranges + * over [0, nr_cores_at_init) with no gaps. No-topo cids have all fields set + * to -1. + * + * @core_cid: first cid of this cid's core (smt-sibling group) + * @core_idx: global index of that core, in [0, nr_cores_at_init) + * @llc_cid: first cid of this cid's LLC + * @llc_idx: global index of that LLC, in [0, nr_llcs_at_init) + * @node_cid: first cid of this cid's NUMA node + * @node_idx: global index of that node, in [0, nr_nodes_at_init) + */ +struct scx_cid_topo { + s32 core_cid; + s32 core_idx; + s32 llc_cid; + s32 llc_idx; + s32 node_cid; + s32 node_idx; +}; + +/* + * cmask: variable-length, base-windowed bitmap over cid space + * ----------------------------------------------------------- + * + * A cmask covers the cid range [base, base + nr_cids). bits[] is aligned to the + * global 64-cid grid: bits[0] spans [base & ~63, (base & ~63) + 64), so the + * first (base & 63) bits of bits[0] are head padding and the trailing bits of + * the last active word past base + nr_cids are tail padding. Both stay zero; + * all mutating helpers preserve that. Words past the last active word are not + * read by any helper and have no constraint. + * + * Grid alignment means two cmasks always address bits[] against the same global + * 64-cid windows, so cross-cmask word ops (AND, OR, ...) reduce to + * + * dst->bits[i] OP= src->bits[i - delta] + * + * with no bit-shifting, regardless of how the two bases relate mod 64. + */ +struct scx_cmask { + u32 base; + u32 nr_cids; + u32 alloc_words; + u64 bits[] __counted_by(alloc_words); +}; + +/* + * Number of u64 words of bits[] storage that covers @nr_cids regardless of base + * alignment. The +1 absorbs up to 63 bits of head padding when base is not + * 64-aligned - always allocating one extra word beats branching on base or + * splitting the compute. The u64 cast keeps the +63 from wrapping when @nr_cids + * is near U32_MAX, so callers bounds-checking the result against @alloc_words + * catch the overflow instead of seeing a small value. + */ +#define SCX_CMASK_NR_WORDS(nr_cids) ((u32)(((u64)(nr_cids) + 63) / 64 + 1)) + +/** + * __SCX_CMASK_DEFINE - Define an on-stack cmask with explicit storage capacity + * @NAME: variable name to define + * @BASE: first cid of the active range + * @NR_CIDS: active range length + * @ALLOC_CIDS: storage capacity in cids, at least @NR_CIDS + * + * @NAME aliases zero-initialized storage with the active range set to + * [BASE, BASE + NR_CIDS). Use scx_cmask_reframe() to reshape later, up to + * @ALLOC_CIDS. + */ +#define __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, ALLOC_CIDS) \ + _DEFINE_FLEX(struct scx_cmask, NAME, bits, SCX_CMASK_NR_WORDS(ALLOC_CIDS), \ + = { .base = (BASE), \ + .nr_cids = (NR_CIDS), \ + .alloc_words = SCX_CMASK_NR_WORDS(ALLOC_CIDS) }) + +/** + * SCX_CMASK_DEFINE - Define an on-stack cmask on tight storage + * @NAME: variable name to define + * @BASE: first cid of the active range + * @NR_CIDS: active range length, also storage capacity + * + * @NAME aliases zero-initialized storage with the active range and storage + * both [BASE, BASE + NR_CIDS). + */ +#define SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS) \ + __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, NR_CIDS) + +/** + * SCX_CMASK_DEFINE_SHARD - Define an on-stack cmask sized to one shard + * @NAME: variable name to define + * @BASE: first cid of the active range + * @NR_CIDS: active range length, must be <= SCX_CID_SHARD_MAX_CPUS + * + * Storage is fixed at SCX_CID_SHARD_MAX_CPUS, active range framed by + * (BASE, NR_CIDS). Passing NR_CIDS > SCX_CID_SHARD_MAX_CPUS leaves the + * cmask claiming more bits than storage holds and subsequent cmask + * operations will overrun. + */ +#define SCX_CMASK_DEFINE_SHARD(NAME, BASE, NR_CIDS) \ + __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, SCX_CID_SHARD_MAX_CPUS) + +#endif /* _KERNEL_SCHED_EXT_TYPES_H */ diff --git a/kernel/sched/ext_arena.c b/kernel/sched/ext_arena.c deleted file mode 100644 index 493c2424f842..000000000000 --- a/kernel/sched/ext_arena.c +++ /dev/null @@ -1,131 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst - * - * scx_arena_pool: kernel-side sub-allocator over BPF-arena pages. - * - * Each chunk added to @sch->arena_pool comes from one - * bpf_arena_alloc_pages_sleepable() call and is registered at the - * kernel-side mapping address. Callers translate to the BPF-arena form - * themselves if needed. - * - * Allocations grow the pool on demand. Underlying arena pages are released - * when the arena map itself is torn down. - * - * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2026 Tejun Heo - */ - -enum scx_arena_consts { - SCX_ARENA_MIN_ORDER = 3, /* 8-byte minimum sub-allocation */ - SCX_ARENA_GROW_PAGES = 4, /* per growth */ -}; - -s32 scx_arena_pool_init(struct scx_sched *sch) -{ - if (!sch->arena_map) - return 0; - - sch->arena_pool = gen_pool_create(SCX_ARENA_MIN_ORDER, NUMA_NO_NODE); - if (!sch->arena_pool) - return -ENOMEM; - return 0; -} - -static void scx_arena_clear_chunk(struct gen_pool *pool, struct gen_pool_chunk *chunk, - void *data) -{ - int order = pool->min_alloc_order; - size_t chunk_sz = chunk->end_addr - chunk->start_addr + 1; - unsigned long end_bit = chunk_sz >> order; - unsigned long b, e; - - for_each_set_bitrange(b, e, chunk->bits, end_bit) - gen_pool_free(pool, chunk->start_addr + (b << order), - (e - b) << order); -} - -/* - * Tear down the pool. Outstanding gen_pool allocations are freed via - * scx_arena_clear_chunk() so gen_pool_destroy() doesn't BUG. The underlying - * arena pages are released when the arena map itself is torn down. - */ -void scx_arena_pool_destroy(struct scx_sched *sch) -{ - if (!sch->arena_pool) - return; - gen_pool_for_each_chunk(sch->arena_pool, scx_arena_clear_chunk, NULL); - gen_pool_destroy(sch->arena_pool); - sch->arena_pool = NULL; -} - -/* - * Grow the pool by @page_cnt pages. bpf_arena_alloc_pages_sleepable() and - * gen_pool_add() (which calls vzalloc(GFP_KERNEL)) require a sleepable - * context. - */ -static int scx_arena_grow(struct scx_sched *sch, u32 page_cnt) -{ - u64 kern_vm_start; - u32 uaddr32; - void *p; - int ret; - - if (!sch->arena_map || !sch->arena_pool) - return -EINVAL; - - p = bpf_arena_alloc_pages_sleepable(sch->arena_map, NULL, - page_cnt, NUMA_NO_NODE, 0); - if (!p) - return -ENOMEM; - - uaddr32 = (u32)(unsigned long)p; - /* arena.o, which defines these, is built only on MMU && 64BIT */ -#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) - kern_vm_start = bpf_arena_map_kern_vm_start(sch->arena_map); -#else - kern_vm_start = 0; -#endif - - ret = gen_pool_add(sch->arena_pool, kern_vm_start + uaddr32, - page_cnt * PAGE_SIZE, NUMA_NO_NODE); - if (ret) { - bpf_arena_free_pages_non_sleepable(sch->arena_map, p, page_cnt); - return ret; - } - return 0; -} - -/* - * Allocate @size bytes from the arena pool. Returns kernel VA on success, NULL - * on failure. May grow the pool via scx_arena_grow() which sleeps. Caller must - * be in a GFP_KERNEL context. - */ -void *scx_arena_alloc(struct scx_sched *sch, size_t size) -{ - unsigned long kern_va; - u32 page_cnt; - - might_sleep(); - - if (!sch->arena_pool) - return NULL; - - while (true) { - kern_va = gen_pool_alloc(sch->arena_pool, size); - if (kern_va) - break; - page_cnt = max_t(u32, SCX_ARENA_GROW_PAGES, - (size + PAGE_SIZE - 1) >> PAGE_SHIFT); - if (scx_arena_grow(sch, page_cnt)) - return NULL; - } - - return (void *)kern_va; -} - -void scx_arena_free(struct scx_sched *sch, void *kern_va, size_t size) -{ - if (sch->arena_pool && kern_va) - gen_pool_free(sch->arena_pool, (unsigned long)kern_va, size); -} diff --git a/kernel/sched/ext_arena.h b/kernel/sched/ext_arena.h deleted file mode 100644 index 4f3610160102..000000000000 --- a/kernel/sched/ext_arena.h +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst - * - * Copyright (c) 2025 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2025 Tejun Heo - */ -#ifndef _KERNEL_SCHED_EXT_ARENA_H -#define _KERNEL_SCHED_EXT_ARENA_H - -struct scx_sched; - -s32 scx_arena_pool_init(struct scx_sched *sch); -void scx_arena_pool_destroy(struct scx_sched *sch); -void *scx_arena_alloc(struct scx_sched *sch, size_t size); -void scx_arena_free(struct scx_sched *sch, void *kern_va, size_t size); - -#endif /* _KERNEL_SCHED_EXT_ARENA_H */ diff --git a/kernel/sched/ext_cid.c b/kernel/sched/ext_cid.c deleted file mode 100644 index 66944a7ef79d..000000000000 --- a/kernel/sched/ext_cid.c +++ /dev/null @@ -1,707 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst - * - * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2026 Tejun Heo - */ -#include - -/* - * cid tables. - * - * Pointers are published once on first enable and never revoked. The default - * mapping is populated before ops.init() runs; scx_bpf_cid_override() commits - * before it returns. As long as the BPF scheduler only uses the tables from - * those points onward, it sees a consistent view. - */ -s16 *scx_cid_to_cpu_tbl; -s16 *scx_cpu_to_cid_tbl; -struct scx_cid_topo *scx_cid_topo; - -#define SCX_CID_TOPO_NEG (struct scx_cid_topo) { \ - .core_cid = -1, .core_idx = -1, .llc_cid = -1, .llc_idx = -1, \ - .node_cid = -1, .node_idx = -1, \ -} - -/* - * Return @cpu's LLC shared_cpu_map. If cacheinfo isn't populated (offline or - * !present), record @cpu in @fallbacks and return its node mask instead - the - * worst that can happen is that the cpu's LLC becomes coarser than reality. - */ -static const struct cpumask *cpu_llc_mask(int cpu, struct cpumask *fallbacks) -{ - struct cpu_cacheinfo *ci = get_cpu_cacheinfo(cpu); - - if (!ci || !ci->info_list || !ci->num_leaves) { - cpumask_set_cpu(cpu, fallbacks); - return cpumask_of_node(cpu_to_node(cpu)); - } - return &ci->info_list[ci->num_leaves - 1].shared_cpu_map; -} - -/* Allocate the cid tables once on first enable; never freed. */ -static s32 scx_cid_arrays_alloc(void) -{ - u32 npossible = num_possible_cpus(); - s16 *cid_to_cpu, *cpu_to_cid; - struct scx_cid_topo *cid_topo; - - if (scx_cid_to_cpu_tbl) - return 0; - - cid_to_cpu = kzalloc_objs(*scx_cid_to_cpu_tbl, npossible, GFP_KERNEL); - cpu_to_cid = kzalloc_objs(*scx_cpu_to_cid_tbl, nr_cpu_ids, GFP_KERNEL); - cid_topo = kmalloc_objs(*scx_cid_topo, npossible, GFP_KERNEL); - - if (!cid_to_cpu || !cpu_to_cid || !cid_topo) { - kfree(cid_to_cpu); - kfree(cpu_to_cid); - kfree(cid_topo); - return -ENOMEM; - } - - WRITE_ONCE(scx_cid_to_cpu_tbl, cid_to_cpu); - WRITE_ONCE(scx_cpu_to_cid_tbl, cpu_to_cid); - WRITE_ONCE(scx_cid_topo, cid_topo); - return 0; -} - -/** - * scx_cid_init - build the cid mapping - * @sch: the scx_sched being initialized; used as the scx_error() target - * - * See "Topological CPU IDs" in ext_cid.h for the model. Walk online cpus by - * intersection at each level (parent_scratch & this_level_mask), which keeps - * containment correct by construction and naturally splits a physical LLC - * straddling two NUMA nodes into two LLC units. The caller must hold - * cpus_read_lock. - */ -s32 scx_cid_init(struct scx_sched *sch) -{ - cpumask_var_t to_walk __free(free_cpumask_var) = CPUMASK_VAR_NULL; - cpumask_var_t node_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; - cpumask_var_t llc_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; - cpumask_var_t core_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; - cpumask_var_t llc_fallback __free(free_cpumask_var) = CPUMASK_VAR_NULL; - cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL; - u32 next_cid = 0; - s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0; - s32 cpu, ret; - - /* CMASK_MAX_WORDS in cid.bpf.h covers NR_CPUS up to 8192 */ - BUILD_BUG_ON(NR_CPUS > 8192); - - lockdep_assert_cpus_held(); - - ret = scx_cid_arrays_alloc(); - if (ret) - return ret; - - if (!zalloc_cpumask_var(&to_walk, GFP_KERNEL) || - !zalloc_cpumask_var(&node_scratch, GFP_KERNEL) || - !zalloc_cpumask_var(&llc_scratch, GFP_KERNEL) || - !zalloc_cpumask_var(&core_scratch, GFP_KERNEL) || - !zalloc_cpumask_var(&llc_fallback, GFP_KERNEL) || - !zalloc_cpumask_var(&online_no_topo, GFP_KERNEL)) - return -ENOMEM; - - /* -1 sentinels for sparse-possible cpu id holes (0 is a valid cid) */ - for (cpu = 0; cpu < nr_cpu_ids; cpu++) - scx_cpu_to_cid_tbl[cpu] = -1; - - cpumask_copy(to_walk, cpu_online_mask); - - while (!cpumask_empty(to_walk)) { - s32 next_cpu = cpumask_first(to_walk); - s32 nid = cpu_to_node(next_cpu); - s32 node_cid = next_cid; - s32 node_idx; - - /* - * No NUMA info: skip and let the tail loop assign a no-topo - * cid. cpumask_of_node(-1) is undefined. - */ - if (nid < 0) { - cpumask_clear_cpu(next_cpu, to_walk); - continue; - } - - node_idx = next_node_idx++; - - /* node_scratch = to_walk & this node */ - cpumask_and(node_scratch, to_walk, cpumask_of_node(nid)); - if (WARN_ON_ONCE(!cpumask_test_cpu(next_cpu, node_scratch))) - return -EINVAL; - - while (!cpumask_empty(node_scratch)) { - s32 ncpu = cpumask_first(node_scratch); - const struct cpumask *llc_mask = cpu_llc_mask(ncpu, llc_fallback); - s32 llc_cid = next_cid; - s32 llc_idx = next_llc_idx++; - - /* llc_scratch = node_scratch & this llc */ - cpumask_and(llc_scratch, node_scratch, llc_mask); - if (WARN_ON_ONCE(!cpumask_test_cpu(ncpu, llc_scratch))) - return -EINVAL; - - while (!cpumask_empty(llc_scratch)) { - s32 lcpu = cpumask_first(llc_scratch); - const struct cpumask *sib = topology_sibling_cpumask(lcpu); - s32 core_cid = next_cid; - s32 core_idx = next_core_idx++; - s32 ccpu; - - /* core_scratch = llc_scratch & this core */ - cpumask_and(core_scratch, llc_scratch, sib); - if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch))) - return -EINVAL; - - for_each_cpu(ccpu, core_scratch) { - s32 cid = next_cid++; - - scx_cid_to_cpu_tbl[cid] = ccpu; - scx_cpu_to_cid_tbl[ccpu] = cid; - scx_cid_topo[cid] = (struct scx_cid_topo){ - .core_cid = core_cid, - .core_idx = core_idx, - .llc_cid = llc_cid, - .llc_idx = llc_idx, - .node_cid = node_cid, - .node_idx = node_idx, - }; - - cpumask_clear_cpu(ccpu, llc_scratch); - cpumask_clear_cpu(ccpu, node_scratch); - cpumask_clear_cpu(ccpu, to_walk); - } - } - } - } - - /* - * No-topo section: any possible cpu without a cid - normally just the - * not-online ones. Collect any currently-online cpus that land here in - * @online_no_topo so we can warn about them at the end. - */ - for_each_cpu(cpu, cpu_possible_mask) { - s32 cid; - - if (__scx_cpu_to_cid(cpu) != -1) - continue; - if (cpu_online(cpu)) - cpumask_set_cpu(cpu, online_no_topo); - - cid = next_cid++; - scx_cid_to_cpu_tbl[cid] = cpu; - scx_cpu_to_cid_tbl[cpu] = cid; - scx_cid_topo[cid] = SCX_CID_TOPO_NEG; - } - - if (!cpumask_empty(llc_fallback)) - pr_warn("scx_cid: cpus without cacheinfo, using node mask as llc: %*pbl\n", - cpumask_pr_args(llc_fallback)); - if (!cpumask_empty(online_no_topo)) - pr_warn("scx_cid: online cpus with no usable topology: %*pbl\n", - cpumask_pr_args(online_no_topo)); - - return 0; -} - -/** - * scx_cmask_clear - Zero every bit in @m's active range - * @m: cmask to clear - * - * Storage past the active range is left as is. - */ -void scx_cmask_clear(struct scx_cmask *m) -{ - u32 nr_words; - - if (!m->nr_cids) - return; - nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1; - memset(m->bits, 0, nr_words * sizeof(u64)); -} - -/** - * scx_cmask_fill - Set every bit in @m's active range - * @m: cmask to fill - * - * Counterpart to scx_cmask_clear(). Storage past the active range is left as is. - */ -void scx_cmask_fill(struct scx_cmask *m) -{ - u32 nr_words, head_bits, tail_bits; - - if (!m->nr_cids) - return; - nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1; - memset(m->bits, 0xff, nr_words * sizeof(u64)); - - /* clear word-0 bits below base */ - head_bits = m->base & 63; - if (head_bits) - m->bits[0] &= ~((1ULL << head_bits) - 1); - - /* clear last-word bits at or past base + nr_cids */ - tail_bits = (m->base + m->nr_cids) & 63; - if (tail_bits) - m->bits[nr_words - 1] &= (1ULL << tail_bits) - 1; -} - -/** - * scx_cpumask_to_cmask - Translate a kernel cpumask into a cmask - * @src: source cpumask - * @dst: cmask to write - * - * Clear @dst's active range and set the bit for each cid whose cpu is in - * @src and lies within that range. Out-of-range cids are silently ignored. - */ -void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst) -{ - s32 cpu; - - scx_cmask_clear(dst); - for_each_cpu(cpu, src) { - s32 cid = __scx_cpu_to_cid(cpu); - - if (cid >= 0) - __scx_cmask_set(cid, dst); - } -} - -__bpf_kfunc_start_defs(); - -/** - * scx_bpf_cid_override - Install an explicit cpu->cid mapping - * @cpu_to_cid: array of nr_cpu_ids s32 entries (cid for each cpu) - * @cpu_to_cid__sz: must be nr_cpu_ids * sizeof(s32) bytes - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * May only be called from ops.init() of the root scheduler. Replace the - * topology-probed cid mapping with the caller-provided one. Each possible cpu - * must map to a unique cid in [0, num_possible_cpus()). Topo info is cleared. - * On invalid input, trigger scx_error() to abort the scheduler. - */ -__bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz, - const struct bpf_prog_aux *aux) -{ - cpumask_var_t seen __free(free_cpumask_var) = CPUMASK_VAR_NULL; - struct scx_sched *sch; - bool alloced; - s32 cpu, cid; - - /* GFP_KERNEL alloc must happen before the rcu read section */ - alloced = zalloc_cpumask_var(&seen, GFP_KERNEL); - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return; - - if (!alloced) { - scx_error(sch, "scx_bpf_cid_override: failed to allocate cpumask"); - return; - } - - if (scx_parent(sch)) { - scx_error(sch, "scx_bpf_cid_override() only allowed from root sched"); - return; - } - - if (cpu_to_cid__sz != nr_cpu_ids * sizeof(s32)) { - scx_error(sch, "scx_bpf_cid_override: expected %zu bytes, got %u", - nr_cpu_ids * sizeof(s32), cpu_to_cid__sz); - return; - } - - for_each_possible_cpu(cpu) { - s32 c = cpu_to_cid[cpu]; - - if (!cid_valid(sch, c)) - return; - if (cpumask_test_and_set_cpu(c, seen)) { - scx_error(sch, "cid %d assigned to multiple cpus", c); - return; - } - scx_cpu_to_cid_tbl[cpu] = c; - scx_cid_to_cpu_tbl[c] = cpu; - } - - /* Invalidate stale topo info - the override carries no topology. */ - for (cid = 0; cid < num_possible_cpus(); cid++) - scx_cid_topo[cid] = SCX_CID_TOPO_NEG; -} - -/** - * scx_bpf_cid_to_cpu - Return the raw CPU id for @cid - * @cid: cid to look up - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Return the raw CPU id for @cid. Trigger scx_error() and return -EINVAL if - * @cid is invalid. The cid<->cpu mapping is static for the lifetime of the - * loaded scheduler, so the BPF side can cache the result to avoid repeated - * kfunc invocations. - */ -__bpf_kfunc s32 scx_bpf_cid_to_cpu(s32 cid, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -EINVAL; - return scx_cid_to_cpu(sch, cid); -} - -/** - * scx_bpf_cpu_to_cid - Return the cid for @cpu - * @cpu: cpu to look up - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Return the cid for @cpu. Trigger scx_error() and return -EINVAL if @cpu is - * invalid. The cid<->cpu mapping is static for the lifetime of the loaded - * scheduler, so the BPF side can cache the result to avoid repeated kfunc - * invocations. - */ -__bpf_kfunc s32 scx_bpf_cpu_to_cid(s32 cpu, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -EINVAL; - return scx_cpu_to_cid(sch, cpu); -} - -/* - * Set ops on cmasks. cmask_walk_op2() shares one walk across mutating - * (and/or/copy/andnot) and predicate (subset/intersects) two-cmask forms; - * cmask_walk_op1() does the same shape over a single cmask range. Every public - * entry passes a compile-time-constant @op; cmask_walk_op{1,2}() and - * cmask_word_op{1,2}() are __always_inline so the inner switch collapses to the - * selected op and cmask_op2_is_pred() folds the predicate early-exit out of - * mutating ops. - * - * Two-cmask ops only touch @dst bits inside the intersection of the two ranges; - * bits outside stay untouched. In particular, scx_cmask_copy() does NOT zero - * @dst bits that lie outside @src's range. - * - * The _RACY variants are otherwise identical to their non-racy counterpart but - * read @src word-by-word via data_race(). Memory ordering with concurrent - * writers is the caller's responsibility. - */ -enum cmask_op2 { - /* mutating */ - CMASK_OP2_AND, - CMASK_OP2_OR, - CMASK_OP2_OR_RACY, - CMASK_OP2_COPY, - CMASK_OP2_COPY_RACY, - CMASK_OP2_ANDNOT, - /* predicates - short-circuit when the per-word result is true */ - CMASK_OP2_SUBSET, - CMASK_OP2_INTERSECTS, -}; - -static __always_inline bool cmask_op2_is_pred(const enum cmask_op2 op) -{ - return op == CMASK_OP2_SUBSET || op == CMASK_OP2_INTERSECTS; -} - -static __always_inline bool cmask_word_op2(u64 *av, const u64 *bp, u64 mask, - const enum cmask_op2 op) -{ - switch (op) { - case CMASK_OP2_AND: - *av &= ~mask | *bp; - return false; - case CMASK_OP2_OR: - *av |= *bp & mask; - return false; - case CMASK_OP2_OR_RACY: - *av |= data_race(*bp) & mask; - return false; - case CMASK_OP2_COPY: - *av = (*av & ~mask) | (*bp & mask); - return false; - case CMASK_OP2_COPY_RACY: - *av = (*av & ~mask) | (data_race(*bp) & mask); - return false; - case CMASK_OP2_ANDNOT: - *av &= ~(*bp & mask); - return false; - case CMASK_OP2_SUBSET: - /* stop on the first bit in @sub not set in @super */ - return (*bp & ~*av) & mask; - case CMASK_OP2_INTERSECTS: - return (*av & *bp) & mask; - } - unreachable(); -} - -/* - * Walk the intersection of [@a_base, @a_base + @a_nr_cids) with [@b_base, - * @b_base + @b_nr_cids) word by word, applying @op. Mutating ops walk all words - * and return false; predicates return true on the first word whose per-word - * test is true. Empty intersection returns false (matches "no bits to consider" - * for both mutate and predicate). - * - * Base/nr_cids are taken as parameters so callers with snapshotted bounds can - * drive the walk with values independent of the cmask's header. - */ -static __always_inline bool cmask_walk_op2(u64 *a_bits, u32 a_base, u32 a_nr_cids, - const u64 *b_bits, u32 b_base, u32 b_nr_cids, - const enum cmask_op2 op) -{ - u32 lo = max(a_base, b_base); - u32 hi = min(a_base + a_nr_cids, b_base + b_nr_cids); - u32 a_word_off = a_base / 64; - u32 b_word_off = b_base / 64; - u32 lo_word = lo / 64; - u32 hi_word = (hi - 1) / 64; - u64 head_mask = GENMASK_U64(63, lo & 63); - u64 tail_mask = GENMASK_U64((hi - 1) & 63, 0); - u32 w; - - if (lo >= hi) - return false; - - if (lo_word == hi_word) - return cmask_word_op2(&a_bits[lo_word - a_word_off], - &b_bits[lo_word - b_word_off], - head_mask & tail_mask, op); - - if (cmask_word_op2(&a_bits[lo_word - a_word_off], - &b_bits[lo_word - b_word_off], head_mask, op) && - cmask_op2_is_pred(op)) - return true; - - for (w = lo_word + 1; w < hi_word; w++) - if (cmask_word_op2(&a_bits[w - a_word_off], - &b_bits[w - b_word_off], ~0ULL, op) && - cmask_op2_is_pred(op)) - return true; - - return cmask_word_op2(&a_bits[hi_word - a_word_off], - &b_bits[hi_word - b_word_off], tail_mask, op); -} - -enum cmask_op1 { - CMASK_OP1_ANY_SET, -}; - -static __always_inline bool cmask_word_op1(const u64 *ap, u64 mask, - const enum cmask_op1 op) -{ - switch (op) { - case CMASK_OP1_ANY_SET: - return *ap & mask; - } - unreachable(); -} - -/* - * Walk [@a_base, @a_base + @a_nr_cids) of @a_bits word by word, applying @op. - * Returns true on the first word whose per-word test is true; returns false if - * no word matches or the range is empty. All current op1s short-circuit on - * per-word true; if a non-predicate op1 lands here, add a cmask_op1_is_pred() - * guard analogous to cmask_op2_is_pred(). - */ -static __always_inline bool cmask_walk_op1(const u64 *a_bits, u32 a_base, - u32 a_nr_cids, - const enum cmask_op1 op) -{ - u32 lo = a_base; - u32 hi = a_base + a_nr_cids; - u32 a_word_off = a_base / 64; - u32 lo_word = lo / 64; - u32 hi_word = (hi - 1) / 64; - u64 head_mask = GENMASK_U64(63, lo & 63); - u64 tail_mask = GENMASK_U64((hi - 1) & 63, 0); - u32 w; - - if (lo >= hi) - return false; - - if (lo_word == hi_word) - return cmask_word_op1(&a_bits[lo_word - a_word_off], - head_mask & tail_mask, op); - - if (cmask_word_op1(&a_bits[lo_word - a_word_off], head_mask, op)) - return true; - for (w = lo_word + 1; w < hi_word; w++) - if (cmask_word_op1(&a_bits[w - a_word_off], ~0ULL, op)) - return true; - return cmask_word_op1(&a_bits[hi_word - a_word_off], tail_mask, op); -} - -void scx_cmask_and(struct scx_cmask *dst, const struct scx_cmask *src) -{ - cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, - src->bits, src->base, src->nr_cids, CMASK_OP2_AND); -} - -void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src) -{ - cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, - src->bits, src->base, src->nr_cids, CMASK_OP2_OR); -} - -/** - * scx_cmask_or_racy - OR @src into @dst, reading @src without locking - * - * @src is read word-by-word through data_race(). Same per-bit independence - * rationale as scx_cmask_copy_racy(). Memory ordering with writers is the - * caller's responsibility. - */ -void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src) -{ - cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, - src->bits, src->base, src->nr_cids, CMASK_OP2_OR_RACY); -} - -void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src) -{ - cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, - src->bits, src->base, src->nr_cids, CMASK_OP2_COPY); -} - -/** - * scx_cmask_copy_racy - Snapshot @src into @dst without locking - * - * @src is read word-by-word through data_race(). Head/tail masking matches - * scx_cmask_copy(). Each bit in a cmask is independent, so partial updates - * just leave some bits fresher than others. Memory ordering with writers is - * the caller's responsibility. - */ -void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src) -{ - cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, - src->bits, src->base, src->nr_cids, CMASK_OP2_COPY_RACY); -} - -void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src) -{ - cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, - src->bits, src->base, src->nr_cids, CMASK_OP2_ANDNOT); -} - -/* - * Return true if @cm has any bit set in [@lo, @hi). Caller must ensure - * [@lo, @hi) is contained in @cm's range. - */ -static bool cmask_any_set_in_range(const struct scx_cmask *cm, u32 lo, u32 hi) -{ - if (lo >= hi) - return false; - return cmask_walk_op1(&cm->bits[lo / 64 - cm->base / 64], lo, hi - lo, - CMASK_OP1_ANY_SET); -} - -/** - * scx_cmask_subset - test whether @sub is a subset of @super - * @sub: cmask to test - * @super: cmask to test against - * - * Return true iff every set bit of @sub is also set in @super. - */ -bool scx_cmask_subset(const struct scx_cmask *sub, const struct scx_cmask *super) -{ - u32 super_end = super->base + super->nr_cids; - u32 sub_end = sub->base + sub->nr_cids; - - /* - * Set bits in @sub outside @super's range can't be in @super, so any - * such bit means not a subset. The walk below only visits words - * common to both ranges, so these need a separate scan. - */ - if (sub->base < super->base && - cmask_any_set_in_range(sub, sub->base, min(super->base, sub_end))) - return false; - if (sub_end > super_end && - cmask_any_set_in_range(sub, max(sub->base, super_end), sub_end)) - return false; - - return !cmask_walk_op2((u64 *)super->bits, super->base, super->nr_cids, - sub->bits, sub->base, sub->nr_cids, CMASK_OP2_SUBSET); -} - -bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b) -{ - return cmask_walk_op2((u64 *)a->bits, a->base, a->nr_cids, - b->bits, b->base, b->nr_cids, CMASK_OP2_INTERSECTS); -} - -/** - * scx_cmask_empty - Test whether @m has no bits set - * @m: cmask to test - * - * Return true iff @m's active range has no bits set. - */ -bool scx_cmask_empty(const struct scx_cmask *m) -{ - return !cmask_any_set_in_range(m, m->base, m->base + m->nr_cids); -} - -/** - * scx_bpf_cid_topo - Copy out per-cid topology info - * @cid: cid to look up - * @out__uninit: where to copy the topology info; fully written by this call - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Fill @out__uninit with the topology info for @cid. Trigger scx_error() if - * @cid is out of range. If @cid is valid but in the no-topo section, all fields - * are set to -1. - */ -__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out__uninit, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch) || !cid_valid(sch, cid)) { - *out__uninit = SCX_CID_TOPO_NEG; - return; - } - - *out__uninit = READ_ONCE(scx_cid_topo)[cid]; -} - -__bpf_kfunc_end_defs(); - -BTF_KFUNCS_START(scx_kfunc_ids_init) -BTF_ID_FLAGS(func, scx_bpf_cid_override, KF_IMPLICIT_ARGS | KF_SLEEPABLE) -BTF_KFUNCS_END(scx_kfunc_ids_init) - -static const struct btf_kfunc_id_set scx_kfunc_set_init = { - .owner = THIS_MODULE, - .set = &scx_kfunc_ids_init, - .filter = scx_kfunc_context_filter, -}; - -BTF_KFUNCS_START(scx_kfunc_ids_cid) -BTF_ID_FLAGS(func, scx_bpf_cid_to_cpu, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cpu_to_cid, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cid_topo, KF_IMPLICIT_ARGS) -BTF_KFUNCS_END(scx_kfunc_ids_cid) - -static const struct btf_kfunc_id_set scx_kfunc_set_cid = { - .owner = THIS_MODULE, - .set = &scx_kfunc_ids_cid, -}; - -int scx_cid_kfunc_init(void) -{ - return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init) ?: - register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_cid) ?: - register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_cid) ?: - register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_cid); -} diff --git a/kernel/sched/ext_cid.h b/kernel/sched/ext_cid.h deleted file mode 100644 index 5745e5785e89..000000000000 --- a/kernel/sched/ext_cid.h +++ /dev/null @@ -1,271 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Topological CPU IDs (cids) - * -------------------------- - * - * Raw cpu numbers are clumsy for sharding work and communication across - * topology units, especially from BPF: the space can be sparse, numerical - * closeness doesn't imply topological closeness (x86 hyperthreading often puts - * SMT siblings far apart), and a range of cpu ids doesn't mean anything. - * Sub-scheds make this acute - cpu allocation, revocation and other state are - * constantly communicated across sub-scheds, and passing whole cpumasks scales - * poorly with cpu count. cpumasks are also awkward in BPF: a variable-length - * kernel type sized for the maximum NR_CPUS (4k), with verbose helper sequences - * for every op. - * - * cids give every cpu a dense, topology-ordered id. CPUs sharing a core, LLC or - * NUMA node get contiguous cid ranges, so a topology unit becomes a (start, - * length) slice of cid space. Communication can pass a slice instead of a - * cpumask, and BPF code can process, for example, a u64 word's worth of cids at - * a time. - * - * The mapping is built once at root scheduler enable time by walking the - * topology of online cpus only. Going by online cpus is out of necessity: - * depending on the arch, topology info isn't reliably available for offline - * cpus. The expected usage model is restarting the scheduler on hotplug events - * so the mapping is rebuilt against the new online set. A scheduler that wants - * to handle hotplug without a restart can provide its own cid and shard mapping - * through the override interface. - * - * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2026 Tejun Heo - */ -#ifndef _KERNEL_SCHED_EXT_CID_H -#define _KERNEL_SCHED_EXT_CID_H - -struct scx_sched; - -/* - * Cid space (total is always num_possible_cpus()) is laid out with - * topology-annotated cids first, then no-topo cids at the tail. The - * topology-annotated block covers the cpus that were online when scx_cid_init() - * ran and remains valid even after those cpus go offline. The tail block covers - * possible-but-not-online cpus and carries all-(-1) topo info (see - * scx_cid_topo); callers detect it via the -1 sentinels. - * - * See the comment above the table definitions in ext_cid.c for the - * memory-ordering and visibility contract. - */ -extern s16 *scx_cid_to_cpu_tbl; -extern s16 *scx_cpu_to_cid_tbl; -extern struct scx_cid_topo *scx_cid_topo; -extern struct btf_id_set8 scx_kfunc_ids_init; - -void scx_cmask_clear(struct scx_cmask *m); -void scx_cmask_fill(struct scx_cmask *m); -void scx_cmask_and(struct scx_cmask *dst, const struct scx_cmask *src); -void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src); -void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src); -void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src); -void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src); -void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src); -bool scx_cmask_subset(const struct scx_cmask *sub, const struct scx_cmask *super); -bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b); -bool scx_cmask_empty(const struct scx_cmask *m); -s32 scx_cid_init(struct scx_sched *sch); -int scx_cid_kfunc_init(void); -void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst); - -/** - * cid_valid - Verify a cid value, to be used on ops input args - * @sch: scx_sched to abort on error - * @cid: cid which came from a BPF ops - * - * Return true if @cid is in [0, num_possible_cpus()). On failure, trigger - * scx_error() and return false. - */ -static inline bool cid_valid(struct scx_sched *sch, s32 cid) -{ - if (likely(cid >= 0 && cid < num_possible_cpus())) - return true; - scx_error(sch, "invalid cid %d", cid); - return false; -} - -/** - * __scx_cid_to_cpu - Unchecked cid->cpu table lookup - * @cid: cid to look up. Must be in [0, num_possible_cpus()). - * - * Intended for callsites that have already validated @cid and that hold a - * non-NULL @sch from scx_prog_sched() - a live sched implies the table has - * been allocated, so no NULL check is needed here. - */ -static inline s32 __scx_cid_to_cpu(s32 cid) -{ - /* READ_ONCE pairs with WRITE_ONCE in scx_cid_arrays_alloc() */ - return READ_ONCE(scx_cid_to_cpu_tbl)[cid]; -} - -/** - * __scx_cpu_to_cid - Unchecked cpu->cid table lookup - * @cpu: cpu to look up. Must be a valid possible cpu id. - * - * Same usage constraints as __scx_cid_to_cpu(). - */ -static inline s32 __scx_cpu_to_cid(s32 cpu) -{ - return READ_ONCE(scx_cpu_to_cid_tbl)[cpu]; -} - -/** - * scx_cid_to_cpu - Translate @cid to its cpu - * @sch: scx_sched for error reporting - * @cid: cid to look up - * - * Return the cpu for @cid or a negative errno on failure. Invalid cid triggers - * scx_error() on @sch. The cid arrays are allocated on first scheduler enable - * and never freed, so the returned cpu is stable for the lifetime of the loaded - * scheduler. - */ -static inline s32 scx_cid_to_cpu(struct scx_sched *sch, s32 cid) -{ - if (!cid_valid(sch, cid)) - return -EINVAL; - return __scx_cid_to_cpu(cid); -} - -/** - * scx_cpu_to_cid - Translate @cpu to its cid - * @sch: scx_sched for error reporting - * @cpu: cpu to look up - * - * Return the cid for @cpu or a negative errno on failure. Invalid cpu triggers - * scx_error() on @sch. Same lifetime guarantee as scx_cid_to_cpu(). - */ -static inline s32 scx_cpu_to_cid(struct scx_sched *sch, s32 cpu) -{ - if (!scx_cpu_valid(sch, cpu, NULL)) - return -EINVAL; - return __scx_cpu_to_cid(cpu); -} - -/** - * scx_is_cid_type - Test whether the active scheduler hierarchy is cid-form - */ -static inline bool scx_is_cid_type(void) -{ - return static_branch_unlikely(&__scx_is_cid_type); -} - -static inline bool __scx_cmask_contains(u32 cid, const struct scx_cmask *m) -{ - return likely(cid >= m->base && cid < m->base + m->nr_cids); -} - -/* Word in bits[] covering @cid. @cid must satisfy __scx_cmask_contains(). */ -static inline u64 *__scx_cmask_word(u32 cid, const struct scx_cmask *m) -{ - return (u64 *)&m->bits[cid / 64 - m->base / 64]; -} - -/** - * __scx_cmask_init - Initialize @m with explicit storage capacity - * @m: cmask to initialize - * @base: first cid of the active range - * @nr_cids: number of cids in the active range - * @alloc_cids: storage capacity in cids, at least @nr_cids - * - * Use when storage is sized larger than the initial active range. All of - * bits[] is zeroed. - */ -static inline void __scx_cmask_init(struct scx_cmask *m, u32 base, u32 nr_cids, - u32 alloc_cids) -{ - if (WARN_ON_ONCE(alloc_cids < nr_cids)) - nr_cids = alloc_cids; - - m->base = base; - m->nr_cids = nr_cids; - m->alloc_words = SCX_CMASK_NR_WORDS(alloc_cids); - memset(m->bits, 0, m->alloc_words * sizeof(u64)); -} - -/** - * scx_cmask_init - Initialize @m on tight storage - * @m: cmask to initialize - * @base: first cid of the active range - * @nr_cids: number of cids in the active range - * - * All of bits[] is zeroed. - */ -static inline void scx_cmask_init(struct scx_cmask *m, u32 base, u32 nr_cids) -{ - __scx_cmask_init(m, base, nr_cids, nr_cids); -} - -/** - * scx_cmask_reframe - Reshape @m's active range without resizing storage - * @m: cmask to reframe - * @base: new active range base - * @nr_cids: new active range length, must fit within @m->alloc_words - * - * Body bits within the new range become garbage - only the head and tail - * words are zeroed to keep the padding invariant. - */ -static inline void scx_cmask_reframe(struct scx_cmask *m, u32 base, u32 nr_cids) -{ - if (WARN_ON_ONCE(SCX_CMASK_NR_WORDS(nr_cids) > m->alloc_words)) - return; - - if (nr_cids) { - u32 last_word = ((base & 63) + nr_cids - 1) / 64; - - m->bits[0] = 0; - m->bits[last_word] = 0; - } - - m->base = base; - m->nr_cids = nr_cids; -} - -static inline void __scx_cmask_set(u32 cid, struct scx_cmask *m) -{ - if (!__scx_cmask_contains(cid, m)) - return; - *__scx_cmask_word(cid, m) |= BIT_U64(cid & 63); -} - -/** - * scx_cmask_test - test whether @cid is set in @m - * @cid: cid to test - * @m: cmask to test - * - * Return %false if @cid is outside @m's active range. Otherwise return the - * bit's value. Read via READ_ONCE so callers can race set/clear writers. - */ -static inline bool scx_cmask_test(u32 cid, const struct scx_cmask *m) -{ - if (!__scx_cmask_contains(cid, m)) - return false; - return READ_ONCE(*__scx_cmask_word(cid, m)) & BIT_U64(cid & 63); -} - -/* - * Words of bits[] the active range spans, 0 if empty. Tighter than the storage - * SCX_CMASK_NR_WORDS() sizes for the worst-case base alignment. - */ -static inline u32 scx_cmask_nr_used_words(const struct scx_cmask *m) -{ - if (!m->nr_cids) - return 0; - return ((m->base & 63) + m->nr_cids - 1) / 64 + 1; -} - -/** - * scx_cmask_for_each_cid - iterate set cids in @m - * @cid: s32 loop var that receives each set cid in turn - * @m: cmask to iterate - * - * Visits set bits within @m's active range in ascending order. Scans only the - * words the active range spans, where head and tail padding is kept zero, so - * no per-cid range check is needed. - */ -#define scx_cmask_for_each_cid(cid, m) \ - for (u64 __bs = (m)->base & ~63u, __wi = 0, \ - __nw = scx_cmask_nr_used_words(m); \ - __wi < __nw; __wi++) \ - for (u64 __w = READ_ONCE((m)->bits[__wi]); \ - __w && ((cid) = __bs + __wi * 64 + __ffs64(__w), true); \ - __w &= __w - 1) - -#endif /* _KERNEL_SCHED_EXT_CID_H */ diff --git a/kernel/sched/ext_idle.c b/kernel/sched/ext_idle.c deleted file mode 100644 index 2077373d8da3..000000000000 --- a/kernel/sched/ext_idle.c +++ /dev/null @@ -1,1511 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst - * - * Built-in idle CPU tracking policy. - * - * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2022 Tejun Heo - * Copyright (c) 2022 David Vernet - * Copyright (c) 2024 Andrea Righi - */ - -/* Enable/disable built-in idle CPU selection policy */ -static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); - -/* Enable/disable per-node idle cpumasks */ -static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_per_node); - -/* Enable/disable LLC aware optimizations */ -static DEFINE_STATIC_KEY_FALSE(scx_selcpu_topo_llc); - -/* Enable/disable NUMA aware optimizations */ -static DEFINE_STATIC_KEY_FALSE(scx_selcpu_topo_numa); - -/* - * cpumasks to track idle CPUs within each NUMA node. - * - * If SCX_OPS_BUILTIN_IDLE_PER_NODE is not enabled, a single global cpumask - * from is used to track all the idle CPUs in the system. - */ -struct scx_idle_cpus { - cpumask_var_t cpu; - cpumask_var_t smt; -}; - -/* - * Global host-wide idle cpumasks (used when SCX_OPS_BUILTIN_IDLE_PER_NODE - * is not enabled). - */ -static struct scx_idle_cpus scx_idle_global_masks; - -/* - * Per-node idle cpumasks. - */ -static struct scx_idle_cpus **scx_idle_node_masks; - -/* - * Local per-CPU cpumasks (used to generate temporary idle cpumasks). - */ -static DEFINE_PER_CPU(cpumask_var_t, local_idle_cpumask); -static DEFINE_PER_CPU(cpumask_var_t, local_llc_idle_cpumask); -static DEFINE_PER_CPU(cpumask_var_t, local_numa_idle_cpumask); - -/* - * Return the idle masks associated to a target @node. - * - * NUMA_NO_NODE identifies the global idle cpumask. - */ -static struct scx_idle_cpus *idle_cpumask(int node) -{ - return node == NUMA_NO_NODE ? &scx_idle_global_masks : scx_idle_node_masks[node]; -} - -/* - * Returns the NUMA node ID associated with a @cpu, or NUMA_NO_NODE if - * per-node idle cpumasks are disabled. - */ -static int scx_cpu_node_if_enabled(int cpu) -{ - if (!static_branch_maybe(CONFIG_NUMA, &scx_builtin_idle_per_node)) - return NUMA_NO_NODE; - - return cpu_to_node(cpu); -} - -static bool scx_idle_test_and_clear_cpu(int cpu) -{ - int node = scx_cpu_node_if_enabled(cpu); - struct cpumask *idle_cpus = idle_cpumask(node)->cpu; - - /* - * SMT mask should be cleared whether we can claim @cpu or not. The SMT - * cluster is not wholly idle either way. This also prevents - * scx_pick_idle_cpu() from getting caught in an infinite loop. - */ - if (sched_smt_active()) { - const struct cpumask *smt = cpu_smt_mask(cpu); - struct cpumask *idle_smts = idle_cpumask(node)->smt; - - /* - * If offline, @cpu is not its own sibling and - * scx_pick_idle_cpu() can get caught in an infinite loop as - * @cpu is never cleared from the idle SMT mask. Ensure that - * @cpu is eventually cleared. - * - * NOTE: Use cpumask_intersects() and cpumask_test_cpu() to - * reduce memory writes, which may help alleviate cache - * coherence pressure. - */ - if (cpumask_intersects(smt, idle_smts)) - cpumask_andnot(idle_smts, idle_smts, smt); - else if (cpumask_test_cpu(cpu, idle_smts)) - __cpumask_clear_cpu(cpu, idle_smts); - } - - return cpumask_test_and_clear_cpu(cpu, idle_cpus); -} - -/* - * Pick an idle CPU in a specific NUMA node. - */ -static s32 pick_idle_cpu_in_node(const struct cpumask *cpus_allowed, int node, u64 flags) -{ - int cpu; - -retry: - if (sched_smt_active()) { - cpu = cpumask_any_and_distribute(idle_cpumask(node)->smt, cpus_allowed); - if (cpu < nr_cpu_ids) - goto found; - - if (flags & SCX_PICK_IDLE_CORE) - return -EBUSY; - } - - cpu = cpumask_any_and_distribute(idle_cpumask(node)->cpu, cpus_allowed); - if (cpu >= nr_cpu_ids) - return -EBUSY; - -found: - if (scx_idle_test_and_clear_cpu(cpu)) - return cpu; - else - goto retry; -} - -#ifdef CONFIG_NUMA -/* - * Tracks nodes that have not yet been visited when searching for an idle - * CPU across all available nodes. - */ -static DEFINE_PER_CPU(nodemask_t, per_cpu_unvisited); - -/* - * Search for an idle CPU across all nodes, excluding @node. - */ -static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, int node, u64 flags) -{ - nodemask_t *unvisited; - s32 cpu = -EBUSY; - - preempt_disable(); - unvisited = this_cpu_ptr(&per_cpu_unvisited); - - /* - * Restrict the search to the online nodes (excluding the current - * node that has been visited already). - */ - nodes_copy(*unvisited, node_states[N_ONLINE]); - node_clear(node, *unvisited); - - /* - * Traverse all nodes in order of increasing distance, starting - * from @node. - * - * This loop is O(N^2), with N being the amount of NUMA nodes, - * which might be quite expensive in large NUMA systems. However, - * this complexity comes into play only when a scheduler enables - * SCX_OPS_BUILTIN_IDLE_PER_NODE and it's requesting an idle CPU - * without specifying a target NUMA node, so it shouldn't be a - * bottleneck is most cases. - * - * As a future optimization we may want to cache the list of nodes - * in a per-node array, instead of actually traversing them every - * time. - */ - for_each_node_numadist(node, *unvisited) { - cpu = pick_idle_cpu_in_node(cpus_allowed, node, flags); - if (cpu >= 0) - break; - } - preempt_enable(); - - return cpu; -} -#else -static inline s32 -pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, int node, u64 flags) -{ - return -EBUSY; -} -#endif - -/* - * Find an idle CPU in the system, starting from @node. - */ -static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed, int node, u64 flags) -{ - s32 cpu; - - /* - * Always search in the starting node first (this is an - * optimization that can save some cycles even when the search is - * not limited to a single node). - */ - cpu = pick_idle_cpu_in_node(cpus_allowed, node, flags); - if (cpu >= 0) - return cpu; - - /* - * Stop the search if we are using only a single global cpumask - * (NUMA_NO_NODE) or if the search is restricted to the first node - * only. - */ - if (node == NUMA_NO_NODE || flags & SCX_PICK_IDLE_IN_NODE) - return -EBUSY; - - /* - * Extend the search to the other online nodes. - */ - return pick_idle_cpu_from_online_nodes(cpus_allowed, node, flags); -} - -/* - * Return the amount of CPUs in the same LLC domain of @cpu (or zero if the LLC - * domain is not defined). - */ -static unsigned int llc_weight(s32 cpu) -{ - struct sched_domain *sd; - - sd = rcu_dereference(per_cpu(sd_llc, cpu)); - if (!sd) - return 0; - - return sd->span_weight; -} - -/* - * Return the cpumask representing the LLC domain of @cpu (or NULL if the LLC - * domain is not defined). - */ -static struct cpumask *llc_span(s32 cpu) -{ - struct sched_domain *sd; - - sd = rcu_dereference(per_cpu(sd_llc, cpu)); - if (!sd) - return NULL; - - return sched_domain_span(sd); -} - -/* - * Return the amount of CPUs in the same NUMA domain of @cpu (or zero if the - * NUMA domain is not defined). - */ -static unsigned int numa_weight(s32 cpu) -{ - struct sched_domain *sd; - struct sched_group *sg; - - sd = rcu_dereference(per_cpu(sd_numa, cpu)); - if (!sd) - return 0; - sg = sd->groups; - if (!sg) - return 0; - - return sg->group_weight; -} - -/* - * Return the cpumask representing the NUMA domain of @cpu (or NULL if the NUMA - * domain is not defined). - */ -static struct cpumask *numa_span(s32 cpu) -{ - struct sched_domain *sd; - struct sched_group *sg; - - sd = rcu_dereference(per_cpu(sd_numa, cpu)); - if (!sd) - return NULL; - sg = sd->groups; - if (!sg) - return NULL; - - return sched_group_span(sg); -} - -/* - * Return true if the LLC domains do not perfectly overlap with the NUMA - * domains, false otherwise. - */ -static bool llc_numa_mismatch(void) -{ - int cpu; - - /* - * We need to scan all online CPUs to verify whether their scheduling - * domains overlap. - * - * While it is rare to encounter architectures with asymmetric NUMA - * topologies, CPU hotplugging or virtualized environments can result - * in asymmetric configurations. - * - * For example: - * - * NUMA 0: - * - LLC 0: cpu0..cpu7 - * - LLC 1: cpu8..cpu15 [offline] - * - * NUMA 1: - * - LLC 0: cpu16..cpu23 - * - LLC 1: cpu24..cpu31 - * - * In this case, if we only check the first online CPU (cpu0), we might - * incorrectly assume that the LLC and NUMA domains are fully - * overlapping, which is incorrect (as NUMA 1 has two distinct LLC - * domains). - */ - for_each_online_cpu(cpu) - if (llc_weight(cpu) != numa_weight(cpu)) - return true; - - return false; -} - -/* - * Initialize topology-aware scheduling. - * - * Detect if the system has multiple LLC or multiple NUMA domains and enable - * cache-aware / NUMA-aware scheduling optimizations in the default CPU idle - * selection policy. - * - * Assumption: the kernel's internal topology representation assumes that each - * CPU belongs to a single LLC domain, and that each LLC domain is entirely - * contained within a single NUMA node. - */ -void scx_idle_update_selcpu_topology(struct sched_ext_ops *ops) -{ - bool enable_llc = false, enable_numa = false; - unsigned int nr_cpus; - s32 cpu = cpumask_first(cpu_online_mask); - - /* - * Enable LLC domain optimization only when there are multiple LLC - * domains among the online CPUs. If all online CPUs are part of a - * single LLC domain, the idle CPU selection logic can choose any - * online CPU without bias. - * - * Note that it is sufficient to check the LLC domain of the first - * online CPU to determine whether a single LLC domain includes all - * CPUs. - */ - rcu_read_lock(); - nr_cpus = llc_weight(cpu); - if (nr_cpus > 0) { - if (nr_cpus < num_online_cpus()) - enable_llc = true; - pr_debug("sched_ext: LLC=%*pb weight=%u\n", - cpumask_pr_args(llc_span(cpu)), llc_weight(cpu)); - } - - /* - * Enable NUMA optimization only when there are multiple NUMA domains - * among the online CPUs and the NUMA domains don't perfectly overlap - * with the LLC domains. - * - * If all CPUs belong to the same NUMA node and the same LLC domain, - * enabling both NUMA and LLC optimizations is unnecessary, as checking - * for an idle CPU in the same domain twice is redundant. - * - * If SCX_OPS_BUILTIN_IDLE_PER_NODE is enabled ignore the NUMA - * optimization, as we would naturally select idle CPUs within - * specific NUMA nodes querying the corresponding per-node cpumask. - */ - if (!(ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE)) { - nr_cpus = numa_weight(cpu); - if (nr_cpus > 0) { - if (nr_cpus < num_online_cpus() && llc_numa_mismatch()) - enable_numa = true; - pr_debug("sched_ext: NUMA=%*pb weight=%u\n", - cpumask_pr_args(numa_span(cpu)), nr_cpus); - } - } - rcu_read_unlock(); - - pr_debug("sched_ext: LLC idle selection %s\n", - str_enabled_disabled(enable_llc)); - pr_debug("sched_ext: NUMA idle selection %s\n", - str_enabled_disabled(enable_numa)); - - if (enable_llc) - static_branch_enable_cpuslocked(&scx_selcpu_topo_llc); - else - static_branch_disable_cpuslocked(&scx_selcpu_topo_llc); - if (enable_numa) - static_branch_enable_cpuslocked(&scx_selcpu_topo_numa); - else - static_branch_disable_cpuslocked(&scx_selcpu_topo_numa); -} - -/* - * Return true if @p can run on all possible CPUs, false otherwise. - */ -static inline bool task_affinity_all(const struct task_struct *p) -{ - return p->nr_cpus_allowed >= num_possible_cpus(); -} - -/* - * Built-in CPU idle selection policy: - * - * 1. Prioritize full-idle cores: - * - always prioritize CPUs from fully idle cores (both logical CPUs are - * idle) to avoid interference caused by SMT. - * - * 2. Reuse the same CPU: - * - prefer the last used CPU to take advantage of cached data (L1, L2) and - * branch prediction optimizations. - * - * 3. Prefer @prev_cpu's SMT sibling: - * - if @prev_cpu is busy and no fully idle core is available, try to - * place the task on an idle SMT sibling of @prev_cpu; keeping the - * task on the same core makes migration cheaper, preserves L1 cache - * locality and reduces wakeup latency. - * - * 4. Pick a CPU within the same LLC (Last-Level Cache): - * - if the above conditions aren't met, pick a CPU that shares the same - * LLC, if the LLC domain is a subset of @cpus_allowed, to maintain - * cache locality. - * - * 5. Pick a CPU within the same NUMA node, if enabled: - * - choose a CPU from the same NUMA node, if the node cpumask is a - * subset of @cpus_allowed, to reduce memory access latency. - * - * 6. Pick any idle CPU within the @cpus_allowed domain. - * - * Step 4 and 5 are performed only if the system has, respectively, - * multiple LLCs / multiple NUMA nodes (see scx_selcpu_topo_llc and - * scx_selcpu_topo_numa) and they don't contain the same subset of CPUs. - * - * If %SCX_OPS_BUILTIN_IDLE_PER_NODE is enabled, the search will always - * begin in @prev_cpu's node and proceed to other nodes in order of - * increasing distance. - * - * Return the picked CPU if idle, or a negative value otherwise. - * - * NOTE: tasks that can only run on 1 CPU are excluded by this logic, because - * we never call ops.select_cpu() for them, see select_task_rq(). - */ -s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, - const struct cpumask *cpus_allowed, u64 flags) -{ - const struct cpumask *llc_cpus = NULL, *numa_cpus = NULL; - const struct cpumask *allowed = cpus_allowed ?: p->cpus_ptr; - int node = scx_cpu_node_if_enabled(prev_cpu); - bool is_prev_allowed; - s32 cpu; - - preempt_disable(); - - /* - * Determine the subset of CPUs usable by @p within @cpus_allowed. - */ - if (allowed != p->cpus_ptr) { - struct cpumask *local_cpus = this_cpu_cpumask_var_ptr(local_idle_cpumask); - - if (task_affinity_all(p)) { - allowed = cpus_allowed; - } else if (cpumask_and(local_cpus, cpus_allowed, p->cpus_ptr)) { - allowed = local_cpus; - } else { - cpu = -EBUSY; - goto out_enable; - } - } - - /* - * Check whether @prev_cpu is still within the allowed set. If not, - * we can still try selecting a nearby CPU. - */ - is_prev_allowed = cpumask_test_cpu(prev_cpu, allowed); - - /* - * This is necessary to protect llc_cpus. - */ - rcu_read_lock(); - - /* - * Determine the subset of CPUs that the task can use in its - * current LLC and node. - * - * If the task can run on all CPUs, use the node and LLC cpumasks - * directly. - */ - if (static_branch_maybe(CONFIG_NUMA, &scx_selcpu_topo_numa)) { - struct cpumask *local_cpus = this_cpu_cpumask_var_ptr(local_numa_idle_cpumask); - const struct cpumask *cpus = numa_span(prev_cpu); - - if (allowed == p->cpus_ptr && task_affinity_all(p)) - numa_cpus = cpus; - else if (cpus && cpumask_and(local_cpus, allowed, cpus)) - numa_cpus = local_cpus; - } - - if (static_branch_maybe(CONFIG_SCHED_MC, &scx_selcpu_topo_llc)) { - struct cpumask *local_cpus = this_cpu_cpumask_var_ptr(local_llc_idle_cpumask); - const struct cpumask *cpus = llc_span(prev_cpu); - - if (allowed == p->cpus_ptr && task_affinity_all(p)) - llc_cpus = cpus; - else if (cpus && cpumask_and(local_cpus, allowed, cpus)) - llc_cpus = local_cpus; - } - - /* - * If WAKE_SYNC, try to migrate the wakee to the waker's CPU. - */ - if (wake_flags & SCX_WAKE_SYNC) { - int waker_node; - - /* - * If the waker's CPU is cache affine and prev_cpu is idle, - * then avoid a migration. - */ - cpu = smp_processor_id(); - if (is_prev_allowed && cpus_share_cache(cpu, prev_cpu) && - scx_idle_test_and_clear_cpu(prev_cpu)) { - cpu = prev_cpu; - goto out_unlock; - } - - /* - * If the waker's local DSQ is empty, and the system is under - * utilized, try to wake up @p to the local DSQ of the waker. - * - * Checking only for an empty local DSQ is insufficient as it - * could give the wakee an unfair advantage when the system is - * oversaturated. - * - * Checking only for the presence of idle CPUs is also - * insufficient as the local DSQ of the waker could have tasks - * piled up on it even if there is an idle core elsewhere on - * the system. - */ - waker_node = scx_cpu_node_if_enabled(cpu); - if (!(current->flags & PF_EXITING) && - cpu_rq(cpu)->scx.local_dsq.nr == 0 && - (!(flags & SCX_PICK_IDLE_IN_NODE) || (waker_node == node)) && - !cpumask_empty(idle_cpumask(waker_node)->cpu)) { - if (cpumask_test_cpu(cpu, allowed)) - goto out_unlock; - } - } - - /* - * If CPU has SMT, any wholly idle CPU is likely a better pick than - * partially idle @prev_cpu. - */ - if (sched_smt_active()) { - /* - * Keep using @prev_cpu if it's part of a fully idle core. - */ - if (is_prev_allowed && - cpumask_test_cpu(prev_cpu, idle_cpumask(node)->smt) && - scx_idle_test_and_clear_cpu(prev_cpu)) { - cpu = prev_cpu; - goto out_unlock; - } - - /* - * Search for any fully idle core in the same LLC domain. - */ - if (llc_cpus) { - cpu = pick_idle_cpu_in_node(llc_cpus, node, SCX_PICK_IDLE_CORE); - if (cpu >= 0) - goto out_unlock; - } - - /* - * Search for any fully idle core in the same NUMA node. - */ - if (numa_cpus) { - cpu = pick_idle_cpu_in_node(numa_cpus, node, SCX_PICK_IDLE_CORE); - if (cpu >= 0) - goto out_unlock; - } - - /* - * Search for any full-idle core usable by the task. - * - * If the node-aware idle CPU selection policy is enabled - * (%SCX_OPS_BUILTIN_IDLE_PER_NODE), the search will always - * begin in prev_cpu's node and proceed to other nodes in - * order of increasing distance. - */ - cpu = scx_pick_idle_cpu(allowed, node, flags | SCX_PICK_IDLE_CORE); - if (cpu >= 0) - goto out_unlock; - - /* - * Give up if we're strictly looking for a full-idle SMT - * core. - */ - if (flags & SCX_PICK_IDLE_CORE) { - cpu = -EBUSY; - goto out_unlock; - } - } - - /* - * Use @prev_cpu if it's idle. - */ - if (is_prev_allowed && scx_idle_test_and_clear_cpu(prev_cpu)) { - cpu = prev_cpu; - goto out_unlock; - } - - /* - * Use @prev_cpu's sibling if it's idle. - */ - if (sched_smt_active()) { - for_each_cpu_and(cpu, cpu_smt_mask(prev_cpu), allowed) { - if (cpu == prev_cpu) - continue; - if (scx_idle_test_and_clear_cpu(cpu)) - goto out_unlock; - } - } - - /* - * Search for any idle CPU in the same LLC domain. - */ - if (llc_cpus) { - cpu = pick_idle_cpu_in_node(llc_cpus, node, 0); - if (cpu >= 0) - goto out_unlock; - } - - /* - * Search for any idle CPU in the same NUMA node. - */ - if (numa_cpus) { - cpu = pick_idle_cpu_in_node(numa_cpus, node, 0); - if (cpu >= 0) - goto out_unlock; - } - - /* - * Search for any idle CPU usable by the task. - * - * If the node-aware idle CPU selection policy is enabled - * (%SCX_OPS_BUILTIN_IDLE_PER_NODE), the search will always begin - * in prev_cpu's node and proceed to other nodes in order of - * increasing distance. - */ - cpu = scx_pick_idle_cpu(allowed, node, flags); - -out_unlock: - rcu_read_unlock(); -out_enable: - preempt_enable(); - - return cpu; -} - -/* - * Initialize global and per-node idle cpumasks. - */ -void scx_idle_init_masks(void) -{ - int i; - - /* Allocate global idle cpumasks */ - BUG_ON(!alloc_cpumask_var(&scx_idle_global_masks.cpu, GFP_KERNEL)); - BUG_ON(!alloc_cpumask_var(&scx_idle_global_masks.smt, GFP_KERNEL)); - - /* Allocate per-node idle cpumasks (use nr_node_ids for non-contiguous NUMA nodes) */ - scx_idle_node_masks = kzalloc_objs(*scx_idle_node_masks, nr_node_ids); - BUG_ON(!scx_idle_node_masks); - - for_each_node(i) { - scx_idle_node_masks[i] = kzalloc_node(sizeof(**scx_idle_node_masks), - GFP_KERNEL, i); - BUG_ON(!scx_idle_node_masks[i]); - - BUG_ON(!alloc_cpumask_var_node(&scx_idle_node_masks[i]->cpu, GFP_KERNEL, i)); - BUG_ON(!alloc_cpumask_var_node(&scx_idle_node_masks[i]->smt, GFP_KERNEL, i)); - } - - /* Allocate local per-cpu idle cpumasks */ - for_each_possible_cpu(i) { - BUG_ON(!alloc_cpumask_var_node(&per_cpu(local_idle_cpumask, i), - GFP_KERNEL, cpu_to_node(i))); - BUG_ON(!alloc_cpumask_var_node(&per_cpu(local_llc_idle_cpumask, i), - GFP_KERNEL, cpu_to_node(i))); - BUG_ON(!alloc_cpumask_var_node(&per_cpu(local_numa_idle_cpumask, i), - GFP_KERNEL, cpu_to_node(i))); - } -} - -static void update_builtin_idle(int cpu, bool idle) -{ - int node = scx_cpu_node_if_enabled(cpu); - struct cpumask *idle_cpus = idle_cpumask(node)->cpu; - - assign_cpu(cpu, idle_cpus, idle); - - if (sched_smt_active()) { - const struct cpumask *smt = cpu_smt_mask(cpu); - struct cpumask *idle_smts = idle_cpumask(node)->smt; - - if (idle) { - /* - * idle_smt handling is racy but that's fine as it's - * only for optimization and self-correcting. - */ - if (!cpumask_subset(smt, idle_cpus)) - return; - cpumask_or(idle_smts, idle_smts, smt); - } else { - cpumask_andnot(idle_smts, idle_smts, smt); - } - } -} - -/* - * Update the idle state of a CPU to @idle. - * - * If @do_notify is true, ops.update_idle() is invoked to notify the scx - * scheduler of an actual idle state transition (idle to busy or vice - * versa). If @do_notify is false, only the idle state in the idle masks is - * refreshed without invoking ops.update_idle(). - * - * This distinction is necessary, because an idle CPU can be "reserved" and - * awakened via scx_bpf_pick_idle_cpu() + scx_bpf_kick_cpu(), marking it as - * busy even if no tasks are dispatched. In this case, the CPU may return - * to idle without a true state transition. Refreshing the idle masks - * without invoking ops.update_idle() ensures accurate idle state tracking - * while avoiding unnecessary updates and maintaining balanced state - * transitions. - */ -void __scx_update_idle(struct rq *rq, bool idle, bool do_notify) -{ - struct scx_sched *sch = scx_root; - int cpu = cpu_of(rq); - - lockdep_assert_rq_held(rq); - - /* - * Update the idle masks: - * - for real idle transitions (do_notify == true) - * - for idle-to-idle transitions (indicated by the previous task - * being the idle thread, managed by pick_task_idle()) - * - * Skip updating idle masks if the previous task is not the idle - * thread, since set_next_task_idle() has already handled it when - * transitioning from a task to the idle thread (calling this - * function with do_notify == true). - * - * In this way we can avoid updating the idle masks twice, - * unnecessarily. - */ - if (static_branch_likely(&scx_builtin_idle_enabled)) - if (do_notify || is_idle_task(rq->curr)) - update_builtin_idle(cpu, idle); - - /* - * Trigger ops.update_idle() only when transitioning from a task to - * the idle thread and vice versa. - * - * Idle transitions are indicated by do_notify being set to true, - * managed by put_prev_task_idle()/set_next_task_idle(). - * - * This must come after builtin idle update so that BPF schedulers can - * create interlocking between ops.update_idle() and ops.enqueue() - - * either enqueue() sees the idle bit or update_idle() sees the task - * that enqueue() queued. - */ - if (SCX_HAS_OP(sch, update_idle) && do_notify && - !scx_bypassing(sch, cpu_of(rq))) - SCX_CALL_OP(sch, update_idle, rq, scx_cpu_arg(cpu_of(rq)), idle); -} - -static void reset_idle_masks(struct sched_ext_ops *ops) -{ - int node; - - /* - * Consider all online cpus idle. Should converge to the actual state - * quickly. - */ - if (!(ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE)) { - cpumask_copy(idle_cpumask(NUMA_NO_NODE)->cpu, cpu_online_mask); - cpumask_copy(idle_cpumask(NUMA_NO_NODE)->smt, cpu_online_mask); - return; - } - - for_each_node(node) { - const struct cpumask *node_mask = cpumask_of_node(node); - - cpumask_and(idle_cpumask(node)->cpu, cpu_online_mask, node_mask); - cpumask_and(idle_cpumask(node)->smt, cpu_online_mask, node_mask); - } -} - -void scx_idle_enable(struct sched_ext_ops *ops) -{ - if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) - static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); - else - static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); - - if (ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE) - static_branch_enable_cpuslocked(&scx_builtin_idle_per_node); - else - static_branch_disable_cpuslocked(&scx_builtin_idle_per_node); - - reset_idle_masks(ops); -} - -void scx_idle_disable(void) -{ - static_branch_disable(&scx_builtin_idle_enabled); - static_branch_disable(&scx_builtin_idle_per_node); -} - -/******************************************************************************** - * Helpers that can be called from the BPF scheduler. - */ - -static int validate_node(struct scx_sched *sch, int node) -{ - if (!static_branch_likely(&scx_builtin_idle_per_node)) { - scx_error(sch, "per-node idle tracking is disabled"); - return -EOPNOTSUPP; - } - - /* Return no entry for NUMA_NO_NODE (not a critical scx error) */ - if (node == NUMA_NO_NODE) - return -ENOENT; - - /* Make sure node is in a valid range */ - if (node < 0 || node >= nr_node_ids) { - scx_error(sch, "invalid node %d", node); - return -EINVAL; - } - - /* Make sure the node is part of the set of possible nodes */ - if (!node_possible(node)) { - scx_error(sch, "unavailable node %d", node); - return -EINVAL; - } - - return node; -} - -__bpf_kfunc_start_defs(); - -static bool check_builtin_idle_enabled(struct scx_sched *sch) -{ - if (static_branch_likely(&scx_builtin_idle_enabled)) - return true; - - scx_error(sch, "built-in idle tracking is disabled"); - return false; -} - -/* - * Determine whether @p is a migration-disabled task in the context of BPF - * code. - * - * We can't simply check whether @p->migration_disabled is set in a - * sched_ext callback, because the BPF prolog (__bpf_prog_enter) may disable - * migration for the current task while running BPF code. - * - * Since the BPF prolog calls migrate_disable() only when CONFIG_PREEMPT_RCU - * is enabled (via rcu_read_lock_dont_migrate()), migration_disabled == 1 for - * the current task is ambiguous only in that case: it could be from the BPF - * prolog rather than a real migrate_disable() call. - * - * Without CONFIG_PREEMPT_RCU, the BPF prolog never calls migrate_disable(), - * so migration_disabled == 1 always means the task is truly - * migration-disabled. - * - * Therefore, when migration_disabled == 1 and CONFIG_PREEMPT_RCU is enabled, - * check whether @p is the current task or not: if it is, then migration was - * not disabled before entering the callback, otherwise migration was disabled. - * - * Returns true if @p is migration-disabled, false otherwise. - */ -static bool is_bpf_migration_disabled(const struct task_struct *p) -{ - if (p->migration_disabled == 1) { - if (IS_ENABLED(CONFIG_PREEMPT_RCU)) - return p != current; - return true; - } - return p->migration_disabled; -} - -static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p, - s32 prev_cpu, u64 wake_flags, - const struct cpumask *allowed, u64 flags) -{ - unsigned long irq_flags; - bool we_locked = false; - s32 cpu; - - if (!scx_cpu_valid(sch, prev_cpu, NULL)) - return -EINVAL; - - if (!check_builtin_idle_enabled(sch)) - return -EBUSY; - - /* - * Accessing p->cpus_ptr / p->nr_cpus_allowed needs either @p's rq - * lock or @p's pi_lock. Three cases: - * - * - inside ops.select_cpu(): try_to_wake_up() holds the wake-up - * task's pi_lock; the wake-up task is recorded in kf_tasks[0] - * by SCX_CALL_OP_TASK_RET(). - * - other rq-locked SCX op: scx_locked_rq() points at the held rq. - * - truly unlocked (UNLOCKED ops, SYSCALL, non-SCX struct_ops): - * nothing held, take pi_lock ourselves. - * - * In the first two cases, BPF schedulers may pass an arbitrary task - * that the held lock doesn't cover. Refuse those. - */ - if (this_rq()->scx.in_select_cpu) { - if (!scx_kf_arg_task_ok(sch, p)) - return -EINVAL; - lockdep_assert_held(&p->pi_lock); - } else if (scx_locked_rq()) { - if (task_rq(p) != scx_locked_rq()) - goto cross_task; - } else { - raw_spin_lock_irqsave(&p->pi_lock, irq_flags); - we_locked = true; - } - - /* - * This may also be called from ops.enqueue(), so we need to handle - * per-CPU tasks as well. For these tasks, we can skip all idle CPU - * selection optimizations and simply check whether the previously - * used CPU is idle and within the allowed cpumask. - */ - if (p->nr_cpus_allowed == 1 || is_bpf_migration_disabled(p)) { - if (cpumask_test_cpu(prev_cpu, allowed ?: p->cpus_ptr) && - scx_idle_test_and_clear_cpu(prev_cpu)) - cpu = prev_cpu; - else - cpu = -EBUSY; - } else { - cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, - allowed ?: p->cpus_ptr, flags); - } - - if (we_locked) - raw_spin_unlock_irqrestore(&p->pi_lock, irq_flags); - - return cpu; - -cross_task: - scx_error(sch, "select_cpu kfunc called cross-task on %s[%d]", - p->comm, p->pid); - return -EINVAL; -} - -/** - * scx_bpf_cpu_node - Return the NUMA node the given @cpu belongs to, or - * trigger an error if @cpu is invalid - * @cpu: target CPU - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - */ -__bpf_kfunc s32 scx_bpf_cpu_node(s32 cpu, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch) || !scx_cpu_valid(sch, cpu, NULL)) - return NUMA_NO_NODE; - return cpu_to_node(cpu); -} - -/** - * scx_bpf_select_cpu_dfl - The default implementation of ops.select_cpu() - * @p: task_struct to select a CPU for - * @prev_cpu: CPU @p was on previously - * @wake_flags: %SCX_WAKE_* flags - * @is_idle: out parameter indicating whether the returned CPU is idle - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Can be called from ops.select_cpu(), ops.enqueue(), or from an unlocked - * context such as a BPF test_run() call, as long as built-in CPU selection - * is enabled: ops.update_idle() is missing or %SCX_OPS_KEEP_BUILTIN_IDLE - * is set. - * - * Returns the picked CPU with *@is_idle indicating whether the picked CPU is - * currently idle and thus a good candidate for direct dispatching. - */ -__bpf_kfunc s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, - u64 wake_flags, bool *is_idle, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - s32 cpu; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -ENODEV; - - cpu = select_cpu_from_kfunc(sch, p, prev_cpu, wake_flags, NULL, 0); - if (cpu >= 0) { - *is_idle = true; - return cpu; - } - *is_idle = false; - return prev_cpu; -} - -struct scx_bpf_select_cpu_and_args { - /* @p and @cpus_allowed can't be packed together as KF_RCU is not transitive */ - s32 prev_cpu; - u64 wake_flags; - u64 flags; -}; - -/** - * __scx_bpf_select_cpu_and - Arg-wrapped CPU selection with cpumask - * @p: task_struct to select a CPU for - * @cpus_allowed: cpumask of allowed CPUs - * @args: struct containing the rest of the arguments - * @args->prev_cpu: CPU @p was on previously - * @args->wake_flags: %SCX_WAKE_* flags - * @args->flags: %SCX_PICK_IDLE* flags - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Wrapper kfunc that takes arguments via struct to work around BPF's 5 argument - * limit. BPF programs should use scx_bpf_select_cpu_and() which is provided - * as an inline wrapper in common.bpf.h. - * - * Can be called from ops.select_cpu(), ops.enqueue(), or from an unlocked - * context such as a BPF test_run() call, as long as built-in CPU selection - * is enabled: ops.update_idle() is missing or %SCX_OPS_KEEP_BUILTIN_IDLE - * is set. - * - * @p, @args->prev_cpu and @args->wake_flags match ops.select_cpu(). - * - * Returns the selected idle CPU, which will be automatically awakened upon - * returning from ops.select_cpu() and can be used for direct dispatch, or - * a negative value if no idle CPU is available. - */ -__bpf_kfunc s32 -__scx_bpf_select_cpu_and(struct task_struct *p, const struct cpumask *cpus_allowed, - struct scx_bpf_select_cpu_and_args *args, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -ENODEV; - - return select_cpu_from_kfunc(sch, p, args->prev_cpu, args->wake_flags, - cpus_allowed, args->flags); -} - -/* - * COMPAT: Will be removed in v6.22. - */ -__bpf_kfunc s32 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64 wake_flags, - const struct cpumask *cpus_allowed, u64 flags) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = rcu_dereference(scx_root); - if (unlikely(!sch)) - return -ENODEV; - -#ifdef CONFIG_EXT_SUB_SCHED - /* - * Disallow if any sub-scheds are attached. There is no way to tell - * which scheduler called us, just error out @p's scheduler. - */ - if (unlikely(!list_empty(&sch->children))) { - scx_error(scx_task_sched(p), "__scx_bpf_select_cpu_and() must be used"); - return -EINVAL; - } -#endif - - return select_cpu_from_kfunc(sch, p, prev_cpu, wake_flags, - cpus_allowed, flags); -} - -/** - * scx_bpf_get_idle_cpumask_node - Get a referenced kptr to the - * idle-tracking per-CPU cpumask of a target NUMA node. - * @node: target NUMA node - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Returns an empty cpumask if idle tracking is not enabled, if @node is - * not valid, or running on a UP kernel. In this case the actual error will - * be reported to the BPF scheduler via scx_error(). - */ -__bpf_kfunc const struct cpumask * -scx_bpf_get_idle_cpumask_node(s32 node, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return cpu_none_mask; - - node = validate_node(sch, node); - if (node < 0) - return cpu_none_mask; - - return idle_cpumask(node)->cpu; -} - -/** - * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking - * per-CPU cpumask. - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Returns an empty mask if idle tracking is not enabled, or running on a - * UP kernel. - */ -__bpf_kfunc const struct cpumask *scx_bpf_get_idle_cpumask(const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return cpu_none_mask; - - if (static_branch_unlikely(&scx_builtin_idle_per_node)) { - scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE enabled"); - return cpu_none_mask; - } - - if (!check_builtin_idle_enabled(sch)) - return cpu_none_mask; - - return idle_cpumask(NUMA_NO_NODE)->cpu; -} - -/** - * scx_bpf_get_idle_smtmask_node - Get a referenced kptr to the - * idle-tracking, per-physical-core cpumask of a target NUMA node. Can be - * used to determine if an entire physical core is free. - * @node: target NUMA node - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Returns an empty cpumask if idle tracking is not enabled, if @node is - * not valid, or running on a UP kernel. In this case the actual error will - * be reported to the BPF scheduler via scx_error(). - */ -__bpf_kfunc const struct cpumask * -scx_bpf_get_idle_smtmask_node(s32 node, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return cpu_none_mask; - - node = validate_node(sch, node); - if (node < 0) - return cpu_none_mask; - - if (sched_smt_active()) - return idle_cpumask(node)->smt; - else - return idle_cpumask(node)->cpu; -} - -/** - * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, - * per-physical-core cpumask. Can be used to determine if an entire physical - * core is free. - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Returns an empty mask if idle tracking is not enabled, or running on a - * UP kernel. - */ -__bpf_kfunc const struct cpumask *scx_bpf_get_idle_smtmask(const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return cpu_none_mask; - - if (static_branch_unlikely(&scx_builtin_idle_per_node)) { - scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE enabled"); - return cpu_none_mask; - } - - if (!check_builtin_idle_enabled(sch)) - return cpu_none_mask; - - if (sched_smt_active()) - return idle_cpumask(NUMA_NO_NODE)->smt; - else - return idle_cpumask(NUMA_NO_NODE)->cpu; -} - -/** - * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to - * either the percpu, or SMT idle-tracking cpumask. - * @idle_mask: &cpumask to use - */ -__bpf_kfunc void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) -{ - /* - * Empty function body because we aren't actually acquiring or releasing - * a reference to a global idle cpumask, which is read-only in the - * caller and is never released. The acquire / release semantics here - * are just used to make the cpumask a trusted pointer in the caller. - */ -} - -/** - * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state - * @cpu: cpu to test and clear idle for - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Returns %true if @cpu was idle and its idle state was successfully cleared. - * %false otherwise. - * - * Unavailable if ops.update_idle() is implemented and - * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. - */ -__bpf_kfunc bool scx_bpf_test_and_clear_cpu_idle(s32 cpu, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return false; - - if (!check_builtin_idle_enabled(sch)) - return false; - - if (!scx_cpu_valid(sch, cpu, NULL)) - return false; - - return scx_idle_test_and_clear_cpu(cpu); -} - -/** - * scx_bpf_pick_idle_cpu_node - Pick and claim an idle cpu from @node - * @cpus_allowed: Allowed cpumask - * @node: target NUMA node - * @flags: %SCX_PICK_IDLE_* flags - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Pick and claim an idle cpu in @cpus_allowed from the NUMA node @node. - * - * Returns the picked idle cpu number on success, or -%EBUSY if no matching - * cpu was found. - * - * The search starts from @node and proceeds to other online NUMA nodes in - * order of increasing distance (unless SCX_PICK_IDLE_IN_NODE is specified, - * in which case the search is limited to the target @node). - * - * Always returns an error if ops.update_idle() is implemented and - * %SCX_OPS_KEEP_BUILTIN_IDLE is not set, or if - * %SCX_OPS_BUILTIN_IDLE_PER_NODE is not set. - */ -__bpf_kfunc s32 scx_bpf_pick_idle_cpu_node(const struct cpumask *cpus_allowed, - s32 node, u64 flags, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -ENODEV; - - node = validate_node(sch, node); - if (node < 0) - return node; - - return scx_pick_idle_cpu(cpus_allowed, node, flags); -} - -/** - * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu - * @cpus_allowed: Allowed cpumask - * @flags: %SCX_PICK_IDLE_CPU_* flags - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Pick and claim an idle cpu in @cpus_allowed. Returns the picked idle cpu - * number on success. -%EBUSY if no matching cpu was found. - * - * Idle CPU tracking may race against CPU scheduling state transitions. For - * example, this function may return -%EBUSY as CPUs are transitioning into the - * idle state. If the caller then assumes that there will be dispatch events on - * the CPUs as they were all busy, the scheduler may end up stalling with CPUs - * idling while there are pending tasks. Use scx_bpf_pick_any_cpu() and - * scx_bpf_kick_cpu() to guarantee that there will be at least one dispatch - * event in the near future. - * - * Unavailable if ops.update_idle() is implemented and - * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. - * - * Always returns an error if %SCX_OPS_BUILTIN_IDLE_PER_NODE is set, use - * scx_bpf_pick_idle_cpu_node() instead. - */ -__bpf_kfunc s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed, - u64 flags, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -ENODEV; - - if (static_branch_maybe(CONFIG_NUMA, &scx_builtin_idle_per_node)) { - scx_error(sch, "per-node idle tracking is enabled"); - return -EBUSY; - } - - if (!check_builtin_idle_enabled(sch)) - return -EBUSY; - - return scx_pick_idle_cpu(cpus_allowed, NUMA_NO_NODE, flags); -} - -/** - * scx_bpf_pick_any_cpu_node - Pick and claim an idle cpu if available - * or pick any CPU from @node - * @cpus_allowed: Allowed cpumask - * @node: target NUMA node - * @flags: %SCX_PICK_IDLE_CPU_* flags - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any - * CPU in @cpus_allowed. Guaranteed to succeed and returns the picked idle cpu - * number if @cpus_allowed is not empty. -%EBUSY is returned if @cpus_allowed is - * empty. - * - * The search starts from @node and proceeds to other online NUMA nodes in - * order of increasing distance (unless %SCX_PICK_IDLE_IN_NODE is specified, - * in which case the search is limited to the target @node, regardless of - * the CPU idle state). - * - * If ops.update_idle() is implemented and %SCX_OPS_KEEP_BUILTIN_IDLE is not - * set, this function can't tell which CPUs are idle and will always pick any - * CPU. - */ -__bpf_kfunc s32 scx_bpf_pick_any_cpu_node(const struct cpumask *cpus_allowed, - s32 node, u64 flags, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - s32 cpu; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -ENODEV; - - node = validate_node(sch, node); - if (node < 0) - return node; - - cpu = scx_pick_idle_cpu(cpus_allowed, node, flags); - if (cpu >= 0) - return cpu; - - if (flags & SCX_PICK_IDLE_IN_NODE) - cpu = cpumask_any_and_distribute(cpumask_of_node(node), cpus_allowed); - else - cpu = cpumask_any_distribute(cpus_allowed); - if (cpu < nr_cpu_ids) - return cpu; - else - return -EBUSY; -} - -/** - * scx_bpf_pick_any_cpu - Pick and claim an idle cpu if available or pick any CPU - * @cpus_allowed: Allowed cpumask - * @flags: %SCX_PICK_IDLE_CPU_* flags - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any - * CPU in @cpus_allowed. Guaranteed to succeed and returns the picked idle cpu - * number if @cpus_allowed is not empty. -%EBUSY is returned if @cpus_allowed is - * empty. - * - * If ops.update_idle() is implemented and %SCX_OPS_KEEP_BUILTIN_IDLE is not - * set, this function can't tell which CPUs are idle and will always pick any - * CPU. - * - * Always returns an error if %SCX_OPS_BUILTIN_IDLE_PER_NODE is set, use - * scx_bpf_pick_any_cpu_node() instead. - */ -__bpf_kfunc s32 scx_bpf_pick_any_cpu(const struct cpumask *cpus_allowed, - u64 flags, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - s32 cpu; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -ENODEV; - - if (static_branch_maybe(CONFIG_NUMA, &scx_builtin_idle_per_node)) { - scx_error(sch, "per-node idle tracking is enabled"); - return -EBUSY; - } - - if (static_branch_likely(&scx_builtin_idle_enabled)) { - cpu = scx_pick_idle_cpu(cpus_allowed, NUMA_NO_NODE, flags); - if (cpu >= 0) - return cpu; - } - - cpu = cpumask_any_distribute(cpus_allowed); - if (cpu < nr_cpu_ids) - return cpu; - else - return -EBUSY; -} - -__bpf_kfunc_end_defs(); - -BTF_KFUNCS_START(scx_kfunc_ids_idle) -BTF_ID_FLAGS(func, scx_bpf_cpu_node, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_IMPLICIT_ARGS | KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_IMPLICIT_ARGS | KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) -BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_IMPLICIT_ARGS | KF_RCU) -BTF_KFUNCS_END(scx_kfunc_ids_idle) - -static const struct btf_kfunc_id_set scx_kfunc_set_idle = { - .owner = THIS_MODULE, - .set = &scx_kfunc_ids_idle, - .filter = scx_kfunc_context_filter, -}; - -/* - * The select_cpu kfuncs internally call task_rq_lock() when invoked from an - * rq-unlocked context, and thus cannot be safely called from arbitrary tracing - * contexts where @p's pi_lock state is unknown. Keep them out of - * BPF_PROG_TYPE_TRACING by registering them in their own set which is exposed - * only to STRUCT_OPS and SYSCALL programs. - * - * These kfuncs are also members of scx_kfunc_ids_unlocked (see ext.c) because - * they're callable from unlocked contexts in addition to ops.select_cpu() and - * ops.enqueue(). - */ -BTF_KFUNCS_START(scx_kfunc_ids_select_cpu) -BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU) -BTF_KFUNCS_END(scx_kfunc_ids_select_cpu) - -static const struct btf_kfunc_id_set scx_kfunc_set_select_cpu = { - .owner = THIS_MODULE, - .set = &scx_kfunc_ids_select_cpu, - .filter = scx_kfunc_context_filter, -}; - -int scx_idle_init(void) -{ - return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_idle) ?: - register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_idle) ?: - register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_idle) ?: - register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_select_cpu) ?: - register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_select_cpu); -} diff --git a/kernel/sched/ext_idle.h b/kernel/sched/ext_idle.h deleted file mode 100644 index 8d169d3bbdf9..000000000000 --- a/kernel/sched/ext_idle.h +++ /dev/null @@ -1,27 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst - * - * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2022 Tejun Heo - * Copyright (c) 2022 David Vernet - * Copyright (c) 2024 Andrea Righi - */ -#ifndef _KERNEL_SCHED_EXT_IDLE_H -#define _KERNEL_SCHED_EXT_IDLE_H - -struct sched_ext_ops; - -extern struct btf_id_set8 scx_kfunc_ids_idle; -extern struct btf_id_set8 scx_kfunc_ids_select_cpu; - -void scx_idle_update_selcpu_topology(struct sched_ext_ops *ops); -void scx_idle_init_masks(void); - -s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, - const struct cpumask *cpus_allowed, u64 flags); -void scx_idle_enable(struct sched_ext_ops *ops); -void scx_idle_disable(void); -int scx_idle_init(void); - -#endif /* _KERNEL_SCHED_EXT_IDLE_H */ diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h deleted file mode 100644 index b04701190b23..000000000000 --- a/kernel/sched/ext_internal.h +++ /dev/null @@ -1,1653 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst - * - * Copyright (c) 2025 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2025 Tejun Heo - */ -#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) -#define SCX_MOFF_IDX(moff) ((moff) / sizeof(void (*)(void))) - -enum scx_exit_kind { - SCX_EXIT_NONE, - SCX_EXIT_DONE, - - SCX_EXIT_UNREG = 64, /* user-space initiated unregistration */ - SCX_EXIT_UNREG_BPF, /* BPF-initiated unregistration */ - SCX_EXIT_UNREG_KERN, /* kernel-initiated unregistration */ - SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ - SCX_EXIT_PARENT, /* parent exiting */ - - SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ - SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ - SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ -}; - -/* - * An exit code can be specified when exiting with scx_bpf_exit() or scx_exit(), - * corresponding to exit_kind UNREG_BPF and UNREG_KERN respectively. The codes - * are 64bit of the format: - * - * Bits: [63 .. 48 47 .. 32 31 .. 0] - * [ SYS ACT ] [ SYS RSN ] [ USR ] - * - * SYS ACT: System-defined exit actions - * SYS RSN: System-defined exit reasons - * USR : User-defined exit codes and reasons - * - * Using the above, users may communicate intention and context by ORing system - * actions and/or system reasons with a user-defined exit code. - */ -enum scx_exit_code { - /* Reasons */ - SCX_ECODE_RSN_HOTPLUG = 1LLU << 32, - SCX_ECODE_RSN_CGROUP_OFFLINE = 2LLU << 32, - - /* Actions */ - SCX_ECODE_ACT_RESTART = 1LLU << 48, -}; - -enum scx_exit_flags { - /* - * ops.exit() may be called even if the loading failed before ops.init() - * finishes successfully. This is because ops.exit() allows rich exit - * info communication. The following flag indicates whether ops.init() - * finished successfully. - */ - SCX_EFLAG_INITIALIZED = 1LLU << 0, -}; - -/* - * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is - * being disabled. - */ -struct scx_exit_info { - /* %SCX_EXIT_* - broad category of the exit reason */ - enum scx_exit_kind kind; - - /* - * CPU that initiated the exit, valid once @kind has been set. - * Negative if the exit path didn't identify a CPU. - */ - s32 exit_cpu; - - /* exit code if gracefully exiting */ - s64 exit_code; - - /* %SCX_EFLAG_* */ - u64 flags; - - /* textual representation of the above */ - const char *reason; - - /* backtrace if exiting due to an error */ - unsigned long *bt; - u32 bt_len; - - /* informational message */ - char *msg; - - /* debug dump */ - char *dump; -}; - -/* sched_ext_ops.flags */ -enum scx_ops_flags { - /* - * Keep built-in idle tracking even if ops.update_idle() is implemented. - */ - SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, - - /* - * By default, if there are no other task to run on the CPU, ext core - * keeps running the current task even after its slice expires. If this - * flag is specified, such tasks are passed to ops.enqueue() with - * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. - */ - SCX_OPS_ENQ_LAST = 1LLU << 1, - - /* - * An exiting task may schedule after PF_EXITING is set. In such cases, - * bpf_task_from_pid() may not be able to find the task and if the BPF - * scheduler depends on pid lookup for dispatching, the task will be - * lost leading to various issues including RCU grace period stalls. - * - * To mask this problem, by default, unhashed tasks are automatically - * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't - * depend on pid lookups and wants to handle these tasks directly, the - * following flag can be used. With %SCX_OPS_TID_TO_TASK, - * scx_bpf_tid_to_task() can find exiting tasks reliably. - */ - SCX_OPS_ENQ_EXITING = 1LLU << 2, - - /* - * If set, only tasks with policy set to SCHED_EXT are attached to - * sched_ext. If clear, SCHED_NORMAL tasks are also included. - */ - SCX_OPS_SWITCH_PARTIAL = 1LLU << 3, - - /* - * A migration disabled task can only execute on its current CPU. By - * default, such tasks are automatically put on the CPU's local DSQ with - * the default slice on enqueue. If this ops flag is set, they also go - * through ops.enqueue(). - * - * A migration disabled task never invokes ops.select_cpu() as it can - * only select the current CPU. Also, p->cpus_ptr will only contain its - * current CPU while p->nr_cpus_allowed keeps tracking p->user_cpus_ptr - * and thus may disagree with cpumask_weight(p->cpus_ptr). - */ - SCX_OPS_ENQ_MIGRATION_DISABLED = 1LLU << 4, - - /* - * Queued wakeup (ttwu_queue) is a wakeup optimization that invokes - * ops.enqueue() on the ops.select_cpu() selected or the wakee's - * previous CPU via IPI (inter-processor interrupt) to reduce cacheline - * transfers. When this optimization is enabled, ops.select_cpu() is - * skipped in some cases (when racing against the wakee switching out). - * As the BPF scheduler may depend on ops.select_cpu() being invoked - * during wakeups, queued wakeup is disabled by default. - * - * If this ops flag is set, queued wakeup optimization is enabled and - * the BPF scheduler must be able to handle ops.enqueue() invoked on the - * wakee's CPU without preceding ops.select_cpu() even for tasks which - * may be executed on multiple CPUs. - */ - SCX_OPS_ALLOW_QUEUED_WAKEUP = 1LLU << 5, - - /* - * If set, enable per-node idle cpumasks. If clear, use a single global - * flat idle cpumask. - */ - SCX_OPS_BUILTIN_IDLE_PER_NODE = 1LLU << 6, - - /* - * If set, %SCX_ENQ_IMMED is assumed to be set on all local DSQ - * enqueues. - */ - SCX_OPS_ALWAYS_ENQ_IMMED = 1LLU << 7, - - /* - * Maintain a mapping from p->scx.tid to task_struct so the BPF - * scheduler can recover task pointers from stored tids via - * scx_bpf_tid_to_task(). - * - * Only the root scheduler turns this on. A sub-sched may set the flag - * to declare a dependency on the lookup; if the root scheduler hasn't - * enabled it, attaching the sub-sched is rejected. - */ - SCX_OPS_TID_TO_TASK = 1LLU << 8, - - SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | - SCX_OPS_ENQ_LAST | - SCX_OPS_ENQ_EXITING | - SCX_OPS_ENQ_MIGRATION_DISABLED | - SCX_OPS_ALLOW_QUEUED_WAKEUP | - SCX_OPS_SWITCH_PARTIAL | - SCX_OPS_BUILTIN_IDLE_PER_NODE | - SCX_OPS_ALWAYS_ENQ_IMMED | - SCX_OPS_TID_TO_TASK, - - /* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */ - __SCX_OPS_INTERNAL_MASK = 0xffLLU << 56, - - SCX_OPS_HAS_CPU_PREEMPT = 1LLU << 56, -}; - -/* argument container for ops.init_task() */ -struct scx_init_task_args { - /* - * Set if ops.init_task() is being invoked on the fork path, as opposed - * to the scheduler transition path. - */ - bool fork; -#ifdef CONFIG_EXT_GROUP_SCHED - /* the cgroup the task is joining */ - struct cgroup *cgroup; -#endif -}; - -/* argument container for ops.exit_task() */ -struct scx_exit_task_args { - /* Whether the task exited before running on sched_ext. */ - bool cancelled; -}; - -/* argument container for ops.cgroup_init() */ -struct scx_cgroup_init_args { - /* the weight of the cgroup [1..10000] */ - u32 weight; - - /* bandwidth control parameters from cpu.max and cpu.max.burst */ - u64 bw_period_us; - u64 bw_quota_us; - u64 bw_burst_us; -}; - -enum scx_cpu_preempt_reason { - /* next task is being scheduled by &sched_class_rt */ - SCX_CPU_PREEMPT_RT, - /* next task is being scheduled by &sched_class_dl */ - SCX_CPU_PREEMPT_DL, - /* next task is being scheduled by &sched_class_stop */ - SCX_CPU_PREEMPT_STOP, - /* unknown reason for SCX being preempted */ - SCX_CPU_PREEMPT_UNKNOWN, -}; - -/* - * Argument container for ops.cpu_acquire(). Currently empty, but may be - * expanded in the future. - */ -struct scx_cpu_acquire_args {}; - -/* argument container for ops.cpu_release() */ -struct scx_cpu_release_args { - /* the reason the CPU was preempted */ - enum scx_cpu_preempt_reason reason; - - /* the task that's going to be scheduled on the CPU */ - struct task_struct *task; -}; - -/* informational context provided to dump operations */ -struct scx_dump_ctx { - enum scx_exit_kind kind; - s64 exit_code; - const char *reason; - u64 at_ns; - u64 at_jiffies; -}; - -/* argument container for ops.sub_attach() */ -struct scx_sub_attach_args { - struct sched_ext_ops *ops; - char *cgroup_path; -}; - -/* argument container for ops.sub_detach() */ -struct scx_sub_detach_args { - struct sched_ext_ops *ops; - char *cgroup_path; -}; - -/** - * struct sched_ext_ops - Operation table for BPF scheduler implementation - * - * A BPF scheduler can implement an arbitrary scheduling policy by - * implementing and loading operations in this table. Note that a userland - * scheduling policy can also be implemented using the BPF scheduler - * as a shim layer. - */ -struct sched_ext_ops { - /** - * @select_cpu: Pick the target CPU for a task which is being woken up - * @p: task being woken up - * @prev_cpu: the cpu @p was on before sleeping - * @wake_flags: SCX_WAKE_* - * - * Decision made here isn't final. @p may be moved to any CPU while it - * is getting dispatched for execution later. However, as @p is not on - * the rq at this point, getting the eventual execution CPU right here - * saves a small bit of overhead down the line. - * - * If an idle CPU is returned, the CPU is kicked and will try to - * dispatch. While an explicit custom mechanism can be added, - * select_cpu() serves as the default way to wake up idle CPUs. - * - * @p may be inserted into a DSQ directly by calling - * scx_bpf_dsq_insert(). If so, the ops.enqueue() will be skipped. - * Directly inserting into %SCX_DSQ_LOCAL will put @p in the local DSQ - * of the CPU returned by this operation. - * - * Note that select_cpu() is never called for tasks that can only run - * on a single CPU or tasks with migration disabled, as they don't have - * the option to select a different CPU. See select_task_rq() for - * details. - */ - s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); - - /** - * @enqueue: Enqueue a task on the BPF scheduler - * @p: task being enqueued - * @enq_flags: %SCX_ENQ_* - * - * @p is ready to run. Insert directly into a DSQ by calling - * scx_bpf_dsq_insert() or enqueue on the BPF scheduler. If not directly - * inserted, the bpf scheduler owns @p and if it fails to dispatch @p, - * the task will stall. - * - * If @p was inserted into a DSQ from ops.select_cpu(), this callback is - * skipped. - */ - void (*enqueue)(struct task_struct *p, u64 enq_flags); - - /** - * @dequeue: Remove a task from the BPF scheduler - * @p: task being dequeued - * @deq_flags: %SCX_DEQ_* - * - * Remove @p from the BPF scheduler. This is usually called to isolate - * the task while updating its scheduling properties (e.g. priority). - * - * The ext core keeps track of whether the BPF side owns a given task or - * not and can gracefully ignore spurious dispatches from BPF side, - * which makes it safe to not implement this method. However, depending - * on the scheduling logic, this can lead to confusing behaviors - e.g. - * scheduling position not being updated across a priority change. - */ - void (*dequeue)(struct task_struct *p, u64 deq_flags); - - /** - * @dispatch: Dispatch tasks from the BPF scheduler and/or user DSQs - * @cpu: CPU to dispatch tasks for - * @prev: previous task being switched out - * - * Called when a CPU's local dsq is empty. The operation should dispatch - * one or more tasks from the BPF scheduler into the DSQs using - * scx_bpf_dsq_insert() and/or move from user DSQs into the local DSQ - * using scx_bpf_dsq_move_to_local(). - * - * The maximum number of times scx_bpf_dsq_insert() can be called - * without an intervening scx_bpf_dsq_move_to_local() is specified by - * ops.dispatch_max_batch. See the comments on top of the two functions - * for more details. - * - * When not %NULL, @prev is an SCX task with its slice depleted. If - * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in - * @prev->scx.flags, it is not enqueued yet and will be enqueued after - * ops.dispatch() returns. To keep executing @prev, return without - * dispatching or moving any tasks. Also see %SCX_OPS_ENQ_LAST. - */ - void (*dispatch)(s32 cpu, struct task_struct *prev); - - /** - * @tick: Periodic tick - * @p: task running currently - * - * This operation is called every 1/HZ seconds on CPUs which are - * executing an SCX task. Setting @p->scx.slice to 0 will trigger an - * immediate dispatch cycle on the CPU. - */ - void (*tick)(struct task_struct *p); - - /** - * @runnable: A task is becoming runnable on its associated CPU - * @p: task becoming runnable - * @enq_flags: %SCX_ENQ_* - * - * This and the following three functions can be used to track a task's - * execution state transitions. A task becomes ->runnable() on a CPU, - * and then goes through one or more ->running() and ->stopping() pairs - * as it runs on the CPU, and eventually becomes ->quiescent() when it's - * done running on the CPU. - * - * @p is becoming runnable on the CPU because it's - * - * - waking up (%SCX_ENQ_WAKEUP) - * - being moved from another CPU - * - being restored after temporarily taken off the queue for an - * attribute change. - * - * This and ->enqueue() are related but not coupled. This operation - * notifies @p's state transition and may not be followed by ->enqueue() - * e.g. when @p is being dispatched to a remote CPU, or when @p is - * being enqueued on a CPU experiencing a hotplug event. Likewise, a - * task may be ->enqueue()'d without being preceded by this operation - * e.g. after exhausting its slice. - */ - void (*runnable)(struct task_struct *p, u64 enq_flags); - - /** - * @running: A task is starting to run on its associated CPU - * @p: task starting to run - * - * Note that this callback may be called from a CPU other than the - * one the task is going to run on. This can happen when a task - * property is changed (i.e., affinity), since scx_next_task_scx(), - * which triggers this callback, may run on a CPU different from - * the task's assigned CPU. - * - * Therefore, always use scx_bpf_task_cpu(@p) to determine the - * target CPU the task is going to use. - * - * See ->runnable() for explanation on the task state notifiers. - */ - void (*running)(struct task_struct *p); - - /** - * @stopping: A task is stopping execution - * @p: task stopping to run - * @runnable: is task @p still runnable? - * - * Note that this callback may be called from a CPU other than the - * one the task was running on. This can happen when a task - * property is changed (i.e., affinity), since dequeue_task_scx(), - * which triggers this callback, may run on a CPU different from - * the task's assigned CPU. - * - * Therefore, always use scx_bpf_task_cpu(@p) to retrieve the CPU - * the task was running on. - * - * See ->runnable() for explanation on the task state notifiers. If - * !@runnable, ->quiescent() will be invoked after this operation - * returns. - */ - void (*stopping)(struct task_struct *p, bool runnable); - - /** - * @quiescent: A task is becoming not runnable on its associated CPU - * @p: task becoming not runnable - * @deq_flags: %SCX_DEQ_* - * - * See ->runnable() for explanation on the task state notifiers. - * - * @p is becoming quiescent on the CPU because it's - * - * - sleeping (%SCX_DEQ_SLEEP) - * - being moved to another CPU - * - being temporarily taken off the queue for an attribute change - * (%SCX_DEQ_SAVE) - * - * This and ->dequeue() are related but not coupled. This operation - * notifies @p's state transition and may not be preceded by ->dequeue() - * e.g. when @p is being dispatched to a remote CPU. - */ - void (*quiescent)(struct task_struct *p, u64 deq_flags); - - /** - * @yield: Yield CPU - * @from: yielding task - * @to: optional yield target task - * - * If @to is NULL, @from is yielding the CPU to other runnable tasks. - * The BPF scheduler should ensure that other available tasks are - * dispatched before the yielding task. Return value is ignored in this - * case. - * - * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf - * scheduler can implement the request, return %true; otherwise, %false. - */ - bool (*yield)(struct task_struct *from, struct task_struct *to); - - /** - * @core_sched_before: Task ordering for core-sched - * @a: task A - * @b: task B - * - * Used by core-sched to determine the ordering between two tasks. See - * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on - * core-sched. - * - * Both @a and @b are runnable and may or may not currently be queued on - * the BPF scheduler. Should return %true if @a should run before @b. - * %false if there's no required ordering or @b should run before @a. - * - * If not specified, the default is ordering them according to when they - * became runnable. - */ - bool (*core_sched_before)(struct task_struct *a, struct task_struct *b); - - /** - * @set_weight: Set task weight - * @p: task to set weight for - * @weight: new weight [1..10000] - * - * Update @p's weight to @weight. - */ - void (*set_weight)(struct task_struct *p, u32 weight); - - /** - * @set_cpumask: Set CPU affinity - * @p: task to set CPU affinity for - * @cpumask: cpumask of cpus that @p can run on - * - * Update @p's CPU affinity to @cpumask. - */ - void (*set_cpumask)(struct task_struct *p, - const struct cpumask *cpumask); - - /** - * @update_idle: Update the idle state of a CPU - * @cpu: CPU to update the idle state for - * @idle: whether entering or exiting the idle state - * - * This operation is called when @rq's CPU goes or leaves the idle - * state. By default, implementing this operation disables the built-in - * idle CPU tracking and the following helpers become unavailable: - * - * - scx_bpf_select_cpu_dfl() - * - scx_bpf_select_cpu_and() - * - scx_bpf_test_and_clear_cpu_idle() - * - scx_bpf_pick_idle_cpu() - * - * The user also must implement ops.select_cpu() as the default - * implementation relies on scx_bpf_select_cpu_dfl(). - * - * Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle - * tracking. - */ - void (*update_idle)(s32 cpu, bool idle); - - /** - * @init_task: Initialize a task to run in a BPF scheduler - * @p: task to initialize for BPF scheduling - * @args: init arguments, see the struct definition - * - * Either we're loading a BPF scheduler or a new task is being forked. - * Initialize @p for BPF scheduling. This operation may block and can - * be used for allocations, and is called exactly once for a task. - * - * Return 0 for success, -errno for failure. An error return while - * loading will abort loading of the BPF scheduler. During a fork, it - * will abort that specific fork. - */ - s32 (*init_task)(struct task_struct *p, struct scx_init_task_args *args); - - /** - * @exit_task: Exit a previously-running task from the system - * @p: task to exit - * @args: exit arguments, see the struct definition - * - * @p is exiting or the BPF scheduler is being unloaded. Perform any - * necessary cleanup for @p. - */ - void (*exit_task)(struct task_struct *p, struct scx_exit_task_args *args); - - /** - * @enable: Enable BPF scheduling for a task - * @p: task to enable BPF scheduling for - * - * Enable @p for BPF scheduling. enable() is called on @p any time it - * enters SCX, and is always paired with a matching disable(). - */ - void (*enable)(struct task_struct *p); - - /** - * @disable: Disable BPF scheduling for a task - * @p: task to disable BPF scheduling for - * - * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. - * Disable BPF scheduling for @p. A disable() call is always matched - * with a prior enable() call. - */ - void (*disable)(struct task_struct *p); - - /** - * @dump: Dump BPF scheduler state on error - * @ctx: debug dump context - * - * Use scx_bpf_dump() to generate BPF scheduler specific debug dump. - */ - void (*dump)(struct scx_dump_ctx *ctx); - - /** - * @dump_cpu: Dump BPF scheduler state for a CPU on error - * @ctx: debug dump context - * @cpu: CPU to generate debug dump for - * @idle: @cpu is currently idle without any runnable tasks - * - * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for - * @cpu. If @idle is %true and this operation doesn't produce any - * output, @cpu is skipped for dump. - */ - void (*dump_cpu)(struct scx_dump_ctx *ctx, s32 cpu, bool idle); - - /** - * @dump_task: Dump BPF scheduler state for a runnable task on error - * @ctx: debug dump context - * @p: runnable task to generate debug dump for - * - * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for - * @p. - */ - void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p); - -#ifdef CONFIG_EXT_GROUP_SCHED - /** - * @cgroup_init: Initialize a cgroup - * @cgrp: cgroup being initialized - * @args: init arguments, see the struct definition - * - * Either the BPF scheduler is being loaded or @cgrp created, initialize - * @cgrp for sched_ext. This operation may block. - * - * Return 0 for success, -errno for failure. An error return while - * loading will abort loading of the BPF scheduler. During cgroup - * creation, it will abort the specific cgroup creation. - */ - s32 (*cgroup_init)(struct cgroup *cgrp, - struct scx_cgroup_init_args *args); - - /** - * @cgroup_exit: Exit a cgroup - * @cgrp: cgroup being exited - * - * Either the BPF scheduler is being unloaded or @cgrp destroyed, exit - * @cgrp for sched_ext. This operation my block. - */ - void (*cgroup_exit)(struct cgroup *cgrp); - - /** - * @cgroup_prep_move: Prepare a task to be moved to a different cgroup - * @p: task being moved - * @from: cgroup @p is being moved from - * @to: cgroup @p is being moved to - * - * Prepare @p for move from cgroup @from to @to. This operation may - * block and can be used for allocations. - * - * Return 0 for success, -errno for failure. An error return aborts the - * migration. - */ - s32 (*cgroup_prep_move)(struct task_struct *p, - struct cgroup *from, struct cgroup *to); - - /** - * @cgroup_move: Commit cgroup move - * @p: task being moved - * @from: cgroup @p is being moved from - * @to: cgroup @p is being moved to - * - * Commit the move. @p is dequeued during this operation. - */ - void (*cgroup_move)(struct task_struct *p, - struct cgroup *from, struct cgroup *to); - - /** - * @cgroup_cancel_move: Cancel cgroup move - * @p: task whose cgroup move is being canceled - * @from: cgroup @p was being moved from - * @to: cgroup @p was being moved to - * - * @p was cgroup_prep_move()'d but failed before reaching cgroup_move(). - * Undo the preparation. - */ - void (*cgroup_cancel_move)(struct task_struct *p, - struct cgroup *from, struct cgroup *to); - - /** - * @cgroup_set_weight: A cgroup's weight is being changed - * @cgrp: cgroup whose weight is being updated - * @weight: new weight [1..10000] - * - * Update @cgrp's weight to @weight. - */ - void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); - - /** - * @cgroup_set_bandwidth: A cgroup's bandwidth is being changed - * @cgrp: cgroup whose bandwidth is being updated - * @period_us: bandwidth control period - * @quota_us: bandwidth control quota - * @burst_us: bandwidth control burst - * - * Update @cgrp's bandwidth control parameters. This is from the cpu.max - * cgroup interface. - * - * @quota_us / @period_us determines the CPU bandwidth @cgrp is entitled - * to. For example, if @period_us is 1_000_000 and @quota_us is - * 2_500_000. @cgrp is entitled to 2.5 CPUs. @burst_us can be - * interpreted in the same fashion and specifies how much @cgrp can - * burst temporarily. The specific control mechanism and thus the - * interpretation of @period_us and burstiness is up to the BPF - * scheduler. - */ - void (*cgroup_set_bandwidth)(struct cgroup *cgrp, - u64 period_us, u64 quota_us, u64 burst_us); - - /** - * @cgroup_set_idle: A cgroup's idle state is being changed - * @cgrp: cgroup whose idle state is being updated - * @idle: whether the cgroup is entering or exiting idle state - * - * Update @cgrp's idle state to @idle. This callback is invoked when - * a cgroup transitions between idle and non-idle states, allowing the - * BPF scheduler to adjust its behavior accordingly. - */ - void (*cgroup_set_idle)(struct cgroup *cgrp, bool idle); - -#endif /* CONFIG_EXT_GROUP_SCHED */ - - /** - * @sub_attach: Attach a sub-scheduler - * @args: argument container, see the struct definition - * - * Return 0 to accept the sub-scheduler. -errno to reject. - */ - s32 (*sub_attach)(struct scx_sub_attach_args *args); - - /** - * @sub_detach: Detach a sub-scheduler - * @args: argument container, see the struct definition - */ - void (*sub_detach)(struct scx_sub_detach_args *args); - - /* - * All online ops must come before ops.cpu_online(). - */ - - /** - * @cpu_online: A CPU became online - * @cpu: CPU which just came up - * - * @cpu just came online. @cpu will not call ops.enqueue() or - * ops.dispatch(), nor run tasks associated with other CPUs beforehand. - */ - void (*cpu_online)(s32 cpu); - - /** - * @cpu_offline: A CPU is going offline - * @cpu: CPU which is going offline - * - * @cpu is going offline. @cpu will not call ops.enqueue() or - * ops.dispatch(), nor run tasks associated with other CPUs afterwards. - */ - void (*cpu_offline)(s32 cpu); - - /* - * All CPU hotplug ops must come before ops.init(). - */ - - /** - * @init: Initialize the BPF scheduler - */ - s32 (*init)(void); - - /** - * @exit: Clean up after the BPF scheduler - * @info: Exit info - * - * ops.exit() is also called on ops.init() failure, which is a bit - * unusual. This is to allow rich reporting through @info on how - * ops.init() failed. - */ - void (*exit)(struct scx_exit_info *info); - - /* - * Data fields must comes after all ops fields. - */ - - /** - * @dispatch_max_batch: Max nr of tasks that dispatch() can dispatch - */ - u32 dispatch_max_batch; - - /** - * @flags: %SCX_OPS_* flags - */ - u64 flags; - - /** - * @timeout_ms: The maximum amount of time, in milliseconds, that a - * runnable task should be able to wait before being scheduled. The - * maximum timeout may not exceed the default timeout of 30 seconds. - * - * Defaults to the maximum allowed timeout value of 30 seconds. - */ - u32 timeout_ms; - - /** - * @exit_dump_len: scx_exit_info.dump buffer length. If 0, the default - * value of 32768 is used. - */ - u32 exit_dump_len; - - /** - * @hotplug_seq: A sequence number that may be set by the scheduler to - * detect when a hotplug event has occurred during the loading process. - * If 0, no detection occurs. Otherwise, the scheduler will fail to - * load if the sequence number does not match @scx_hotplug_seq on the - * enable path. - */ - u64 hotplug_seq; - - /** - * @cgroup_id: When >1, attach the scheduler as a sub-scheduler on the - * specified cgroup. - */ - u64 sub_cgroup_id; - - /** - * @name: BPF scheduler's name - * - * Must be a non-zero valid BPF object name including only isalnum(), - * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the - * BPF scheduler is enabled. - */ - char name[SCX_OPS_NAME_LEN]; - - /* internal use only, must be NULL */ - void __rcu *priv; - - /* - * Deprecated callbacks. Kept at the end of the struct so the cid-form - * struct (sched_ext_ops_cid) can omit them without affecting the - * shared field offsets. Use SCX_ENQ_IMMED instead. Sitting past - * SCX_OPI_END means has_op doesn't cover them, so SCX_HAS_OP() cannot - * be used; callers must test sch->ops.cpu_acquire / cpu_release - * directly. - */ - - /** - * @cpu_acquire: A CPU is becoming available to the BPF scheduler - * @cpu: The CPU being acquired by the BPF scheduler. - * @args: Acquire arguments, see the struct definition. - * - * A CPU that was previously released from the BPF scheduler is now once - * again under its control. Deprecated; use SCX_ENQ_IMMED instead. - */ - void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); - - /** - * @cpu_release: A CPU is taken away from the BPF scheduler - * @cpu: The CPU being released by the BPF scheduler. - * @args: Release arguments, see the struct definition. - * - * The specified CPU is no longer under the control of the BPF - * scheduler. This could be because it was preempted by a higher - * priority sched_class, though there may be other reasons as well. The - * caller should consult @args->reason to determine the cause. - * Deprecated; use SCX_ENQ_IMMED instead. - */ - void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); -}; - -/** - * struct sched_ext_ops_cid - cid-form alternative to struct sched_ext_ops - * - * Mirrors struct sched_ext_ops with cpu/cpumask substituted with cid/cmask - * where applicable. Layout up to and including @priv matches sched_ext_ops - * byte-for-byte (verified by BUILD_BUG_ON checks at scx_init() time) so - * shared field offsets work for both struct types in bpf_scx_init_member() - * and bpf_scx_check_member(). The deprecated cpu_acquire/cpu_release - * callbacks at the tail of sched_ext_ops are omitted here entirely. - * - * Differences from sched_ext_ops: - * - select_cpu -> select_cid (returns cid) - * - dispatch -> dispatch (cpu arg is now cid) - * - update_idle -> update_idle (cpu arg is now cid) - * - set_cpumask -> set_cmask (cmask instead of cpumask) - * - cpu_online -> cid_online - * - cpu_offline -> cid_offline - * - dump_cpu -> dump_cid - * - cpu_acquire/cpu_release -> not present (deprecated in sched_ext_ops) - * - * BPF schedulers using this type cannot call cpu-form scx_bpf_* kfuncs; - * use the cid-form variants instead. Enforced at BPF verifier time via - * scx_kfunc_context_filter() branching on prog->aux->st_ops. - * - * See sched_ext_ops for callback documentation. - */ -struct sched_ext_ops_cid { - s32 (*select_cid)(struct task_struct *p, s32 prev_cid, u64 wake_flags); - void (*enqueue)(struct task_struct *p, u64 enq_flags); - void (*dequeue)(struct task_struct *p, u64 deq_flags); - void (*dispatch)(s32 cid, struct task_struct *prev); - void (*tick)(struct task_struct *p); - void (*runnable)(struct task_struct *p, u64 enq_flags); - void (*running)(struct task_struct *p); - void (*stopping)(struct task_struct *p, bool runnable); - void (*quiescent)(struct task_struct *p, u64 deq_flags); - bool (*yield)(struct task_struct *from, struct task_struct *to); - bool (*core_sched_before)(struct task_struct *a, - struct task_struct *b); - void (*set_weight)(struct task_struct *p, u32 weight); - void (*set_cmask)(struct task_struct *p, - const struct scx_cmask *cmask); - void (*update_idle)(s32 cid, bool idle); - s32 (*init_task)(struct task_struct *p, - struct scx_init_task_args *args); - void (*exit_task)(struct task_struct *p, - struct scx_exit_task_args *args); - void (*enable)(struct task_struct *p); - void (*disable)(struct task_struct *p); - void (*dump)(struct scx_dump_ctx *ctx); - void (*dump_cid)(struct scx_dump_ctx *ctx, s32 cid, bool idle); - void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p); -#ifdef CONFIG_EXT_GROUP_SCHED - s32 (*cgroup_init)(struct cgroup *cgrp, - struct scx_cgroup_init_args *args); - void (*cgroup_exit)(struct cgroup *cgrp); - s32 (*cgroup_prep_move)(struct task_struct *p, - struct cgroup *from, struct cgroup *to); - void (*cgroup_move)(struct task_struct *p, - struct cgroup *from, struct cgroup *to); - void (*cgroup_cancel_move)(struct task_struct *p, - struct cgroup *from, struct cgroup *to); - void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); - void (*cgroup_set_bandwidth)(struct cgroup *cgrp, - u64 period_us, u64 quota_us, u64 burst_us); - void (*cgroup_set_idle)(struct cgroup *cgrp, bool idle); -#endif /* CONFIG_EXT_GROUP_SCHED */ - s32 (*sub_attach)(struct scx_sub_attach_args *args); - void (*sub_detach)(struct scx_sub_detach_args *args); - void (*cid_online)(s32 cid); - void (*cid_offline)(s32 cid); - s32 (*init)(void); - void (*exit)(struct scx_exit_info *info); - - /* Data fields - must match sched_ext_ops layout exactly */ - u32 dispatch_max_batch; - u64 flags; - u32 timeout_ms; - u32 exit_dump_len; - u64 hotplug_seq; - u64 sub_cgroup_id; - char name[SCX_OPS_NAME_LEN]; - - /* internal use only, must be NULL */ - void __rcu *priv; - - /* layout end anchor for the BUILD_BUG_ON in scx_init(); keep last */ - char __end[0]; -}; - -enum scx_opi { - SCX_OPI_BEGIN = 0, - SCX_OPI_NORMAL_BEGIN = 0, - SCX_OPI_NORMAL_END = SCX_OP_IDX(cpu_online), - SCX_OPI_CPU_HOTPLUG_BEGIN = SCX_OP_IDX(cpu_online), - SCX_OPI_CPU_HOTPLUG_END = SCX_OP_IDX(init), - SCX_OPI_END = SCX_OP_IDX(init), -}; - -/* - * Collection of event counters. Event types are placed in descending order. - */ -struct scx_event_stats { - /* - * If ops.select_cpu() returns a CPU which can't be used by the task, - * the core scheduler code silently picks a fallback CPU. - */ - s64 SCX_EV_SELECT_CPU_FALLBACK; - - /* - * When dispatching to a local DSQ, the CPU may have gone offline in - * the meantime. In this case, the task is bounced to the global DSQ. - */ - s64 SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE; - - /* - * If SCX_OPS_ENQ_LAST is not set, the number of times that a task - * continued to run because there were no other tasks on the CPU. - */ - s64 SCX_EV_DISPATCH_KEEP_LAST; - - /* - * If SCX_OPS_ENQ_EXITING is not set, the number of times that a task - * is dispatched to a local DSQ when exiting. - */ - s64 SCX_EV_ENQ_SKIP_EXITING; - - /* - * If SCX_OPS_ENQ_MIGRATION_DISABLED is not set, the number of times a - * migration disabled task skips ops.enqueue() and is dispatched to its - * local DSQ. - */ - s64 SCX_EV_ENQ_SKIP_MIGRATION_DISABLED; - - /* - * The number of times a task, enqueued on a local DSQ with - * SCX_ENQ_IMMED, was re-enqueued because the CPU was not available for - * immediate execution. - */ - s64 SCX_EV_REENQ_IMMED; - - /* - * The number of times a reenq of local DSQ caused another reenq of - * local DSQ. This can happen when %SCX_ENQ_IMMED races against a higher - * priority class task even if the BPF scheduler always satisfies the - * prerequisites for %SCX_ENQ_IMMED at the time of enqueue. However, - * that scenario is very unlikely and this count going up regularly - * indicates that the BPF scheduler is handling %SCX_ENQ_REENQ - * incorrectly causing recursive reenqueues. - */ - s64 SCX_EV_REENQ_LOCAL_REPEAT; - - /* - * Total number of times a task's time slice was refilled with the - * default value (SCX_SLICE_DFL). - */ - s64 SCX_EV_REFILL_SLICE_DFL; - - /* - * The total duration of bypass modes in nanoseconds. - */ - s64 SCX_EV_BYPASS_DURATION; - - /* - * The number of tasks dispatched in the bypassing mode. - */ - s64 SCX_EV_BYPASS_DISPATCH; - - /* - * The number of times the bypassing mode has been activated. - */ - s64 SCX_EV_BYPASS_ACTIVATE; - - /* - * The number of times the scheduler attempted to insert a task that it - * doesn't own into a DSQ. Such attempts are ignored. - * - * As BPF schedulers are allowed to ignore dequeues, it's difficult to - * tell whether such an attempt is from a scheduler malfunction or an - * ignored dequeue around sub-sched enabling. If this count keeps going - * up regardless of sub-sched enabling, it likely indicates a bug in the - * scheduler. - */ - s64 SCX_EV_INSERT_NOT_OWNED; - - /* - * The number of times tasks from bypassing descendants are scheduled - * from sub_bypass_dsq's. - */ - s64 SCX_EV_SUB_BYPASS_DISPATCH; -}; - -struct scx_sched; - -enum scx_sched_pcpu_flags { - SCX_SCHED_PCPU_BYPASSING = 1LLU << 0, -}; - -/* dispatch buf */ -struct scx_dsp_buf_ent { - struct task_struct *task; - unsigned long qseq; - u64 dsq_id; - u64 enq_flags; -}; - -struct scx_dsp_ctx { - struct rq *rq; - u32 cursor; - u32 nr_tasks; - struct scx_dsp_buf_ent buf[]; -}; - -struct scx_deferred_reenq_local { - struct list_head node; - u64 flags; - u64 seq; - u32 cnt; -}; - -struct scx_sched_pcpu { - struct scx_sched *sch; - u64 flags; /* protected by rq lock */ - - /* - * The event counters are in a per-CPU variable to minimize the - * accounting overhead. A system-wide view on the event counter is - * constructed when requested by scx_bpf_events(). - */ - struct scx_event_stats event_stats; - - struct scx_deferred_reenq_local deferred_reenq_local; - struct scx_dispatch_q bypass_dsq; -#ifdef CONFIG_EXT_SUB_SCHED - u32 bypass_host_seq; -#endif - - /* must be the last entry - contains flex array */ - struct scx_dsp_ctx dsp_ctx; -}; - -struct scx_sched_pnode { - struct scx_dispatch_q global_dsq; -}; - -struct scx_sched { - /* - * cpu-form and cid-form ops share field offsets up to .priv (verified - * by BUILD_BUG_ON in scx_init()). The anonymous union lets the kernel - * access either view of the same storage without function-pointer - * casts: use .ops for cpu-form and shared fields, .ops_cid for the - * cid-renamed callbacks (set_cmask, select_cid, cid_online, ...). - */ - union { - struct sched_ext_ops ops; - struct sched_ext_ops_cid ops_cid; - }; - bool is_cid_type; /* true if registered via bpf_sched_ext_ops_cid */ - - /* - * Arena map auto-discovered from member progs at struct_ops attach. - * cid-form schedulers must use exactly one arena across all member - * progs. NULL on cpu-form. - * - * @arena_pool sub-allocates @arena_map. Each gen_pool chunk is added - * at the kernel-side mapping address. @arena_kern_base is the start - * of the arena's kern_vm range. See scx_arena_to_kaddr() and - * scx_kaddr_to_arena(). - */ - struct bpf_map *arena_map; - struct gen_pool *arena_pool; - uintptr_t arena_kern_base; - - /* - * Per-CPU arena cmask used by scx_call_op_set_cpumask() to hand a cmask - * to ops_cid.set_cmask(). The kernel writes through the stored kern_va - * and hands BPF its arena pointer via scx_kaddr_to_arena(). - */ - struct scx_cmask * __percpu *set_cmask_scratch; - - DECLARE_BITMAP(has_op, SCX_OPI_END); - - /* - * Dispatch queues. - * - * The global DSQ (%SCX_DSQ_GLOBAL) is split per-node for scalability. - * This is to avoid live-locking in bypass mode where all tasks are - * dispatched to %SCX_DSQ_GLOBAL and all CPUs consume from it. If - * per-node split isn't sufficient, it can be further split. - */ - struct rhashtable dsq_hash; - struct scx_sched_pnode **pnode; - struct scx_sched_pcpu __percpu *pcpu; - - u64 slice_dfl; - u64 bypass_timestamp; - s32 bypass_depth; - - /* bypass dispatch path enable state, see bypass_dsp_enabled() */ - unsigned long bypass_dsp_claim; - atomic_t bypass_dsp_enable_depth; - - bool aborting; - bool dump_disabled; /* protected by scx_dump_lock */ - u32 dsp_max_batch; - s32 level; - - /* - * Updates to the following warned bitfields can race causing RMW issues - * but it doesn't really matter. - */ - bool warned_zero_slice:1; - bool warned_deprecated_rq:1; - bool warned_unassoc_progs:1; - - struct list_head all; - -#ifdef CONFIG_EXT_SUB_SCHED - struct rhash_head hash_node; - - struct list_head children; - struct list_head sibling; - struct cgroup *cgrp; - char *cgrp_path; - struct kset *sub_kset; - - bool sub_attached; -#endif /* CONFIG_EXT_SUB_SCHED */ - - /* - * The maximum amount of time in jiffies that a task may be runnable - * without being scheduled on a CPU. If this timeout is exceeded, it - * will trigger scx_error(). - */ - unsigned long watchdog_timeout; - - atomic_t exit_kind; - struct scx_exit_info *exit_info; - - struct kobject kobj; - - struct kthread_worker *helper; - struct irq_work disable_irq_work; - struct kthread_work disable_work; - struct timer_list bypass_lb_timer; - cpumask_var_t bypass_lb_donee_cpumask; - cpumask_var_t bypass_lb_resched_cpumask; - struct rcu_work rcu_work; - - /* all ancestors including self */ - struct scx_sched *ancestors[]; -}; - -/** - * scx_arena_to_kaddr - Translate a BPF-arena pointer to its kernel address - * @sch: scheduler whose arena hosts @bpf_ptr - * @bpf_ptr: BPF-arena pointer, only the low 32 bits are used - * - * The (u32) cast normalizes any input into the arena's 4 GiB kern_vm range, - * which combined with scratch-page fault recovery makes the returned pointer - * safe to dereference up to GUARD_SZ / 2 past the intended object. Accesses - * larger than GUARD_SZ / 2 must be explicitly bounds-checked. - */ -static inline void *scx_arena_to_kaddr(struct scx_sched *sch, const void *bpf_ptr) -{ - return (void *)(sch->arena_kern_base + (u32)(uintptr_t)bpf_ptr); -} - -/** - * scx_kaddr_to_arena - Translate a kernel arena address to its BPF form - * @sch: scheduler whose arena hosts @kaddr - * @kaddr: kernel-side arena address, supplied by trusted kernel code - */ -static inline void *scx_kaddr_to_arena(struct scx_sched *sch, const void *kaddr) -{ - return (void *)((uintptr_t)kaddr - sch->arena_kern_base); -} - -enum scx_wake_flags { - /* expose select WF_* flags as enums */ - SCX_WAKE_FORK = WF_FORK, - SCX_WAKE_TTWU = WF_TTWU, - SCX_WAKE_SYNC = WF_SYNC, -}; - -enum scx_enq_flags { - /* expose select ENQUEUE_* flags as enums */ - SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, - SCX_ENQ_HEAD = ENQUEUE_HEAD, - SCX_ENQ_CPU_SELECTED = ENQUEUE_RQ_SELECTED, - - /* high 32bits are SCX specific */ - - /* - * Set the following to trigger preemption when calling - * scx_bpf_dsq_insert() with a local dsq as the target. The slice of the - * current task is cleared to zero and the CPU is kicked into the - * scheduling path. Implies %SCX_ENQ_HEAD. - */ - SCX_ENQ_PREEMPT = 1LLU << 32, - - /* - * Only allowed on local DSQs. Guarantees that the task either gets - * on the CPU immediately and stays on it, or gets reenqueued back - * to the BPF scheduler. It will never linger on a local DSQ or be - * silently put back after preemption. - * - * The protection persists until the next fresh enqueue - it - * survives SAVE/RESTORE cycles, slice extensions and preemption. - * If the task can't stay on the CPU for any reason, it gets - * reenqueued back to the BPF scheduler. - * - * Exiting and migration-disabled tasks bypass ops.enqueue() and - * are placed directly on a local DSQ without IMMED protection - * unless %SCX_OPS_ENQ_EXITING and %SCX_OPS_ENQ_MIGRATION_DISABLED - * are set respectively. - */ - SCX_ENQ_IMMED = 1LLU << 33, - - /* - * The task being enqueued was previously enqueued on a DSQ, but was - * removed and is being re-enqueued. See SCX_TASK_REENQ_* flags to find - * out why a given task is being reenqueued. - */ - SCX_ENQ_REENQ = 1LLU << 40, - - /* - * The task being enqueued is the only task available for the cpu. By - * default, ext core keeps executing such tasks but when - * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with the - * %SCX_ENQ_LAST flag set. - * - * The BPF scheduler is responsible for triggering a follow-up - * scheduling event. Otherwise, Execution may stall. - */ - SCX_ENQ_LAST = 1LLU << 41, - - /* high 8 bits are internal */ - __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, - - SCX_ENQ_CLEAR_OPSS = 1LLU << 56, - SCX_ENQ_DSQ_PRIQ = 1LLU << 57, - SCX_ENQ_NESTED = 1LLU << 58, - SCX_ENQ_GDSQ_FALLBACK = 1LLU << 59, /* fell back to global DSQ */ -}; - -enum scx_deq_flags { - /* expose select DEQUEUE_* flags as enums */ - SCX_DEQ_SLEEP = DEQUEUE_SLEEP, - - /* high 32bits are SCX specific */ - - /* - * The generic core-sched layer decided to execute the task even though - * it hasn't been dispatched yet. Dequeue from the BPF side. - */ - SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, - - /* - * The task is being dequeued due to a property change (e.g., - * sched_setaffinity(), sched_setscheduler(), set_user_nice(), - * etc.). - */ - SCX_DEQ_SCHED_CHANGE = 1LLU << 33, -}; - -enum scx_reenq_flags { - /* low 16bits determine which tasks should be reenqueued */ - SCX_REENQ_ANY = 1LLU << 0, /* all tasks */ - - __SCX_REENQ_FILTER_MASK = 0xffffLLU, - - __SCX_REENQ_USER_MASK = SCX_REENQ_ANY, - - /* bits 32-35 used by task_should_reenq() */ - SCX_REENQ_TSR_RQ_OPEN = 1LLU << 32, - SCX_REENQ_TSR_NOT_FIRST = 1LLU << 33, - - __SCX_REENQ_TSR_MASK = 0xfLLU << 32, -}; - -enum scx_pick_idle_cpu_flags { - SCX_PICK_IDLE_CORE = 1LLU << 0, /* pick a CPU whose SMT siblings are also idle */ - SCX_PICK_IDLE_IN_NODE = 1LLU << 1, /* pick a CPU in the same target NUMA node */ -}; - -enum scx_kick_flags { - /* - * Kick the target CPU if idle. Guarantees that the target CPU goes - * through at least one full scheduling cycle before going idle. If the - * target CPU can be determined to be currently not idle and going to go - * through a scheduling cycle before going idle, noop. - */ - SCX_KICK_IDLE = 1LLU << 0, - - /* - * Preempt the current task and execute the dispatch path. If the - * current task of the target CPU is an SCX task, its ->scx.slice is - * cleared to zero before the scheduling path is invoked so that the - * task expires and the dispatch path is invoked. - */ - SCX_KICK_PREEMPT = 1LLU << 1, - - /* - * The scx_bpf_kick_cpu() call will return after the current SCX task of - * the target CPU switches out. This can be used to implement e.g. core - * scheduling. This has no effect if the current task on the target CPU - * is not on SCX. - */ - SCX_KICK_WAIT = 1LLU << 2, -}; - -enum scx_tg_flags { - SCX_TG_ONLINE = 1U << 0, - SCX_TG_INITED = 1U << 1, -}; - -enum scx_enable_state { - SCX_ENABLING, - SCX_ENABLED, - SCX_DISABLING, - SCX_DISABLED, -}; - -static const char *scx_enable_state_str[] = { - [SCX_ENABLING] = "enabling", - [SCX_ENABLED] = "enabled", - [SCX_DISABLING] = "disabling", - [SCX_DISABLED] = "disabled", -}; - -/* - * Task Ownership State Machine (sched_ext_entity->ops_state) - * - * The sched_ext core uses this state machine to track task ownership - * between the SCX core and the BPF scheduler. This allows the BPF - * scheduler to dispatch tasks without strict ordering requirements, while - * the SCX core safely rejects invalid dispatches. - * - * State Transitions - * - * .------------> NONE (owned by SCX core) - * | | ^ - * | enqueue | | direct dispatch - * | v | - * | QUEUEING -------' - * | | - * | enqueue | - * | completes | - * | v - * | QUEUED (owned by BPF scheduler) - * | | - * | dispatch | - * | | - * | v - * | DISPATCHING - * | | - * | dispatch | - * | completes | - * `---------------' - * - * State Descriptions - * - * - %SCX_OPSS_NONE: - * Task is owned by the SCX core. It's either on a run queue, running, - * or being manipulated by the core scheduler. The BPF scheduler has no - * claim on this task. - * - * - %SCX_OPSS_QUEUEING: - * Transitional state while transferring a task from the SCX core to - * the BPF scheduler. The task's rq lock is held during this state. - * Since QUEUEING is both entered and exited under the rq lock, dequeue - * can never observe this state (it would be a BUG). When finishing a - * dispatch, if the task is still in %SCX_OPSS_QUEUEING the completion - * path busy-waits for it to leave this state (via wait_ops_state()) - * before retrying. - * - * - %SCX_OPSS_QUEUED: - * Task is owned by the BPF scheduler. It's on a DSQ (dispatch queue) - * and the BPF scheduler is responsible for dispatching it. A QSEQ - * (queue sequence number) is embedded in this state to detect - * dispatch/dequeue races: if a task is dequeued and re-enqueued, the - * QSEQ changes and any in-flight dispatch operations targeting the old - * QSEQ are safely ignored. - * - * - %SCX_OPSS_DISPATCHING: - * Transitional state while transferring a task from the BPF scheduler - * back to the SCX core. This state indicates the BPF scheduler has - * selected the task for execution. When dequeue needs to take the task - * off a DSQ and it is still in %SCX_OPSS_DISPATCHING, the dequeue path - * busy-waits for it to leave this state (via wait_ops_state()) before - * proceeding. Exits to %SCX_OPSS_NONE when dispatch completes. - * - * Memory Ordering - * - * Transitions out of %SCX_OPSS_QUEUEING and %SCX_OPSS_DISPATCHING into - * %SCX_OPSS_NONE or %SCX_OPSS_QUEUED must use atomic_long_set_release() - * and waiters must use atomic_long_read_acquire(). This ensures proper - * synchronization between concurrent operations. - * - * Cross-CPU Task Migration - * - * When moving a task in the %SCX_OPSS_DISPATCHING state, we can't simply - * grab the target CPU's rq lock because a concurrent dequeue might be - * waiting on %SCX_OPSS_DISPATCHING while holding the source rq lock - * (deadlock). - * - * The sched_ext core uses a "lock dancing" protocol coordinated by - * p->scx.holding_cpu. When moving a task to a different rq: - * - * 1. Verify task can be moved (CPU affinity, migration_disabled, etc.) - * 2. Set p->scx.holding_cpu to the current CPU - * 3. Set task state to %SCX_OPSS_NONE; dequeue waits while DISPATCHING - * is set, so clearing DISPATCHING first prevents the circular wait - * (safe to lock the rq we need) - * 4. Unlock the current CPU's rq - * 5. Lock src_rq (where the task currently lives) - * 6. Verify p->scx.holding_cpu == current CPU, if not, dequeue won the - * race (dequeue clears holding_cpu to -1 when it takes the task), in - * this case migration is aborted - * 7. If src_rq == dst_rq: clear holding_cpu and enqueue directly - * into dst_rq's local DSQ (no lock swap needed) - * 8. Otherwise: call move_remote_task_to_local_dsq(), which releases - * src_rq, locks dst_rq, and performs the deactivate/activate - * migration cycle (dst_rq is held on return) - * 9. Unlock dst_rq and re-lock the current CPU's rq to restore - * the lock state expected by the caller - * - * If any verification fails, abort the migration. - * - * This state tracking allows the BPF scheduler to try to dispatch any task - * at any time regardless of its state. The SCX core can safely - * reject/ignore invalid dispatches, simplifying the BPF scheduler - * implementation. - */ -enum scx_ops_state { - SCX_OPSS_NONE, /* owned by the SCX core */ - SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ - SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ - SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ - - /* - * QSEQ brands each QUEUED instance so that, when dispatch races - * dequeue/requeue, the dispatcher can tell whether it still has a claim - * on the task being dispatched. - * - * As some 32bit archs can't do 64bit store_release/load_acquire, - * p->scx.ops_state is atomic_long_t which leaves 30 bits for QSEQ on - * 32bit machines. The dispatch race window QSEQ protects is very narrow - * and runs with IRQ disabled. 30 bits should be sufficient. - */ - SCX_OPSS_QSEQ_SHIFT = 2, -}; - -/* Use macros to ensure that the type is unsigned long for the masks */ -#define SCX_OPSS_STATE_MASK ((1LU << SCX_OPSS_QSEQ_SHIFT) - 1) -#define SCX_OPSS_QSEQ_MASK (~SCX_OPSS_STATE_MASK) - -extern struct scx_sched __rcu *scx_root; -DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); - -/* - * True when the currently loaded scheduler hierarchy is cid-form. All scheds - * in a hierarchy share one form, so this single key tells callsites which - * view to use without per-sch dereferences. Use scx_is_cid_type() to test. - */ -DECLARE_STATIC_KEY_FALSE(__scx_is_cid_type); - -int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id); - -bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where); - -__printf(5, 0) bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind, - s64 exit_code, s32 exit_cpu, const char *fmt, - va_list args); -__printf(5, 6) bool __scx_exit(struct scx_sched *sch, enum scx_exit_kind kind, - s64 exit_code, s32 exit_cpu, const char *fmt, ...); - -#define scx_exit(sch, kind, exit_code, fmt, args...) \ - __scx_exit(sch, kind, exit_code, raw_smp_processor_id(), fmt, ##args) -#define scx_error(sch, fmt, args...) \ - scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) -#define scx_verror(sch, fmt, args) \ - scx_vexit((sch), SCX_EXIT_ERROR, 0, raw_smp_processor_id(), fmt, args) - -/* - * Return the rq currently locked from an scx callback, or NULL if no rq is - * locked. - */ -static inline struct rq *scx_locked_rq(void) -{ - return __this_cpu_read(scx_locked_rq_state); -} - -static inline bool scx_bypassing(struct scx_sched *sch, s32 cpu) -{ - return unlikely(per_cpu_ptr(sch->pcpu, cpu)->flags & - SCX_SCHED_PCPU_BYPASSING); -} - -#ifdef CONFIG_EXT_SUB_SCHED -/** - * scx_task_sched - Find scx_sched scheduling a task - * @p: task of interest - * - * Return @p's scheduler instance. Must be called with @p's pi_lock or rq lock - * held. - */ -static inline struct scx_sched *scx_task_sched(const struct task_struct *p) -{ - return rcu_dereference_protected(p->scx.sched, - lockdep_is_held(&p->pi_lock) || - lockdep_is_held(__rq_lockp(task_rq(p)))); -} - -/** - * scx_task_sched_rcu - Find scx_sched scheduling a task - * @p: task of interest - * - * Return @p's scheduler instance. The returned scx_sched is RCU protected. - */ -static inline struct scx_sched *scx_task_sched_rcu(const struct task_struct *p) -{ - return rcu_dereference_all(p->scx.sched); -} - -/** - * scx_task_on_sched - Is a task on the specified sched? - * @sch: sched to test against - * @p: task of interest - * - * Returns %true if @p is on @sch, %false otherwise. - */ -static inline bool scx_task_on_sched(struct scx_sched *sch, - const struct task_struct *p) -{ - return rcu_access_pointer(p->scx.sched) == sch; -} - -/** - * scx_prog_sched - Find scx_sched associated with a BPF prog - * @aux: aux passed in from BPF to a kfunc - * - * To be called from kfuncs. Return the scheduler instance associated with the - * BPF program given the implicit kfunc argument aux. The returned scx_sched is - * RCU protected. - */ -static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) -{ - struct sched_ext_ops *ops; - struct scx_sched *root; - - ops = bpf_prog_get_assoc_struct_ops(aux); - if (likely(ops)) - return rcu_dereference_all(ops->priv); - - root = rcu_dereference_all(scx_root); - if (root) { - /* - * COMPAT-v6.19: Schedulers built before sub-sched support was - * introduced may have unassociated non-struct_ops programs. - */ - if (!root->ops.sub_attach) - return root; - - if (!root->warned_unassoc_progs) { - printk_deferred(KERN_WARNING "sched_ext: Unassociated program %s (id %d)\n", - aux->name, aux->id); - root->warned_unassoc_progs = true; - } - } - - return NULL; -} -#else /* CONFIG_EXT_SUB_SCHED */ -static inline struct scx_sched *scx_task_sched(const struct task_struct *p) -{ - return rcu_dereference_protected(scx_root, - lockdep_is_held(&p->pi_lock) || - lockdep_is_held(__rq_lockp(task_rq(p)))); -} - -static inline struct scx_sched *scx_task_sched_rcu(const struct task_struct *p) -{ - return rcu_dereference_all(scx_root); -} - -static inline bool scx_task_on_sched(struct scx_sched *sch, - const struct task_struct *p) -{ - return true; -} - -static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) -{ - return rcu_dereference_all(scx_root); -} -#endif /* CONFIG_EXT_SUB_SCHED */ diff --git a/kernel/sched/ext_types.h b/kernel/sched/ext_types.h deleted file mode 100644 index 8b3527e21fca..000000000000 --- a/kernel/sched/ext_types.h +++ /dev/null @@ -1,144 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Early sched_ext type definitions. - * - * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2026 Tejun Heo - */ -#ifndef _KERNEL_SCHED_EXT_TYPES_H -#define _KERNEL_SCHED_EXT_TYPES_H - -enum scx_consts { - SCX_DSP_DFL_MAX_BATCH = 32, - SCX_DSP_MAX_LOOPS = 32, - SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, - - /* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */ - SCX_TID_CHUNK = 1024, - - SCX_EXIT_BT_LEN = 64, - SCX_EXIT_MSG_LEN = 1024, - SCX_EXIT_DUMP_DFL_LEN = 32768, - - SCX_CPUPERF_ONE = SCHED_CAPACITY_SCALE, - - /* - * Iterating all tasks may take a while. Periodically drop - * scx_tasks_lock to avoid causing e.g. CSD and RCU stalls. - */ - SCX_TASK_ITER_BATCH = 32, - - SCX_BYPASS_HOST_NTH = 2, - - SCX_BYPASS_LB_DFL_INTV_US = 500 * USEC_PER_MSEC, - SCX_BYPASS_LB_DONOR_PCT = 125, - SCX_BYPASS_LB_MIN_DELTA_DIV = 4, - SCX_BYPASS_LB_BATCH = 256, - - SCX_REENQ_LOCAL_MAX_REPEAT = 256, - - SCX_SUB_MAX_DEPTH = 4, -}; - -/* - * Per-cid topology info. For each topology level (core, LLC, node), records - * the first cid in the unit and its global index. Global indices are - * consecutive integers assigned in cid-walk order, so e.g. core_idx ranges - * over [0, nr_cores_at_init) with no gaps. No-topo cids have all fields set - * to -1. - * - * @core_cid: first cid of this cid's core (smt-sibling group) - * @core_idx: global index of that core, in [0, nr_cores_at_init) - * @llc_cid: first cid of this cid's LLC - * @llc_idx: global index of that LLC, in [0, nr_llcs_at_init) - * @node_cid: first cid of this cid's NUMA node - * @node_idx: global index of that node, in [0, nr_nodes_at_init) - */ -struct scx_cid_topo { - s32 core_cid; - s32 core_idx; - s32 llc_cid; - s32 llc_idx; - s32 node_cid; - s32 node_idx; -}; - -/* - * cmask: variable-length, base-windowed bitmap over cid space - * ----------------------------------------------------------- - * - * A cmask covers the cid range [base, base + nr_cids). bits[] is aligned to the - * global 64-cid grid: bits[0] spans [base & ~63, (base & ~63) + 64), so the - * first (base & 63) bits of bits[0] are head padding and the trailing bits of - * the last active word past base + nr_cids are tail padding. Both stay zero; - * all mutating helpers preserve that. Words past the last active word are not - * read by any helper and have no constraint. - * - * Grid alignment means two cmasks always address bits[] against the same global - * 64-cid windows, so cross-cmask word ops (AND, OR, ...) reduce to - * - * dst->bits[i] OP= src->bits[i - delta] - * - * with no bit-shifting, regardless of how the two bases relate mod 64. - */ -struct scx_cmask { - u32 base; - u32 nr_cids; - u32 alloc_words; - u64 bits[] __counted_by(alloc_words); -}; - -/* - * Number of u64 words of bits[] storage that covers @nr_cids regardless of base - * alignment. The +1 absorbs up to 63 bits of head padding when base is not - * 64-aligned - always allocating one extra word beats branching on base or - * splitting the compute. The u64 cast keeps the +63 from wrapping when @nr_cids - * is near U32_MAX, so callers bounds-checking the result against @alloc_words - * catch the overflow instead of seeing a small value. - */ -#define SCX_CMASK_NR_WORDS(nr_cids) ((u32)(((u64)(nr_cids) + 63) / 64 + 1)) - -/** - * __SCX_CMASK_DEFINE - Define an on-stack cmask with explicit storage capacity - * @NAME: variable name to define - * @BASE: first cid of the active range - * @NR_CIDS: active range length - * @ALLOC_CIDS: storage capacity in cids, at least @NR_CIDS - * - * @NAME aliases zero-initialized storage with the active range set to - * [BASE, BASE + NR_CIDS). Use scx_cmask_reframe() to reshape later, up to - * @ALLOC_CIDS. - */ -#define __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, ALLOC_CIDS) \ - _DEFINE_FLEX(struct scx_cmask, NAME, bits, SCX_CMASK_NR_WORDS(ALLOC_CIDS), \ - = { .base = (BASE), \ - .nr_cids = (NR_CIDS), \ - .alloc_words = SCX_CMASK_NR_WORDS(ALLOC_CIDS) }) - -/** - * SCX_CMASK_DEFINE - Define an on-stack cmask on tight storage - * @NAME: variable name to define - * @BASE: first cid of the active range - * @NR_CIDS: active range length, also storage capacity - * - * @NAME aliases zero-initialized storage with the active range and storage - * both [BASE, BASE + NR_CIDS). - */ -#define SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS) \ - __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, NR_CIDS) - -/** - * SCX_CMASK_DEFINE_SHARD - Define an on-stack cmask sized to one shard - * @NAME: variable name to define - * @BASE: first cid of the active range - * @NR_CIDS: active range length, must be <= SCX_CID_SHARD_MAX_CPUS - * - * Storage is fixed at SCX_CID_SHARD_MAX_CPUS, active range framed by - * (BASE, NR_CIDS). Passing NR_CIDS > SCX_CID_SHARD_MAX_CPUS leaves the - * cmask claiming more bits than storage holds and subsequent cmask - * operations will overrun. - */ -#define SCX_CMASK_DEFINE_SHARD(NAME, BASE, NR_CIDS) \ - __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, SCX_CID_SHARD_MAX_CPUS) - -#endif /* _KERNEL_SCHED_EXT_TYPES_H */ diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index c7c2dea65edd..56acf502ba26 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -4211,6 +4211,6 @@ DEFINE_CLASS(sched_change, struct sched_change_ctx *, DEFINE_CLASS_IS_UNCONDITIONAL(sched_change) -#include "ext.h" +#include "ext/ext.h" #endif /* _KERNEL_SCHED_SCHED_H */ -- cgit v1.2.3