1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
|
/* SPDX-License-Identifier: GPL-2.0 */
/*
* BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
*
* Inline definitions layered on top of internal.h and cid.h.
*
* Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
* Copyright (c) 2026 Tejun Heo <tj@kernel.org>
*/
#ifndef _KERNEL_SCHED_EXT_INLINES_H
#define _KERNEL_SCHED_EXT_INLINES_H
#include "internal.h"
#include "cid.h"
/* what dispatch concluded, consumed by the pick that follows */
enum scx_dsp_verdict {
SCX_DSP_NONE, /* nothing to run */
SCX_DSP_LOCAL, /* local DSQ has tasks */
SCX_DSP_PREV, /* keep running @prev */
SCX_DSP_RETRY, /* pick helpers only: restart the pick */
};
/*
* One user of this function is scx_bpf_sub_dispatch() which can be called
* recursively as sub-sched dispatches nest. Always inline to reduce stack usage
* from the call frame.
*/
static __always_inline enum scx_dsp_verdict
scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
struct task_struct *prev, bool nested)
{
struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
int nr_loops = SCX_DSP_MAX_LOOPS;
s32 cpu = cpu_of(rq);
bool prev_on_sch = (prev->sched_class == &ext_sched_class) &&
scx_task_on_sched(sch, prev);
if (scx_consume_global_dsq(sch, rq))
return SCX_DSP_LOCAL;
if (scx_bypass_dsp_enabled(sch)) {
/* if @sch is bypassing, only the bypass DSQs are active */
if (scx_bypassing(sch, cpu)) {
if (scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0))
return SCX_DSP_LOCAL;
return SCX_DSP_NONE;
}
#ifdef CONFIG_EXT_SUB_SCHED
/*
* If @sch isn't bypassing but its children are, @sch is
* responsible for making forward progress for both its own
* tasks that aren't bypassing and the bypassing descendants'
* tasks. The following implements a simple built-in behavior -
* let each CPU try to run the bypass DSQ every Nth time.
*
* Later, if necessary, we can add an ops flag to suppress the
* auto-consumption and a kfunc to consume the bypass DSQ and,
* so that the BPF scheduler can fully control scheduling of
* bypassed tasks.
*/
struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) &&
scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) {
__scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1);
return SCX_DSP_LOCAL;
}
#endif /* CONFIG_EXT_SUB_SCHED */
}
if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq))
return SCX_DSP_NONE;
dspc->rq = rq;
/*
* The dispatch loop. Because scx_flush_dispatch_buf() may drop the rq
* lock, the local DSQ might still end up empty after a successful
* ops.dispatch(). If the local DSQ is empty even after ops.dispatch()
* produced some tasks, retry. The BPF scheduler may depend on this
* looping behavior to simplify its implementation.
*/
do {
dspc->nr_tasks = 0;
#ifdef CONFIG_EXT_SUB_SCHED
/* stash @prev so that nested invocations can access it */
if (!nested)
rq->scx.sub_dispatch_prev = prev;
#endif
SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu),
prev_on_sch ? prev : NULL);
#ifdef CONFIG_EXT_SUB_SCHED
if (!nested)
rq->scx.sub_dispatch_prev = NULL;
#endif
scx_flush_dispatch_buf(sch, rq);
if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice)
return SCX_DSP_PREV;
if (rq->scx.local_dsq.nr)
return SCX_DSP_LOCAL;
if (scx_consume_global_dsq(sch, rq))
return SCX_DSP_LOCAL;
/*
* ops.dispatch() can trap us in this loop by repeatedly
* dispatching ineligible tasks. Break out once in a while to
* allow the watchdog to run. As IRQ can't be enabled in
* dispatch, we want to complete this scheduling cycle and then
* start a new one. IOW, we want to call resched_curr() on the
* next, most likely idle, task, not the current one. Use
* __scx_bpf_kick_cpu() for deferred kicking.
*/
if (unlikely(!--nr_loops)) {
scx_kick_cpu(sch, cpu, 0);
break;
}
} while (dspc->nr_tasks);
/*
* Prevent the CPU from going idle while bypassed descendants have tasks
* queued. Without this fallback, bypassed tasks could stall if the host
* scheduler's ops.dispatch() doesn't yield any tasks.
*/
if (scx_bypass_dsp_enabled(sch) &&
scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0))
return SCX_DSP_LOCAL;
return SCX_DSP_NONE;
}
#endif /* _KERNEL_SCHED_EXT_INLINES_H */
|
