1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
|
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Shared definitions between scx_qmap.bpf.c and scx_qmap.c.
*
* The scheduler keeps all state in a single BPF arena map. struct
* qmap_arena is the one object that lives at the base of the arena and is
* mmap'd into userspace so the loader can read counters directly.
*
* Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
* Copyright (c) 2026 Tejun Heo <tj@kernel.org>
*/
#ifndef __SCX_QMAP_H
#define __SCX_QMAP_H
#ifdef __BPF__
#include <scx/bpf_arena_common.bpf.h>
#else
#include <linux/types.h>
#include <scx/bpf_arena_common.h>
#endif
#define MAX_SUB_SCHEDS 8
#define MAX_PARTS (MAX_SUB_SCHEDS + 1) /* participants: children + self */
/*
* cpu_ctxs[] is sized to a fixed cap so the layout is shared between BPF and
* userspace. Keep this in sync with NR_CPUS used by the BPF side.
*/
#define SCX_QMAP_MAX_CPUS 1024
/*
* An owner id identifies who holds a cid: a child slot in [0, MAX_SUB_SCHEDS),
* CID_SELF for this node, CID_NONE for a cid not currently held, or CID_SHARED
* for a cid in the round-robin pool (its live holder is rr_slots[rr_pos]). Used
* by the partition's cid_owner[].
*/
#define CID_SELF (-1)
#define CID_NONE (-2)
#define CID_SHARED (-3)
/* -C cid-override test modes. Selects cid_override_mode in scx_qmap.bpf.c. */
enum qmap_cid_override {
QMAP_CID_OVR_OFF = 0, /* disabled */
QMAP_CID_OVR_SHUFFLE = 1, /* valid reversed cpu->cid mapping */
QMAP_CID_OVR_BAD_DUP = 2, /* invalid: duplicate cid assignment */
QMAP_CID_OVR_BAD_RANGE = 3, /* invalid: out-of-range cid */
QMAP_CID_OVR_BAD_MONO = 4, /* invalid: non-monotonic shard_start */
};
struct cpu_ctx {
u64 dsp_idx; /* dispatch index */
u64 dsp_cnt; /* remaining count */
u32 avg_weight;
u32 cpuperf_target;
};
struct qmap_fifo {
struct task_ctx __arena *head;
struct task_ctx __arena *tail;
s32 idx;
};
/* -J fault-injection modes. Selects inject_mode in struct qmap_arena. */
enum qmap_inject {
QMAP_INJ_OFF = 0,
QMAP_INJ_WRONG_CID = 1, /* dispatch to a cid we don't hold */
QMAP_INJ_INIT_FAIL = 2, /* fail init_task for "qmfail*" comms */
QMAP_INJ_CGRP_INIT_FAIL = 3, /* fail cpuctl_init for "qmfail*" cgroups */
};
/*
* scx_cmask's are embedded in struct qmap_arena with inline backing storage.
* The bpf side uses &field.mask with the normal cmask_* helpers. Userspace
* doesn't have access to the type definition and sees same-sized opaque words.
* _Static_assert()'s in .bpf.c ensure that they are in sync.
*/
#define QMAP_CMASK_WORDS (((SCX_QMAP_MAX_CPUS) + 63) / 64 + 1)
struct qmap_cmask {
#ifdef __BPF__
union {
struct scx_cmask mask;
u64 words[QMAP_CMASK_WORDS + 2];
};
#else
u64 words[QMAP_CMASK_WORDS + 2];
#endif
};
/* Opaque to userspace; defined in scx_qmap.bpf.c. */
struct task_ctx;
/* per-direct-child state for the sub-scheduler */
struct sub_sched_ctx {
u64 cgroup_id;
u32 weight; /* cpu.weight, seeded at attach, then set_weight */
u64 nr_dsps;
struct qmap_cmask granted_cids; /* cids granted excl to this child */
struct qmap_cmask prev_granted; /* last grant, for delta calculation */
};
/*
* compute_partition() builds the following from this node's held caps, and
* apply_partition()/rr_advance() execute it. Userspace only reads for the
* hierarchy display.
*/
struct qmap_partition {
u32 nr_excl; /* number of excl-held (delegatable) cids */
s32 cid_owner[SCX_QMAP_MAX_CPUS]; /* per cid: owner id, or CID_NONE */
s32 shared_cids[MAX_PARTS]; /* the round-robin cid pool */
u32 nr_shared; /* number of shared_cids entries */
u64 rr_slots[MAX_PARTS]; /* rotation order: holder cgroup_id, 0 = self */
u32 nr_rr; /* number of rr_slots entries */
u32 rr_pos; /* current rotation index */
};
struct qmap_arena {
/* userspace-visible stats */
u64 nr_enqueued, nr_dispatched, nr_reenqueued, nr_reenqueued_cid0;
u64 nr_dequeued, nr_ddsp_from_enq;
u64 nr_core_sched_execed;
u64 nr_expedited_local, nr_expedited_remote;
u64 nr_expedited_lost, nr_expedited_from_timer;
u64 nr_highpri_queued;
u32 test_error_cnt;
u32 cpuperf_min, cpuperf_avg, cpuperf_max;
u32 cpuperf_target_min, cpuperf_target_avg, cpuperf_target_max;
/* kernel-side runtime state */
u64 core_sched_head_seqs[5];
u64 core_sched_tail_seqs[5];
struct cpu_ctx cpu_ctxs[SCX_QMAP_MAX_CPUS];
/* cid-override test input, populated by the loader before attach */
__s32 cid_override_cpu_to_cid[SCX_QMAP_MAX_CPUS];
__s32 cid_override_shard_start[SCX_QMAP_MAX_CPUS];
/* task_ctx slab; allocated and threaded by qmap_init() */
struct task_ctx __arena *task_ctxs;
struct task_ctx __arena *task_free_head;
/* five priority FIFOs, each a doubly-linked list through task_ctx */
struct qmap_fifo fifos[5];
/*
* Hierarchical sub-scheduling state. See the design comment at the top
* of scx_qmap.bpf.c.
*/
u32 nr_cids; /* cid count, cached at init */
/* bpf-owned partition: read by userspace for display */
struct qmap_partition part;
struct sub_sched_ctx sub_sched_ctxs[MAX_SUB_SCHEDS]; /* per-child context */
u64 nr_sub_scheds; /* number of attached children */
/* bpf-internal per-cid state */
u8 cid_shared[SCX_QMAP_MAX_CPUS]; /* per cid: 1 if held shared (ENQ_IMMED-only) */
/* allocated cid-time, charged per owner by account_alloc() */
u64 alloc_ns[MAX_SUB_SCHEDS]; /* per child slot */
u64 self_alloc_ns;
u64 alloc_ts; /* last accounting timestamp */
u64 alloc_window_ns; /* total accounted time, the alloc denominator */
/* bpf-internal cmasks (embedded, see struct qmap_cmask) */
struct qmap_cmask self_cids; /* cids this node runs its own tasks on */
struct qmap_cmask idle_cids; /* idle state of all cids regardless of delegation */
struct qmap_cmask rr_cids; /* the shared pool, as a mask for grant/revoke */
/* scratch cmasks */
struct qmap_cmask to_revoke_cids; /* delta cids to revoke */
struct qmap_cmask to_grant_cids; /* delta cids to grant */
struct qmap_cmask prev_rr_cids; /* previous shared pool, to clear stale grants */
struct qmap_cmask held_excl; /* cids held excl (ENQ): delegatable */
struct qmap_cmask held_shared; /* cids held shared (ENQ_IMMED only): self-local */
/* bpf -> userspace: stats */
u64 nr_reenq_cap; /* SCX_TASK_REENQ_CAP bounces */
u64 nr_reenq_immed; /* SCX_TASK_REENQ_IMMED bounces */
u64 nr_inject_attempts; /* fault-injection: dispatches to an unheld cid */
u64 nr_rescue_dsp; /* SCX_ENQ_RESCUE dispatch attempts */
u32 inject_mode; /* fault-injection mode (QMAP_INJ_*) */
};
#endif /* __SCX_QMAP_H */
|
